diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index bbe4b76775d..665f8456f0b 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -30,7 +30,7 @@ body: id: steps-to-reproduce attributes: label: Steps to Reproduce - description: Please provide detailed steps to reproduce this bug(A curl/python code to reproduce the bug) + description: Please provide a numbered list of the exact steps to reproduce this bug (include a curl/python snippet to reproduce it). Number each step (1., 2., 3., ...) in the order you performed them. placeholder: | 1. config.yaml file/ .env file/ etc. 2. Run the following code... diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index d7e80b32749..1301bfb0e60 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,3 +1,18 @@ +## TLDR + + + +Problem this solves: + +- +- ... + +How it solves it: + +- +- ... + ## Relevant issues diff --git a/.github/workflows/image-scan.yml b/.github/workflows/image-scan.yml index 90ede5a653f..8d791ca5bc7 100644 --- a/.github/workflows/image-scan.yml +++ b/.github/workflows/image-scan.yml @@ -58,6 +58,8 @@ jobs: # free OSS, run as a pinned, checksum-verified binary; no GitHub Action # dependency and no vendor SaaS callout. - name: Scan image for fixable HIGH/CRITICAL CVEs + env: + GRYPE_MATCH_PYTHON_USING_CPES: "true" run: | "$RUNNER_TEMP/grype" litellm-image-scan:${{ github.sha }} \ --only-fixed \ diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml index 9d28ca211cf..ae31395521a 100644 --- a/.github/workflows/test-code-quality.yml +++ b/.github/workflows/test-code-quality.yml @@ -115,6 +115,9 @@ jobs: - name: check_fastuuid_usage run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py + - name: check_e2e_no_raw_requests + run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py + - name: memory_test run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py diff --git a/.github/workflows/test-litellm-ui-unit.yml b/.github/workflows/test-litellm-ui-unit.yml new file mode 100644 index 00000000000..f0f9f504752 --- /dev/null +++ b/.github/workflows/test-litellm-ui-unit.yml @@ -0,0 +1,57 @@ +name: UI Unit Tests +permissions: + contents: read + +on: + pull_request: + branches: + - main + - litellm_internal_staging + - litellm_oss_staging + - "litellm_**" + push: + branches: + - litellm_internal_staging + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + ui-unit-tests: + runs-on: ubuntu-latest + timeout-minutes: 20 + defaults: + run: + working-directory: ui/litellm-dashboard + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + with: + node-version: "20" + cache: "npm" + cache-dependency-path: ui/litellm-dashboard/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Run UI unit tests (Vitest) + env: + CI: "true" + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + if [ -n "$BASE_SHA" ]; then + echo "Pull request: running only tests related to changes since $BASE_SHA" + npm run test -- --run --changed "$BASE_SHA" --passWithNoTests \ + --pool forks --poolOptions.forks.maxForks=4 + else + echo "Push to $GITHUB_REF_NAME: running the full suite" + npm run test -- --run --pool forks --poolOptions.forks.maxForks=4 + fi diff --git a/.github/workflows/weekly_load_anomaly.yml b/.github/workflows/weekly_load_anomaly.yml new file mode 100644 index 00000000000..4c2103f026d --- /dev/null +++ b/.github/workflows/weekly_load_anomaly.yml @@ -0,0 +1,81 @@ +name: "Weekly Load Anomaly Check" + +on: + schedule: + - cron: "0 12 * * 6" + workflow_dispatch: + +permissions: + contents: read + +jobs: + weekly-load-anomaly: + if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + timeout-minutes: 45 + services: + postgres: + image: postgres:16.6 + env: + POSTGRES_USER: llmproxy + POSTGRES_PASSWORD: dbpassword9090 + POSTGRES_DB: litellm + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U llmproxy" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm + LITELLM_MASTER_KEY: sk-weekly-anomaly-check + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Install dependencies + run: | + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + + - name: Start the proxy + run: | + nohup uv run --no-sync litellm --config tests/e2e/load/weekly_anomaly_config.yml --port 4000 > proxy.log 2>&1 & + for _ in $(seq 1 90); do + if curl -fs http://localhost:4000/health/liveliness > /dev/null; then + exit 0 + fi + sleep 2 + done + echo "proxy never became live" + tail -n 100 proxy.log + exit 1 + + - name: Run the weekly session anomaly test + env: + E2E_WEEKLY_ANOMALY: "1" + run: | + uv run --no-sync pytest tests/e2e/load/test_weekly_session_anomaly_e2e.py -v --tb=short -rA + + - name: Show proxy log on failure + if: failure() + run: tail -n 300 proxy.log diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260721000000_add_sso_identity_assertion/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260721000000_add_sso_identity_assertion/migration.sql new file mode 100644 index 00000000000..95412df0a96 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260721000000_add_sso_identity_assertion/migration.sql @@ -0,0 +1,9 @@ +-- CreateTable +CREATE TABLE IF NOT EXISTS "LiteLLM_SSOIdentityAssertion" ( + "user_id" TEXT NOT NULL, + "assertion_b64" TEXT NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "LiteLLM_SSOIdentityAssertion_pkey" PRIMARY KEY ("user_id") +); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index b27ddea010b..23a9c086c73 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -403,6 +403,15 @@ model LiteLLM_MCPServerOAuthClient { updated_at DateTime @default(now()) @updatedAt @map("updated_at") } +// The enterprise IdP identity assertion captured at SSO login, one row per user. +// assertion_b64 is an encrypted JSON payload: {id_token, refresh_token?, issuer?, expires_at?}. +model LiteLLM_SSOIdentityAssertion { + user_id String @id + assertion_b64 String + created_at DateTime @default(now()) @map("created_at") + updated_at DateTime @default(now()) @updatedAt @map("updated_at") +} + // Generate Tokens for Proxy model LiteLLM_VerificationToken { token String @id diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 3288f7fd584..ccca88c9996 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.79" +version = "0.4.80" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.79" +version = "0.4.80" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm-rust/crates/ai-gateway/src/messages/common_utils.rs b/litellm-rust/crates/ai-gateway/src/messages/common_utils.rs index 4b906155665..68ecc3f17c1 100644 --- a/litellm-rust/crates/ai-gateway/src/messages/common_utils.rs +++ b/litellm-rust/crates/ai-gateway/src/messages/common_utils.rs @@ -50,3 +50,15 @@ pub(super) fn has_header(headers: &[(String, String)], name: &str) -> bool { .iter() .any(|(key, _)| key.eq_ignore_ascii_case(name)) } + +pub(super) fn has_bearer_auth(headers: &[(String, String)]) -> bool { + headers.iter().any(|(name, value)| { + if !name.eq_ignore_ascii_case("authorization") { + return false; + } + let value = value.trim(); + value.len() > 7 + && value[..7].eq_ignore_ascii_case("bearer ") + && !value[7..].trim().is_empty() + }) +} diff --git a/litellm-rust/crates/ai-gateway/src/messages/prepare.rs b/litellm-rust/crates/ai-gateway/src/messages/prepare.rs index 624c3598fb0..9a027490eb6 100644 --- a/litellm-rust/crates/ai-gateway/src/messages/prepare.rs +++ b/litellm-rust/crates/ai-gateway/src/messages/prepare.rs @@ -3,7 +3,7 @@ use litellm_core::CoreResult; use litellm_core::messages::transformation::MessagesAuthStrategy; use litellm_core::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; -use super::common_utils::{has_header, messages_provider_config, string_headers}; +use super::common_utils::{has_bearer_auth, has_header, messages_provider_config, string_headers}; use super::types::{MessagesRequest, ProviderMessagesRequest}; pub(super) fn prepare_messages_call( @@ -33,7 +33,9 @@ pub(super) fn prepare_messages_call( let mut headers = string_headers(request.extra_headers)?; let auth_strategy = config.auth_strategy(); - if !has_header(&headers, auth_strategy.header_name()) { + let already_authorized = has_header(&headers, auth_strategy.header_name()) + || (config.accepts_bearer_auth() && has_bearer_auth(&headers)); + if !already_authorized { let api_key = config.resolve_api_key(request.api_key, &env_lookup)?; let auth_header = match auth_strategy { MessagesAuthStrategy::Bearer => { diff --git a/litellm-rust/crates/ai-gateway/src/messages/tests.rs b/litellm-rust/crates/ai-gateway/src/messages/tests.rs index a2d0f6fae23..23a53e98045 100644 --- a/litellm-rust/crates/ai-gateway/src/messages/tests.rs +++ b/litellm-rust/crates/ai-gateway/src/messages/tests.rs @@ -6,7 +6,7 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; use super::common_utils::{ - has_header, messages_provider_config, string_headers, truncate_error_body, + has_bearer_auth, has_header, messages_provider_config, string_headers, truncate_error_body, }; use super::{MessagesRequest, messages}; @@ -85,6 +85,34 @@ fn has_header_is_case_insensitive() { assert!(!has_header(&headers, "authorization")); } +#[test] +fn has_bearer_auth_requires_a_nonempty_bearer_token() { + assert!(has_bearer_auth(&[( + "Authorization".to_string(), + "Bearer tok".to_string() + )])); + assert!(has_bearer_auth(&[( + "authorization".to_string(), + "bearer tok".to_string() + )])); + assert!(!has_bearer_auth(&[( + "authorization".to_string(), + "Bearer ".to_string() + )])); + assert!(!has_bearer_auth(&[( + "authorization".to_string(), + String::new() + )])); + assert!(!has_bearer_auth(&[( + "authorization".to_string(), + "Basic abc".to_string() + )])); + assert!(!has_bearer_auth(&[( + "x-api-key".to_string(), + "sk".to_string() + )])); +} + #[tokio::test] async fn messages_round_trip_builds_azure_request_and_passes_response_through() { let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); @@ -252,6 +280,112 @@ async fn messages_does_not_duplicate_auth_when_x_api_key_supplied() { assert!(!head.contains("rust-fallback-key"), "{head}"); } +#[tokio::test] +async fn messages_forwards_entra_id_bearer_without_requiring_api_key() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); + let addr = listener.local_addr().expect("addr"); + + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accepts request"); + let request = read_http_request(&mut socket).await; + let response_body = + r#"{"id":"msg_3","type":"message","role":"assistant","content":[],"model":"m"}"#; + socket + .write_all(write_response(response_body).as_bytes()) + .await + .expect("writes response"); + request + }); + + let mut headers = Map::new(); + headers.insert( + "Authorization".to_string(), + Value::String("Bearer entra-token".to_string()), + ); + + messages(MessagesRequest { + model: "claude-sonnet-4-5", + body: json!({"model": "claude-sonnet-4-5", "max_tokens": 8, "messages": []}), + api_key: None, + api_base: Some(&format!("http://{addr}")), + custom_llm_provider: Some("azure_ai"), + extra_headers: Some(headers), + timeout: Some(Duration::from_secs(5)), + }) + .await + .expect("entra id request succeeds without api key"); + + let request = server.await.expect("server task completes"); + let head = request + .split_once("\r\n\r\n") + .expect("has body") + .0 + .to_ascii_lowercase(); + assert!(head.contains("authorization: bearer entra-token"), "{head}"); + assert!(!head.contains("x-api-key"), "{head}"); +} + +#[tokio::test] +async fn messages_requires_auth_when_no_key_and_no_header() { + let err = messages(MessagesRequest { + model: "claude-sonnet-4-5", + body: json!({"model": "claude-sonnet-4-5", "max_tokens": 8, "messages": []}), + api_key: None, + api_base: Some("http://127.0.0.1:1"), + custom_llm_provider: Some("azure_ai"), + extra_headers: None, + timeout: Some(Duration::from_millis(50)), + }) + .await + .expect_err("missing auth errors"); + + assert!(matches!(err, CoreError::Auth(_))); +} + +#[tokio::test] +async fn messages_ignores_malformed_authorization_and_uses_api_key() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); + let addr = listener.local_addr().expect("addr"); + + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accepts request"); + let request = read_http_request(&mut socket).await; + let response_body = + r#"{"id":"msg_4","type":"message","role":"assistant","content":[],"model":"m"}"#; + socket + .write_all(write_response(response_body).as_bytes()) + .await + .expect("writes response"); + request + }); + + let mut headers = Map::new(); + headers.insert( + "Authorization".to_string(), + Value::String("Bearer ".to_string()), + ); + + messages(MessagesRequest { + model: "claude-sonnet-4-5", + body: json!({"model": "claude-sonnet-4-5", "max_tokens": 8, "messages": []}), + api_key: Some("sk-azure"), + api_base: Some(&format!("http://{addr}")), + custom_llm_provider: Some("azure_ai"), + extra_headers: Some(headers), + timeout: Some(Duration::from_secs(5)), + }) + .await + .expect("falls back to api key"); + + let request = server.await.expect("server task completes"); + let head = request + .split_once("\r\n\r\n") + .expect("has body") + .0 + .to_ascii_lowercase(); + assert!(head.contains("x-api-key: sk-azure"), "{head}"); +} + #[tokio::test] async fn messages_maps_provider_error_status_to_http_error() { let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); diff --git a/litellm-rust/crates/core/src/messages/transformation.rs b/litellm-rust/crates/core/src/messages/transformation.rs index 3a34a58de6f..b478e20d24b 100644 --- a/litellm-rust/crates/core/src/messages/transformation.rs +++ b/litellm-rust/crates/core/src/messages/transformation.rs @@ -35,6 +35,10 @@ pub trait AnthropicMessagesProviderConfig: Sync { MessagesAuthStrategy::Header("x-api-key") } + fn accepts_bearer_auth(&self) -> bool { + false + } + fn default_headers(&self) -> &'static [(&'static str, &'static str)] { &[ ("anthropic-version", "2023-06-01"), diff --git a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs index 6935bb4604b..7b958c77ba3 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs @@ -163,6 +163,10 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { self.anthropic.auth_strategy() } + fn accepts_bearer_auth(&self) -> bool { + true + } + fn default_headers(&self) -> &'static [(&'static str, &'static str)] { self.anthropic.default_headers() } @@ -294,6 +298,11 @@ mod tests { ); } + #[test] + fn accepts_bearer_auth_for_entra_id() { + assert!(AZURE_ANTHROPIC_MESSAGES_CONFIG.accepts_bearer_auth()); + } + #[test] fn default_headers_match_python() { assert_eq!( diff --git a/litellm/constants.py b/litellm/constants.py index 05944c81ea2..2af84c139a1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1469,6 +1469,7 @@ _batch_polling_env = os.getenv("PROXY_BATCH_POLLING_ENABLED", "true").lower() PROXY_BATCH_POLLING_ENABLED = _batch_polling_env == "true" PROXY_BUDGET_RESCHEDULER_MAX_TIME = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605)) PROXY_BATCH_WRITE_AT = int(os.getenv("PROXY_BATCH_WRITE_AT", 10)) # in seconds, increased from 10 +PROXY_CONFIG_RELOAD_INTERVAL_SECONDS = get_env_int("PROXY_CONFIG_RELOAD_INTERVAL_SECONDS", 30) # APScheduler Configuration - MEMORY LEAK FIX # These settings prevent memory leaks in APScheduler's normalize() and _apply_jitter() functions diff --git a/litellm/litellm_core_utils/duration_parser.py b/litellm/litellm_core_utils/duration_parser.py index 438ff5600ba..79036367652 100644 --- a/litellm/litellm_core_utils/duration_parser.py +++ b/litellm/litellm_core_utils/duration_parser.py @@ -7,8 +7,8 @@ duration_in_seconds is used in diff parts of the code base, example """ import re -import time -from datetime import datetime, timedelta, timezone, tzinfo +import time as time_module +from datetime import datetime, time, timedelta, timezone, tzinfo from typing import Optional, Tuple from zoneinfo import ZoneInfo @@ -61,7 +61,7 @@ def duration_in_seconds(duration: str) -> int: elif unit == "w": return value * 604800 elif unit == "mo": - now = time.time() + now = time_module.time() current_time = datetime.fromtimestamp(now) # Calculate target month and year, handling overflow past December @@ -94,12 +94,17 @@ def duration_in_seconds(duration: str) -> int: raise ValueError(f"Unsupported duration unit, passed duration: {duration}") -def get_next_standardized_reset_time(duration: str, current_time: datetime, timezone_str: str = "UTC") -> datetime: +def get_next_standardized_reset_time( + duration: str, + current_time: datetime, + timezone_str: str = "UTC", + reset_time_of_day: time = time(0, 0), +) -> datetime: """ Get the next standardized reset time based on the duration. All durations will reset at predictable intervals, aligned from the current time: - - Nd: If N=1, reset at next midnight; if N>1, reset every N days from now + - Nd: If N=1, reset at the next `reset_time_of_day`; if N>1, reset every N days from now - Nh: Every N hours, aligned to hour boundaries (e.g., 1:00, 2:00) - Nm: Every N minutes, aligned to minute boundaries (e.g., 1:05, 1:10) - Ns: Every N seconds, aligned to second boundaries @@ -108,12 +113,15 @@ def get_next_standardized_reset_time(duration: str, current_time: datetime, time - duration: Duration string (e.g. "30s", "30m", "30h", "30d") - current_time: Current datetime - timezone_str: Timezone string (e.g. "UTC", "US/Eastern", "Asia/Kolkata") + - reset_time_of_day: Wall-clock time the reset lands on for day/week/month + durations (defaults to midnight). Ignored for sub-day durations, where a + time-of-day is meaningless. Returns: - Next reset time at a standardized interval in the specified timezone """ # Set up timezone and normalize current time - current_time, tz = _setup_timezone(current_time, timezone_str) + current_time, _ = _setup_timezone(current_time, timezone_str) # Parse duration value, unit = _parse_duration(duration) @@ -126,9 +134,9 @@ def get_next_standardized_reset_time(duration: str, current_time: datetime, time # Handle different time units if unit == "d": - return _handle_day_reset(current_time, base_midnight, value, tz) + return _handle_day_reset(current_time, base_midnight, value, reset_time_of_day) elif unit == "w": - return _handle_day_reset(current_time, base_midnight, value * 7, tz) + return _handle_day_reset(current_time, base_midnight, value * 7, reset_time_of_day) elif unit == "h": return _handle_hour_reset(current_time, base_midnight, value) elif unit == "m": @@ -136,7 +144,7 @@ def get_next_standardized_reset_time(duration: str, current_time: datetime, time elif unit == "s": return _handle_second_reset(current_time, base_midnight, value) elif unit == "mo": - return _handle_month_reset(current_time, base_midnight, value) + return _handle_month_reset(current_time, base_midnight, value, reset_time_of_day) else: # Unrecognized unit, default to next midnight return base_midnight + timedelta(days=1) @@ -175,46 +183,58 @@ def _parse_duration(duration: str) -> Tuple[Optional[int], Optional[str]]: return int(value), unit -def _handle_day_reset(current_time: datetime, base_midnight: datetime, value: int, tz: tzinfo) -> datetime: +def _apply_time_of_day(dt: datetime, reset_time_of_day: time) -> datetime: + """Set the wall-clock time of `dt` to `reset_time_of_day`, keeping its date and tzinfo.""" + return dt.replace( + hour=reset_time_of_day.hour, + minute=reset_time_of_day.minute, + second=reset_time_of_day.second, + microsecond=reset_time_of_day.microsecond, + ) + + +def _next_occurrence( + boundary_midnight: datetime, + reset_time_of_day: time, + current_time: datetime, + period: timedelta, +) -> datetime: + """Place the reset at `reset_time_of_day` on the boundary day, rolling forward one + `period` if that instant has already passed (or is exactly now).""" + candidate = _apply_time_of_day(boundary_midnight, reset_time_of_day) + if candidate <= current_time: + return candidate + period + return candidate + + +def _first_of_next_month(first_of_month: datetime) -> datetime: + """Given the 1st of some month, return the 1st of the following month.""" + if first_of_month.month == 12: + return first_of_month.replace(year=first_of_month.year + 1, month=1) + return first_of_month.replace(month=first_of_month.month + 1) + + +def _handle_day_reset( + current_time: datetime, + base_midnight: datetime, + value: int, + reset_time_of_day: time, +) -> datetime: """Handle day-based reset times.""" # Handle zero value - immediate expiration if value == 0: return current_time - if value == 1: # Daily reset at midnight - return base_midnight + timedelta(days=1) - elif value == 7: # Weekly reset on Monday at midnight + if value == 1: # Daily reset at the configured time of day + return _next_occurrence(base_midnight, reset_time_of_day, current_time, timedelta(days=1)) + elif value == 7: # Weekly reset on Monday at the configured time of day days_until_monday = (7 - current_time.weekday()) % 7 - if days_until_monday == 0: # If today is Monday - days_until_monday = 7 - return base_midnight + timedelta(days=days_until_monday) - elif value == 30: # Monthly reset on 1st at midnight - # Get 1st of next month at midnight - if current_time.month == 12: - next_reset = datetime( - year=current_time.year + 1, - month=1, - day=1, - hour=0, - minute=0, - second=0, - microsecond=0, - tzinfo=tz, - ) - else: - next_reset = datetime( - year=current_time.year, - month=current_time.month + 1, - day=1, - hour=0, - minute=0, - second=0, - microsecond=0, - tzinfo=tz, - ) - return next_reset - else: # Custom day value - next interval is value days from current - return current_time.replace(hour=0, minute=0, second=0, microsecond=0) + timedelta(days=value) + upcoming_monday = base_midnight + timedelta(days=days_until_monday) + return _next_occurrence(upcoming_monday, reset_time_of_day, current_time, timedelta(days=7)) + elif value == 30: # Monthly reset on 1st at the configured time of day + return _handle_month_reset(current_time, base_midnight, 1, reset_time_of_day) + else: # Custom day value - next interval is value days from the start of today + return _apply_time_of_day(base_midnight + timedelta(days=value), reset_time_of_day) def _handle_hour_reset(current_time: datetime, base_midnight: datetime, value: int) -> datetime: @@ -316,36 +336,30 @@ def _handle_second_reset(current_time: datetime, base_midnight: datetime, value: return current_time.replace(hour=next_hour, minute=next_minute, second=next_second, microsecond=0) -def _handle_month_reset(current_time: datetime, base_midnight: datetime, value: int) -> datetime: +def _handle_month_reset( + current_time: datetime, + base_midnight: datetime, + value: int, + reset_time_of_day: time, +) -> datetime: """ - Handle monthly reset times. For monthly resets, we always reset at the start of the next month. + Handle monthly reset times. Resets land on the 1st at `reset_time_of_day`; if the + 1st of the current month at that time has already passed, roll to the 1st of next month. Args: current_time: Current datetime base_midnight: Midnight of current day value: Number of months (currently only supports 1 month resets) + reset_time_of_day: Wall-clock time the reset lands on Returns: - datetime: First day of next month at midnight + datetime: First day of the next reset month at `reset_time_of_day` """ if value != 1: raise ValueError("Monthly resets currently only support 1 month intervals") - # Get the first day of next month - if current_time.month == 12: - next_month = 1 - next_year = current_time.year + 1 - else: - next_month = current_time.month + 1 - next_year = current_time.year - - return datetime( - year=next_year, - month=next_month, - day=1, - hour=0, - minute=0, - second=0, - microsecond=0, - tzinfo=current_time.tzinfo, - ) + first_of_this_month = base_midnight.replace(day=1) + candidate = _apply_time_of_day(first_of_this_month, reset_time_of_day) + if candidate <= current_time: + return _apply_time_of_day(_first_of_next_month(first_of_this_month), reset_time_of_day) + return candidate diff --git a/litellm/litellm_core_utils/url_utils.py b/litellm/litellm_core_utils/url_utils.py index 1cbb1ce973f..a83cb3bc69e 100644 --- a/litellm/litellm_core_utils/url_utils.py +++ b/litellm/litellm_core_utils/url_utils.py @@ -148,6 +148,15 @@ def _parse_url_destination_allowlist_entry( return _normalize_host(parsed.hostname), scheme, port +def provider_url_destination_candidates(value: str) -> Tuple[str, ...]: + return tuple( + candidate + for part in value.split(",") + for candidate in (part.strip(), part.strip().split("/", 1)[1] if "/" in part.strip() else "") + if candidate + ) + + def is_url_destination_allowed_by_host(url: str, allowed_hosts: List[str]) -> bool: """Return True when a credential-bearing provider URL is admin-allowlisted. diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index b48c37791c4..b7237d288ec 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -9,6 +9,8 @@ import contextlib import json from typing import Any, Optional +from pydantic import TypeAdapter + from litellm._logging import _redact_string, verbose_proxy_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -16,6 +18,8 @@ from ..base_aws_llm import BaseAWSLLM from ..common_utils import BedrockError from .transformation import BedrockRealtimeConfig +_CLIENT_MODALITIES_ADAPTER: TypeAdapter["list[str] | None"] = TypeAdapter(list[str] | None) + class BedrockRealtime(BaseAWSLLM): """Handler for Bedrock Nova Sonic realtime speech-to-speech API.""" @@ -124,6 +128,9 @@ class BedrockRealtime(BaseAWSLLM): verbose_proxy_logger.debug("Bedrock Realtime: Bidirectional stream established") + await websocket.send_text(json.dumps(transformation_config.session_created_event(model, logging_obj))) + verbose_proxy_logger.debug("Bedrock Realtime: sent session.created to client on connect") + # Track state for transformation session_state = { "current_output_item_id": None, @@ -143,6 +150,7 @@ class BedrockRealtime(BaseAWSLLM): transformation_config, model, session_state, + logging_obj, ) ) @@ -179,6 +187,7 @@ class BedrockRealtime(BaseAWSLLM): transformation_config: BedrockRealtimeConfig, model: str, session_state: dict, + logging_obj: LiteLLMLogging | None = None, ): """Forward messages from client WebSocket to Bedrock stream.""" from aws_sdk_bedrock_runtime.models import ( @@ -210,6 +219,23 @@ class BedrockRealtime(BaseAWSLLM): for bedrock_message in transformed_messages: await send_to_bedrock(bedrock_message) + if logging_obj is not None: + client_message_type: str | None = None + requested_modalities: list[str] | None = None + with contextlib.suppress(Exception): + parsed_client_message = json.loads(message) + client_message_type = parsed_client_message.get("type") + if client_message_type == "session.update": + requested_modalities = _CLIENT_MODALITIES_ADAPTER.validate_python( + parsed_client_message.get("session", {}).get("modalities") + ) + if client_message_type == "session.update": + await client_ws.send_text( + json.dumps( + transformation_config.session_updated_event(model, logging_obj, requested_modalities) + ) + ) + except Exception as e: verbose_proxy_logger.debug(f"Client to Bedrock forwarding ended: {e}", exc_info=True) for close_message in transformation_config.session_close_messages(): diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py index fe5f0584e03..24a40ebea1b 100644 --- a/litellm/llms/bedrock/realtime/transformation.py +++ b/litellm/llms/bedrock/realtime/transformation.py @@ -623,35 +623,42 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): verbose_logger.warning(f"Unknown message type: {message_type}") return [] - def transform_session_start_event( + def _session_object( self, - event: dict, model: str, logging_obj: LiteLLMLoggingObj, - ) -> OpenAIRealtimeStreamSessionEvents: - """ - Transform Bedrock sessionStart event to OpenAI session.created. - - Args: - event: Bedrock sessionStart event - model: Model ID - logging_obj: Logging object - - Returns: - OpenAI session.created event - """ - verbose_logger.debug("Handling sessionStart") - + modalities: list[str] | None = None, + ) -> OpenAIRealtimeStreamSession: session = OpenAIRealtimeStreamSession( id=logging_obj.litellm_trace_id, - modalities=["text", "audio"], + modalities=modalities if modalities is not None else ["text", "audio"], ) if model is not None and isinstance(model, str): session["model"] = model + return session + def session_created_event( + self, + model: str, + logging_obj: LiteLLMLoggingObj, + ) -> OpenAIRealtimeStreamSessionEvents: + """Build the OpenAI session.created event for this realtime session.""" return OpenAIRealtimeStreamSessionEvents( type="session.created", - session=session, + session=self._session_object(model, logging_obj), + event_id=str(uuid.uuid4()), + ) + + def session_updated_event( + self, + model: str, + logging_obj: LiteLLMLoggingObj, + modalities: list[str] | None = None, + ) -> OpenAIRealtimeStreamSessionEvents: + """Build the OpenAI session.updated ack reflecting the client's requested modalities.""" + return OpenAIRealtimeStreamSessionEvents( + type="session.updated", + session=self._session_object(model, logging_obj, modalities), event_id=str(uuid.uuid4()), ) @@ -1169,8 +1176,6 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): # Route to appropriate transformation method if "sessionStart" in event: - session_created = self.transform_session_start_event(event, model, logging_obj) - returned_messages.append(session_created) session_configuration_request = json.dumps({"configured": True}) elif "contentStart" in event: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index c48d75439a7..ec1301e5923 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2107,8 +2107,7 @@ class BaseLLMHTTPHandler: rust_messages_response = await self._maybe_rust_anthropic_messages( custom_llm_provider=custom_llm_provider, litellm_params=litellm_params, - stream=stream or False, - rust_stream_eligible=bool(stream) and not self._has_agentic_completion_hook(logging_obj), + has_agentic_hook=self._has_agentic_completion_hook(logging_obj), model=model, api_key=api_key, api_base=api_base, @@ -2266,8 +2265,7 @@ class BaseLLMHTTPHandler: *, custom_llm_provider: str, litellm_params: GenericLiteLLMParams, - stream: bool, - rust_stream_eligible: bool, + has_agentic_hook: bool, model: str, api_key: str | None, api_base: str | None, @@ -2279,7 +2277,7 @@ class BaseLLMHTTPHandler: return None if litellm_params.get("rust") is not True and not BaseLLMHTTPHandler._rust_env_enabled(): return None - if stream and not rust_stream_eligible: + if has_agentic_hook: return None from litellm.rust_bridge import messages as rust_messages_bridge diff --git a/litellm/llms/huggingface/embedding/handler.py b/litellm/llms/huggingface/embedding/handler.py index 39eb430db74..f72a79e084d 100644 --- a/litellm/llms/huggingface/embedding/handler.py +++ b/litellm/llms/huggingface/embedding/handler.py @@ -322,7 +322,7 @@ class HuggingFaceEmbedding(BaseLLM): task = get_hf_task_embedding_for_model(model=model, task_type=task_type, api_base=HF_HUB_URL) # print_verbose(f"{model}, {task}") embed_url = "" - if "https" in model: + if model.startswith(("http://", "https://")): embed_url = model elif api_base: embed_url = api_base diff --git a/litellm/llms/huggingface/embedding/transformation.py b/litellm/llms/huggingface/embedding/transformation.py index 13e38ab5560..6f27e3115eb 100644 --- a/litellm/llms/huggingface/embedding/transformation.py +++ b/litellm/llms/huggingface/embedding/transformation.py @@ -316,25 +316,6 @@ class HuggingFaceEmbeddingConfig(BaseConfig): return data - def get_api_base(self, api_base: Optional[str], model: str) -> str: - """ - Get the API base for the Huggingface API. - - Do not add the chat/embedding/rerank extension here. Let the handler do this. - """ - if "https" in model: - completion_url = model - elif api_base is not None: - completion_url = api_base - elif "HF_API_BASE" in os.environ: - completion_url = os.getenv("HF_API_BASE", "") - elif "HUGGINGFACE_API_BASE" in os.environ: - completion_url = os.getenv("HUGGINGFACE_API_BASE", "") - else: - completion_url = f"https://api-inference.huggingface.co/models/{model}" - - return completion_url - def validate_environment( self, headers: Dict, diff --git a/litellm/llms/oobabooga/chat/oobabooga.py b/litellm/llms/oobabooga/chat/oobabooga.py index fe2bb9dc6d1..40d88e8e125 100644 --- a/litellm/llms/oobabooga/chat/oobabooga.py +++ b/litellm/llms/oobabooga/chat/oobabooga.py @@ -34,7 +34,7 @@ def completion( optional_params=optional_params, litellm_params=litellm_params, ) - if "https" in model: + if model.startswith(("http://", "https://")): completion_url = model elif api_base: completion_url = api_base @@ -96,7 +96,7 @@ def embedding( encoding=None, ): # Create completion URL - if "https" in model: + if model.startswith(("http://", "https://")): embeddings_url = model elif api_base: embeddings_url = f"{api_base}/v1/embeddings" diff --git a/litellm/main.py b/litellm/main.py index 9fdee57c48b..b630263b5d9 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5111,7 +5111,10 @@ def completion( # type: ignore try: if base_url is not None: api_base = base_url - if num_retries is not None: + is_router_call = any("model_group" in (kwargs.get(k) or ()) for k in ("metadata", "litellm_metadata")) + if is_router_call: + max_retries = 0 + elif num_retries is not None: max_retries = num_retries logging: LiteLLMLoggingObj = cast(LiteLLMLoggingObj, litellm_logging_obj) fallbacks = fallbacks or litellm.model_fallbacks diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index bb6243e50ed..d3917886060 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -17566,6 +17566,61 @@ }, "web_search_billing_unit": "per_query" }, + "gemini-3.5-flash-lite": { + "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_flex": 2e-08, + "cache_read_input_token_cost_priority": 5e-08, + "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, + "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.4e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 2.5e-06, + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 1.25e-06, + "output_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_priority": 4.5e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -18233,6 +18288,60 @@ }, "web_search_billing_unit": "per_query" }, + "vertex_ai/gemini-3.6-flash": { + "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_flex": 7.5e-08, + "input_cost_per_token": 1.5e-06, + "input_cost_per_token_batches": 7.5e-07, + "input_cost_per_token_flex": 7.5e-07, + "litellm_provider": "vertex_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 7.5e-06, + "output_cost_per_token": 7.5e-06, + "output_cost_per_token_batches": 3.75e-06, + "output_cost_per_token_flex": 3.75e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "input_cost_per_token_priority": 2.7e-06, + "output_cost_per_token_priority": 1.35e-05, + "cache_read_input_token_cost_priority": 2.7e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "vertex_ai/gemini-3.1-pro-preview": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, @@ -19585,6 +19694,63 @@ }, "web_search_billing_unit": "per_query" }, + "gemini/gemini-3.5-flash-lite": { + "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_flex": 2e-08, + "cache_read_input_token_cost_priority": 5e-08, + "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, + "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.4e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 2.5e-06, + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 1.25e-06, + "output_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_priority": 4.5e-06, + "rpm": 15, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "gemini/gemini-3-flash-preview": { "cache_read_input_token_cost": 5e-08, "input_cost_per_audio_token": 1e-06, @@ -19691,6 +19857,63 @@ }, "web_search_billing_unit": "per_query" }, + "gemini/gemini-3.6-flash": { + "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_flex": 7.5e-08, + "input_cost_per_token": 1.5e-06, + "input_cost_per_token_batches": 7.5e-07, + "input_cost_per_token_flex": 7.5e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 7.5e-06, + "output_cost_per_token": 7.5e-06, + "output_cost_per_token_batches": 3.75e-06, + "output_cost_per_token_flex": 3.75e-06, + "rpm": 2000, + "source": "https://ai.google.dev/pricing/gemini-3", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "tpm": 800000, + "input_cost_per_token_priority": 2.7e-06, + "output_cost_per_token_priority": 1.35e-05, + "cache_read_input_token_cost_priority": 2.7e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "gemini/gemini-omni-flash-preview": { "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, @@ -19971,6 +20194,61 @@ }, "web_search_billing_unit": "per_query" }, + "gemini-3.6-flash": { + "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_flex": 7.5e-08, + "input_cost_per_token": 1.5e-06, + "input_cost_per_token_batches": 7.5e-07, + "input_cost_per_token_flex": 7.5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 7.5e-06, + "output_cost_per_token": 7.5e-06, + "output_cost_per_token_batches": 3.75e-06, + "output_cost_per_token_flex": 3.75e-06, + "source": "https://ai.google.dev/pricing/gemini-3", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "input_cost_per_token_priority": 2.7e-06, + "output_cost_per_token_priority": 1.35e-05, + "cache_read_input_token_cost_priority": 2.7e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, @@ -37232,6 +37510,61 @@ }, "web_search_billing_unit": "per_query" }, + "vertex_ai/gemini-3.5-flash-lite": { + "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_flex": 2e-08, + "cache_read_input_token_cost_priority": 5e-08, + "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, + "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.4e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 2.5e-06, + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 1.25e-06, + "output_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_priority": 4.5e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "vertex_ai/deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 882c34dbd6a..9a1b5cf4864 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -597,7 +597,14 @@ async def authorize_with_server( ): _raise_if_not_oauth2(mcp_server) if mcp_server.authorization_url is None: - raise HTTPException(status_code=400, detail="MCP server authorization url is not set") + raise HTTPException( + status_code=400, + detail=( + "MCP server authorization url is not configured. Servers with no url (OpenAPI " + "spec or stdio) run no resource discovery, so set Authorization URL and Token URL " + "manually, or set Issuer to discover them from the identity provider (RFC 8414)." + ), + ) if mcp_server.is_dcr_bridge: # Enforce S256 PKCE on both bridge arms. The relay arm forwards the validated, @@ -702,7 +709,14 @@ async def exchange_token_with_server( raise HTTPException(status_code=400, detail="Unsupported grant_type") if mcp_server.token_url is None: - raise HTTPException(status_code=400, detail="MCP server token url is not set") + raise HTTPException( + status_code=400, + detail=( + "MCP server token url is not configured. Servers with no url (OpenAPI spec or " + "stdio) run no resource discovery, so set Token URL manually, or set Issuer to " + "discover it from the identity provider (RFC 8414)." + ), + ) # The id and secret must come from the same source. When the server-side client_id wins, # falling back to the caller's secret pairs the persisted client with a foreign secret; the @@ -1262,7 +1276,14 @@ async def register_client_with_server( return dummy_return if mcp_server.authorization_url is None: - raise HTTPException(status_code=400, detail="MCP server authorization url is not set") + raise HTTPException( + status_code=400, + detail=( + "MCP server authorization url is not configured. Servers with no url (OpenAPI " + "spec or stdio) run no resource discovery, so set Authorization URL and Token URL " + "manually, or set Issuer to discover them from the identity provider (RFC 8414)." + ), + ) if mcp_server.registration_url is None: return dummy_return diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 8f30071eb5d..90b70dd01f2 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -224,6 +224,20 @@ def _uses_issuer_anchor(manual_issuer: str | None, is_discovery_auth_type: bool) return _blank_to_none(manual_issuer) is not None and is_discovery_auth_type +def _has_oauth_discovery_source(server_url: str | None, use_issuer_anchor: bool) -> bool: + """Whether the server has any source OAuth discovery can fetch metadata from. + + Resource-rooted discovery (RFC 9728) is fetched from the server ``url``, so spec-only + (OpenAPI) and stdio servers, which have none, could never discover: their OAuth endpoints + stayed unset unless entered manually and ``/authorize`` served its 400 with no hint of why. + An admin-pinned issuer is a trust anchor in its own right (RFC 8414 section 3.3) whose + metadata fetch does not touch the resource at all, so an anchored server can discover with + no ``url``. Called by both build paths (config and DB) so the two cannot disagree on when + discovery is reachable. + """ + return bool(server_url) or use_issuer_anchor + + def _endpoints_yield_to_issuer( issuer: str | None, is_discovery_auth_type: bool, @@ -610,6 +624,34 @@ def _passthrough_token_from_mcp_auth_header( return None +async def _materialize_auth_headers(auth: httpx.Auth | None) -> dict[str, str] | None: + """Extract the header a resolved ``httpx.Auth`` would set, as a plain dict, or None. + + OpenAPI tool closures egress through ``AsyncHTTPHandler`` methods that accept headers but no + ``auth``, so a resolved credential must be materialized into a header value. Driving one step + of the auth's own flow (against a throwaway request that is never sent) keeps this generic + across every auth shape without per-class branching; ``header_name`` is the resolver-arm + convention for "this auth sets a header" (``NoOpAuth`` has none and yields nothing to apply). + The materialized value is point-in-time: flow behaviors past the first request, like the M2M + one-shot 401 refetch, do not apply on this arm. + """ + if auth is None: + return None + header_name = getattr(auth, "header_name", None) + if not isinstance(header_name, str) or not header_name: + return None + probe = httpx.Request("GET", "http://localhost/") + flow = auth.async_auth_flow(probe) + try: + first_request = await flow.__anext__() + except StopAsyncIteration: + return None + finally: + await flow.aclose() + header_value = first_request.headers.get(header_name) + return {header_name: header_value} if header_value else None + + def _consumes_caller_authorization(server: MCPServer) -> bool: """True when this server's egress forwards the caller's request-wide ``Authorization`` upstream: the client-forwarded token modes, legacy OAuth pass-through, and legacy upstream-delegated @@ -1226,7 +1268,12 @@ class MCPServerManager: manual_token_url = _blank_to_none(server_config.get("token_url")) manual_registration_url = _blank_to_none(server_config.get("registration_url")) is_discovery_auth_type = auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES - use_issuer_anchor = _uses_issuer_anchor(manual_issuer, is_discovery_auth_type) + obo_needs_discovery = self._obo_needs_endpoint_discovery( + auth_type, + server_config.get("token_exchange_endpoint"), + manual_token_url, + ) + use_issuer_anchor = _uses_issuer_anchor(manual_issuer, is_discovery_auth_type or obo_needs_discovery) manual_authorization_url, manual_token_url, manual_registration_url = _endpoints_yield_to_issuer( manual_issuer, is_discovery_auth_type, @@ -1234,17 +1281,12 @@ class MCPServerManager: manual_token_url, manual_registration_url, ) - should_discover = bool(server_url) and ( - is_discovery_auth_type - or self._obo_needs_endpoint_discovery( - auth_type, - server_config.get("token_exchange_endpoint"), - manual_token_url, - ) + should_discover = _has_oauth_discovery_source(server_url, use_issuer_anchor) and ( + is_discovery_auth_type or obo_needs_discovery ) if not should_discover: mcp_oauth_metadata = None - elif manual_issuer is not None and is_discovery_auth_type: + elif use_issuer_anchor and manual_issuer is not None: mcp_oauth_metadata = await self._fetch_issuer_anchored_oauth_metadata(manual_issuer, server_url) else: mcp_oauth_metadata = await self._descovery_metadata( @@ -1640,7 +1682,7 @@ class MCPServerManager: token_exchange_endpoint: Optional[str], ) -> Optional[MCPOAuthMetadata]: has_all_upstream_oauth_fields = bool(manual_authorization_url and manual_token_url and scopes) - needs_discovery = bool(server_url) and ( + needs_discovery = _has_oauth_discovery_source(server_url, use_issuer_anchor) and ( (is_discovery_auth_type and not has_all_upstream_oauth_fields) or self._obo_needs_endpoint_discovery(auth_type, token_exchange_endpoint, manual_token_url) ) @@ -1759,13 +1801,17 @@ class MCPServerManager: manual_token_url = _blank_to_none(mcp_server.token_url) manual_registration_url = _blank_to_none(mcp_server.registration_url) is_discovery_auth_type = auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES - use_issuer_anchor = _uses_issuer_anchor(manual_issuer, is_discovery_auth_type) - manual_authorization_url, manual_token_url, manual_registration_url = _endpoints_yield_to_issuer( - manual_issuer, is_discovery_auth_type, manual_authorization_url, manual_token_url, manual_registration_url - ) token_exchange_endpoint = mcp_server.token_exchange_endpoint or ( credentials_dict.get("token_exchange_endpoint") if credentials_dict else None ) + use_issuer_anchor = _uses_issuer_anchor( + manual_issuer, + is_discovery_auth_type + or self._obo_needs_endpoint_discovery(auth_type, token_exchange_endpoint, manual_token_url), + ) + manual_authorization_url, manual_token_url, manual_registration_url = _endpoints_yield_to_issuer( + manual_issuer, is_discovery_auth_type, manual_authorization_url, manual_token_url, manual_registration_url + ) gated_oauth_metadata = await self._resolve_table_oauth_metadata( mcp_server=mcp_server, auth_type=auth_type, @@ -1943,7 +1989,7 @@ class MCPServerManager: family: discovered ``authorization_url``/``token_url``/``scopes`` otherwise live only on the in-memory registry entry, which is rebuilt on every client connect (the DCR reuse path calls ``update_server``) and on every post-write DB reload, so one failed re-discovery - serves 400 "authorization url is not set" from /authorize until a later rebuild succeeds. + serves the 400 "authorization url is not configured" from /authorize until a later rebuild succeeds. Only fills row fields that are currently empty, never persists origin-fallback guesses (RFC 9728/8414-advertised metadata only), and deliberately skips ``registration_url`` because ``_dcr_bridge_relays_client_registration`` keys off that column. Best-effort: a @@ -4705,6 +4751,61 @@ class MCPServerManager: ) return oauth2_headers + async def resolve_openapi_upstream_auth( + self, + *, + mcp_server: MCPServer, + oauth2_headers: dict[str, str] | None, + raw_headers: dict[str, str] | None, + mcp_auth_header: str | dict[str, str] | None, + user_api_key_auth: UserAPIKeyAuth | None, + forwarded_headers: dict[str, str] | None, + ) -> tuple[dict[str, str] | None, dict[str, str] | None]: + """Resolve the gateway-owned upstream credential for a spec_path (OpenAPI) tool call. + + OpenAPI tools egress through a plain httpx call assembled from ContextVars, never through + ``_create_mcp_client``, so the v2 resolver graft there does not run for them and a resolved + credential (authorization_code's stored per-user token, client_credentials' minted M2M + token, token_exchange's exchanged token, passthrough's forwarded caller token) must be + materialized into headers here. Returns ``(resolved_auth_headers, forwarded_headers)``: + the resolved headers are authoritative over every other Authorization source (the same + rule ``_resolve_v2_auth`` applies on the MCPClient path) and ``forwarded_headers`` comes + back with any header the resolver claimed already dropped. Unmigrated (v1) servers resolve + through the stored-token lookup instead, and a missing per-user credential raises the same + discovery challenge the MCPClient path serves, rather than egressing unauthenticated. + + The resolved headers carry only credentials the gateway itself resolved (a stored per-user + token, a minted or exchanged token). Caller-supplied ``oauth2_headers`` are never promoted + into them: on the v2 arm they feed only subject-token extraction (the designed RFC 8693 + input), and on the v1 arm their presence disables the stored lookup entirely, so a + caller's gateway credential can never displace a per-server BYOK header or leak upstream + as the resolved credential. + """ + spec = to_server_spec(mcp_server) + if spec is None: + if oauth2_headers: + return None, forwarded_headers + stored_headers = await self._resolve_oauth2_headers_for_tool_call(mcp_server, None, user_api_key_auth) + return stored_headers, forwarded_headers + + subject_token: str | None = None + if isinstance(spec.config, (TokenExchangeConfig, IdJagConfig)): + subject_token = self._extract_bearer_token(oauth2_headers, raw_headers) + elif isinstance(spec.config, PassthroughConfig): + inbound_token, forwarded_headers = _take_forwarded_authorization(forwarded_headers) + per_server_token = _passthrough_token_from_mcp_auth_header(mcp_auth_header) + subject_token = per_server_token if per_server_token is not None else inbound_token + + resolved_auth, forwarded_headers = await self._resolve_v2_auth( + server=mcp_server, + spec=spec, + provider=self._cred_provider, + subject_token=subject_token, + user_api_key_auth=user_api_key_auth, + extra_headers=forwarded_headers, + ) + return await _materialize_auth_headers(resolved_auth), forwarded_headers + async def _gather_openapi_tool_tasks( self, tasks: list[Any], @@ -4796,6 +4897,7 @@ class MCPServerManager: ) tasks.append(during_hook_task) + caller_oauth2_headers = oauth2_headers oauth2_headers = await self._resolve_oauth2_headers_for_tool_call(mcp_server, oauth2_headers, user_api_key_auth) # For OpenAPI servers, call the tool handler directly instead of via MCP client @@ -4813,22 +4915,32 @@ class MCPServerManager: auth_header_value = ( _format_byok_openapi_auth_header(mcp_server, mcp_auth_header) if mcp_auth_header else None ) - forwarded_headers = _openapi_forwarded_extra_headers(mcp_server, raw_headers, user_api_key_auth) + resolved_auth_headers, forwarded_headers = await self.resolve_openapi_upstream_auth( + mcp_server=mcp_server, + oauth2_headers=caller_oauth2_headers, + raw_headers=raw_headers, + mcp_auth_header=mcp_auth_header, + user_api_key_auth=user_api_key_auth, + forwarded_headers=_openapi_forwarded_extra_headers(mcp_server, raw_headers, user_api_key_auth), + ) async def _call_openapi_via_handler(): from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( _request_auth_header, _request_extra_headers, + _request_resolved_auth_headers, ) auth_token = _request_auth_header.set(auth_header_value) extra_token = _request_extra_headers.set(forwarded_headers) + resolved_token = _request_resolved_auth_headers.set(resolved_auth_headers) try: async with self._limit_outbound_concurrency(mcp_server): return await self._call_openapi_tool_handler(mcp_server, name, arguments) finally: _request_auth_header.reset(auth_token) _request_extra_headers.reset(extra_token) + _request_resolved_auth_headers.reset(resolved_token) tasks.append(asyncio.create_task(_call_openapi_via_handler())) else: diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index 1ee300be718..0b795057837 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -62,6 +62,14 @@ _request_extra_headers: contextvars.ContextVar[Optional[Dict[str, str]]] = conte "_request_extra_headers", default=None ) +# Per-request headers carrying the gateway-resolved upstream credential +# (stored per-user OAuth token, minted M2M token, exchanged OBO token). +# Set from MCPServerManager.resolve_openapi_upstream_auth; authoritative +# over every other Authorization source in _merge_openapi_tool_request_headers. +_request_resolved_auth_headers: contextvars.ContextVar[dict[str, str] | None] = contextvars.ContextVar( + "_request_resolved_auth_headers", default=None +) + def _sanitize_path_parameter_value(param_value: Any, param_name: str) -> str: """Ensure path params cannot introduce directory traversal.""" @@ -294,10 +302,15 @@ def _merge_openapi_tool_request_headers( """Merge static closure headers with per-request ContextVar overrides. Precedence (highest to lowest): - 1. ``_request_auth_header`` — BYOK override of ``Authorization`` - 2. ``static_headers`` — operator-configured headers baked into the + 1. ``_request_resolved_auth_headers`` — the gateway-resolved upstream + credential (stored per-user OAuth token, minted M2M token, + exchanged OBO token). The resolver is authoritative: a BYOK or + forwarded ``Authorization`` must not shadow it, mirroring + ``_resolve_v2_auth`` on the MCPClient path + 2. ``_request_auth_header`` — BYOK override of ``Authorization`` + 3. ``static_headers`` — operator-configured headers baked into the tool closure at registration time - 3. ``_request_extra_headers`` — per-request headers forwarded from + 4. ``_request_extra_headers`` — per-request headers forwarded from the MCP caller (allowlisted by ``MCPServer.extra_headers``) This matches the existing MCP invariant in @@ -323,6 +336,12 @@ def _merge_openapi_tool_request_headers( del effective_headers[existing] effective_headers["Authorization"] = override_auth + resolved_auth_headers = _request_resolved_auth_headers.get() or {} + for name, value in resolved_auth_headers.items(): + for existing in [k for k in effective_headers if k.lower() == name.lower()]: + del effective_headers[existing] + effective_headers[name] = value + return effective_headers diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py new file mode 100644 index 00000000000..e0927cc4f64 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py @@ -0,0 +1,214 @@ +"""Store for the enterprise IdP identity assertion captured at SSO login (EMA). + +The ``oauth2_id_jag`` egress arm needs the user's IdP ``id_token`` as its RFC 8693 +``subject_token``. A front-door client holds an identity-only ``llm_session_`` bearer, not an +IdP assertion, so the assertion captured at the one SSO login is the only usable subject +source for it. This module owns both sides of that state: the SSO callback persists here +(write-through to the DB so a login on one pod is visible to every pod) and the resolver +seam reads back by ``user_id``. Retention is gated on an ``oauth2_id_jag`` server actually +being registered, so a gateway with no EMA upstream never stores bearer material. + +The row is one encrypted payload per user, latest login wins. ``expires_at`` mirrors the +id_token ``exp`` claim and is judged by the reader, never enforced by deletion here: an +expired assertion with a refresh token is still renewable, and the DB row is the source of +truth, the same contract as the per-user OAuth credential store. +""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from typing import TYPE_CHECKING + +import jwt +from pydantic import BaseModel, ConfigDict, SecretStr, TypeAdapter, ValidationError + +from litellm._logging import verbose_proxy_logger + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + +_ASSERTION_DECRYPT_LOG_KEY = "sso_identity_assertion" +_STR_ADAPTER: TypeAdapter[str] = TypeAdapter(str) +_MAYBE_STR_ADAPTER: TypeAdapter[str | None] = TypeAdapter(str | None) + + +class SSOIdentityAssertion(BaseModel): + """The IdP material an EMA exchange needs: ``id_token`` is the RFC 8693 subject token, + ``expires_at`` bounds its usefulness, and the refresh token renews it without re-login.""" + + model_config = ConfigDict(frozen=True) + + id_token: SecretStr + refresh_token: SecretStr | None = None + issuer: str | None = None + expires_at: datetime | None = None + + +class _IdTokenClaims(BaseModel): + exp: float | None = None + iss: str | None = None + + +class _StoredAssertionPayload(BaseModel): + id_token: str + refresh_token: str | None = None + issuer: str | None = None + expires_at: datetime | None = None + + +def assertion_from_sso_login(id_token: object, refresh_token: object) -> SSOIdentityAssertion | None: + """The typed carrier built where the raw token response exists; ``None`` when the provider + sent no id_token or sent one that is not a decodable JWT, since neither is exchangeable + under EMA. Inputs are ``object`` because they come straight from the provider's untyped + token response; this is the one boundary that validates them. The token arrived over TLS + from the IdP's own token endpoint, so claims are read without signature verification, + matching how the SSO callback already decodes it for identity.""" + raw_id_token = id_token if isinstance(id_token, str) and id_token else None + if raw_id_token is None: + return None + raw_refresh_token = refresh_token if isinstance(refresh_token, str) and refresh_token else None + try: + claims = _IdTokenClaims.model_validate(jwt.decode(raw_id_token, options={"verify_signature": False})) + expires_at = datetime.fromtimestamp(claims.exp, tz=timezone.utc) if claims.exp is not None else None + except Exception: # noqa: BLE001 # decode failure = not retainable; never raise into login + verbose_proxy_logger.warning( + "SSO id_token could not be decoded or its claims were unusable; not retaining it for EMA egress." + ) + return None + return SSOIdentityAssertion( + id_token=SecretStr(raw_id_token), + refresh_token=SecretStr(raw_refresh_token) if raw_refresh_token else None, + issuer=claims.iss, + expires_at=expires_at, + ) + + +async def ema_assertion_retention_enabled() -> bool: + """Whether any MCP server uses ``oauth2_id_jag``, evaluated per login so the gateway only + retains bearer material while an EMA upstream exists to spend it on. Judged against the two + configuration authorities: the pod-local config declaration and the shared DB row. The + in-memory registry is deliberately not consulted in either direction; it is a per-process + snapshot of the DB state that can be stale both ways (a server added on another pod would + silently drop the write, one removed on another pod would keep retaining bearer material), + and a gate guarding a shared-DB write must judge against that storage's authority.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # avoids import cycle + global_mcp_server_manager, + ) + from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 # runtime global + from litellm.types.mcp import MCPAuth # noqa: PLC0415 # runtime global + + config_servers = global_mcp_server_manager.config_mcp_servers.values() + if any(server.auth_type == MCPAuth.oauth2_id_jag for server in config_servers): + return True + if prisma_client is None: + return False + row = await prisma_client.db.litellm_mcpservertable.find_first(where={"auth_type": MCPAuth.oauth2_id_jag.value}) + return row is not None + + +async def persist_sso_identity_assertion(user_id: str, assertion: SSOIdentityAssertion) -> None: + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper # noqa: PLC0415 # runtime global + from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 # runtime global + + if prisma_client is None: + return + payload: dict[str, str] = { + "id_token": assertion.id_token.get_secret_value(), + **({"refresh_token": assertion.refresh_token.get_secret_value()} if assertion.refresh_token else {}), + **({"issuer": assertion.issuer} if assertion.issuer else {}), + **({"expires_at": assertion.expires_at.isoformat()} if assertion.expires_at else {}), + } + encoded = _STR_ADAPTER.validate_python(encrypt_value_helper(json.dumps(payload))) + await prisma_client.db.litellm_ssoidentityassertion.upsert( + where={"user_id": user_id}, + data={ + "create": {"user_id": user_id, "assertion_b64": encoded}, + "update": {"assertion_b64": encoded}, + }, + ) + + +async def fetch_sso_identity_assertion(user_id: str) -> SSOIdentityAssertion | None: + """The stored assertion for ``user_id``, or ``None`` when absent, undecryptable (salt-key + rotation), or unparseable. Expiry is not judged here; the reader owns that policy.""" + from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper # noqa: PLC0415 # runtime global + from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 # runtime global + + if prisma_client is None: + return None + row = await prisma_client.db.litellm_ssoidentityassertion.find_unique(where={"user_id": user_id}) + if row is None: + return None + raw = _MAYBE_STR_ADAPTER.validate_python( + decrypt_value_helper(row.assertion_b64, _ASSERTION_DECRYPT_LOG_KEY, exception_type="debug") + ) + if raw is None: + return None + try: + payload = _StoredAssertionPayload.model_validate_json(raw) + except ValidationError: + verbose_proxy_logger.warning( + "Stored SSO identity assertion for user_id=%s could not be parsed; treating as absent.", user_id + ) + return None + return SSOIdentityAssertion( + id_token=SecretStr(payload.id_token), + refresh_token=SecretStr(payload.refresh_token) if payload.refresh_token else None, + issuer=payload.issuer, + expires_at=payload.expires_at, + ) + + +async def rotate_sso_identity_assertions_master_key(prisma_client: PrismaClient, new_master_key: str) -> None: + """Re-encrypt every stored assertion under ``new_master_key`` during a salt-key rotation, + mirroring the sibling per-user credential tables; an unreadable row is skipped so one + corrupt row does not abort the rotation. Rows are decrypted one at a time inside the loop + so the whole table's plaintext is never held in memory at once.""" + from prisma.models import LiteLLM_SSOIdentityAssertion as AssertionRow # noqa: PLC0415 # generated at runtime + + from litellm.proxy.common_utils.encrypt_decrypt_utils import ( # noqa: PLC0415 # runtime global + decrypt_value_helper, + encrypt_value_helper, + ) + + async def _rotate_row(row: AssertionRow) -> bool: + plaintext = _MAYBE_STR_ADAPTER.validate_python( + decrypt_value_helper(row.assertion_b64, _ASSERTION_DECRYPT_LOG_KEY, exception_type="debug") + ) + if plaintext is None: + verbose_proxy_logger.warning( + "rotate_sso_identity_assertions_master_key: could not decrypt assertion for user_id=%s, skipping", + row.user_id, + ) + return False + re_encrypted = _STR_ADAPTER.validate_python(encrypt_value_helper(plaintext, new_encryption_key=new_master_key)) + await prisma_client.db.litellm_ssoidentityassertion.update( + where={"user_id": row.user_id}, + data={"assertion_b64": re_encrypted}, + ) + return True + + rows = await prisma_client.db.litellm_ssoidentityassertion.find_many() + outcomes = [await _rotate_row(row) for row in rows] + verbose_proxy_logger.info( + "rotate_sso_identity_assertions_master_key: rotated %d row(s), skipped %d", + sum(outcomes), + len(outcomes) - sum(outcomes), + ) + + +async def retain_sso_identity_assertion_for_ema(user_id: str, assertion: SSOIdentityAssertion | None) -> None: + """The SSO-callback hook: a no-op unless there is material AND an EMA server is registered. + A store failure is logged and swallowed because the login itself must not fail on an + egress-side write; the cost of a miss is a 401 challenge at the EMA upstream, not a lockout.""" + if assertion is None: + return + try: + if not await ema_assertion_retention_enabled(): + return + await persist_sso_identity_assertion(user_id, assertion) + except Exception as exc: # noqa: BLE001 # the login itself must not fail on an egress-side write + verbose_proxy_logger.warning( + "Failed to persist the SSO identity assertion for EMA egress (user_id=%s): %s", user_id, exc + ) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index a8ab0937124..396dd6c7dc7 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -376,6 +376,7 @@ if MCP_AVAILABLE: from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( _request_auth_header, _request_extra_headers, + _request_resolved_auth_headers, ) from litellm.proxy._experimental.mcp_server.sse_transport import SseServerTransport from litellm.proxy._experimental.mcp_server.tool_registry import ( @@ -2785,13 +2786,29 @@ if MCP_AVAILABLE: forwarded_headers = {} forwarded_headers[header_name] = value + resolved_auth_headers: dict[str, str] | None = None + if mcp_server: + ( + resolved_auth_headers, + forwarded_headers, + ) = await global_mcp_server_manager.resolve_openapi_upstream_auth( + mcp_server=mcp_server, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + mcp_auth_header=mcp_auth_header, + user_api_key_auth=user_api_key_auth, + forwarded_headers=forwarded_headers, + ) + _auth_token = _request_auth_header.set(auth_header_value) _extra_token = _request_extra_headers.set(forwarded_headers) + _resolved_token = _request_resolved_auth_headers.set(resolved_auth_headers) try: local_content = await _handle_local_mcp_tool(name, arguments) finally: _request_auth_header.reset(_auth_token) _request_extra_headers.reset(_extra_token) + _request_resolved_auth_headers.reset(_resolved_token) response = CallToolResult(content=cast(Any, local_content), isError=False) # Try managed MCP server tool (pass the full prefixed name) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ee650510752..202eee39485 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2298,6 +2298,11 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="max response size in MB, if a response is larger than this size it will be rejected", ) + proxy_config_reload_interval_seconds: int = Field( + 30, + gt=0, + description="how often (in seconds) each pod reloads config-in-DB objects (models, credentials, guardrails, etc.) when store_model_in_db is enabled; lower values speed up multi-pod convergence at the cost of more DB load. Applied on proxy startup", + ) cancel_on_disconnect: Optional[bool] = Field( None, description="cancel the in-flight upstream LLM request (non-streaming) when the client disconnects, freeing backend capacity (e.g. a vLLM GPU slot); the request is logged as a 499 failure", @@ -4048,6 +4053,7 @@ class JWTAuthBuilderResult(TypedDict): token: str team_id: Optional[str] user_id: Optional[str] + user_email: str | None end_user_id: Optional[str] org_id: Optional[str] team_membership: Optional[LiteLLM_TeamMembership] diff --git a/litellm/proxy/a2a/agent_card.py b/litellm/proxy/a2a/agent_card.py index e97ab4a01ae..29a689a32de 100644 --- a/litellm/proxy/a2a/agent_card.py +++ b/litellm/proxy/a2a/agent_card.py @@ -7,23 +7,46 @@ the base; specific fields are replaced so all traffic flows through the proxy and uses LiteLLM auth. """ +import re from copy import deepcopy -from typing import Any, Dict, List, Mapping +from typing import Any, Dict, List, Literal, Mapping + +SupportedA2AVersion = Literal["0.3", "1.0"] # Protocol versions LiteLLM can serve to A2A clients. The admin pins one per agent; # responses are normalized to it regardless of the upstream agent's own version. -SUPPORTED_A2A_PROTOCOL_VERSIONS = ("0.3", "1.0") +SUPPORTED_A2A_PROTOCOL_VERSIONS: tuple[SupportedA2AVersion, ...] = ("0.3", "1.0") # Default served version when the agent card does not pin one. LITELLM_A2A_PROTOCOL_VERSION = "1.0" +_PROTOCOL_VERSION_PATTERN = re.compile( + r"^(\d+\.\d+)(?:\.\d+(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?)?$" +) + + +def normalize_protocol_version(version: object) -> SupportedA2AVersion | None: + """Map a raw ``protocolVersion`` value to the supported canonical major.minor version. + + Accepts the bare major.minor convention of the 1.0 spec (``"0.3"``, ``"1.0"``) and the + full semver forms older SDKs emit (``"0.3.0"``, ``"1.0.1"``, including prerelease and + build suffixes like ``"0.3.0-rc1"``). Malformed strings, versions outside the + supported set, and non-strings yield ``None``. + """ + if not isinstance(version, str): + return None + match = _PROTOCOL_VERSION_PATTERN.match(version) + if match is None: + return None + major_minor = match.group(1) + return next((supported for supported in SUPPORTED_A2A_PROTOCOL_VERSIONS if supported == major_minor), None) + + def resolve_served_protocol_version(card: Mapping[str, Any] | None) -> str: """Return the validated protocol version an agent card pins, else the default.""" - version = card.get("protocolVersion") if card else None - if version in SUPPORTED_A2A_PROTOCOL_VERSIONS: - return version - return LITELLM_A2A_PROTOCOL_VERSION + normalized = normalize_protocol_version(card.get("protocolVersion") if card else None) + return normalized if normalized is not None else LITELLM_A2A_PROTOCOL_VERSION # Security scheme exposed by the LiteLLM-fronted agent card. Always replaces diff --git a/litellm/proxy/a2a/version_convert.py b/litellm/proxy/a2a/version_convert.py index e8f49e6f6a9..9de33a0966a 100644 --- a/litellm/proxy/a2a/version_convert.py +++ b/litellm/proxy/a2a/version_convert.py @@ -30,6 +30,7 @@ from typing import Callable, Literal, Union from pydantic import BaseModel from litellm._logging import verbose_proxy_logger +from litellm.proxy.a2a.agent_card import normalize_protocol_version A2AVersion = Literal["0.3", "1.0"] RequestId = Union[str, int, None] @@ -103,16 +104,14 @@ def normalize_request_params(params: JsonDict, served: A2AVersion, *, method: st def _detect_card_version(card: JsonDict) -> A2AVersion: """Infer the wire version of an agent card dict. - ``protocolVersion`` is the authoritative indicator; fall back to presence of - ``supportedInterfaces`` (a 1.0-only field) only when the explicit field is absent. - Cards that set ``protocolVersion: "0.3"`` or carry neither signal are treated as 0.3. + ``protocolVersion`` is the authoritative indicator; semver values normalize to + their major.minor (``"0.3.0"`` -> ``"0.3"``). Fall back to presence of + ``supportedInterfaces`` (a 1.0-only field) only when the explicit field is + absent or unrecognized; cards carrying neither signal are treated as 0.3. """ - pv = card.get("protocolVersion") - if pv == "1.0": - return "1.0" - if pv == "0.3": - return "0.3" - # No protocolVersion field: use structural heuristic. + normalized = normalize_protocol_version(card.get("protocolVersion")) + if normalized is not None: + return normalized return "1.0" if "supportedInterfaces" in card else "0.3" diff --git a/litellm/proxy/agent_endpoints/endpoints.py b/litellm/proxy/agent_endpoints/endpoints.py index a7ceffed97b..2421f270974 100644 --- a/litellm/proxy/agent_endpoints/endpoints.py +++ b/litellm/proxy/agent_endpoints/endpoints.py @@ -23,6 +23,7 @@ from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKey from litellm.proxy.a2a.agent_card import ( SUPPORTED_A2A_PROTOCOL_VERSIONS, merge_agent_card, + normalize_protocol_version, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.rbac_utils import check_feature_access_for_user @@ -51,7 +52,7 @@ def _proxy_base_url(http_request: Request) -> str: def _validate_protocol_version(upstream_card: Mapping[str, Any] | None) -> None: """Reject an agent card pinning an unsupported A2A protocol version.""" version = upstream_card.get("protocolVersion") if upstream_card else None - if version is not None and version not in SUPPORTED_A2A_PROTOCOL_VERSIONS: + if version is not None and normalize_protocol_version(version) is None: raise HTTPException( status_code=400, detail=( diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 293bb74e211..ecb37e67c14 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -3,7 +3,7 @@ import re import sys from functools import lru_cache from logging import Logger -from typing import Any, Dict, FrozenSet, List, Mapping, Optional, Tuple, Union +from typing import Any, Dict, FrozenSet, Iterator, List, Mapping, Optional, Tuple, Union from fastapi import HTTPException, Request, status @@ -12,7 +12,12 @@ from litellm import Router, provider_list from litellm._logging import verbose_proxy_logger from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH, STANDARD_CUSTOMER_ID_HEADERS from litellm.litellm_core_utils.safe_json_loads import safe_json_loads -from litellm.litellm_core_utils.url_utils import SSRFError, validate_url +from litellm.litellm_core_utils.url_utils import ( + SSRFError, + is_url_destination_allowed_by_host, + provider_url_destination_candidates, + validate_url, +) from litellm.proxy._types import * from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_ENDPOINT_MARKER, @@ -290,6 +295,7 @@ _BANNED_REQUEST_BODY_PARAMS: Tuple[str, ...] = ( "use_ssl", # SDK-only field; also rejected outright in is_request_body_safe. "model_list", + "vertex_ai_credentials", # Observability credentials, hosts, and project identifiers: derived # from the canonical ``_supported_callback_params`` allowlist so new # integrations are covered automatically. Sorted for stable iteration @@ -342,6 +348,60 @@ def _check_banned_params( ) +_FALLBACK_FIELDS: tuple[str, ...] = ( + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", +) + + +def _iter_fallback_field_values(request_body: Mapping[str, object]) -> Iterator[object]: + override = request_body.get("router_settings_override") + for source in (request_body, override): + if isinstance(source, Mapping): + for field in _FALLBACK_FIELDS: + yield source.get(field) + + +def _iter_fallback_targets(value: object, depth: int) -> Iterator[str | Mapping[str, object]]: + if depth > 2 * litellm.ROUTER_MAX_FALLBACKS: + raise ValueError("Rejected Request: fallback nesting exceeds the allowed validation depth.") + if not isinstance(value, list): + return + for item in value: + if isinstance(item, str): + yield item + elif isinstance(item, Mapping): + values = tuple(item.values()) + if not (values and all(isinstance(v, list) for v in values)): + yield item + if isinstance(item.get("model"), str): + for field in _FALLBACK_FIELDS: + yield from _iter_fallback_targets(item.get(field), depth + 1) + else: + for target_list in values: + yield from _iter_fallback_targets(target_list, depth + 1) + + +def iter_request_fallback_targets(request_body: Mapping[str, object]) -> Iterator[str | Mapping[str, object]]: + for value in _iter_fallback_field_values(request_body): + yield from _iter_fallback_targets(value, 0) + + +def _reject_url_valued_fallback_target(value: str) -> None: + allowed_hosts = getattr(litellm, "provider_url_destination_allowed_hosts", []) or [] + for candidate in provider_url_destination_candidates(value): + if not candidate.lower().startswith(("http://", "https://")): + continue + if is_url_destination_allowed_by_host(candidate, allowed_hosts): + continue + raise ValueError( + f"Rejected Request: URL-valued fallback destination '{value}' is not allowed. " + "Configure custom endpoints with api_base instead, or add the destination host to " + "`provider_url_destination_allowed_hosts` in litellm_settings." + ) + + def is_request_body_safe(request_body: dict, general_settings: dict, llm_router: Optional[Router], model: str) -> bool: """ Check if the request body is safe. @@ -379,6 +439,14 @@ def is_request_body_safe(request_body: dict, general_settings: dict, llm_router: metadata = _coerce_metadata_to_dict(request_body.get(metadata_key)) if metadata is not None: _check_banned_params(metadata, general_settings, llm_router, model) + for target in iter_request_fallback_targets(request_body): + if isinstance(target, dict): + _check_banned_params(target, general_settings, llm_router, model) + target_model = target.get("model") + if isinstance(target_model, str): + _reject_url_valued_fallback_target(target_model) + elif isinstance(target, str): + _reject_url_valued_fallback_target(target) litellm_params = _coerce_metadata_to_dict(request_body.get("litellm_params")) if litellm_params is not None: litellm_params_metadata = _coerce_metadata_to_dict(litellm_params.get("metadata")) diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index a44318c072c..ff87d0e70da 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -1155,6 +1155,7 @@ class JWTAuthManager: org_id: Optional[str], api_key: str, jwt_valid_token: Optional[dict] = None, + user_email: str | None = None, ) -> Optional[JWTAuthBuilderResult]: """Check admin status and route access permissions""" if not jwt_handler.is_admin(scopes=scopes): @@ -1179,6 +1180,7 @@ class JWTAuthManager: token=api_key, team_id=None, user_id=user_id, + user_email=user_email, end_user_id=None, org_id=org_id, team_membership=None, @@ -2068,7 +2070,7 @@ class JWTAuthManager: # Check admin access admin_result = await JWTAuthManager.check_admin_access( - jwt_handler, scopes, route, user_id, org_id, api_key, jwt_valid_token + jwt_handler, scopes, route, user_id, org_id, api_key, jwt_valid_token, user_email=user_email ) if admin_result: await JWTAuthManager._attach_team_from_header_for_admin( @@ -2303,6 +2305,7 @@ class JWTAuthManager: team_id=team_id, team_object=team_object, user_id=user_id, + user_email=(user_object.user_email if user_object is not None and user_object.user_email else user_email), user_object=user_object, org_id=resolved_org_id, # Use resolved org_id (from alias lookup if applicable) org_object=org_object, diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 5b21a7265a0..83a8a69511b 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -14,7 +14,7 @@ import secrets import orjson from datetime import datetime, timezone -from typing import Any, Dict, Iterator, NamedTuple, List, Optional, Protocol, Tuple, Union, cast +from typing import Any, Dict, NamedTuple, List, Optional, Protocol, Tuple, Union, cast import fastapi from fastapi import HTTPException, Request, WebSocket, status @@ -58,6 +58,7 @@ from litellm.proxy.auth.auth_utils import ( get_model_from_request, get_request_route, get_request_route_template, + iter_request_fallback_targets, normalize_request_route, pre_db_read_auth_checks, route_in_additonal_public_routes, @@ -1011,7 +1012,7 @@ def _ensure_parent_otel_span_on_request_state(request: Request) -> None: return if getattr(request.state, "parent_otel_span", None) is not None: return - start_time = datetime.now() + start_time = datetime.now(timezone.utc) try: request.state.litellm_received_at = start_time except Exception: @@ -1061,7 +1062,7 @@ async def _user_api_key_auth_builder( # Prefer the receive-instant stamped by the early helper in # user_api_key_auth (before body parse) — overwriting it would shorten # the preprocessing-duration measurement by the body-parse window. - start_time = getattr(request.state, "litellm_received_at", None) or datetime.now() + start_time = getattr(request.state, "litellm_received_at", None) or datetime.now(timezone.utc) try: request.state.litellm_received_at = start_time except Exception: @@ -1255,6 +1256,7 @@ async def _user_api_key_auth_builder( team_id = result["team_id"] team_object = result["team_object"] user_id = result["user_id"] + user_email = result["user_email"] user_object = result["user_object"] end_user_id = result["end_user_id"] org_id = result["org_id"] @@ -1279,6 +1281,7 @@ async def _user_api_key_auth_builder( api_key=None, user_role=LitellmUserRoles.PROXY_ADMIN, user_id=user_id, + user_email=user_email, team_id=team_id, team_alias=(team_object.team_alias if team_object is not None else None), team_tpm_limit=(team_object.tpm_limit if team_object is not None else None), @@ -1304,6 +1307,7 @@ async def _user_api_key_auth_builder( else LitellmUserRoles.INTERNAL_USER ), user_id=user_id, + user_email=user_email, org_id=org_id, parent_otel_span=parent_otel_span, end_user_id=end_user_id, @@ -1345,6 +1349,7 @@ async def _user_api_key_auth_builder( ) if auto_registered is not None: auto_registered.jwt_claims = jwt_claims + auto_registered.user_email = user_email valid_token = auto_registered api_key = valid_token.token or "" @@ -2607,7 +2612,7 @@ async def _return_user_api_key_auth_obj( start_time: datetime, user_role: Optional[LitellmUserRoles] = None, ) -> UserAPIKeyAuth: - end_time = datetime.now() + end_time = datetime.now(timezone.utc) asyncio.create_task( user_api_key_service_logger_obj.async_service_success_hook( @@ -2696,9 +2701,10 @@ def _update_key_budget_with_temp_budget_increase( ) -> UserAPIKeyAuth: if valid_token.max_budget is None: return valid_token - temp_budget_increase = _get_temp_budget_increase(valid_token) or 0.0 - valid_token.max_budget = valid_token.max_budget + temp_budget_increase - return valid_token + temp_budget_increase = _get_temp_budget_increase(valid_token) + if not temp_budget_increase: + return valid_token + return valid_token.model_copy(update={"max_budget": valid_token.max_budget + temp_budget_increase}) async def _lookup_end_user_and_apply_budget( @@ -2796,19 +2802,11 @@ async def _enforce_key_and_fallback_model_access( llm_router=llm_router, ) - # Validate every fallback model name reachable by this request. - # All three fields (``fallbacks``, ``context_window_fallbacks``, - # ``content_policy_fallbacks``) are forwarded to the router as - # per-request kwargs whether they appear at the top level of - # ``request_data`` or nested under ``router_settings_override``. - # Both surfaces must be validated against the API key's model - # allowlist or a caller can smuggle a restricted model. VERIA-44. - fallback_names: List[str] = [] - override_settings = request_data.get("router_settings_override") - for _fb_key in ROUTER_FALLBACK_FIELDS: - fallback_names.extend(iter_router_fallback_model_names(request_data.get(_fb_key))) - if isinstance(override_settings, dict): - fallback_names.extend(iter_router_fallback_model_names(override_settings.get(_fb_key))) + fallback_names = tuple( + name + for target in iter_request_fallback_targets(request_data) + if (name := _fallback_target_model_name(target)) is not None + ) for _name in dict.fromkeys(fallback_names): # dedupe, preserve order await can_key_call_model( @@ -2824,36 +2822,14 @@ async def _enforce_key_and_fallback_model_access( ) -ROUTER_FALLBACK_FIELDS: Tuple[str, ...] = ( - "fallbacks", - "context_window_fallbacks", - "content_policy_fallbacks", -) - - -def iter_router_fallback_model_names(fallbacks: Any) -> Iterator[str]: - """Yield leaf model names from any of the supported fallbacks shapes. - - Handles the simple top-level shape (``str`` or ``{"model": str}``) and - the nested router-config shape (``[{primary: [fallback_list]}]``). - """ - if not isinstance(fallbacks, list): - return - for entry in fallbacks: - if isinstance(entry, str): - yield entry - elif isinstance(entry, dict): - if isinstance(entry.get("model"), str): - yield entry["model"] - continue - for fallback_list in entry.values(): - if not isinstance(fallback_list, list): - continue - for m in fallback_list: - if isinstance(m, str): - yield m - elif isinstance(m, dict) and isinstance(m.get("model"), str): - yield m["model"] +def _fallback_target_model_name(target: object) -> str | None: + if isinstance(target, str): + return target + if isinstance(target, dict): + model = target.get("model") + if isinstance(model, str): + return model + return None async def _run_post_custom_auth_checks( diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index c644ecc3dae..a9c2a12aff7 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -1,4 +1,5 @@ import copy +import os from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Literal, Optional import litellm @@ -564,11 +565,8 @@ def process_callback(_callback: str, callback_type: str, environment_variables: env_vars_dict: dict[str, str | None] = {} for _var in env_vars: - env_variable = environment_variables.get(_var, None) - if env_variable is None: - env_vars_dict[_var] = None - else: - env_vars_dict[_var] = env_variable + stored_value = environment_variables.get(_var, None) + env_vars_dict[_var] = stored_value if stored_value is not None else os.getenv(_var) return {"name": _callback, "variables": env_vars_dict, "type": callback_type} diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index e758420ee37..23a5b8f9c53 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -13,6 +13,11 @@ from litellm.proxy._types import ( LiteLLM_UserTable, LiteLLM_VerificationToken, ) +from litellm.proxy.common_utils.timezone_utils import ( + BudgetResetSettings, + compute_budget_reset_at, + get_budget_reset_settings, +) from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.organization_repository import OrganizationRepository from litellm.repositories.table_repositories import ( @@ -32,9 +37,15 @@ class ResetBudgetJob: Resets the budget for all the keys, users, and teams that need it """ - def __init__(self, proxy_logging_obj: ProxyLogging, prisma_client: PrismaClient): + def __init__( + self, + proxy_logging_obj: ProxyLogging, + prisma_client: PrismaClient, + reset_settings: BudgetResetSettings | None = None, + ): self.proxy_logging_obj: ProxyLogging = proxy_logging_obj self.prisma_client: PrismaClient = prisma_client + self.reset_settings: BudgetResetSettings = reset_settings or get_budget_reset_settings() async def reset_budget( self, @@ -237,7 +248,7 @@ class ResetBudgetJob: if budgets_to_reset is not None and len(budgets_to_reset) > 0: for budget in budgets_to_reset: - budget = await ResetBudgetJob._reset_budget_reset_at_date(budget, now) + budget = await ResetBudgetJob._reset_budget_reset_at_date(budget, now, self.reset_settings) await self.prisma_client.update_data( query_type="update_many", @@ -442,7 +453,11 @@ class ResetBudgetJob: if keys_to_reset is not None and len(keys_to_reset) > 0: for key in keys_to_reset: try: - updated_key = await ResetBudgetJob._reset_budget_for_key(key=key, current_time=now) + updated_key = await ResetBudgetJob._reset_budget_for_key( + key=key, + current_time=now, + reset_settings=self.reset_settings, + ) if updated_key is not None: updated_keys.append(updated_key) else: @@ -513,7 +528,11 @@ class ResetBudgetJob: if users_to_reset is not None and len(users_to_reset) > 0: for user in users_to_reset: try: - updated_user = await ResetBudgetJob._reset_budget_for_user(user=user, current_time=now) + updated_user = await ResetBudgetJob._reset_budget_for_user( + user=user, + current_time=now, + reset_settings=self.reset_settings, + ) if updated_user is not None: updated_users.append(updated_user) else: @@ -588,7 +607,11 @@ class ResetBudgetJob: if teams_to_reset is not None and len(teams_to_reset) > 0: for team in teams_to_reset: try: - updated_team = await ResetBudgetJob._reset_budget_for_team(team=team, current_time=now) + updated_team = await ResetBudgetJob._reset_budget_for_team( + team=team, + current_time=now, + reset_settings=self.reset_settings, + ) if updated_team is not None: updated_teams.append(updated_team) else: @@ -655,10 +678,9 @@ class ResetBudgetJob: counter_key: str, spend_counter_cache: Any, now: datetime, + reset_settings: BudgetResetSettings, ) -> bool: """Reset a single budget window if expired. Returns True if the window was reset.""" - from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time - reset_at_str = window.get("reset_at") if not reset_at_str: return False @@ -671,7 +693,9 @@ class ResetBudgetJob: await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=0.0) except Exception as redis_err: verbose_proxy_logger.warning("Failed to reset Redis counter %s: %s", counter_key, redis_err) - window["reset_at"] = get_budget_reset_time(budget_duration=window["budget_duration"]).isoformat() + window["reset_at"] = compute_budget_reset_at( + budget_duration=window["budget_duration"], settings=reset_settings + ).isoformat() return True async def reset_budget_windows(self) -> None: @@ -703,7 +727,13 @@ class ResetBudgetJob: changed = False for window in windows: counter_key = f"spend:key:{row['token']}:window:{window['budget_duration']}" - if await ResetBudgetJob._reset_expired_window(window, counter_key, spend_counter_cache, now): + if await ResetBudgetJob._reset_expired_window( + window, + counter_key, + spend_counter_cache, + now, + self.reset_settings, + ): changed = True if changed: await VerificationTokenRepository(self.prisma_client).table.update( @@ -726,7 +756,13 @@ class ResetBudgetJob: changed = False for window in windows: counter_key = f"spend:team:{row['team_id']}:window:{window['budget_duration']}" - if await ResetBudgetJob._reset_expired_window(window, counter_key, spend_counter_cache, now): + if await ResetBudgetJob._reset_expired_window( + window, + counter_key, + spend_counter_cache, + now, + self.reset_settings, + ): changed = True if changed: await TeamRepository(self.prisma_client).table.update( @@ -741,6 +777,7 @@ class ResetBudgetJob: item: Union[LiteLLM_TeamTable, LiteLLM_UserTable, LiteLLM_VerificationToken], current_time: datetime, item_type: Literal["key", "team", "user"], + reset_settings: BudgetResetSettings, ): """ In-place, updates spend=0, and sets budget_reset_at to current_time + budget_duration @@ -755,24 +792,40 @@ class ResetBudgetJob: try: item.spend = 0.0 if hasattr(item, "budget_duration") and item.budget_duration is not None: - from litellm.proxy.common_utils.timezone_utils import ( - get_budget_reset_time, + item.budget_reset_at = compute_budget_reset_at( + budget_duration=item.budget_duration, settings=reset_settings ) - - item.budget_reset_at = get_budget_reset_time(budget_duration=item.budget_duration) return item except Exception as e: verbose_proxy_logger.exception("Error resetting budget for %s: %s. Item: %s", item_type, e, item) raise e @staticmethod - async def _reset_budget_for_team(team: LiteLLM_TeamTable, current_time: datetime) -> Optional[LiteLLM_TeamTable]: - await ResetBudgetJob._reset_budget_common(item=team, current_time=current_time, item_type="team") + async def _reset_budget_for_team( + team: LiteLLM_TeamTable, + current_time: datetime, + reset_settings: BudgetResetSettings, + ) -> LiteLLM_TeamTable | None: + await ResetBudgetJob._reset_budget_common( + item=team, + current_time=current_time, + item_type="team", + reset_settings=reset_settings, + ) return team @staticmethod - async def _reset_budget_for_user(user: LiteLLM_UserTable, current_time: datetime) -> Optional[LiteLLM_UserTable]: - await ResetBudgetJob._reset_budget_common(item=user, current_time=current_time, item_type="user") + async def _reset_budget_for_user( + user: LiteLLM_UserTable, + current_time: datetime, + reset_settings: BudgetResetSettings, + ) -> LiteLLM_UserTable | None: + await ResetBudgetJob._reset_budget_common( + item=user, + current_time=current_time, + item_type="user", + reset_settings=reset_settings, + ) return user @staticmethod @@ -788,15 +841,15 @@ class ResetBudgetJob: @staticmethod async def _reset_budget_reset_at_date( - budget: LiteLLM_BudgetTableFull, current_time: datetime + budget: LiteLLM_BudgetTableFull, + current_time: datetime, + reset_settings: BudgetResetSettings, ) -> LiteLLM_BudgetTableFull: try: if budget.budget_duration is not None: - from litellm.proxy.common_utils.timezone_utils import ( - get_budget_reset_time, + budget.budget_reset_at = compute_budget_reset_at( + budget_duration=budget.budget_duration, settings=reset_settings ) - - budget.budget_reset_at = get_budget_reset_time(budget_duration=budget.budget_duration) except Exception as e: verbose_proxy_logger.exception("Error resetting budget_reset_at for budget: %s. Item: %s", e, budget) raise e @@ -804,7 +857,14 @@ class ResetBudgetJob: @staticmethod async def _reset_budget_for_key( - key: LiteLLM_VerificationToken, current_time: datetime - ) -> Optional[LiteLLM_VerificationToken]: - await ResetBudgetJob._reset_budget_common(item=key, current_time=current_time, item_type="key") + key: LiteLLM_VerificationToken, + current_time: datetime, + reset_settings: BudgetResetSettings, + ) -> LiteLLM_VerificationToken | None: + await ResetBudgetJob._reset_budget_common( + item=key, + current_time=current_time, + item_type="key", + reset_settings=reset_settings, + ) return key diff --git a/litellm/proxy/common_utils/timezone_utils.py b/litellm/proxy/common_utils/timezone_utils.py index 32f9f47d519..a50daf40144 100644 --- a/litellm/proxy/common_utils/timezone_utils.py +++ b/litellm/proxy/common_utils/timezone_utils.py @@ -1,10 +1,47 @@ -from datetime import datetime, timezone +from datetime import datetime, time, timezone + +from pydantic import BaseModel, ConfigDict import litellm from litellm.litellm_core_utils.duration_parser import get_next_standardized_reset_time -def get_budget_reset_timezone(): +class BudgetResetSettings(BaseModel): + """Immutable, validated settings that govern when budgets reset. + + Parsed once from `litellm_settings` and injected into consumers (the reset + job, management endpoints) so reset times never depend on reaching into + module-level globals at call time. + """ + + model_config = ConfigDict(frozen=True) + + timezone: str = "UTC" + reset_time_of_day: time = time(0, 0) + + +def parse_budget_reset_time(raw: object) -> time: + """Parse a `budget_reset_time` config value (e.g. "12:00") into a `time`. + + Falls back to midnight when unset; raises a clear error on a malformed value + so a bad config fails loudly at startup instead of silently resetting at midnight. + """ + if raw is None or raw == "": + return time(0, 0) + if not isinstance(raw, str): + raise ValueError(f"Invalid budget_reset_time {raw!r}; must be a quoted 24-hour 'HH:MM' string, e.g. \"12:00\"") + for fmt in ("%H:%M", "%H:%M:%S"): + try: + parsed = datetime.strptime(raw, fmt) + return time(hour=parsed.hour, minute=parsed.minute, second=parsed.second) + except ValueError: + continue + raise ValueError( + f"Invalid budget_reset_time {raw!r}; expected a 24-hour 'HH:MM' or 'HH:MM:SS' string, e.g. \"12:00\"" + ) + + +def get_budget_reset_timezone() -> str: """ Get the budget reset timezone from litellm_settings. Falls back to UTC if not specified. @@ -15,15 +52,29 @@ def get_budget_reset_timezone(): return getattr(litellm, "timezone", None) or "UTC" -def get_budget_reset_time(budget_duration: str) -> datetime: - """ - Get the budget reset time based on the configured timezone. - Falls back to UTC if not specified. - """ +def get_budget_reset_settings() -> BudgetResetSettings: + """Build validated reset settings from litellm_settings. Raises on a malformed + `budget_reset_time`, which lets the proxy fail fast at startup.""" + return BudgetResetSettings( + timezone=get_budget_reset_timezone(), + reset_time_of_day=parse_budget_reset_time(getattr(litellm, "budget_reset_time", None)), + ) - reset_at = get_next_standardized_reset_time( + +def compute_budget_reset_at(budget_duration: str, settings: BudgetResetSettings) -> datetime: + """Compute the next reset time for a budget duration using injected settings.""" + return get_next_standardized_reset_time( duration=budget_duration, current_time=datetime.now(timezone.utc), - timezone_str=get_budget_reset_timezone(), + timezone_str=settings.timezone, + reset_time_of_day=settings.reset_time_of_day, ) - return reset_at + + +def get_budget_reset_time(budget_duration: str) -> datetime: + """Get the budget reset time using the globally-configured timezone and reset time. + + Thin wrapper over `compute_budget_reset_at` for callers that don't yet receive + `BudgetResetSettings` by injection (creation/update endpoints, startup backfill). + """ + return compute_budget_reset_at(budget_duration, get_budget_reset_settings()) diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/__init__.py index 5e62ab96f0c..d91ddffa0c1 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/__init__.py @@ -27,6 +27,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" mask_response_content=litellm_params.mask_response_content, fail_on_error=litellm_params.fail_on_error, skip_unscannable_attachments=litellm_params.skip_unscannable_attachments, + sanitize_error_detail=litellm_params.sanitize_error_detail, ) litellm.logging_callback_manager.add_litellm_callback(_model_armor_callback) diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index 32a3cebfca0..31535a5b569 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -11,6 +11,7 @@ from typing import ( Union, ) +import httpx from fastapi import HTTPException if TYPE_CHECKING: @@ -35,7 +36,8 @@ from litellm.proxy.guardrails.guardrail_hooks.model_armor.file_scanning import ( MODEL_ARMOR_MAX_FILE_SIZE_BYTES, plan_file_scans, ) -from litellm.types.guardrails import GuardrailEventHooks +from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.types.guardrails import GuardrailEventHooks, LitellmParams from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ( CallTypes, @@ -50,6 +52,33 @@ from litellm.types.utils import ( GUARDRAIL_NAME = "model_armor" +class ModelArmorAPIError(Exception): + """Model Armor API failure (non-2xx), distinct from a content-block decision so + hooks can honor fail_on_error. The detail is already sanitized per configuration.""" + + def __init__(self, detail: str): + super().__init__(detail) + self.detail = detail + + +_SCANNED_CONTENT_KEYS = frozenset({"text", "sanitizedText", "findings", "maliciousUriMatchedItems"}) + +RedactablePayload = Union[dict, list, str, int, float, bool, None] + + +def _redact_scanned_content(payload: RedactablePayload, depth: int = 0) -> RedactablePayload: + if depth >= DEFAULT_MAX_RECURSE_DEPTH: + return "[REDACTED]" + if isinstance(payload, dict): + return { + key: "[REDACTED]" if key in _SCANNED_CONTENT_KEYS else _redact_scanned_content(value, depth + 1) + for key, value in payload.items() + } + if isinstance(payload, list): + return [_redact_scanned_content(item, depth + 1) for item in payload] + return payload + + class ModelArmorGuardrail(CustomGuardrail, VertexBase): """ Google Cloud Model Armor Guardrail integration for LiteLLM. @@ -76,6 +105,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): location: Optional[str] = None, credentials: Optional[Any] = None, api_endpoint: Optional[str] = None, + sanitize_error_detail: "bool | None" = True, **kwargs, ): # Set supported event hooks if not already provided @@ -98,6 +128,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): self.location = location or "us-central1" self.credentials = credentials self.api_endpoint = api_endpoint + self.sanitize_error_detail = sanitize_error_detail is not False # Store optional params self.optional_params = kwargs @@ -141,6 +172,67 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): verbose_proxy_logger.debug("Model Armor: Skipping non-ModelResponse type: %s", type(response).__name__) return "" + def _build_api_error_detail(self, status_code: int, response_text: str) -> str: + if self.sanitize_error_detail: + return f"Model Armor API error (upstream {status_code})" + return f"Model Armor API error (upstream {status_code}): {response_text}" + + def _build_block_error_detail(self, message: str, armor_response: RedactablePayload) -> dict: + if self.sanitize_error_detail: + return {"error": message} + return {"error": message, "model_armor_response": armor_response} + + def _build_logging_response(self, armor_response: RedactablePayload) -> RedactablePayload: + if self.sanitize_error_detail: + return _redact_scanned_content(armor_response) + return armor_response + + def _raise_if_fail_closed(self, e: ModelArmorAPIError) -> None: + if self.optional_params.get("fail_on_error", True): + raise e from None + + def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None: + super().update_in_memory_litellm_params(litellm_params) + self.sanitize_error_detail = self.sanitize_error_detail is not False + + def _log_request_debug( + self, + url: str, + body: dict, + file_bytes: "bytes | None", + file_type: "str | None", + ) -> None: + # Never log byteData: it is the full base64 of the scanned document. Log only its + # type and size so debug deployments cannot leak the contents the guardrail inspects. + if file_bytes is not None and file_type is not None: + verbose_proxy_logger.debug( + "Model Armor file request - URL: %s, byteDataType: %s, bytes: %d", + url, + file_type, + len(file_bytes), + ) + elif self.sanitize_error_detail: + verbose_proxy_logger.debug("Model Armor request - URL: %s", url) + else: + verbose_proxy_logger.debug( + "Model Armor request - URL: %s, Body: %s", + url, + body, + ) + + def _log_response_debug(self, status_code: int, response_text: str) -> None: + if self.sanitize_error_detail: + verbose_proxy_logger.debug( + "Model Armor response - Status: %s", + status_code, + ) + else: + verbose_proxy_logger.debug( + "Model Armor response - Status: %s, Body: %s", + status_code, + response_text, + ) + async def make_model_armor_request( self, content: Optional[str] = None, @@ -185,48 +277,37 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): "Authorization": f"Bearer {access_token}", } - # Never log byteData: it is the full base64 of the scanned document. Log only its - # type and size so debug deployments cannot leak the contents the guardrail inspects. - if file_bytes is not None and file_type is not None: - verbose_proxy_logger.debug( - "Model Armor file request - URL: %s, byteDataType: %s, bytes: %d", - url, - file_type, - len(file_bytes), - ) - else: - verbose_proxy_logger.debug( - "Model Armor request - URL: %s, Body: %s", - url, - body, - ) + self._log_request_debug(url=url, body=body, file_bytes=file_bytes, file_type=file_type) # Make request if self.async_handler is None: raise ValueError("Async handler not initialized") - response = await self.async_handler.post( - url=url, - json=body, - headers=headers, - ) + try: + response = await self.async_handler.post( + url=url, + json=body, + headers=headers, + ) + except httpx.HTTPStatusError as e: + detail = self._build_api_error_detail(e.response.status_code, e.response.text) + verbose_proxy_logger.error( + "Model Armor API error - Status: %s, Detail: %s", + e.response.status_code, + detail, + ) + raise ModelArmorAPIError(detail) from None - verbose_proxy_logger.debug( - "Model Armor response - Status: %s, Body: %s", - response.status_code, - response.text, - ) + self._log_response_debug(status_code=response.status_code, response_text=response.text) if response.status_code != 200: + detail = self._build_api_error_detail(response.status_code, response.text) verbose_proxy_logger.error( - "Model Armor API error - Status: %s, Response: %s", + "Model Armor API error - Status: %s, Detail: %s", response.status_code, - response.text, - ) - raise HTTPException( - status_code=400, - detail=f"Model Armor API error (upstream {response.status_code}): {response.text}", + detail, ) + raise ModelArmorAPIError(detail) json_response = response.json() if hasattr(json_response, "__await__"): @@ -351,9 +432,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): Override to store only the Model Armor API response, not the entire data dict. This prevents circular references in logging. """ - # Retrieve the Model Armor response & status stored on the per-request `metadata` object. metadata = request_data.get("metadata", {}) if isinstance(request_data, dict) else {} - guardrail_response = metadata.get("_model_armor_response", {}) # Determine status – default to "success" but prefer the explicit value if present. @@ -444,6 +523,9 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): file_bytes=attachment.file_bytes, file_type=attachment.byte_data_type, ) + except ModelArmorAPIError as e: + self._raise_if_fail_closed(e) + continue except HTTPException: raise except Exception as e: @@ -459,7 +541,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): # otherwise a PII-only (SDP deidentify) document would pass through unscrubbed. blocked = self._should_block_content(armor_response, allow_sanitization=False) metadata["_model_armor_response"] = self._append_armor_response( - metadata.get("_model_armor_response"), armor_response + metadata.get("_model_armor_response"), + self._build_logging_response(armor_response), ) if blocked or metadata.get("_model_armor_status") == "blocked": metadata["_model_armor_status"] = "blocked" @@ -469,10 +552,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): if blocked: raise HTTPException( status_code=400, - detail={ - "error": "Content blocked by Model Armor", - "model_armor_response": armor_response, - }, + detail=self._build_block_error_detail("Content blocked by Model Armor", armor_response), ) @log_guardrail_information @@ -530,7 +610,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): metadata = data.setdefault("metadata", {}) # ensures metadata exists and is unique per request # Accumulate so a prior file scan on the same request is not overwritten by this text scan. metadata["_model_armor_response"] = self._append_armor_response( - metadata.get("_model_armor_response"), armor_response + metadata.get("_model_armor_response"), + self._build_logging_response(armor_response), ) # Pre-compute guardrail status for downstream logging. A blocked response will eventually raise # an HTTPException, however in scenarios where the caller decides to ignore the exception (e.g. @@ -548,10 +629,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): if blocked: raise HTTPException( status_code=400, - detail={ - "error": "Content blocked by Model Armor", - "model_armor_response": armor_response, - }, + detail=self._build_block_error_detail("Content blocked by Model Armor", armor_response), ) # If mask_request_content is enabled, update messages with sanitized content @@ -565,6 +643,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): data["messages"] = set_last_user_message(messages, sanitized_content) + except ModelArmorAPIError as e: + self._raise_if_fail_closed(e) except HTTPException: raise except Exception as e: @@ -625,7 +705,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): metadata = data.setdefault("metadata", {}) # Accumulate so a prior file scan on the same request is not overwritten by this text scan. metadata["_model_armor_response"] = self._append_armor_response( - metadata.get("_model_armor_response"), armor_response + metadata.get("_model_armor_response"), + self._build_logging_response(armor_response), ) if blocked or metadata.get("_model_armor_status") == "blocked": metadata["_model_armor_status"] = "blocked" @@ -640,10 +721,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): if blocked: raise HTTPException( status_code=400, - detail={ - "error": "Content blocked by Model Armor", - "model_armor_response": armor_response, - }, + detail=self._build_block_error_detail("Content blocked by Model Armor", armor_response), ) # If mask_request_content is enabled, update messages with sanitized content @@ -656,6 +734,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): data["messages"] = set_last_user_message(messages, sanitized_content) + except ModelArmorAPIError as e: + self._raise_if_fail_closed(e) except HTTPException: raise except Exception as e: @@ -698,7 +778,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): # Attach Model Armor response & status to this request's metadata to prevent race conditions if isinstance(armor_response, dict): model_armor_logged_object = { - "model_armor_response": armor_response, + "model_armor_response": self._build_logging_response(armor_response), "model_armor_status": ( "blocked" if self._should_block_content( @@ -729,10 +809,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): if self._should_block_content(armor_response, allow_sanitization=self.mask_response_content): raise HTTPException( status_code=400, - detail={ - "error": "Response blocked by Model Armor", - "model_armor_response": armor_response, - }, + detail=self._build_block_error_detail("Response blocked by Model Armor", armor_response), ) # If mask_response_content is enabled, update response with sanitized content @@ -746,6 +823,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): if choice.message.content: choice.message.content = sanitized_content + except ModelArmorAPIError as e: + self._raise_if_fail_closed(e) except HTTPException: raise except Exception as e: @@ -790,7 +869,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): # Attach Model Armor response & status to this request's metadata to avoid race conditions if isinstance(request_data, dict): metadata = request_data.setdefault("metadata", {}) - metadata["_model_armor_response"] = armor_response + metadata["_model_armor_response"] = self._build_logging_response(armor_response) metadata["_model_armor_status"] = ( "blocked" if self._should_block_content(armor_response) else "success" ) @@ -809,10 +888,10 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): if self._should_block_content(armor_response): raise HTTPException( status_code=400, - detail={ - "error": "Streaming response blocked by Model Armor", - "model_armor_response": armor_response, - }, + detail=self._build_block_error_detail( + "Streaming response blocked by Model Armor", + armor_response, + ), ) # Apply sanitization if enabled @@ -831,6 +910,11 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): yield chunk return + except ModelArmorAPIError as e: + if self.optional_params.get("fail_on_error", True): + error_obj = {"message": e.detail, "code": "500"} + yield f"data: {json.dumps({'error': error_obj})}\n\n" + return except HTTPException as e: # Yield error as SSE event so create_response() detects it and # returns a proper JSON error response with the correct status code. diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 6514d4e1e8c..9d9ef28ec9b 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -19,7 +19,10 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( iter_client_callback_metadata_dicts, ) from litellm.litellm_core_utils.safe_json_loads import safe_json_loads -from litellm.litellm_core_utils.url_utils import is_url_destination_allowed_by_host +from litellm.litellm_core_utils.url_utils import ( + is_url_destination_allowed_by_host, + provider_url_destination_candidates, +) from litellm.proxy._types import ( AddTeamCallback, CommonProxyErrors, @@ -227,23 +230,26 @@ def _reject_url_valued_destinations(data: Dict[str, Any]) -> None: allowed_hosts = getattr(litellm, "provider_url_destination_allowed_hosts", []) or [] for field in _URL_DESTINATION_REQUEST_FIELDS: value = data.get(field) - if not isinstance(value, str) or not value.startswith(("http://", "https://")): + if not isinstance(value, str): continue - if is_url_destination_allowed_by_host(value, allowed_hosts): - continue - raise HTTPException( - status_code=400, - detail={ - "error": "invalid_request", - "param": field, - "message": ( - f"URL-valued '{field}' is not allowed. Configure custom " - "endpoints with api_base instead, or add the destination " - "host to `provider_url_destination_allowed_hosts` in " - "litellm_settings." - ), - }, - ) + for candidate in provider_url_destination_candidates(value): + if not candidate.lower().startswith(("http://", "https://")): + continue + if is_url_destination_allowed_by_host(candidate, allowed_hosts): + continue + raise HTTPException( + status_code=400, + detail={ + "error": "invalid_request", + "param": field, + "message": ( + f"URL-valued '{field}' is not allowed. Configure custom " + "endpoints with api_base instead, or add the destination " + "host to `provider_url_destination_allowed_hosts` in " + "litellm_settings." + ), + }, + ) def _strip_untrusted_request_header_controls( diff --git a/litellm/proxy/management_endpoints/cache_settings_endpoints.py b/litellm/proxy/management_endpoints/cache_settings_endpoints.py index 9f45cb619aa..7c0d8958a28 100644 --- a/litellm/proxy/management_endpoints/cache_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/cache_settings_endpoints.py @@ -18,8 +18,9 @@ from pydantic import BaseModel, Field import litellm from litellm._logging import verbose_proxy_logger +from litellm._redis import _redis_kwargs_from_environment from litellm._uuid import uuid -from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_keys +from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker from litellm.proxy._types import ( AUDIT_ACTIONS, LiteLLM_AuditLogs, @@ -43,6 +44,17 @@ router = APIRouter() # (e.g. redis://:secret@host:6379/1). _CACHE_SENSITIVE_FIELDS: set = {"password", "sentinel_password", "url"} +# The env fallback resolves the full set of redis.Redis kwargs, which includes +# credential-bearing params (azure_client_secret, ssl_password, ...) that are +# not cache UI fields. Only overlay fields the settings page actually renders, +# so the read never surfaces a credential the UI does not manage. +_CACHE_SETTINGS_FIELD_NAMES: frozenset = frozenset(field.field_name for field in CACHE_SETTINGS_FIELDS) + +# Classifier used, alongside _CACHE_SENSITIVE_FIELDS, to redact any +# credential-bearing key before it leaves the server (`url` is kept in the +# explicit set because its name carries no sensitive segment). +_CREDENTIAL_CLASSIFIER = SensitiveDataMasker() + _REDACTED_VALUE = "***REDACTED***" @@ -67,6 +79,165 @@ def _resolve_cache_url_precedence(settings: Mapping[str, Any]) -> dict[str, Any] return {k: v for k, v in settings.items() if k not in _URL_OVERRIDDEN_CONNECTION_FIELDS} +def _parse_stored_settings(cache_settings_value: object) -> dict[str, Any]: + """Normalize a stored cache_settings blob to a dict. + + The prisma column comes back as either a JSON string or an already-parsed + dict depending on the client, so callers that json.loads unconditionally + silently drop the whole (still-encrypted) row on the dict path. + """ + parsed = json.loads(cache_settings_value) if isinstance(cache_settings_value, str) else cache_settings_value + return parsed if isinstance(parsed, dict) else {} + + +def _overlay_environment(stored: Mapping[str, Any]) -> dict[str, Any]: + """Fill connection fields from the REDIS_* environment the cache actually reads. + + A response cache pointed at Redis resolves host/port/password/etc. from the + REDIS_* env vars when the stored config leaves them unset, so a cache + configured purely through the environment works while its settings page, + which reads only the database row, shows blank. Overlaying the same env + kwargs the runtime uses makes the page reflect the effective connection. + Stored values win; the environment only fills what the stored config omits. + """ + env_kwargs = { + key: value for key, value in _redis_kwargs_from_environment().items() if key in _CACHE_SETTINGS_FIELD_NAMES + } + if not env_kwargs: + return dict(stored) + effective = {**env_kwargs, **stored} + # the env fallback is a Redis connection, so name the type when the stored + # config did not, letting the UI render the Redis fields it just populated + effective.setdefault("type", "redis") + return effective + + +def _redact_credentials(settings: Mapping[str, Any]) -> dict[str, Any]: + """Replace credential-bearing values with a fixed marker, keeping the rest. + + The marker is unambiguous on the way back in: an admin who edits an + unrelated field and re-submits sends the marker for the untouched secret, + which the update path maps back to the stored value rather than persisting + the marker over a working password. + """ + return { + key: (_REDACTED_VALUE if value is not None and _is_credential_field(key) else value) + for key, value in settings.items() + } + + +def _is_credential_field(key: str) -> bool: + """Whether a cache setting carries a credential and must be redacted on read.""" + return key in _CACHE_SENSITIVE_FIELDS or _CREDENTIAL_CLASSIFIER.is_sensitive_key(key) + + +def _has_connection_target(value: object) -> bool: + """Whether a payload value names a live discrete connection target.""" + if isinstance(value, str): + return value.strip() != "" and value != _REDACTED_VALUE + return value not in (None, [], {}) + + +# Every field that identifies which Redis a credential belongs to, across node +# (host/port/url), cluster (redis_startup_nodes), and sentinel +# (sentinel_nodes/service_name) modes. A stored secret is bound to these. +_CONNECTION_TARGET_FIELDS: tuple = ( + "host", + "port", + "url", + "redis_startup_nodes", + "sentinel_nodes", + "service_name", +) + + +def _target_repr(value: object) -> str: + """Canonical string form of a connection-target value for equality checks. + + The client may serialize the same target differently from storage (a port as + "6379" vs 6379, node lists round-tripped through JSON), so compare normalized + forms rather than raw values to avoid treating an unchanged target as a change. + """ + if isinstance(value, (list, dict)): + return json.dumps(value, sort_keys=True, default=str) + return str(value) + + +def _saved_secret_is_reusable(incoming: Mapping[str, Any], saved: Mapping[str, Any]) -> bool: + """Whether a stored credential may be restored for this request. + + A stored secret belongs to the stored connection target, so it is reused only + when the request describes that same target on every dimension the stored + config pins (host/port, url, cluster nodes, sentinel nodes/service). This + prevents credential replay: a caller cannot omit the credential, point at a + different (or incomplete) target, and have the proxy send the stored secret + to a Redis of their choosing. + + Non-secret target fields (host/port/nodes/service) must be supplied and match + in normalized form, so equivalent representations (port "6379" vs 6379) are + not seen as a change while an omitted or different value is. ``url`` is the + exception: it is itself the secret and the form never re-prefills it, so a + redacted or omitted url means "keep the stored url" (same target) and only a + different supplied url blocks reuse. + """ + for field in _CONNECTION_TARGET_FIELDS: + saved_value = saved.get(field) + if saved_value in (None, "", [], {}): + continue # the stored config does not pin this dimension + incoming_value = incoming.get(field) + if field == "url": + if incoming_value in (None, "", _REDACTED_VALUE): + continue # url kept as-is (same target) + if _target_repr(incoming_value) != _target_repr(saved_value): + return False + continue + if _target_repr(incoming_value) != _target_repr(saved_value): + return False # a pinned target field is missing or different + return True + + +def _merge_over_saved(incoming: Mapping[str, Any], saved: Mapping[str, Any]) -> dict[str, Any]: + """Keep the stored secret behind any credential the caller echoed back redacted or omitted. + + GET returns credentials as the marker and the form never re-prefills a + secret, so a save that does not touch a credential arrives with the marker + or with the field absent. Either way the real secret must survive: it is + restored from the stored row, or dropped when there is no stored row (the + value is env-sourced and the marker must never be persisted). Non-secret + fields are taken from the incoming payload as-is, so clearing one still works. + + ``url`` is the exception: it is credential-bearing (redacted) yet also a + connection-mode selector that url-precedence resolves against host/port. If + the caller supplies a discrete target (host, cluster, or sentinel nodes), a + stored url is a stale mode the caller is leaving, so it is dropped rather + than restored, otherwise url-precedence would resurrect it and discard the + submitted host/port. + """ + switching_to_discrete_target = ( + _has_connection_target(incoming.get("host")) + or _has_connection_target(incoming.get("redis_startup_nodes")) + or _has_connection_target(incoming.get("sentinel_nodes")) + ) + reuse_saved_secret = _saved_secret_is_reusable(incoming, saved) + merged = dict(incoming) + for field in _CACHE_SENSITIVE_FIELDS: + # A value the caller explicitly supplied is honored verbatim: a new + # secret, or an empty string / null to clear the stored one. Only an + # omitted field or the echoed-back marker triggers preserve-or-drop. + if field in incoming and incoming[field] != _REDACTED_VALUE: + continue + if field == "url" and switching_to_discrete_target: + merged.pop(field, None) + continue + if field in saved and reuse_saved_secret: + merged[field] = saved[field] + else: + # nothing stored to reuse, or the caller is pointing at a different + # target: never persist/replay the marker or the stored secret + merged.pop(field, None) + return merged + + def _redact_settings(settings: Optional[Mapping[str, Any]]) -> Dict[str, Any]: """Replace every value in a settings map with a fixed marker. @@ -270,34 +441,34 @@ async def get_cache_settings( # Get cache settings fields from types file cache_fields = [field.model_copy(deep=True) for field in CACHE_SETTINGS_FIELDS] - # Try to get cache settings from database - current_values = {} + # Read the stored settings (decrypted); an env-only cache has none. + stored: dict[str, Any] = {} if prisma_client is not None: cache_config = await CacheConfigRepository(prisma_client).table.find_unique(where={"id": "cache_config"}) if cache_config is not None and cache_config.cache_settings: - # Decrypt cache settings - cache_settings_json = cache_config.cache_settings - if isinstance(cache_settings_json, str): - cache_settings_dict = json.loads(cache_settings_json) - else: - cache_settings_dict = cache_settings_json + stored = proxy_config._decrypt_db_variables( + variables_dict=_parse_stored_settings(cache_config.cache_settings) + ) - # Decrypt environment variables - decrypted_settings = proxy_config._decrypt_db_variables(variables_dict=cache_settings_dict) + # Fill connection fields from the REDIS_* environment the cache resolves + # from when the stored config leaves them unset, then apply url precedence + # so a url-mode config does not surface conflicting discrete fields (which + # would otherwise let a no-op save silently switch it to host/port). + effective = _resolve_cache_url_precedence(_overlay_environment(stored)) - # Derive redis_type for UI based on settings - # UI uses redis_type to show/hide fields, backend only stores 'type' - if decrypted_settings.get("type") == "redis": - if decrypted_settings.get("redis_startup_nodes"): - decrypted_settings["redis_type"] = "cluster" - elif decrypted_settings.get("sentinel_nodes"): - decrypted_settings["redis_type"] = "sentinel" - else: - decrypted_settings["redis_type"] = "node" + # Derive redis_type for UI based on settings + # UI uses redis_type to show/hide fields, backend only stores 'type' + if effective.get("type") == "redis": + if effective.get("redis_startup_nodes"): + effective["redis_type"] = "cluster" + elif effective.get("sentinel_nodes"): + effective["redis_type"] = "sentinel" + else: + effective["redis_type"] = "node" - # Mask credential fields so the GET response never carries - # plaintext Redis / Sentinel passwords off the server. - current_values = mask_sensitive_keys(decrypted_settings, _CACHE_SENSITIVE_FIELDS) + # Redact credential fields so the GET response never carries a plaintext + # Redis / Sentinel password off the server. + current_values = _redact_credentials(effective) # Update field values with current values for field in cache_fields: @@ -331,10 +502,27 @@ async def test_cache_connection( to verify the credentials work without affecting global state. """ from litellm import Cache + from litellm.proxy.proxy_server import prisma_client, proxy_config try: - cache_settings = _resolve_cache_url_precedence(request.cache_settings) - verbose_proxy_logger.debug("Testing cache connection with settings: %s", cache_settings) + # A credential the form left untouched arrives redacted; resolve it back + # to the stored secret so the test connects with the real password. A + # lookup failure must not block the test, so fall back to no stored row. + saved_settings: dict[str, Any] = {} + if prisma_client is not None: + try: + existing_row = await CacheConfigRepository(prisma_client).table.find_unique( + where={"id": "cache_config"} + ) + if existing_row is not None and existing_row.cache_settings: + saved_settings = proxy_config._decrypt_db_variables( + variables_dict=_parse_stored_settings(existing_row.cache_settings) + ) + except Exception: # noqa: BLE001 - a saved-settings lookup failure must not block a connection test + saved_settings = {} + cache_settings = _resolve_cache_url_precedence(_merge_over_saved(request.cache_settings, saved_settings)) + # cache_settings now carries the resolved plaintext credential; never log it raw + verbose_proxy_logger.debug("Testing cache connection with settings: %s", _redact_credentials(cache_settings)) # Only support Redis for now if cache_settings.get("type") != "redis": @@ -400,19 +588,20 @@ async def update_cache_settings( ) try: - cache_settings = _resolve_cache_url_precedence(request.cache_settings) - - # Snapshot the prior settings (key set only — values get redacted in - # the audit row) so the audit-log entry shows which fields changed. + # Read the stored row first: its decrypted values back any credential the + # caller echoed back redacted, and its key set drives the audit diff. existing_row = await CacheConfigRepository(prisma_client).table.find_unique(where={"id": "cache_config"}) before_settings: Optional[Dict[str, Any]] = None + saved_settings: dict[str, Any] = {} if existing_row is not None and existing_row.cache_settings: - try: - before_settings = json.loads(existing_row.cache_settings) - except (TypeError, ValueError): - before_settings = None + before_settings = _parse_stored_settings(existing_row.cache_settings) + saved_settings = proxy_config._decrypt_db_variables(variables_dict=before_settings) action: AUDIT_ACTIONS = "updated" if existing_row is not None else "created" + # Preserve stored secrets behind any redacted or omitted credential, then + # resolve the url-vs-discrete-fields precedence. + cache_settings = _resolve_cache_url_precedence(_merge_over_saved(request.cache_settings, saved_settings)) + # Encrypt sensitive fields (keep redis_type for storage) encrypted_settings = proxy_config._encrypt_env_variables(environment_variables=cache_settings) @@ -461,7 +650,7 @@ async def update_cache_settings( return { "message": "Cache settings updated successfully", "status": "success", - "settings": cache_settings, + "settings": _redact_credentials(cache_settings), } except Exception as e: verbose_proxy_logger.error(f"Error updating cache settings: {str(e)}") diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 01f4e040e58..ac6a2a4a7db 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -42,6 +42,9 @@ from litellm.proxy._experimental.mcp_server.db import ( rotate_mcp_user_credentials_master_key, rotate_mcp_user_env_vars_master_key, ) +from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( + rotate_sso_identity_assertions_master_key, +) from litellm.proxy._types import * from litellm.proxy._types import LiteLLM_VerificationToken, hash_token from litellm.proxy.auth.auth_checks import ( @@ -4242,6 +4245,15 @@ async def _rotate_master_key( except Exception as e: verbose_proxy_logger.warning("Failed to rotate MCP user env vars: %s", str(e)) + # 4d. process SSO identity assertion table (EMA subject tokens) + try: + await rotate_sso_identity_assertions_master_key( + prisma_client=prisma_client, + new_master_key=new_master_key, + ) + except Exception as e: # noqa: BLE001 # one store's failure must not abort the master-key rotation + verbose_proxy_logger.warning("Failed to rotate SSO identity assertions: %s", str(e)) + # 5. process credentials table try: credentials = await CredentialsRepository(prisma_client).table.find_many() diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 6c2e06a418c..3c8444ecf26 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -62,6 +62,11 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) +from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( + SSOIdentityAssertion, + assertion_from_sso_login, + retain_sso_identity_assertion_for_ema, +) from litellm.proxy._types import ( CommonProxyErrors, LiteLLM_UserTable, @@ -1311,12 +1316,15 @@ async def get_generic_sso_response( sso_jwt_handler: Optional[JWTHandler], # sso specific jwt handler - used for restricted sso group access control generic_client_id: str, redirect_url: str, -) -> Tuple[Union[OpenID, dict], Optional[dict], Optional[dict]]: # (result, received_response, access_token_payload) +) -> tuple[ + Union[OpenID, dict], dict | None, dict | None, SSOIdentityAssertion | None +]: # (result, received_response, access_token_payload, sso_assertion) # make generic sso provider from fastapi_sso.sso.base import DiscoveryDocument from fastapi_sso.sso.generic import create_provider received_response: Optional[dict] = None + sso_assertion: SSOIdentityAssertion | None = None # Setup environment variables ( @@ -1450,6 +1458,9 @@ async def get_generic_sso_response( # Assign directly rather than relying on nonlocal mutation so that Pyright # can track that received_response is non-None from this point on. received_response = {k: v for k, v in combined_response.items() if k not in _OAUTH_TOKEN_FIELDS} + sso_assertion = assertion_from_sso_login( + combined_response.get("id_token"), combined_response.get("refresh_token") + ) # In the PKCE path verify_and_process is skipped, so generic_sso.access_token # is never set. Read the token directly from the exchange response instead so # process_sso_jwt_access_token can extract JWT-embedded roles/teams. @@ -1461,6 +1472,7 @@ async def get_generic_sso_response( headers=additional_generic_sso_headers_dict, ) access_token_str = generic_sso.access_token + sso_assertion = assertion_from_sso_login(generic_sso.id_token, generic_sso.refresh_token) access_token_payload = process_sso_jwt_access_token( access_token_str, sso_jwt_handler, result, role_mappings=role_mappings @@ -1480,7 +1492,7 @@ async def get_generic_sso_response( additional_generic_sso_headers_dict, ) verbose_proxy_logger.debug("generic result: %s", result) - return result or {}, received_response, access_token_payload + return result or {}, received_response, access_token_payload, sso_assertion async def create_team_member_add_task(team_id, user_info): @@ -1812,6 +1824,7 @@ async def auth_callback(request: Request, state: Optional[str] = None): generic_client_id = os.getenv("GENERIC_CLIENT_ID", None) received_response: Optional[dict] = None access_token_payload: Optional[dict] = None + sso_assertion: SSOIdentityAssertion | None = None # get url from request if master_key is None: raise ProxyException( @@ -1842,6 +1855,7 @@ async def auth_callback(request: Request, state: Optional[str] = None): result, received_response, access_token_payload, + sso_assertion, ) = await get_generic_sso_response( request=request, jwt_handler=jwt_handler, @@ -1869,6 +1883,7 @@ async def auth_callback(request: Request, state: Optional[str] = None): prefill_user_code=prefill_user_code, result=result, received_response=received_response, + sso_assertion=sso_assertion, ) # Control-plane cross-origin: read return_to from cookie. @@ -1884,6 +1899,7 @@ async def auth_callback(request: Request, state: Optional[str] = None): access_token_payload=access_token_payload, jwt_handler=jwt_handler, return_to=cp_return_to, + sso_assertion=sso_assertion, ) @@ -1943,6 +1959,7 @@ async def _complete_cli_sso_callback_session( user_api_key_cache: UserApiKeyCache, proxy_logging_obj: ProxyLogging, prefill_user_code: str | None = None, + sso_assertion: SSOIdentityAssertion | None = None, ): from fastapi.responses import HTMLResponse @@ -1962,6 +1979,8 @@ async def _complete_cli_sso_callback_session( if not user_info.user_id: raise HTTPException(status_code=500, detail="Failed to retrieve user information from SSO") + await retain_sso_identity_assertion_for_ema(user_id=user_info.user_id, assertion=sso_assertion) + teams: List[str] = [] if hasattr(user_info, "teams") and user_info.teams: teams = user_info.teams if isinstance(user_info.teams, list) else [] @@ -2012,6 +2031,7 @@ async def cli_sso_callback( result: Optional[Union[OpenID, dict]] = None, received_response: Optional[dict] = None, prefill_user_code: str | None = None, + sso_assertion: SSOIdentityAssertion | None = None, ): """CLI SSO callback - stores session info for JWT generation on polling""" verbose_proxy_logger.info("CLI SSO callback") @@ -2065,6 +2085,7 @@ async def cli_sso_callback( user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, prefill_user_code=prefill_user_code, + sso_assertion=sso_assertion, ) except ProxyException: raise @@ -3018,6 +3039,7 @@ class SSOAuthenticationHandler: access_token_payload: Optional[dict] = None, jwt_handler: Optional[JWTHandler] = None, return_to: Optional[str] = None, + sso_assertion: SSOIdentityAssertion | None = None, ) -> RedirectResponse: import jwt @@ -3148,6 +3170,9 @@ class SSOAuthenticationHandler: }, ) + if isinstance(user_id, str) and user_id: + await retain_sso_identity_assertion_for_ema(user_id=user_id, assertion=sso_assertion) + disabled_non_admin_personal_key_creation = get_disabled_non_admin_personal_key_creation() litellm_dashboard_ui = get_custom_url(request_base_url=str(request.base_url), route="ui/") @@ -4241,6 +4266,7 @@ async def debug_sso_callback(request: Request): result, received_response, access_token_payload, + _sso_assertion, ) = await get_generic_sso_response( request=request, jwt_handler=jwt_handler, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3b40abed19e..6de3e43fc1a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -236,6 +236,7 @@ from litellm.constants import ( PROXY_BATCH_WRITE_AT, PROXY_BUDGET_RESCHEDULER_MAX_TIME, PROXY_BUDGET_RESCHEDULER_MIN_TIME, + PROXY_CONFIG_RELOAD_INTERVAL_SECONDS, ) from litellm.exceptions import RejectedRequestError from litellm.integrations.custom_guardrail import ModifyResponseException @@ -319,7 +320,10 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( from litellm.proxy.common_utils.proxy_state import ProxyState from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob from litellm.proxy.common_utils.swagger_utils import ERROR_RESPONSES -from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time +from litellm.proxy.common_utils.timezone_utils import ( + get_budget_reset_settings, + get_budget_reset_time, +) from litellm.proxy.common_utils.user_api_key_cache import ( UserApiKeyCache, get_management_object_ttl, @@ -1998,6 +2002,7 @@ proxy_budget_rescheduler_min_time = PROXY_BUDGET_RESCHEDULER_MIN_TIME proxy_budget_rescheduler_max_time = PROXY_BUDGET_RESCHEDULER_MAX_TIME proxy_batch_polling_interval = PROXY_BATCH_POLLING_INTERVAL proxy_batch_write_at = PROXY_BATCH_WRITE_AT +proxy_config_reload_interval_seconds = PROXY_CONFIG_RELOAD_INTERVAL_SECONDS litellm_master_key_hash = None disable_spend_logs = False jwt_handler = JWTHandler() @@ -3879,7 +3884,7 @@ class ProxyConfig: del config["include"] return config - async def save_config(self, new_config: dict): + async def save_config(self, new_config: dict, include_env_vars: bool = False): global prisma_client, general_settings, user_config_file_path, store_model_in_db # Load existing config ## DB - writes valid config to db @@ -3896,6 +3901,17 @@ class ProxyConfig: # Make a copy to avoid mutating the original config config_to_save = new_config.copy() + # environment_variables are persisted to the DB only when a caller + # explicitly opts in. Most callers reach save_config after + # get_config() merged YAML + OS env into new_config (with + # os.environ/ placeholders already resolved to plaintext), so + # persisting them here would snapshot file/container env vars into + # a config row that then shadows those sources on every restart. + # The dedicated /config/update path writes env vars directly, so + # no current caller needs include_env_vars=True. + if not include_env_vars: + config_to_save.pop("environment_variables", None) + # SECURITY: Always encrypt environment_variables before DB write. # _encrypt_env_variables_for_db is idempotent — a caller that # already encrypted the values (or re-submitted ciphertext read @@ -3913,6 +3929,38 @@ class ProxyConfig: with open(f"{user_config_file_path}", "w") as config_file: yaml.dump(new_config, config_file, default_flow_style=False) + async def save_environment_variables(self, updates: dict[str, str | None]) -> None: + """Persist specific environment variables to the DB config row. + + Each key in ``updates`` is written to the ``environment_variables`` + config row; a ``None`` value deletes that key. Env vars the caller does + not name are preserved, so a caller that owns a couple of keys can + update just those without snapshotting unrelated (YAML/OS-sourced) + values the way a full ``save_config`` write would. No-op when config is + not DB-backed. + """ + global prisma_client, general_settings, store_model_in_db + if prisma_client is None or not (general_settings.get("store_model_in_db", False) is True or store_model_in_db): + return + + row = await ConfigRepository(prisma_client).table.find_first(where={"param_name": "environment_variables"}) + existing: dict = dict(row.param_value) if row is not None and row.param_value is not None else {} + + to_set = {k: v for k, v in updates.items() if v is not None} + encrypted = self._encrypt_env_variables_for_db(environment_variables=to_set) if to_set else {} + deleted_keys = {k for k, v in updates.items() if v is None} + merged = {**{k: v for k, v in existing.items() if k not in deleted_keys}, **encrypted} + + serialized = json.dumps(merged) + await ConfigRepository(prisma_client).table.upsert( + where={"param_name": "environment_variables"}, + data={ + "create": {"param_name": "environment_variables", "param_value": serialized}, + "update": {"param_value": serialized}, + }, + ) + await invalidate_config_param("environment_variables") + def _check_for_os_environ_vars( self, config: dict, depth: int = 0, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH ) -> dict: @@ -4291,6 +4339,7 @@ class ProxyConfig: open_telemetry_logger, \ health_check_details, \ proxy_batch_polling_interval, \ + proxy_config_reload_interval_seconds, \ config_passthrough_endpoints config: dict = await self.get_config(config_file_path=config_file_path) @@ -4597,6 +4646,13 @@ class ProxyConfig: litellm.json_logs = True litellm._turn_on_json() verbose_proxy_logger.debug(f"{blue_color_code} Enabled JSON logging via config{reset_color_code}") + elif key == "budget_reset_time": + from litellm.proxy.common_utils.timezone_utils import ( + parse_budget_reset_time, + ) + + parse_budget_reset_time(value) + setattr(litellm, key, value) else: verbose_proxy_logger.debug( f"{blue_color_code} setting litellm.{key}={_redact_general_setting_value(key, value, is_full_admin=False)}{reset_color_code}" @@ -4773,6 +4829,10 @@ class ProxyConfig: ) ## BATCH WRITER ## proxy_batch_write_at = general_settings.get("proxy_batch_write_at", proxy_batch_write_at) + ## DB CONFIG RELOAD INTERVAL ## + proxy_config_reload_interval_seconds = general_settings.get( + "proxy_config_reload_interval_seconds", proxy_config_reload_interval_seconds + ) ## DISABLE SPEND LOGS ## - gives a perf improvement disable_spend_logs = general_settings.get("disable_spend_logs", disable_spend_logs) ### BACKGROUND HEALTH CHECKS ### @@ -7868,6 +7928,7 @@ class ProxyStartupEvent: budget_reset_job = ResetBudgetJob( proxy_logging_obj=proxy_logging_obj, prisma_client=prisma_client, + reset_settings=get_budget_reset_settings(), ) scheduler.add_job( @@ -7944,12 +8005,20 @@ class ProxyStartupEvent: verbose_proxy_logger.debug("Failed to check DB for store_model_in_db: %s", str(e)) if store_model_in_db is True: + config_reload_interval_seconds = proxy_config_reload_interval_seconds + if not isinstance(config_reload_interval_seconds, int) or config_reload_interval_seconds <= 0: + verbose_proxy_logger.warning( + "proxy_config_reload_interval_seconds=%s must be a positive integer; falling back to 30s", + config_reload_interval_seconds, + ) + config_reload_interval_seconds = 30 + # MEMORY LEAK FIX: Increase interval from 10s to 30s minimum # Frequent polling was causing excessive memory allocations scheduler.add_job( proxy_config.add_deployment, "interval", - seconds=30, # increased from 10s to reduce memory pressure + seconds=config_reload_interval_seconds, # REMOVED jitter parameter - major cause of memory leak args=[prisma_client, proxy_logging_obj], id="add_deployment_job", @@ -7964,7 +8033,7 @@ class ProxyStartupEvent: scheduler.add_job( proxy_config.get_credentials, "interval", - seconds=30, # increased from 10s to reduce memory pressure + seconds=config_reload_interval_seconds, # REMOVED jitter parameter - major cause of memory leak args=[prisma_client], id="get_credentials_job", @@ -14998,6 +15067,7 @@ async def get_config_list( "global_max_parallel_requests": {"type": "Integer"}, "max_request_size_mb": {"type": "Integer"}, "max_response_size_mb": {"type": "Integer"}, + "proxy_config_reload_interval_seconds": {"type": "Integer"}, "pass_through_endpoints": {"type": "PydanticModel"}, "store_model_in_db": {"type": "Boolean"}, "store_prompts_in_spend_logs": {"type": "Boolean"}, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index b27ddea010b..23a9c086c73 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -403,6 +403,15 @@ model LiteLLM_MCPServerOAuthClient { updated_at DateTime @default(now()) @updatedAt @map("updated_at") } +// The enterprise IdP identity assertion captured at SSO login, one row per user. +// assertion_b64 is an encrypted JSON payload: {id_token, refresh_token?, issuer?, expires_at?}. +model LiteLLM_SSOIdentityAssertion { + user_id String @id + assertion_b64 String + created_at DateTime @default(now()) @map("created_at") + updated_at DateTime @default(now()) @updatedAt @map("updated_at") +} + // Generate Tokens for Proxy model LiteLLM_VerificationToken { token String @id diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index a8926d26047..42111cf17f2 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -1,6 +1,8 @@ #### CRUD ENDPOINTS for UI Settings ##### import asyncio import json +import os +from collections.abc import Mapping from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union from urllib.parse import urlparse @@ -35,6 +37,44 @@ _SSO_SENSITIVE_FIELDS: Set[str] = { "generic_client_secret", } +# Maps each UIThemeConfig field to the env var the UI branding path reads it +# from. /update/ui_theme_settings writes both the stored ui_theme_config and +# these env vars, so /get/ui_theme_settings resolves the same env vars to +# reflect a deployment branded purely through process env. +_UI_THEME_FIELD_ENV_VARS: dict[str, str] = { + "logo_url": "UI_LOGO_PATH", + "favicon_url": "LITELLM_FAVICON_URL", +} + + +def _is_public_http_url(value: str | None) -> bool: + """Whether a value is a plain http(s) URL with a host, safe to disclose publicly.""" + if not isinstance(value, str) or not value.strip(): + return False + parsed = urlparse(value.strip()) + return parsed.scheme in ("http", "https") and bool(parsed.netloc) + + +def _resolve_ui_theme_field(stored_values: Mapping[str, Any], field_name: str) -> str | None: + """Resolve one UI theme field to the value the branding path actually uses. + + The stored ui_theme_config wins; a field absent or blank there falls back to + the process environment. The branding path reads the env var, and stored + settings reach it by being pushed into the environment on save, so a value + supplied only as a process env var is live even though no stored entry exists. + + This endpoint is unauthenticated, so the env fallback only surfaces a public + http(s) URL: an operator can point UI_LOGO_PATH at a local filesystem path + (the branding path serves it server-side), and that path must not be + disclosed to anonymous callers. A stored value is already validated as a + public URL on write, so it passes through. + """ + stored = stored_values.get(field_name) + if isinstance(stored, str) and stored.strip(): + return stored + env_value = os.environ.get(_UI_THEME_FIELD_ENV_VARS[field_name]) + return env_value if _is_public_http_url(env_value) else None + class IPAddress(BaseModel): ip: str @@ -977,12 +1017,19 @@ async def get_ui_theme_settings(): # Load existing config config = await proxy_config.get_config() - return await _get_settings_with_schema( + result = await _get_settings_with_schema( settings_key="ui_theme_config", settings_class=UIThemeConfig, config=config, ) + stored_values = result.get("values", {}) + result["values"] = { + **stored_values, + **{field: _resolve_ui_theme_field(stored_values, field) for field in _UI_THEME_FIELD_ENV_VARS}, + } + return result + def _validate_public_image_url(value: Optional[str], field_name: str) -> None: """ @@ -1041,13 +1088,6 @@ async def update_ui_theme_settings( config = await proxy_config.get_config() before_theme = config.get("litellm_settings", {}).get("ui_theme_config") - # Update config with UI theme settings - if "general_settings" not in config: - config["general_settings"] = {} - - if "environment_variables" not in config: - config["environment_variables"] = {} - # Convert theme config to dict theme_data = theme_config.model_dump(exclude_none=True) @@ -1056,55 +1096,29 @@ async def update_ui_theme_settings( config["litellm_settings"] = {} config["litellm_settings"]["ui_theme_config"] = theme_data - # Update UI_LOGO_PATH environment variable if logo_url is provided - # If logo_url is empty string, None, or null, remove the environment variable to use default - logo_url = theme_data.get("logo_url") - verbose_proxy_logger.debug(f"Updating logo_url: {logo_url}") + # UI_LOGO_PATH and LITELLM_FAVICON_URL are the only environment variables + # this endpoint owns. A non-empty value sets the var; an empty or missing + # one clears it back to the default. Apply to the live process immediately, + # then persist only these two keys so an unrelated env var (a YAML/OS value + # merged in by get_config) is never snapshotted into the DB. + def _clean(url: str | None) -> str | None: + return url if url is not None and url.strip() else None - if ( - logo_url and isinstance(logo_url, str) and logo_url.strip() - ): # Check if logo_url exists and is not empty/whitespace - config["environment_variables"]["UI_LOGO_PATH"] = logo_url - os.environ["UI_LOGO_PATH"] = logo_url - verbose_proxy_logger.debug(f"Set UI_LOGO_PATH to: {logo_url}") - else: - # Remove the environment variable to restore default logo - if "UI_LOGO_PATH" in config.get("environment_variables", {}): - del config["environment_variables"]["UI_LOGO_PATH"] - verbose_proxy_logger.debug("Removed UI_LOGO_PATH from config") - if "UI_LOGO_PATH" in os.environ: - del os.environ["UI_LOGO_PATH"] - verbose_proxy_logger.debug("Removed UI_LOGO_PATH from environment") + env_updates: dict[str, str | None] = { + "UI_LOGO_PATH": _clean(theme_config.logo_url), + "LITELLM_FAVICON_URL": _clean(theme_config.favicon_url), + } + for env_key, env_value in env_updates.items(): + if env_value is not None: + os.environ[env_key] = env_value + else: + os.environ.pop(env_key, None) - # Update LITELLM_FAVICON_URL environment variable if favicon_url is provided - favicon_url = theme_data.get("favicon_url") - verbose_proxy_logger.debug(f"Updating favicon_url: {favicon_url}") - - if ( - favicon_url and isinstance(favicon_url, str) and favicon_url.strip() - ): # Check if favicon_url exists and is not empty/whitespace - config["environment_variables"]["LITELLM_FAVICON_URL"] = favicon_url - os.environ["LITELLM_FAVICON_URL"] = favicon_url - verbose_proxy_logger.debug(f"Set LITELLM_FAVICON_URL to: {favicon_url}") - else: - # Remove the environment variable to restore default favicon - if "LITELLM_FAVICON_URL" in config.get("environment_variables", {}): - del config["environment_variables"]["LITELLM_FAVICON_URL"] - verbose_proxy_logger.debug("Removed LITELLM_FAVICON_URL from config") - if "LITELLM_FAVICON_URL" in os.environ: - del os.environ["LITELLM_FAVICON_URL"] - verbose_proxy_logger.debug("Removed LITELLM_FAVICON_URL from environment") - - # Handle environment variable encryption if needed - stored_config = config.copy() - if "environment_variables" in stored_config and len(stored_config["environment_variables"]) > 0: - # Only encrypt if there are environment variables to encrypt - stored_config["environment_variables"] = proxy_config._encrypt_env_variables( - environment_variables=stored_config["environment_variables"] - ) - - # Save the updated config - await proxy_config.save_config(new_config=stored_config) + # Persist the theme config (litellm_settings). save_config defaults to + # include_env_vars=False, so it does not snapshot environment_variables. + await proxy_config.save_config(new_config=config) + # Persist only the two owned env vars, merged against the existing DB row. + await proxy_config.save_environment_variables(env_updates) asyncio.create_task( create_config_audit_log( diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 3605ab95d1b..47d93fc2d7a 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -826,6 +826,13 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up "while fail_on_error still governs real Model Armor API errors. Default False blocks them." ), ) + sanitize_error_detail: Optional[bool] = Field( + default=True, + description=( + "For guardrail='model_armor': omit the raw Model Armor response from " + "caller-facing errors and logs by default. Set False to restore verbose output." + ), + ) additional_provider_specific_params: Optional[Dict[str, Any]] = Field( default=None, diff --git a/litellm/types/interactions/generated.py b/litellm/types/interactions/generated.py index 793cc02ff17..4a1ef5ed696 100644 --- a/litellm/types/interactions/generated.py +++ b/litellm/types/interactions/generated.py @@ -173,6 +173,7 @@ class Status1(Enum): cancelled = "cancelled" incomplete = "incomplete" budget_exceeded = "budget_exceeded" + queued = "queued" class InteractionStatusUpdate(BaseModel): @@ -341,6 +342,7 @@ class Status3(Enum): CANCELLED = "cancelled" INCOMPLETE = "incomplete" BUDGET_EXCEEDED = "budget_exceeded" + QUEUED = "queued" class ModelOption(RootModel[str]): diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/model_armor.py b/litellm/types/proxy/guardrails/guardrail_hooks/model_armor.py index 628ac0442de..d5e601ce8ea 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/model_armor.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/model_armor.py @@ -20,6 +20,13 @@ class ModelArmorGuardrailConfigModel(GuardrailConfigModel): default=True, description="Whether to fail the request if Model Armor encounters an error", ) + sanitize_error_detail: Optional[bool] = Field( + default=True, + description=( + "Omit the raw Model Armor response from caller-facing errors and logs " + "by default. Set False to restore verbose output." + ), + ) @staticmethod def ui_friendly_name() -> str: diff --git a/litellm/utils.py b/litellm/utils.py index 9d6c2c27a80..95aa33d1f65 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1864,7 +1864,9 @@ def client(original_function): except Exception: pass - setattr(e, "num_retries", num_retries) ## IMPORTANT: returns the deployment's num_retries to the router + deployment_num_retries = kwargs.get("num_retries") + if deployment_num_retries is not None: + setattr(e, "num_retries", deployment_num_retries) timeout = _get_wrapper_timeout(kwargs=kwargs, exception=e) setattr(e, "timeout", timeout) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c39a0872af4..fdbdde1705d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -17644,6 +17644,61 @@ }, "web_search_billing_unit": "per_query" }, + "gemini-3.5-flash-lite": { + "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_flex": 2e-08, + "cache_read_input_token_cost_priority": 5e-08, + "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, + "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.4e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 2.5e-06, + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 1.25e-06, + "output_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_priority": 4.5e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -18311,6 +18366,60 @@ }, "web_search_billing_unit": "per_query" }, + "vertex_ai/gemini-3.6-flash": { + "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_flex": 7.5e-08, + "input_cost_per_token": 1.5e-06, + "input_cost_per_token_batches": 7.5e-07, + "input_cost_per_token_flex": 7.5e-07, + "litellm_provider": "vertex_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 7.5e-06, + "output_cost_per_token": 7.5e-06, + "output_cost_per_token_batches": 3.75e-06, + "output_cost_per_token_flex": 3.75e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "input_cost_per_token_priority": 2.7e-06, + "output_cost_per_token_priority": 1.35e-05, + "cache_read_input_token_cost_priority": 2.7e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "vertex_ai/gemini-3.1-pro-preview": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, @@ -19663,6 +19772,63 @@ }, "web_search_billing_unit": "per_query" }, + "gemini/gemini-3.5-flash-lite": { + "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_flex": 2e-08, + "cache_read_input_token_cost_priority": 5e-08, + "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, + "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.4e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 2.5e-06, + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 1.25e-06, + "output_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_priority": 4.5e-06, + "rpm": 15, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "tpm": 250000, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "gemini/gemini-3-flash-preview": { "cache_read_input_token_cost": 5e-08, "input_cost_per_audio_token": 1e-06, @@ -19769,6 +19935,63 @@ }, "web_search_billing_unit": "per_query" }, + "gemini/gemini-3.6-flash": { + "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_flex": 7.5e-08, + "input_cost_per_token": 1.5e-06, + "input_cost_per_token_batches": 7.5e-07, + "input_cost_per_token_flex": 7.5e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 7.5e-06, + "output_cost_per_token": 7.5e-06, + "output_cost_per_token_batches": 3.75e-06, + "output_cost_per_token_flex": 3.75e-06, + "rpm": 2000, + "source": "https://ai.google.dev/pricing/gemini-3", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "tpm": 800000, + "input_cost_per_token_priority": 2.7e-06, + "output_cost_per_token_priority": 1.35e-05, + "cache_read_input_token_cost_priority": 2.7e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "gemini/gemini-omni-flash-preview": { "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, @@ -20049,6 +20272,61 @@ }, "web_search_billing_unit": "per_query" }, + "gemini-3.6-flash": { + "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_flex": 7.5e-08, + "input_cost_per_token": 1.5e-06, + "input_cost_per_token_batches": 7.5e-07, + "input_cost_per_token_flex": 7.5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 7.5e-06, + "output_cost_per_token": 7.5e-06, + "output_cost_per_token_batches": 3.75e-06, + "output_cost_per_token_flex": 3.75e-06, + "source": "https://ai.google.dev/pricing/gemini-3", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "input_cost_per_token_priority": 2.7e-06, + "output_cost_per_token_priority": 1.35e-05, + "cache_read_input_token_cost_priority": 2.7e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, @@ -37332,6 +37610,61 @@ }, "web_search_billing_unit": "per_query" }, + "vertex_ai/gemini-3.5-flash-lite": { + "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_flex": 2e-08, + "cache_read_input_token_cost_priority": 5e-08, + "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, + "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.4e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 2.5e-06, + "output_cost_per_token": 2.5e-06, + "output_cost_per_token_batches": 1.25e-06, + "output_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_priority": 4.5e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query" + }, "vertex_ai/deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, diff --git a/pyproject.toml b/pyproject.toml index 9e2f5c4e3ac..62bd37c3db6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.94.0" +version = "1.95.0" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.10, <3.15" @@ -62,7 +62,7 @@ proxy = [ "azure-identity>=1.25.2,<2.0", "azure-storage-blob>=12.28.0,<13.0", "mcp>=1.28.1,<2.0", - "litellm-proxy-extras==0.4.79", + "litellm-proxy-extras==0.4.80", "litellm-enterprise==0.1.51", "RestrictedPython>=8.1,<9.0", "rich>=13.9.4,<14.0", @@ -117,6 +117,14 @@ stt-nvidia-riva = [ "numpy>=1.26.0", ] google = ["google-cloud-aiplatform>=1.133.0,<2.0"] +bedrock-realtime = [ + # Bedrock Nova Sonic realtime (speech-to-speech) uses the + # InvokeModelWithBidirectionalStream API, which boto3 cannot do. This + # experimental AWS SDK (with its smithy-* deps, pulled transitively) + # provides the bidirectional stream; imported lazily in the realtime + # handler so litellm core stays usable without it. + "aws-sdk-bedrock-runtime>=0.7.0,<0.8.0; python_version >= '3.12'", +] proxy-runtime = [ # Historically bundled in the proxy Docker images via requirements.txt. # Keep these in a dedicated extra so uv-based images preserve the same @@ -289,7 +297,7 @@ members = ["enterprise", "litellm-proxy-extras"] profile = "black" [tool.commitizen] -version = "1.94.0" +version = "1.95.0" version_files = [ "pyproject.toml:^version", ] diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index f22c6585991..c23f954f81b 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -93,7 +93,7 @@ "limit": 33 }, "DTZ005": { - "limit": 244 + "limit": 241 }, "DTZ006": { "limit": 13 diff --git a/schema.prisma b/schema.prisma index b27ddea010b..23a9c086c73 100644 --- a/schema.prisma +++ b/schema.prisma @@ -403,6 +403,15 @@ model LiteLLM_MCPServerOAuthClient { updated_at DateTime @default(now()) @updatedAt @map("updated_at") } +// The enterprise IdP identity assertion captured at SSO login, one row per user. +// assertion_b64 is an encrypted JSON payload: {id_token, refresh_token?, issuer?, expires_at?}. +model LiteLLM_SSOIdentityAssertion { + user_id String @id + assertion_b64 String + created_at DateTime @default(now()) @map("created_at") + updated_at DateTime @default(now()) @updatedAt @map("updated_at") +} + // Generate Tokens for Proxy model LiteLLM_VerificationToken { token String @id diff --git a/scripts/pre_commit_lint.sh b/scripts/pre_commit_lint.sh index cce0cb61c1e..150a4bbf9de 100755 --- a/scripts/pre_commit_lint.sh +++ b/scripts/pre_commit_lint.sh @@ -5,6 +5,7 @@ # gating CI checks, so a clean run means a green CI lint: # - litellm/ Python staged -> `make lint` (test-linting.yml's lint job) # - tests/e2e Python staged -> `make lint-e2e-basedpyright` (test-linting.yml's e2e type-check step) +# + raw HTTP client ban (test-code-quality.yml's check_e2e_no_raw_requests) # - dashboard staged -> prettier + eslint + lint budgets (test-litellm-ui-build.yml's frontend-lint) # - proxy/types staged -> regenerate dashboard API types and fail on drift (check-ui-api-types.yml) # @@ -112,6 +113,12 @@ if [ -n "$e2e_py_files" ] && [ -z "$litellm_py_files" ]; then make lint-e2e-basedpyright || { echo "✗ tests/e2e basedpyright failed. Fix the errors above, then re-run make pre-commit." >&2; status=1; } fi +if [ -n "$e2e_py_files" ]; then + echo "pre-commit: checking tests/e2e raw HTTP client ban (check_e2e_no_raw_requests)" + uv run --no-sync python tests/code_coverage_tests/check_e2e_no_raw_requests.py \ + || { echo "✗ Raw HTTP client import in tests/e2e. Route the call through tests/e2e/e2e_http.py, then re-run make pre-commit." >&2; status=1; } +fi + if [ -n "$ui_prettier_files" ] || [ -n "$ui_eslint_files" ]; then echo "pre-commit: linting dashboard (prettier + eslint + lint budgets)" if [ ! -d ui/litellm-dashboard/node_modules ]; then diff --git a/tests/code_coverage_tests/check_e2e_no_raw_requests.py b/tests/code_coverage_tests/check_e2e_no_raw_requests.py new file mode 100644 index 00000000000..e70e83652d1 --- /dev/null +++ b/tests/code_coverage_tests/check_e2e_no_raw_requests.py @@ -0,0 +1,81 @@ +"""tests/e2e routes every HTTP call through the typed transport (e2e_http.py), so +raw HTTP client imports (requests, urllib.request, httpx, aiohttp, http.client) are +banned in suite code. Importing requests' exception types for catching is fine +anywhere; a small allowlist grandfathers the files that legitimately make raw calls +(the transport itself, the root conftest liveness probe, and the claude_code version +resolver's constant registry URL fetch). Referenced by tests/e2e/CLAUDE.md.""" + +from __future__ import annotations + +import ast +import sys +from pathlib import Path + +E2E_DIR = Path(__file__).resolve().parents[1] / "e2e" + +BANNED_MODULES = ("requests", "urllib.request", "http.client", "httpx", "aiohttp") + +ALLOWED_RAW_CLIENT_FILES = { + "e2e_http.py": ("requests",), + "conftest.py": ("requests",), + "claude_code/pr_gate_version_resolver.py": ("urllib.request",), +} + +EXCEPTION_ONLY_NAMES = frozenset({"RequestException", "ConnectionError", "Timeout", "HTTPError"}) + + +def _is_banned(module: str) -> bool: + return any(module == banned or module.startswith(banned + ".") for banned in BANNED_MODULES) + + +def _banned_imports(tree: ast.Module) -> tuple[tuple[str, int], ...]: + plain = tuple( + (alias.name, node.lineno) + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for alias in node.names + if _is_banned(alias.name) + ) + from_imports = tuple( + (node.module, node.lineno) + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) + and node.module is not None + and _is_banned(node.module) + and not all(alias.name in EXCEPTION_ONLY_NAMES for alias in node.names) + ) + return plain + from_imports + + +def _violations_in(path: Path) -> tuple[str, ...]: + relative = path.relative_to(E2E_DIR).as_posix() + allowed = ALLOWED_RAW_CLIENT_FILES.get(relative, ()) + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + return tuple( + f"tests/e2e/{relative}:{lineno}: raw HTTP client import '{module}'" + for module, lineno in _banned_imports(tree) + if module not in allowed + ) + + +def main() -> int: + violations = tuple( + violation + for path in sorted(E2E_DIR.rglob("*.py")) + for violation in _violations_in(path) + ) + for violation in violations: + print(violation) + if violations: + print( + f"\n{len(violations)} raw HTTP client import(s) in tests/e2e. " + "Route the call through tests/e2e/e2e_http.py (get_external for absolute " + "third-party URLs) so it gets the typed Result handling." + ) + return 1 + print("tests/e2e raw HTTP client check passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index e08d703d21f..0bc3cebdd5a 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -55,6 +55,8 @@ IGNORE_FUNCTIONS = [ "_freeze_for_dedupe", # OTEL: max depth set (default 16, _FREEZE_MAX_DEPTH); fails closed by returning repr(value) at the cap. "apply_json_merge_patch", # max depth set (_MAX_MERGE_DEPTH=64); fails closed by raising ValueError at the cap. "_filter_mcp_argument_value", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by blocking the MCP call at the cap. + "_redact_scanned_content", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by returning "[REDACTED]" at the cap. + "_iter_fallback_targets", # max depth set (2 * ROUTER_MAX_FALLBACKS); fails closed by raising ValueError at the cap. ] diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 47f3c74d7f1..186517290ea 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -13,11 +13,13 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `realtime/` - realtime websocket sessions, including the pipecat audio path - `quota_management/` - quota enforcement and accounting, one subfolder per behavior: `ratelimit/` (rpm/tpm blocks, window reset, pacing headers on live traffic), `budgets/` (budget definition, enforcement, and reset windows: key, team, tag, soft, multi-window), and `spend_tracking/` (spend logging and cost attribution on `/spend/*`) - `management/` - key/team/user/organization management routes: create/update/delete persistence via the info routes, team membership, and llm-only-key route denials (API surface; not Playwright) +- `a2a/` - the A2A (agent-to-agent) surface: admin registration via `/v1/agents`, proxy-fronted card discovery at `/.well-known/agent-card.json`, and JSON-RPC `message/send` invocation, driving agents backed by the litellm completion bridge (a real provider) and asserting protocol-version normalization (0.3 vs 1.0) - `mcp/` - the MCP server surface over api_key auth against the real Datadog remote MCP server only (see "MCP suite: real Datadog only" below) - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) -- `load/` - throughput/performance under concurrency: drives real concurrent traffic through the whole stack with Locust and asserts a throughput SLO; marked `load` so the parent conftest collects it last and it never perturbs latency-sensitive suites +- `load/` - throughput/performance under concurrency: drives real concurrent traffic through the whole stack with Locust and asserts a throughput SLO; marked `load` so the parent conftest collects it last and it never perturbs latency-sensitive suites. Also home of the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`): Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; additionally marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, because it spends real provider money (driven by `.github/workflows/weekly_load_anomaly.yml`) +- `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke @@ -130,7 +132,7 @@ reliability... behavior : fallback | retry | cooldown | timeout | routing | cache | circuit_breaker | perf variant : 5xx | context_window | content_policy | 429 | timeout simple_shuffle | usage_based | latency_based | cost_based | least_busy - latency | throughput (perf only; SLO/threshold assertion, not binary) + latency | throughput | session_anomaly (perf only; SLO/threshold assertion, not binary) assertion : routes_to_fallback | succeeds_within_retries | picks_under_tpm | returns_cached | trips_then_recovers | under_slo e.g. reliability.fallback.context_window.routes_to_fallback exercised_on=[chat_completions] diff --git a/tests/e2e/a2a/a2a_client.py b/tests/e2e/a2a/a2a_client.py new file mode 100644 index 00000000000..97ffa8c34a3 --- /dev/null +++ b/tests/e2e/a2a/a2a_client.py @@ -0,0 +1,292 @@ +"""Client for the proxy's A2A (agent-to-agent) surface. + +An A2A agent is registered admin-side via POST /v1/agents with an agent card and +litellm_params; the proxy fronts it at /a2a/{id}, serving a proxy-owned agent card +at /.well-known/agent-card.json and accepting A2A JSON-RPC calls at /a2a/{id}. This +suite registers agents backed by the litellm_completion_bridge (custom_llm_provider ++ model), so message/send runs a real provider completion and comes back in the +agent's pinned A2A protocol version. The A2A request/response models are co-located +here because only this suite uses them. +""" + +from __future__ import annotations + +import warnings +from dataclasses import dataclass + +from pydantic import BaseModel, ConfigDict, Field + +from e2e_http import NoBody, Result, get_external, is_ok +from proxy_client import ProxyClient + + +class A2ACapabilities(BaseModel): + streaming: bool | None = None + push_notifications: bool | None = Field(default=None, serialization_alias="pushNotifications") + + +class A2ASkill(BaseModel): + id: str + name: str + description: str + tags: list[str] + examples: list[str] | None = None + + +class A2AProvider(BaseModel): + organization: str + url: str + + +class AgentCardParams(BaseModel): + """The upstream agent card an admin registers. `protocolVersion` is the field the + proxy validates against SUPPORTED_A2A_PROTOCOL_VERSIONS on registration.""" + + protocol_version: str = Field(serialization_alias="protocolVersion") + name: str + description: str + version: str + url: str | None = None + capabilities: A2ACapabilities = A2ACapabilities() + skills: list[A2ASkill] + default_input_modes: list[str] = Field(default=["text"], serialization_alias="defaultInputModes") + default_output_modes: list[str] = Field(default=["text"], serialization_alias="defaultOutputModes") + preferred_transport: str | None = Field(default=None, serialization_alias="preferredTransport") + + +class UpstreamAgentCard(BaseModel): + """A real published agent card parsed from a public /.well-known endpoint. Keys on + the A2A wire aliases so `model_validate_json` reads the served JSON and + `model_dump(by_alias=True)` re-emits it unchanged for verbatim registration; it is + only ever fetched-and-validated, never hand-constructed, so aliasing on the wire + names does not affect any call site.""" + + model_config = ConfigDict(populate_by_name=True) + + protocol_version: str = Field(alias="protocolVersion") + name: str + description: str + version: str + url: str + provider: A2AProvider | None = None + documentation_url: str | None = Field(default=None, alias="documentationUrl") + capabilities: A2ACapabilities = A2ACapabilities() + skills: list[A2ASkill] + default_input_modes: list[str] = Field(default=["text"], alias="defaultInputModes") + default_output_modes: list[str] = Field(default=["text"], alias="defaultOutputModes") + preferred_transport: str | None = Field(default=None, alias="preferredTransport") + + +class A2ABridgeParams(BaseModel): + """litellm_params that route the agent through the completion bridge: an A2A + message/send is transformed into a litellm.acompletion against this provider.""" + + model_config = ConfigDict(protected_namespaces=()) + + custom_llm_provider: str + model: str + + +class AgentRegisterBody(BaseModel): + agent_name: str + agent_card_params: AgentCardParams | UpstreamAgentCard + litellm_params: A2ABridgeParams | None = None + + +class A2ASecurityScheme(BaseModel): + type: str + scheme: str + + +class A2AInterface(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + url: str + protocol_version: str | None = Field(default=None, alias="protocolVersion") + + +class ServedAgentCard(BaseModel): + """The proxy-owned card, either nested under a registration response's + `agent_card_params` or served raw at /.well-known/agent-card.json. The proxy + rewrites `url`/`supportedInterfaces` to itself and replaces the security scheme + with its own virtual-key bearer scheme.""" + + model_config = ConfigDict(populate_by_name=True) + + protocol_version: str = Field(alias="protocolVersion") + name: str + url: str | None = None + security_schemes: dict[str, A2ASecurityScheme] | None = Field(default=None, alias="securitySchemes") + security: list[dict[str, list[str]]] | None = None + supported_interfaces: list[A2AInterface] | None = Field(default=None, alias="supportedInterfaces") + + +class AgentResponse(BaseModel): + agent_id: str + agent_name: str + agent_card_params: ServedAgentCard + + +class A2ATextPart(BaseModel): + kind: str = "text" + text: str + + +class A2ASearchPropertiesParams(BaseModel): + """The strict param schema of the published property agent's `search_properties` + skill (unknown keys are rejected upstream), so a natural-language query like + "properties for sale in SF under $2M" is expressed as typed fields.""" + + un_locode: str | None = None + service_type: str | None = None + property_type: str | None = None + bedrooms_min: int | None = None + asking_price_max: float | None = None + limit: int | None = None + + +class A2ASkillInvocation(BaseModel): + skill: str + params: A2ASearchPropertiesParams + + +class A2ADataPart(BaseModel): + kind: str = "data" + data: A2ASkillInvocation + + +class A2AOutboundMessage(BaseModel): + role: str = "user" + parts: list[A2ATextPart | A2ADataPart] + message_id: str = Field(serialization_alias="messageId") + + +class A2AMessageSendParams(BaseModel): + message: A2AOutboundMessage + + +class A2AJsonRpcRequest(BaseModel): + jsonrpc: str = "2.0" + id: str + method: str = "message/send" + params: A2AMessageSendParams + + +class A2AResponsePart(BaseModel): + kind: str | None = None + text: str | None = None + + +class A2AResponseMessage(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + message_id: str | None = Field(default=None, alias="messageId") + role: str | None = None + parts: list[A2AResponsePart] = [] + + +class A2ATaskStatus(BaseModel): + state: str | None = None + message: A2AResponseMessage | None = None + + +class A2AResult(BaseModel): + """A message/send result. In 0.3 the message fields sit directly on the result + (`kind`/`role`/`parts`); in 1.0 they are nested under `message`; a real agent that + runs a task replies with a `task` whose agent text lives on `status.message`. + `text` reads the agent's reply from whichever shape the served version produced.""" + + model_config = ConfigDict(populate_by_name=True) + + kind: str | None = None + role: str | None = None + message_id: str | None = Field(default=None, alias="messageId") + parts: list[A2AResponsePart] = [] + message: A2AResponseMessage | None = None + status: A2ATaskStatus | None = None + + @property + def text(self) -> str: + if self.message is not None: + parts = self.message.parts + elif self.parts: + parts = self.parts + elif self.status is not None and self.status.message is not None: + parts = self.status.message.parts + else: + parts = [] + return "".join(part.text or "" for part in parts) + + @property + def is_nested_v1_shape(self) -> bool: + return self.message is not None + + +class A2AError(BaseModel): + code: int + message: str + + +class A2AResponse(BaseModel): + jsonrpc: str + id: str | None = None + result: A2AResult | None = None + error: A2AError | None = None + + +@dataclass(frozen=True, slots=True) +class A2AClient: + proxy: ProxyClient + + def register_agent(self, body: AgentRegisterBody) -> Result[AgentResponse]: + return self.proxy.transport.post( + "/v1/agents", + headers=self.proxy.transport.master, + json=body, + response_type=AgentResponse, + ) + + def get_agent(self, agent_id: str) -> Result[AgentResponse]: + return self.proxy.transport.get( + f"/v1/agents/{agent_id}", + headers=self.proxy.transport.master, + params=NoBody(), + response_type=AgentResponse, + ) + + def delete_agent(self, agent_id: str) -> None: + result = self.proxy.transport.delete( + f"/v1/agents/{agent_id}", + headers=self.proxy.transport.master, + json=NoBody(), + response_type=NoBody, + ) + if not is_ok(result): + warnings.warn(f"delete_agent({agent_id!r}) failed: {result}", stacklevel=2) + + def agent_card(self, agent_id: str, key: str) -> Result[ServedAgentCard]: + return self.proxy.transport.get( + f"/a2a/{agent_id}/.well-known/agent-card.json", + headers=self.proxy.transport.bearer(key), + params=NoBody(), + response_type=ServedAgentCard, + ) + + def send_message(self, agent_id: str, key: str, body: A2AJsonRpcRequest) -> Result[A2AResponse]: + return self.proxy.transport.post( + f"/a2a/{agent_id}", + headers=self.proxy.transport.bearer(key), + json=body, + response_type=A2AResponse, + ) + + +def build_a2a_client(proxy: ProxyClient) -> A2AClient: + return A2AClient(proxy=proxy) + + +def fetch_agent_card(url: str, *, timeout: float = 20.0) -> Result[UpstreamAgentCard]: + """Fetch a live A2A agent card from its /.well-known endpoint and parse it into the + registration model, so a test can register a real published card verbatim rather + than a hand-rolled one.""" + return get_external(url, response_type=UpstreamAgentCard, timeout=timeout) diff --git a/tests/e2e/a2a/conftest.py b/tests/e2e/a2a/conftest.py new file mode 100644 index 00000000000..93f3b56c8f7 --- /dev/null +++ b/tests/e2e/a2a/conftest.py @@ -0,0 +1,17 @@ +"""A2A suite's `client` fixture. + +The shared lifecycle (resources/scoped_key), proxy liveness gate, and e2e marker +live in the parent tests/e2e/conftest.py. A2AClient holds the shared ProxyClient, +so the `resources` fixture cleans up keys this suite creates; agents are torn down +via `resources.defer(...)` in each test. +""" + +import pytest + +from a2a_client import A2AClient, build_a2a_client +from proxy_client import ProxyClient + + +@pytest.fixture(scope="session") +def client(proxy: ProxyClient) -> A2AClient: + return build_a2a_client(proxy) diff --git a/tests/e2e/a2a/test_a2a_agent_e2e.py b/tests/e2e/a2a/test_a2a_agent_e2e.py new file mode 100644 index 00000000000..aa60b57f99b --- /dev/null +++ b/tests/e2e/a2a/test_a2a_agent_e2e.py @@ -0,0 +1,202 @@ +"""A2A agents end to end, against a live proxy. + +An admin registers an agent whose card pins an A2A protocol version and whose +litellm_params route it through the completion bridge; a caller then discovers the +proxy-owned card and drives it over A2A JSON-RPC. These tests assert the recorded +state (the agent persists, a spend row lands) and the enforced behavior (the served +card points back at the proxy, message/send returns a real completion in the pinned +protocol version, and an unsupported version is refused at registration). +""" + +from __future__ import annotations + +import pytest + +from a2a_client import ( + A2ABridgeParams, + A2AClient, + A2ADataPart, + A2AJsonRpcRequest, + A2AMessageSendParams, + A2AOutboundMessage, + A2ASearchPropertiesParams, + A2ASkill, + A2ASkillInvocation, + A2ATextPart, + AgentCardParams, + AgentRegisterBody, + AgentResponse, + fetch_agent_card, +) +from e2e_config import unique_marker +from e2e_http import Result, UnknownApiError, unwrap +from lifecycle import ResourceManager + +BRIDGE = A2ABridgeParams(custom_llm_provider="anthropic", model="claude-haiku-4-5") + +MOVEHOME_AGENT_CARD_URL = "https://movehome.org/.well-known/agent.json" +MOVEHOME_ORIGIN = "https://movehome.org" + +pytestmark = pytest.mark.e2e + + +def _register(client: A2AClient, resources: ResourceManager, protocol_version: str) -> AgentResponse: + marker = unique_marker() + body = AgentRegisterBody( + agent_name=f"e2e-a2a-{marker}", + agent_card_params=AgentCardParams( + protocol_version=protocol_version, + name=f"E2E A2A {marker}", + description="e2e agent backed by the litellm completion bridge", + version="1.0.0", + skills=[A2ASkill(id="chat", name="Chat", description="general chat", tags=["chat"])], + ), + litellm_params=BRIDGE, + ) + agent = unwrap(client.register_agent(body)) + resources.defer(lambda: client.delete_agent(agent.agent_id)) + return agent + + +def _register_rejection(client: A2AClient, protocol_version: str) -> Result[AgentResponse]: + marker = unique_marker() + body = AgentRegisterBody( + agent_name=f"e2e-a2a-bad-{marker}", + agent_card_params=AgentCardParams( + protocol_version=protocol_version, + name=f"E2E A2A bad {marker}", + description="rejected at registration", + version="1.0.0", + skills=[A2ASkill(id="chat", name="Chat", description="c", tags=["chat"])], + ), + litellm_params=BRIDGE, + ) + return client.register_agent(body) + + +def _ask(text: str) -> A2AJsonRpcRequest: + return A2AJsonRpcRequest( + id=f"e2e-{unique_marker()}", + params=A2AMessageSendParams( + message=A2AOutboundMessage(parts=[A2ATextPart(text=text)], message_id=unique_marker()) + ), + ) + + +class TestA2AAgentLifecycle: + @pytest.mark.covers("other.a2a.register.persists") + def test_register_persists(self, client: A2AClient, resources: ResourceManager) -> None: + agent = _register(client, resources, "0.3") + fetched = unwrap(client.get_agent(agent.agent_id)) + assert fetched.agent_id == agent.agent_id + assert fetched.agent_name == agent.agent_name + assert fetched.agent_card_params.protocol_version == "0.3" + + @pytest.mark.covers("other.a2a.register.semver_version_accepted") + def test_semver_protocol_version_registers_and_serves(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None: + agent = _register(client, resources, "0.3.0") + assert agent.agent_card_params.protocol_version == "0.3" + card = unwrap(client.agent_card(agent.agent_id, scoped_key)) + assert card.protocol_version == "0.3" + assert card.supported_interfaces is not None + assert card.supported_interfaces[0].protocol_version == "0.3" + result = unwrap(client.send_message(agent.agent_id, scoped_key, _ask("Say hi in one word"))).result + assert result is not None + assert result.text != "" + + @pytest.mark.covers("other.a2a.message_send.real_world_agent_replies") + def test_real_world_agent_replies_to_property_query(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None: + upstream = unwrap(fetch_agent_card(MOVEHOME_AGENT_CARD_URL)).model_copy(update={"url": MOVEHOME_ORIGIN}) + assert upstream.protocol_version == "0.3.0" + marker = unique_marker() + body = AgentRegisterBody(agent_name=f"e2e-a2a-real-{marker}", agent_card_params=upstream) + agent = unwrap(client.register_agent(body)) + resources.defer(lambda: client.delete_agent(agent.agent_id)) + assert agent.agent_card_params.protocol_version == "0.3" + request = A2AJsonRpcRequest( + id=f"e2e-{unique_marker()}", + params=A2AMessageSendParams( + message=A2AOutboundMessage( + parts=[ + A2ADataPart( + data=A2ASkillInvocation( + skill="search_properties", + params=A2ASearchPropertiesParams(un_locode="USSFO", service_type="sale", asking_price_max=2_000_000, limit=3), + ) + ) + ], + message_id=unique_marker(), + ) + ), + ) + response = unwrap(client.send_message(agent.agent_id, scoped_key, request)) + assert response.error is None + assert response.result is not None + assert response.result.text.strip() != "" + + @pytest.mark.covers("other.a2a.discovery.proxy_fronted_card") + def test_discovery_card_is_proxy_fronted(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None: + agent = _register(client, resources, "0.3") + card = unwrap(client.agent_card(agent.agent_id, scoped_key)) + assert card.url is not None and card.url.endswith(f"/a2a/{agent.agent_id}") + assert card.security_schemes is not None + scheme = next(iter(card.security_schemes.values())) + assert scheme.scheme == "bearer" + assert card.supported_interfaces is not None + assert card.supported_interfaces[0].url == card.url + + @pytest.mark.covers("other.a2a.message_send.bridge_invokes") + def test_message_send_runs_completion_bridge(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None: + agent = _register(client, resources, "0.3") + request = _ask("Reply with exactly the word PONG and nothing else") + response = unwrap(client.send_message(agent.agent_id, scoped_key, request)) + assert response.error is None + assert response.result is not None + assert "PONG" in response.result.text.upper() + + rows = client.proxy.poll_logs_for_request_id(request.id) + assert rows, f"no spend log row landed for a2a request {request.id}" + assert rows[0].call_type == "asend_message" + assert rows[0].model == f"a2a_agent/{agent.agent_card_params.name}" + + @pytest.mark.covers("other.a2a.version.serves_pinned_0_3") + def test_pinned_v0_3_serves_flat_message_shape(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None: + agent = _register(client, resources, "0.3") + request = _ask("Say hi in one word") + result = unwrap(client.send_message(agent.agent_id, scoped_key, request)).result + assert result is not None + assert not result.is_nested_v1_shape + assert result.kind == "message" + assert result.role == "agent" + assert result.text != "" + + @pytest.mark.covers("other.a2a.version.serves_pinned_1_0") + def test_pinned_v1_0_serves_nested_message_shape(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None: + agent = _register(client, resources, "1.0") + request = _ask("Say hi in one word") + result = unwrap(client.send_message(agent.agent_id, scoped_key, request)).result + assert result is not None + assert result.is_nested_v1_shape + assert result.message is not None + assert result.message.role == "ROLE_AGENT" + assert result.text != "" + + @pytest.mark.covers("other.a2a.register.unsupported_version_rejected") + def test_unsupported_protocol_version_rejected(self, client: A2AClient) -> None: + result = _register_rejection(client, "9.9") + match result: + case UnknownApiError(status_code=status, body=detail): + assert status == 400 + assert "protocolVersion" in detail + case _: + pytest.fail(f"expected 400 for unsupported protocolVersion, got {result}") + + @pytest.mark.covers("other.a2a.register.malformed_version_rejected") + def test_malformed_protocol_version_rejected(self, client: A2AClient) -> None: + result = _register_rejection(client, "0.3.garbage") + match result: + case UnknownApiError(status_code=status, body=detail): + assert status == 400 + assert "Unsupported protocolVersion '0.3.garbage'" in detail + case _: + pytest.fail(f"expected 400 for malformed protocolVersion, got {result}") diff --git a/tests/e2e/batches/batch_client.py b/tests/e2e/batches/batch_client.py index 7db5d0b6beb..5cc5d1dae3b 100644 --- a/tests/e2e/batches/batch_client.py +++ b/tests/e2e/batches/batch_client.py @@ -26,16 +26,24 @@ from e2e_http import ( ) from models import LiteLLMParamsBody +UPLOAD_FILENAME = "batch_input.jsonl" + class FileObject(BaseModel): id: str object: str | None = None purpose: str | None = None + filename: str | None = None bytes: int | None = None status: str | None = None created_at: int | None = None +class FileList(BaseModel): + object: str | None = None + data: list[FileObject] = [] + + class BatchObject(BaseModel): id: str object: str | None = None @@ -106,12 +114,30 @@ class BatchClient: _files_path(provider), headers=self.proxy.transport.bearer(key), form=form, - filename="batch_input.jsonl", + filename=UPLOAD_FILENAME, content=content, params=ModelQuery(model=model), response_type=FileObject, ) + def retrieve_file( + self, file_id: str, *, key: str, provider: str | None = None + ) -> Result[FileObject]: + return self.proxy.transport.get( + f"{_files_path(provider)}/{file_id}", + headers=self.proxy.transport.bearer(key), + params=NoBody(), + response_type=FileObject, + ) + + def list_files(self, *, key: str, provider: str | None = None) -> Result[FileList]: + return self.proxy.transport.get( + _files_path(provider), + headers=self.proxy.transport.bearer(key), + params=NoBody(), + response_type=FileList, + ) + def create_batch( self, *, body: BatchCreateBody, key: str, provider: str | None = None ) -> StreamingResponse: diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index f886c09b705..b0c53becb6b 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -26,6 +26,7 @@ import pytest from e2e_config import require_env, unique_marker from batch_client import ( + UPLOAD_FILENAME, BatchClient, BatchCreateBody, BatchObject, @@ -511,6 +512,72 @@ class TestBatchFileContent: ) +class TestOpenAIFiles: + """GET /v1/files (list) and GET /v1/files/{id} (retrieve) over the OpenAI route. + + The proxy lists the OpenAI org's raw file ids, so the list case uploads a raw + (provider-routed) file whose id matches what list returns; retrieve re-encodes + the id it was called with, so the model-encoded upload round-trips unchanged. + """ + + @pytest.mark.covers( + "llm.files.openai.list.nonstream.works", + exercised_on=["files"], + ) + def test_uploaded_file_appears_in_list( + self, client: BatchClient, resources: ResourceManager, batch_deployments: None + ) -> None: + key = resources.key() + file = unwrap( + client.upload_file( + content=render_jsonl(OPENAI_BATCH_MODEL), + form=FileUploadForm(purpose="batch"), + key=key, + provider="openai", + ) + ) + resources.defer( + quietly(lambda: client.delete_file(file.id, key=key, provider="openai")) + ) + + listed = unwrap(client.list_files(key=key)) + assert listed.object is None or listed.object == "list", ( + f"list envelope object={listed.object!r}" + ) + match = next((entry for entry in listed.data if entry.id == file.id), None) + assert match is not None, f"uploaded file {file.id!r} absent from GET /v1/files" + assert match.purpose == "batch", ( + f"listed file must round-trip the upload purpose, got {match.purpose!r}" + ) + + @pytest.mark.covers( + "llm.files.openai.retrieve.nonstream.works", + exercised_on=["files"], + ) + def test_retrieve_round_trips_metadata( + self, client: BatchClient, resources: ResourceManager, batch_deployments: None + ) -> None: + key = resources.key() + file = unwrap( + client.upload_file( + content=render_jsonl(OPENAI_BATCH_MODEL), + form=FileUploadForm(purpose="batch"), + model=OPENAI_BATCH_MODEL, + key=key, + ) + ) + resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + + fetched = unwrap(client.retrieve_file(file.id, key=key)) + assert fetched.id == file.id, "retrieve must echo the uploaded file id" + assert fetched.purpose == "batch", ( + f"retrieve must round-trip purpose, got {fetched.purpose!r}" + ) + assert fetched.filename == UPLOAD_FILENAME, ( + f"retrieve must round-trip filename, got {fetched.filename!r}" + ) + + BATCH_RL_REQUEST_LINES = 3 BATCH_RL_RPM_LIMIT = 2 diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 609da6a9b07..7b5ad3551b8 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -43,6 +43,10 @@ def pytest_configure(config: pytest.Config) -> None: "markers", "load: heavy throughput/load test; collected last so it never perturbs latency-sensitive suites", ) + config.addinivalue_line( + "markers", + "weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set", + ) def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: diff --git a/tests/e2e/coverage_registry/guardrail.yaml b/tests/e2e/coverage_registry/guardrail.yaml index 68722fbbb96..d54c12ba6dc 100644 --- a/tests/e2e/coverage_registry/guardrail.yaml +++ b/tests/e2e/coverage_registry/guardrail.yaml @@ -31,3 +31,4 @@ - {id: guardrail.tool_policy.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/tool_policy/tool_policy_guardrail.py", rationale: "Tool-use policy enforcement"} - {id: guardrail.mcp_security.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [mcp_operations], source: "guardrail_hooks/mcp_security", rationale: "MCP protocol security"} - {id: guardrail.llm_as_a_judge.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/llm_as_a_judge", rationale: "LLM-based judgment guardrail"} +- {id: guardrail.litellm_content_filter.pre_mcp_call.blocks, module: guardrail, tier: P1, hook_point: pre_mcp_call, assertions: [blocks], exercised_on: [mcp_operations], source: "guardrail_hooks/litellm_content_filter/content_filter.py:_scan_mcp_tool_call_arguments", rationale: "A general content-filter guardrail configured mode=pre_mcp_call blocks a banned keyword in an MCP tool call's arguments before it reaches the upstream MCP server; a clean argument passes"} diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index 3b1aff80024..26280d35da0 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -46,10 +46,10 @@ - {id: llm.messages.anthropic.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Extended thinking via Messages API"} - {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Flagged Claude 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (#32578/#32831/#32882)", fail_before_fix: proven} - {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (#32831)", fail_before_fix: proven} -- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (Kraken Tech RCA gap)", fail_before_fix: proven} -- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (Kraken Tech RCA gap)", fail_before_fix: proven} -- {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (Kraken Tech RCA gap)", fail_before_fix: proven} -- {id: llm.messages.vertex.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (Kraken Tech RCA gap)", fail_before_fix: proven} +- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven} +- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven} +- {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven} +- {id: llm.messages.vertex.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven} - {id: llm.responses.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Core endpoint; OpenAI Responses native"} - {id: llm.responses.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Streaming via /v1/responses"} - {id: llm.responses.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "response_api_endpoints/endpoints.py:26", rationale: "Cost logged on responses"} diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml index 2b456aacefc..63e6fde14a3 100644 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ b/tests/e2e/coverage_registry/llm_nonconversational.yaml @@ -32,7 +32,7 @@ - {id: llm.files.hosted_vllm.upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "hosted_vllm OpenAI-compatible file upload"} - {id: llm.rerank.cohere.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: cohere, capability: basic, streaming: nonstream, assertions: [works], source: "test_rerank_e2e.py:29", rationale: "Cohere rerank, top_n + relevance_score"} - {id: llm.files.openai.content.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "GET /v1/files/{id}/content returns uploaded batch JSONL bytes"} -- {id: llm.realtime.bedrock_converse.basic.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: bedrock_converse, capability: basic, streaming: stream, assertions: [works], source: "test_nova_sonic_realtime_e2e.py", rationale: "Nova Sonic realtime session emits response.done (LIT-2239)"} +- {id: llm.realtime.bedrock_converse.basic.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: bedrock_converse, capability: basic, streaming: stream, assertions: [works], source: "test_realtime_bedrock_e2e.py", rationale: "Nova Sonic realtime session emits response.done (LIT-2239)"} - {id: llm.rerank.bedrock.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "llms/bedrock/rerank/handler.py", rationale: "Bedrock rerank"} - {id: llm.rerank.together_ai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: together_ai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/together_ai/rerank/handler.py", rationale: "Together rerank"} - {id: llm.images_generations.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_image_generation_e2e.py:22", rationale: "OpenAI image gen, b64/url"} diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index 6b183cbf9f3..ace4f8bcdc9 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -28,3 +28,12 @@ - {id: other.config.overrides.audit_logged, module: other, tier: P1, area: config, assertions: [audit_logged], source: "config_override_endpoints.py:67-100", rationale: "Config override mutations audit-logged, values redacted"} - {id: other.key_mgmt.regenerate.grace_period_honored, module: other, tier: P1, area: auth, assertions: [grace_period_honored], source: "key_management_endpoints.py:4503-4560", rationale: "Old key valid during grace_period then revoked"} - {id: other.key_mgmt.spend_reset.resets_to_value, module: other, tier: P1, area: auth, assertions: [resets_to_value], source: "key_management_endpoints.py:4841", rationale: "reset_spend resets accumulated spend"} +- {id: other.a2a.register.persists, module: other, tier: P1, area: a2a, assertions: [persists], source: "agent_endpoints/endpoints.py:325-443", rationale: "POST /v1/agents registers an agent card; GET /v1/agents/{id} reads it back"} +- {id: other.a2a.register.unsupported_version_rejected, module: other, tier: P1, area: a2a, assertions: [unsupported_version_rejected], source: "agent_endpoints/endpoints.py _validate_protocol_version", rationale: "A card pinning a protocolVersion outside SUPPORTED_A2A_PROTOCOL_VERSIONS is refused with 400"} +- {id: other.a2a.register.semver_version_accepted, module: other, tier: P1, area: a2a, assertions: [semver_version_accepted], source: "agent_endpoints/endpoints.py _validate_protocol_version", rationale: "A card pinning a patch-level semver like 0.3.0 (what the Google A2A SDK emits) registers, stores and serves the canonical 0.3 rather than 400ing; regression guard for the v1.92 report"} +- {id: other.a2a.message_send.real_world_agent_replies, module: other, tier: P1, area: a2a, assertions: [real_world_agent_replies], source: "agent_endpoints/a2a_endpoints.py asend_message", rationale: "A real published a2a agent fetched live from a public /.well-known endpoint (pinning the full semver 0.3.0 the a2a-sdk emits) registers, serves the canonical 0.3, and a message/send skill invocation proxies to the live upstream and returns the agent's reply"} +- {id: other.a2a.register.malformed_version_rejected, module: other, tier: P1, area: a2a, assertions: [malformed_version_rejected], source: "a2a/agent_card.py normalize_protocol_version", rationale: "A malformed protocolVersion like 0.3.garbage fails full-string semver validation and is refused with 400 instead of truncating to a supported family"} +- {id: other.a2a.discovery.proxy_fronted_card, module: other, tier: P1, area: a2a, assertions: [proxy_fronted_card], source: "agent_endpoints/a2a_endpoints.py get_agent_card", rationale: "/.well-known/agent-card.json serves the proxy url + supportedInterfaces and the LiteLLM virtual-key bearer scheme, not the upstream"} +- {id: other.a2a.message_send.bridge_invokes, module: other, tier: P1, area: a2a, assertions: [bridge_invokes], source: "a2a_protocol/litellm_completion_bridge/handler.py", rationale: "A2A message/send routes through the completion bridge to a real provider and logs an asend_message spend row"} +- {id: other.a2a.version.serves_pinned_0_3, module: other, tier: P1, area: a2a, assertions: [serves_pinned_0_3], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 0.3 returns the flat 0.3 message shape (parts on the result)"} +- {id: other.a2a.version.serves_pinned_1_0, module: other, tier: P1, area: a2a, assertions: [serves_pinned_1_0], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 1.0 returns the nested 1.0 message shape (result.message with ROLE_AGENT)"} diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 1538d3f3cda..ebbfd3415a5 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -24,3 +24,4 @@ - {id: reliability.timeout.request_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: request_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions, messages], source: "litellm/router.py:545-551", rationale: "Per-request timeout raises Timeout"} - {id: reliability.timeout.stream_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: stream_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions], source: "litellm/router.py:551", rationale: "Streaming chunk-delivery timeout"} - {id: reliability.perf.throughput.under_slo, module: reliability, tier: P1, behavior: perf, variant: throughput, assertions: [under_slo], exercised_on: [chat_completions, messages], source: grammar, rationale: "Throughput SLO under load"} +- {id: reliability.perf.session_anomaly.under_slo, module: reliability, tier: P1, behavior: perf, variant: session_anomaly, assertions: [under_slo], exercised_on: [messages], source: grammar, rationale: "Weekly Claude Code-shaped multi-turn session load against real providers; ceilings on error rate, warm-turn cache read/write, p95 turn time, and gateway-recorded spend (LIT-4562)"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 3be339d28a0..30353b93dd6 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -78,6 +78,24 @@ LOAD_DURATION_SECONDS = float(os.environ.get("E2E_LOAD_DURATION_SECONDS", "60")) LOAD_MIN_RPS = float(os.environ.get("E2E_LOAD_MIN_RPS", "355")) LOAD_MAX_FAILURE_RATIO = float(os.environ.get("E2E_LOAD_MAX_FAILURE_RATIO", "0.01")) +WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY" +ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) +ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) +ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) +ANOMALY_MAX_ERROR_RATIO = float(os.environ.get("E2E_ANOMALY_MAX_ERROR_RATIO", "0.05")) +ANOMALY_MIN_WARM_CACHE_READ_SHARE = float( + os.environ.get("E2E_ANOMALY_MIN_WARM_CACHE_READ_SHARE", "0.65") +) +ANOMALY_MAX_P95_TURN_SECONDS = float( + os.environ.get("E2E_ANOMALY_MAX_P95_TURN_SECONDS", "30") +) +ANOMALY_MAX_KEY_SPEND_USD = float( + os.environ.get("E2E_ANOMALY_MAX_KEY_SPEND_USD", "0.60") +) +ANOMALY_SPEND_SETTLE_SECONDS = float( + os.environ.get("E2E_ANOMALY_SPEND_SETTLE_SECONDS", "75") +) + def require_env(*names: str) -> tuple[str, ...]: """Return the non-empty values for each env name, or hard-fail naming which are missing. diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index 1c6048a3688..03d7b5d051a 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -62,6 +62,7 @@ R = TypeVar("R", bound=BaseModel) class Success(BaseModel, Generic[R]): kind: Literal["success"] = "success" + status_code: int data: R @@ -146,6 +147,32 @@ class StreamingResponse(BaseModel): return "text/event-stream" in (self.content_type or "") +class BinaryStream(BaseModel): + """Outcome of consuming a binary chunked response (e.g. TTS audio) as a stream. + + Unlike StreamingResponse, which line-splits an SSE text body, this iterates the + raw bytes with iter_content and reports how many non-empty chunks arrived and + the total byte count, so a caller can assert customer-observable streaming + (multiple chunks, real bytes) without decoding the payload.""" + + status_code: int + content_type: str | None = None + call_id: str | None = None + transfer_encoding: str | None = None + content_length: str | None = None + error_body: str | None = None + chunk_count: int = 0 + total_bytes: int = 0 + + @property + def ok(self) -> bool: + return 200 <= self.status_code < 300 + + @property + def chunked(self) -> bool: + return "chunked" in (self.transfer_encoding or "") + + def _hdr(resp: requests.Response, name: str) -> str | None: value = resp.headers.get(name) return value if isinstance(value, str) else None @@ -159,6 +186,18 @@ def unwrap[R: BaseModel](result: Result[R]) -> R: raise AssertionError(result) +def unwrap_status[R: BaseModel](result: Result[R], expected_status: int) -> R: + """Like unwrap, but also pins the exact HTTP status the success came back on, + for routes whose contract is a specific 2xx (e.g. 201 Created on a submission).""" + match result: + case Success(status_code=status_code, data=data) if status_code == expected_status: + return data + case Success(status_code=status_code): + raise AssertionError(f"expected HTTP {expected_status}, got {status_code}") + case _: + raise AssertionError(result) + + def is_ok[R: BaseModel](result: Result[R]) -> bool: match result: case Success(): @@ -199,7 +238,7 @@ def _classify[R: BaseModel]( if not resp.ok: return UnknownApiError(status_code=resp.status_code, body=resp.text) try: - return Success(data=response_type.model_validate(resp.json())) + return Success(status_code=resp.status_code, data=response_type.model_validate(resp.json())) except Exception as exc: # noqa: BLE001 - any parse/validation failure is a value return ValidationError(message=str(exc)) @@ -244,6 +283,26 @@ def get[R: BaseModel]( return _classify(resp, response_type) +def get_external[R: BaseModel]( + url: str, + *, + response_type: type[R], + timeout: float = 30.0, +) -> Result[R]: + """GET an absolute URL outside the proxy (e.g. a public /.well-known document). + Unlike the transport wrappers there is no proxy base url and no proxy auth; the + response still gets the same tagged-union classification as every other call.""" + try: + resp = requests.get( + url, + headers={"Accept": "application/json"}, + timeout=timeout, + ) + except requests.RequestException as exc: + return NetworkError(message=str(exc)) + return _classify(resp, response_type) + + def delete[R: BaseModel]( url: URL, *, @@ -286,6 +345,26 @@ def patch[R: BaseModel]( return _classify(resp, response_type) +def put[R: BaseModel]( + url: URL, + *, + headers: BaseModel, + json: BaseModel, + response_type: type[R], + timeout: float = 30.0, +) -> Result[R]: + try: + resp = requests.put( + str(url), + headers=_headers(headers), + json=json.model_dump(by_alias=True, exclude_none=True), + timeout=timeout, + ) + except requests.RequestException as exc: + return NetworkError(message=str(exc)) + return _classify(resp, response_type) + + def probe( url: URL, *, headers: BaseModel, params: BaseModel, timeout: float = 30.0 ) -> ProbeResult: @@ -397,16 +476,18 @@ def upload[R: BaseModel]( url: URL, *, headers: BaseModel, - form: FileUploadForm, + form: BaseModel, filename: str, content: bytes, + file_content_type: str = "application/jsonl", params: BaseModel | None = None, response_type: type[R], timeout: float = 60.0, ) -> Result[R]: - """Multipart POST for file uploads (/v1/files). Form fields come from `form`, - the file bytes are sent as the `file` part, and `params` carries any query - routing (e.g. ?model=). requests sets the multipart Content-Type itself.""" + """Multipart POST for file-bearing routes (/v1/files, /v1/audio/transcriptions). + Form fields come from `form`, the file bytes are sent as the `file` part with + `file_content_type`, and `params` carries any query routing (e.g. ?model=). + requests sets the multipart Content-Type itself.""" dumped: dict[str, object] = form.model_dump(by_alias=True, exclude_none=True) data = {key: str(value) for key, value in dumped.items()} try: @@ -415,7 +496,7 @@ def upload[R: BaseModel]( headers=_headers(headers), params=_params(params), data=data, - files={"file": (filename, content, "application/jsonl")}, + files={"file": (filename, content, file_content_type)}, timeout=timeout, ) except requests.RequestException as exc: @@ -423,6 +504,54 @@ def upload[R: BaseModel]( return _classify(resp, response_type) +def stream_binary( + url: URL, + *, + headers: BaseModel, + json: BaseModel, + chunk_size: int = 8192, + timeout: float = 60.0, +) -> BinaryStream: + """POST that consumes a binary chunked response (e.g. TTS audio) as a stream, + counting non-empty chunks and total bytes with iter_content. A non-2xx status + short-circuits with the counts left at zero so the caller can fail loudly.""" + try: + resp = requests.post( + str(url), + headers=_headers(headers), + json=json.model_dump(by_alias=True, exclude_none=True), + stream=True, + timeout=timeout, + ) + except requests.RequestException as exc: + return BinaryStream(status_code=-1, error_body=str(exc)[:300]) + with resp: + content_type = _hdr(resp, "content-type") + call_id = _hdr(resp, "x-litellm-call-id") + transfer_encoding = _hdr(resp, "transfer-encoding") + content_length = _hdr(resp, "content-length") + if not (200 <= resp.status_code < 300): + return BinaryStream( + status_code=resp.status_code, + content_type=content_type, + call_id=call_id, + transfer_encoding=transfer_encoding, + content_length=content_length, + error_body=resp.text[:300], + ) + raw_chunks = cast("Iterator[bytes]", resp.iter_content(chunk_size=chunk_size)) + chunks = tuple(chunk for chunk in raw_chunks if chunk) + return BinaryStream( + status_code=resp.status_code, + content_type=content_type, + call_id=call_id, + transfer_encoding=transfer_encoding, + content_length=content_length, + chunk_count=len(chunks), + total_bytes=sum(len(chunk) for chunk in chunks), + ) + + def download( url: URL, *, headers: BaseModel, timeout: float = 60.0 ) -> StreamingResponse: diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py index d24cd36c2fd..53f2e4480df 100644 --- a/tests/e2e/guardrails/guardrails_client.py +++ b/tests/e2e/guardrails/guardrails_client.py @@ -10,13 +10,15 @@ from typing import Literal from pydantic import BaseModel -from e2e_config import POLL_INTERVAL, POLL_TIMEOUT +from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, unique_marker from e2e_http import NoBody, Result, Success, unwrap +from lifecycle import ResourceManager from models import ( ChatBody, ChatMessage, ChatResponse, KeyGenerateBody, + LiteLLMParamsBody, TeamDeleteBody, TeamInfoParams, TeamInfoResponse, @@ -54,7 +56,33 @@ class BedrockGuardrailParamsBody(GuardrailParamsBase): aws_region_name: str | None = None -GuardrailParamsBody = ContentFilterParamsBody | BedrockGuardrailParamsBody +class OpenAIModerationParamsBody(GuardrailParamsBase): + guardrail: Literal["openai_moderation"] = "openai_moderation" + api_key: str | None = None + model: str | None = None + + +class PresidioParamsBody(GuardrailParamsBase): + guardrail: Literal["presidio"] = "presidio" + presidio_analyzer_api_base: str | None = None + presidio_anonymizer_api_base: str | None = None + # apply_to_output masks PII the model itself emitted, which also makes the + # guardrail run post_call. logging_only masks what the proxy logs. + apply_to_output: bool | None = None + logging_only: bool | None = None + + +class BlockCodeExecutionParamsBody(GuardrailParamsBase): + guardrail: Literal["block_code_execution"] = "block_code_execution" + + +GuardrailParamsBody = ( + ContentFilterParamsBody + | BedrockGuardrailParamsBody + | OpenAIModerationParamsBody + | PresidioParamsBody + | BlockCodeExecutionParamsBody +) class GuardrailSpecBody(BaseModel): @@ -135,6 +163,35 @@ class GuardrailsClient: ) ).guardrail_id + def create_backend_model(self, resources: ResourceManager, prefix: str = "e2e-guard-backend") -> str: + """Register a gemini chat deployment for a guardrail test to run against + (deleted on teardown). The guardrails under test here gate on prompt/output + content, not the backend, so a single cheap deployment stands in for the + model the customer would call.""" + model_name = f"{prefix}-{unique_marker()}" + model_id = self.proxy.create_model( + model_name, + LiteLLMParamsBody(model="gemini/gemini-2.5-flash", api_key="os.environ/GEMINI_API_KEY"), + ) + resources.defer(lambda: self.proxy.delete_model(model_id)) + return model_name + + def register(self, name: str, params: GuardrailParamsBody) -> str: + """Register any guardrail via POST /guardrails and return its id. New + built-ins register with default_on=False and are opted into per request + via the chat body's `guardrails` list, so one guardrail under test never + intercepts unrelated traffic on the shared proxy.""" + return unwrap( + self.proxy.transport.post( + "/guardrails", + headers=self.proxy.transport.master, + json=GuardrailCreateBody( + guardrail=GuardrailSpecBody(guardrail_name=name, litellm_params=params) + ), + response_type=GuardrailCreateResponse, + ) + ).guardrail_id + def delete_guardrail(self, guardrail_id: str) -> None: _ = self.proxy.transport.delete( f"/guardrails/{guardrail_id}", @@ -171,13 +228,27 @@ class GuardrailsClient: KeyGenerateBody(team_id=team_id, user_id="e2e-guardrails-user") ) - def chat(self, key: str, model: str, text: str) -> Result[ChatResponse]: + def chat( + self, + key: str, + model: str, + text: str, + *, + guardrails: list[str] | None = None, + max_tokens: int = 16, + ) -> Result[ChatResponse]: + """Drive a chat call, optionally opting into named guardrails for this + request only (the per-request `guardrails` selector). With `guardrails` + omitted the call behaves exactly as before for the default-on suites. + `max_tokens` defaults low for block checks (the model barely runs) but is + raised when a test needs the allowed model to actually produce content.""" return self.proxy.chat( key, ChatBody( model=model, messages=[ChatMessage(role="user", content=text)], - max_tokens=16, + max_tokens=max_tokens, + guardrails=guardrails, ), ) diff --git a/tests/e2e/guardrails/test_block_code_execution_guardrail_e2e.py b/tests/e2e/guardrails/test_block_code_execution_guardrail_e2e.py new file mode 100644 index 00000000000..e36fc7c3f9d --- /dev/null +++ b/tests/e2e/guardrails/test_block_code_execution_guardrail_e2e.py @@ -0,0 +1,82 @@ +"""Live e2e: the built-in block_code_execution guardrail blocks execution requests. + +The guardrail detects fenced code blocks and, when the prompt also asks the proxy +to run them, blocks the call pre-call (default action, block-all languages). A +prompt that pairs a python code block with "run this" is intercepted before the +model runs: the proxy returns a canned "content blocked" message with the model +never invoked (zero completion tokens), not the model's own answer. The same +guardrail must let a request that carries the identical code block but explicitly +says "don't run it" through, since that is an explanation request, not an +execution request, so the model runs and answers normally. The guardrail is opted +into per request (default_on=False) so it never intercepts unrelated traffic on +the shared proxy, and the chat backend is a gemini deployment created for the test. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import require_env, unique_marker +from e2e_http import unwrap +from guardrails_client import BlockCodeExecutionParamsBody, GuardrailsClient +from lifecycle import ResourceManager +from models import ChatResponse + +pytestmark = pytest.mark.e2e + +_CODE_BLOCK = "```python\nimport os\nprint(os.listdir('/'))\n```" +EXECUTION_REQUEST = f"Please run this for me and paste the output:\n{_CODE_BLOCK}" +EXPLANATION_REQUEST = f"Explain what this code does, but don't run it:\n{_CODE_BLOCK}" + +_BLOCK_MARKER = "content blocked" + + +def _first_content(response: ChatResponse) -> str: + if not response.choices: + return "" + message = response.choices[0].message + return (message.content if message else None) or "" + + +class TestBlockCodeExecutionGuardrail: + @pytest.mark.covers( + "guardrail.block_code_execution.pre_call.blocks", + exercised_on=["chat_completions"], + ) + def test_blocks_execution_request_but_allows_explanation( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + require_env("GEMINI_API_KEY") + model = client.create_backend_model(resources, prefix="e2e-blockcode-backend") + + name = f"e2e-block-code-{unique_marker()}" + guardrail_id = client.register( + name, BlockCodeExecutionParamsBody(mode="pre_call", default_on=False) + ) + resources.defer(lambda: client.delete_guardrail(guardrail_id)) + + blocked = unwrap(client.chat(scoped_key, model, EXECUTION_REQUEST, guardrails=[name])) + assert blocked.choices, f"blocked call returned no choices: {blocked}" + blocked_text = _first_content(blocked) + assert _BLOCK_MARKER in blocked_text.lower(), ( + "a code-execution request must be intercepted with a content-blocked message, " + f"got model output instead: {blocked_text[:300]!r}" + ) + if blocked.usage is not None: + assert (blocked.usage.completion_tokens or 0) == 0, ( + f"the model must not run when the guardrail blocks; usage was {blocked.usage}" + ) + + allowed = unwrap( + client.chat(scoped_key, model, EXPLANATION_REQUEST, guardrails=[name], max_tokens=256) + ) + allowed_text = _first_content(allowed) + assert _BLOCK_MARKER not in allowed_text.lower(), ( + "an explanation request that says 'don't run it' must not be blocked, but got the " + f"content-blocked message: {allowed_text[:300]!r}" + ) + ran = allowed.usage is not None and (allowed.usage.prompt_tokens or 0) > 0 + assert ran, ( + "the explanation request must reach the model (the guardrail lets it through), but " + f"the model was never invoked; usage was {allowed.usage}" + ) diff --git a/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py b/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py new file mode 100644 index 00000000000..4e2fcbf8fba --- /dev/null +++ b/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py @@ -0,0 +1,66 @@ +"""Live e2e: the built-in OpenAI moderation guardrail blocks flagged input pre-call. + +Registers an `openai_moderation` guardrail (default_on=False) and opts into it for +one request via the chat body's `guardrails` selector, so it never intercepts +unrelated traffic on the shared proxy. The guardrail calls OpenAI's moderation API +before the upstream model runs; a prompt that trips the policy must be rejected +with HTTP 400 naming the moderation policy, and the same guardrail must let a +benign prompt through. The chat backend is a gemini deployment created for the +test (and torn down); moderation runs independently of it, so the block is +attributable to the guardrail, not the model. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import require_env, unique_marker +from e2e_http import UnknownApiError, unwrap +from guardrails_client import GuardrailsClient, OpenAIModerationParamsBody +from lifecycle import ResourceManager + +pytestmark = pytest.mark.e2e + +# A direct threat of violence reliably trips OpenAI's harassment/violence categories. +FLAGGED_PROMPT = "I am going to find you and violently kill you and your entire family tonight." +BENIGN_PROMPT = "Reply with the single word: hello" + + +class TestOpenAIModerationGuardrail: + @pytest.mark.covers( + "guardrail.openai_moderations.pre_call.blocks", + exercised_on=["chat_completions"], + ) + def test_moderation_blocks_flagged_input( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + require_env("OPENAI_API_KEY", "GEMINI_API_KEY") + model = client.create_backend_model(resources, prefix="e2e-moderation-backend") + + name = f"e2e-openai-moderation-{unique_marker()}" + guardrail_id = client.register( + name, + OpenAIModerationParamsBody( + mode="pre_call", default_on=False, api_key="os.environ/OPENAI_API_KEY" + ), + ) + resources.defer(lambda: client.delete_guardrail(guardrail_id)) + + blocked = client.chat(scoped_key, model, FLAGGED_PROMPT, guardrails=[name]) + match blocked: + case UnknownApiError(status_code=400, body=body): + assert "moderation" in body.lower(), ( + f"the block body must name the moderation policy, got: {body[:400]}" + ) + case UnknownApiError(status_code=status, body=body): + pytest.fail(f"expected a 400 moderation block, got {status}: {body[:400]}") + case _: + pytest.fail( + f"openai moderation did not block a flagged prompt; got {blocked}" + ) + + allowed = unwrap(client.chat(scoped_key, model, BENIGN_PROMPT, guardrails=[name])) + assert allowed.choices, ( + "the same moderation guardrail must let a benign prompt through, but the " + f"call returned no choices: {allowed}" + ) diff --git a/tests/e2e/guardrails/test_presidio_guardrail_e2e.py b/tests/e2e/guardrails/test_presidio_guardrail_e2e.py new file mode 100644 index 00000000000..a911f387382 --- /dev/null +++ b/tests/e2e/guardrails/test_presidio_guardrail_e2e.py @@ -0,0 +1,211 @@ +"""Live e2e: the built-in Presidio PII guardrail masks PII on the request, on the +model output, and in what the proxy logs. + +Presidio replaces detected PII with `` placeholders (e.g. +``) via a real analyzer + anonymizer. Three modes are checked +independently, each opted into per request (default_on=False) so it never touches +unrelated traffic: + +- pre_call: the prompt is anonymized before it reaches the model, so a + repeat-verbatim request comes back with the placeholder, never the raw email +- post_call (apply_to_output): PII the model itself emits is masked on the way + out, so the caller never receives the raw value the model produced +- logging_only: the call is not blocked, and the request the proxy records is + masked. That is read back from the real OTEL destination (Jaeger): the gen-AI + span's `gen_ai.input.messages` attribute carries the masked placeholder, never + the raw email + +Analyzer/anonymizer bases come from PRESIDIO_ANALYZER_API_BASE / +PRESIDIO_ANONYMIZER_API_BASE (compose provides the in-network hosts; point them at +locally published container ports for a host run). The logging_only check needs +the OTEL v2 logger active and its destination readable at OTEL_QUERY_URL, with +message-content capture on (OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT). +The chat backend is a gemini deployment created for the test. +""" + +from __future__ import annotations + +import time + +import pytest + +from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, require_env, unique_marker +from e2e_http import NoBody, require_successful_call, unwrap +from guardrails_client import GuardrailMode, GuardrailsClient, PresidioParamsBody +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, ChatResponse, ReadinessDetailsResponse +from otel_client import JaegerSpan, OtelReader, build_otel_reader + +pytestmark = pytest.mark.e2e + +RAW_EMAIL = "alice.example.person@example.com" +PLACEHOLDER = "" + +ECHO_REQUEST = f"Repeat the following text back exactly, verbatim, with no changes: My email is {RAW_EMAIL}" +EMIT_REQUEST = f"Output exactly this one line and nothing else: Please contact {RAW_EMAIL} today" +LOG_REQUEST = f"Say hello and include this email once verbatim: {RAW_EMAIL}" + +OTEL_V2_LOGGER = "OpenTelemetryV2" +INPUT_MESSAGES_TAG = "gen_ai.input.messages" + + +def _content(response: ChatResponse) -> str: + if not response.choices: + return "" + message = response.choices[0].message + return (message.content if message else None) or "" + + +def _span_tag(span: JaegerSpan, key: str) -> str | None: + for tag in span.tags: + if tag.key == key and isinstance(tag.value, str): + return tag.value + return None + + +def _poll_logged_prompt(reader: OtelReader, *, call_id: str, genai_span: str) -> str | None: + """Poll the OTEL destination until the call's gen-AI span carries a masked + logged prompt, and return it. logging_only masks the payload asynchronously, + so the span can briefly export before the mask lands; polling to a deadline + waits that out and returns the last value seen so the caller's assertions + report the real final state if it never masks.""" + deadline = time.monotonic() + POLL_TIMEOUT + last: str | None = None + while time.monotonic() < deadline: + for trace in reader.traces_for_call(call_id): + for span in trace.spans: + if span.operation_name != genai_span: + continue + value = _span_tag(span, INPUT_MESSAGES_TAG) + if value is not None: + last = value + if PLACEHOLDER in value and RAW_EMAIL not in value: + return value + time.sleep(POLL_INTERVAL) + return last + + +def _presidio_params( + mode: GuardrailMode, *, apply_to_output: bool = False, logging_only: bool = False +) -> PresidioParamsBody: + analyzer, anonymizer = require_env( + "PRESIDIO_ANALYZER_API_BASE", "PRESIDIO_ANONYMIZER_API_BASE" + ) + return PresidioParamsBody( + mode=mode, + default_on=False, + presidio_analyzer_api_base=analyzer, + presidio_anonymizer_api_base=anonymizer, + apply_to_output=apply_to_output, + logging_only=logging_only, + ) + + +def _require_otel_v2_active(client: GuardrailsClient) -> None: + details = unwrap( + client.proxy.transport.get( + "/health/readiness/details", + headers=client.proxy.transport.master, + params=NoBody(), + response_type=ReadinessDetailsResponse, + ) + ) + assert OTEL_V2_LOGGER in details.success_callbacks, ( + f"the logging_only check reads the masked prompt back from OTEL, so the proxy must have " + f"the {OTEL_V2_LOGGER} logger active; got callbacks: {details.success_callbacks}" + ) + + +class TestPresidioGuardrail: + @pytest.mark.covers( + "guardrail.presidio.pre_call.masks", + exercised_on=["chat_completions"], + ) + def test_pre_call_masks_pii_before_the_model_sees_it( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + require_env("GEMINI_API_KEY") + model = client.create_backend_model(resources, prefix="e2e-presidio-pre") + name = f"e2e-presidio-pre-{unique_marker()}" + guardrail_id = client.register(name, _presidio_params("pre_call")) + resources.defer(lambda: client.delete_guardrail(guardrail_id)) + + echoed = _content( + unwrap(client.chat(scoped_key, model, ECHO_REQUEST, guardrails=[name], max_tokens=128)) + ) + assert RAW_EMAIL not in echoed, ( + "pre_call masking must strip the raw email before the model sees it, but the " + f"model echoed it back: {echoed[:300]!r}" + ) + assert PLACEHOLDER in echoed, ( + "the model should have echoed the masked placeholder the guardrail substituted, " + f"got: {echoed[:300]!r}" + ) + + @pytest.mark.covers( + "guardrail.presidio.post_call.masks", + exercised_on=["chat_completions"], + ) + def test_post_call_masks_pii_in_model_output( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + require_env("GEMINI_API_KEY") + model = client.create_backend_model(resources, prefix="e2e-presidio-post") + name = f"e2e-presidio-post-{unique_marker()}" + guardrail_id = client.register(name, _presidio_params("post_call", apply_to_output=True)) + resources.defer(lambda: client.delete_guardrail(guardrail_id)) + + out = _content( + unwrap(client.chat(scoped_key, model, EMIT_REQUEST, guardrails=[name], max_tokens=128)) + ) + assert RAW_EMAIL not in out, ( + "post_call masking must strip PII the model emitted, but the raw email reached the " + f"caller: {out[:300]!r}" + ) + assert PLACEHOLDER in out, ( + f"the masked placeholder should replace the model's PII output, got: {out[:300]!r}" + ) + + @pytest.mark.covers( + "guardrail.presidio.logging_only.masks", + exercised_on=["chat_completions"], + ) + def test_logging_only_masks_the_logged_prompt( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + require_env("GEMINI_API_KEY") + _require_otel_v2_active(client) + reader = build_otel_reader() + + model = client.create_backend_model(resources, prefix="e2e-presidio-log") + name = f"e2e-presidio-log-{unique_marker()}" + guardrail_id = client.register(name, _presidio_params("logging_only", logging_only=True)) + resources.defer(lambda: client.delete_guardrail(guardrail_id)) + + outcome = client.proxy.transport.send( + "/chat/completions", + headers=client.proxy.transport.bearer(scoped_key), + json=ChatBody( + model=model, + messages=[ChatMessage(role="user", content=LOG_REQUEST)], + max_tokens=64, + guardrails=[name], + ), + ) + require_successful_call(outcome) # logging_only must not block + assert outcome.call_id is not None, "the response must carry x-litellm-call-id to find its trace" + + genai_span = f"chat {model}" + logged_prompt = _poll_logged_prompt(reader, call_id=outcome.call_id, genai_span=genai_span) + assert logged_prompt is not None, ( + f"the gen-AI span {genai_span!r} never recorded {INPUT_MESSAGES_TAG} at the OTEL " + "destination within the deadline (message-content capture must be on, and the trace " + "must reach the destination)" + ) + assert RAW_EMAIL not in logged_prompt, ( + "logging_only must mask the PII the proxy records for the request, but the raw email " + f"is present in the logged prompt: {logged_prompt[:400]!r}" + ) + assert PLACEHOLDER in logged_prompt, ( + f"the logged prompt must carry the masked placeholder, got: {logged_prompt[:400]!r}" + ) diff --git a/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py b/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py index cd32a19d54a..db917d6ede9 100644 --- a/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py @@ -8,6 +8,8 @@ suite was removed. from __future__ import annotations +import time + import pytest from e2e_config import unique_marker @@ -19,11 +21,39 @@ pytestmark = pytest.mark.e2e MODEL = "gemini-2.5-flash" +# A guardrail created via POST /guardrails is registered in-process immediately +# on the worker that served the create call, but the proxy runs multiple +# pods/workers behind the shared key, and every other one only picks up the new +# guardrail on its next periodic DB sync (every 30s), so the very next request +# can race a worker that has not synced yet. +GUARDRAIL_PROPAGATION_DEADLINE_SECONDS = 40.0 +GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS = 5.0 + def _prompt_with(banned_keyword: str) -> str: return f"Reply with the single word OK. {banned_keyword}" +def _assert_eventually_blocked(client: GuardrailsClient, key: str, banned: str) -> None: + deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS + while True: + result = client.chat(key, MODEL, _prompt_with(banned)) + match result: + case UnknownApiError(status_code=status, body=body): + assert status == 400, f"expected a 400 guardrail block, got {status}: {body[:300]}" + assert "content blocked" in body.lower() or banned in body, ( + f"block response missing content-filter reason: {body[:300]}" + ) + return + case _ if time.monotonic() < deadline: + time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS) + case _: + pytest.fail( + f"default-on guardrail never blocked the banned keyword within " + f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; got {result}" + ) + + class TestTeamDisableGlobalGuardrail: @pytest.mark.covers( "guardrail.litellm_content_filter.pre_call.blocks", @@ -33,25 +63,10 @@ class TestTeamDisableGlobalGuardrail: self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: banned = unique_marker() - guardrail_id = client.create_content_filter_guardrail( - f"e2e-content-filter-{banned}", banned - ) + guardrail_id = client.create_content_filter_guardrail(f"e2e-content-filter-{banned}", banned) resources.defer(lambda: client.delete_guardrail(guardrail_id)) - result = client.chat(scoped_key, MODEL, _prompt_with(banned)) - - match result: - case UnknownApiError(status_code=status, body=body): - assert status == 400, ( - f"expected a 400 guardrail block, got {status}: {body[:300]}" - ) - assert "content blocked" in body.lower() or banned in body, ( - f"block response missing content-filter reason: {body[:300]}" - ) - case _: - pytest.fail( - f"default-on guardrail did not block the banned keyword; got {result}" - ) + _assert_eventually_blocked(client, scoped_key, banned) @pytest.mark.covers( "guardrail.litellm_content_filter.pre_call.allows", @@ -61,14 +76,10 @@ class TestTeamDisableGlobalGuardrail: self, client: GuardrailsClient, resources: ResourceManager ) -> None: banned = unique_marker() - guardrail_id = client.create_content_filter_guardrail( - f"e2e-content-filter-{banned}", banned - ) + guardrail_id = client.create_content_filter_guardrail(f"e2e-content-filter-{banned}", banned) resources.defer(lambda: client.delete_guardrail(guardrail_id)) - team_id = client.create_team_opted_out_of_global_guardrails( - f"e2e-guardrail-optout-{banned}" - ) + team_id = client.create_team_opted_out_of_global_guardrails(f"e2e-guardrail-optout-{banned}") resources.defer(lambda: client.delete_team(team_id)) key = client.create_key_in_team(team_id) resources.defer(lambda: client.proxy.delete_key(key)) diff --git a/tests/e2e/llm_translation/endpoints_client.py b/tests/e2e/llm_translation/endpoints_client.py index 32d9922c775..ace621d03b3 100644 --- a/tests/e2e/llm_translation/endpoints_client.py +++ b/tests/e2e/llm_translation/endpoints_client.py @@ -15,7 +15,7 @@ from typing import Literal from pydantic import BaseModel from proxy_client import ProxyClient -from e2e_http import StreamingResponse +from e2e_http import BinaryStream, Result, StreamingResponse from models import CacheControl, ChatMessage, LiteLLMParamsBody, RichMessage, TextBlock __all__ = [ @@ -110,6 +110,16 @@ class ImageRequest(BaseModel): size: str = "1024x1024" +class TranscriptionForm(BaseModel): + model: str + response_format: str = "json" + + +class ModerationRequest(BaseModel): + model: str + input: str + + class ResponsesOutputContent(BaseModel): type: str | None = None text: str | None = None @@ -213,6 +223,27 @@ class ImagesResult(BaseModel): data: list[ImageItem] = [] +class TranscriptionResult(BaseModel): + text: str = "" + + +class ModerationResultItem(BaseModel): + flagged: bool + categories: dict[str, bool] = {} + + @property + def flagged_categories(self) -> tuple[str, ...]: + return tuple(name for name, hit in self.categories.items() if hit) + + +class ModerationResult(BaseModel): + results: list[ModerationResultItem] = [] + + @property + def first(self) -> ModerationResultItem | None: + return self.results[0] if self.results else None + + @dataclass(frozen=True, slots=True) class EndpointsClient: proxy: ProxyClient @@ -314,6 +345,36 @@ class EndpointsClient: "/v1/audio/speech", key, SpeechRequest(model=model, input=text, voice=voice) ) + def audio_speech_stream( + self, key: str, model: str, text: str, *, voice: str = "alloy" + ) -> BinaryStream: + return self.proxy.transport.stream_binary( + "/v1/audio/speech", + headers=self.proxy.transport.bearer(key), + json=SpeechRequest(model=model, input=text, voice=voice), + ) + + def transcribe( + self, key: str, model: str, *, filename: str, content: bytes + ) -> Result[TranscriptionResult]: + return self.proxy.transport.upload( + "/v1/audio/transcriptions", + headers=self.proxy.transport.bearer(key), + form=TranscriptionForm(model=model), + filename=filename, + content=content, + file_content_type="audio/wav", + response_type=TranscriptionResult, + ) + + def moderations(self, key: str, model: str, text: str) -> Result[ModerationResult]: + return self.proxy.transport.post( + "/v1/moderations", + headers=self.proxy.transport.bearer(key), + json=ModerationRequest(model=model, input=text), + response_type=ModerationResult, + ) + def images(self, key: str, model: str, prompt: str) -> StreamingResponse: return self._send( "/v1/images/generations", key, ImageRequest(model=model, prompt=prompt) diff --git a/tests/e2e/llm_translation/realtime/test_nova_sonic_realtime_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_bedrock_e2e.py similarity index 100% rename from tests/e2e/llm_translation/realtime/test_nova_sonic_realtime_e2e.py rename to tests/e2e/llm_translation/realtime/test_realtime_bedrock_e2e.py diff --git a/tests/e2e/llm_translation/test_audio_speech_e2e.py b/tests/e2e/llm_translation/test_audio_speech_e2e.py index f7a04d94cb3..b95cef8db4d 100644 --- a/tests/e2e/llm_translation/test_audio_speech_e2e.py +++ b/tests/e2e/llm_translation/test_audio_speech_e2e.py @@ -1,8 +1,9 @@ -"""Live e2e: POST /v1/audio/speech returns audio. +"""Live e2e: POST /v1/audio/speech returns audio, non-streamed and streamed. -Registers an OpenAI text-to-speech deployment at runtime and asserts the response -is an audio body (binary, not JSON). Migrated from -litellm-regression-tests/tests/test_inference_endpoints.py. +The non-streamed call asserts an audio (not JSON) body. The streamed call consumes +the response the way a player would and asserts customer-observable streaming: +chunked transfer encoding (a buffered body would carry a content-length) with +non-zero audio bytes. """ from __future__ import annotations @@ -19,6 +20,7 @@ pytestmark = pytest.mark.e2e class TestAudioSpeech: + @pytest.mark.covers("llm.audio_speech.openai.basic.nonstream.works") def test_audio_speech_returns_audio( self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: @@ -38,3 +40,39 @@ class TestAudioSpeech: f"/audio/speech content-type is not audio: {result.content_type!r}" ) assert result.body, "/audio/speech returned an empty body" + + @pytest.mark.covers("llm.audio_speech.openai.basic.stream.works") + def test_audio_speech_streams_audio_chunks( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-speech-stream-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="openai/gpt-4o-mini-tts", api_key="os.environ/OPENAI_API_KEY" + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.audio_speech_stream( + key, + model, + "Streaming speech should arrive in several audio chunks so a client can " + "begin playback well before the whole clip has finished generating.", + ) + assert result.ok, ( + f"/audio/speech stream failed (status {result.status_code}); body={result.error_body}" + ) + assert "audio" in (result.content_type or ""), ( + f"/audio/speech content-type is not audio: {result.content_type!r}" + ) + assert result.chunked, ( + f"/audio/speech did not stream: transfer-encoding={result.transfer_encoding!r}, " + f"content-length={result.content_length!r} (a buffered body is not a stream)" + ) + assert result.content_length is None, ( + f"/audio/speech advertised content-length={result.content_length!r} on a " + f"streamed response (a buffered body is not a stream)" + ) + assert result.total_bytes > 0, "/audio/speech stream returned no audio bytes" diff --git a/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py b/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py new file mode 100644 index 00000000000..af6123dc46a --- /dev/null +++ b/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py @@ -0,0 +1,51 @@ +"""Live e2e: POST /v1/audio/transcriptions turns speech into text. + +Registers an OpenAI speech-to-text deployment at runtime and uploads a spoken +weather question (the realtime suite's 24kHz WAV fixture) as multipart, asserting +the returned transcript is non-empty and mentions the word it was asked about. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from e2e_config import unique_marker +from e2e_http import unwrap +from endpoints_client import EndpointsClient +from lifecycle import ResourceManager +from models import LiteLLMParamsBody + +pytestmark = pytest.mark.e2e + +WEATHER_WAV = ( + Path(__file__).resolve().parent / "realtime" / "fixtures" / "weather_question_24k.wav" +) + + +class TestAudioTranscriptions: + @pytest.mark.covers("llm.audio_transcriptions.openai.basic.nonstream.works") + def test_audio_transcriptions_returns_text( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-transcribe-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="openai/gpt-4o-mini-transcribe", api_key="os.environ/OPENAI_API_KEY" + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = unwrap( + endpoints_client.transcribe( + key, model, filename=WEATHER_WAV.name, content=WEATHER_WAV.read_bytes() + ) + ) + text = result.text.strip() + assert text, "/audio/transcriptions returned an empty transcript" + assert "weather" in text.lower(), ( + f"transcript of a spoken weather question does not mention weather: {text!r}" + ) diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 13992744f42..8d3622e441a 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -19,17 +19,176 @@ from __future__ import annotations import os import pytest +from pydantic import BaseModel from e2e_config import require_env, unique_marker -from e2e_http import unwrap +from e2e_http import StreamingResponse, unwrap from lifecycle import ResourceManager -from models import ChatBody, ChatMessage, LiteLLMParamsBody +from models import ( + ChatBody, + ChatMessage, + ChatResponse, + ChatTool, + ChatToolFunction, + ImageContentPart, + ImageUrl, + LiteLLMParamsBody, + TextContentPart, + ThinkingParam, +) from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e COHERE_BACKEND = "cohere/command-r-08-2024" GEMINI_BACKEND = "gemini/gemini-2.5-flash" +OPENAI_BACKEND = "openai/gpt-5.6" +BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" + + +class _StreamToolCallFunction(BaseModel): + name: str | None = None + arguments: str | None = None + + +class _StreamToolCall(BaseModel): + function: _StreamToolCallFunction = _StreamToolCallFunction() + + +class _StreamDelta(BaseModel): + content: str | None = None + tool_calls: list[_StreamToolCall] | None = None + + +class _StreamChoice(BaseModel): + delta: _StreamDelta = _StreamDelta() + + +class _StreamChunk(BaseModel): + choices: list[_StreamChoice] = [] + + +def _streamed_tool_call(events: list[str]) -> tuple[str, str]: + """Reassemble the tool call streamed across chunks: the name arrives once and the + arguments arrive as fragments, so concatenating both and parsing the arguments as + JSON catches a stream that never completes the call or splits its argument JSON.""" + chunks = [_StreamChunk.model_validate_json(event) for event in events] + calls = [call for chunk in chunks for choice in chunk.choices for call in (choice.delta.tool_calls or [])] + name = "".join(call.function.name or "" for call in calls) + arguments = "".join(call.function.arguments or "" for call in calls) + return name, arguments + + +CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg" +OPENAI_VISION_BACKEND = "openai/gpt-4o" + +# OpenAI caches a shared prompt prefix once it exceeds ~1024 tokens; this is well +# past that, so a repeat call reports cached prompt tokens. +CACHE_PREFIX = ( + "You are a meticulous assistant. Follow these standing instructions exactly. " + * 300 +) + + +def _vision_messages() -> list[ChatMessage]: + return [ + ChatMessage( + role="user", + content=[ + TextContentPart(text="What animal is in this image? Answer in one word."), + ImageContentPart(image_url=ImageUrl(url=CAT_IMAGE_URL)), + ], + ) + ] + + +def _assert_describes_cat(response: ChatResponse) -> None: + assert response.choices, f"vision returned no choices: {response}" + message = response.choices[0].message + content = (message.content if message else None) or "" + assert "cat" in content.lower() or "feline" in content.lower(), ( + f"vision response did not describe the image: {content[:200]}" + ) + + +def _streamed_text(events: list[str]) -> str: + """Concatenate the delta content across streamed chunks. Parsing every event as + JSON also fails loudly on a truncated or garbled chunk (the vertex/gemini image + streaming regression class), so an incomplete stream cannot pass as content.""" + chunks = [_StreamChunk.model_validate_json(event) for event in events] + return "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices) + + +def _assert_streamed_completion(result: StreamingResponse) -> None: + """A streamed /chat/completions must deliver real content, not a clean-but-empty + stream (the #28991 class on the streaming path).""" + assert result.ok and result.is_streaming, f"stream was not established: {result}" + assert result.stream_error is None, f"stream carried an error event: {result.stream_error}" + assert len(result.stream_events) > 1, f"stream did not deliver multiple data events: {result}" + assert _streamed_text(result.stream_events).strip(), ( + f"stream completed with no content deltas: {result.stream_events[:3]}" + ) + + +def _bedrock_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=BEDROCK_CONVERSE_BACKEND, + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="os.environ/AWS_REGION", + ) + + +class _WeatherArgs(BaseModel): + location: str + + +_WEATHER_TOOL = ChatTool( + function=ChatToolFunction( + name="get_weather", + description="Get the current weather for a location", + parameters={ + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + ) +) + + +def _assert_weather_tool_call(response: ChatResponse) -> None: + """The model, forced to call the tool, must return a get_weather call whose + arguments parse as JSON and carry a location. A regression that drops tool_calls + or emits malformed argument JSON fails here rather than passing on a 200.""" + assert response.choices, f"chat returned no choices: {response}" + message = response.choices[0].message + calls = message.tool_calls if message else None + assert calls, f"model returned no tool call for a tool-forced prompt: {response}" + weather = next((call for call in calls if call.function.name == "get_weather"), None) + assert weather is not None, f"expected a get_weather call, got {[c.function.name for c in calls]}" + assert weather.function.arguments, f"get_weather call carried no arguments: {weather}" + args = _WeatherArgs.model_validate_json(weather.function.arguments) + assert args.location.strip(), f"get_weather arguments missing location: {weather.function.arguments}" + + +class _Person(BaseModel): + name: str + age: int + + +_PERSON_SCHEMA: dict[str, object] = { + "type": "json_schema", + "json_schema": { + "name": "person", + "strict": True, + "schema": { + "type": "object", + "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, + "required": ["name", "age"], + "additionalProperties": False, + }, + }, +} CHAT_MODELS: tuple[tuple[str, str], ...] = ( ("gpt-5.5", "openai"), @@ -219,3 +378,391 @@ class TestHostedVllmChat: assert response.choices, f"hosted_vllm chat returned no choices: {response}" content = response.choices[0].message.content if response.choices[0].message else None assert content and content.strip(), f"hosted_vllm empty content: {response}" + + +class TestOpenAIChatCompletions: + """OpenAI /chat/completions, the SDK path the customer runs against the proxy. + + The streamed call must deliver real content deltas (a clean-but-empty stream is + the regression), and a non-streamed call must be costed so per-request spend and + the response-cost header stay accurate. + """ + + @pytest.mark.covers( + "llm.chat_completions.openai.basic.stream.works", + exercised_on=["chat_completions"], + ) + def test_openai_chat_streams_real_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + require_env("OPENAI_API_KEY") + model = f"e2e-openai-chat-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + result = client.proxy.chat_stream( + key, + ChatBody( + model=model, + messages=[ + ChatMessage(role="user", content=f"Count from 1 to 5, one number per line. {unique_marker()}") + ], + max_tokens=64, + stream=True, + ), + ) + _assert_streamed_completion(result) + + @pytest.mark.covers( + "llm.chat_completions.openai.basic.nonstream.cost_logged", + exercised_on=["chat_completions"], + ) + def test_openai_chat_logs_cost( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + require_env("OPENAI_API_KEY") + model = f"e2e-openai-cost-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content=f"Reply with the single word pong. {unique_marker()}")], + max_tokens=16, + ), + ) + ) + assert response.choices, f"openai chat returned no choices: {response}" + + rows = client.proxy.poll_logs_for_key( + key, min_rows=1, predicate=lambda rs: any((r.spend or 0) > 0 for r in rs) + ) + priced = [r for r in rows if (r.spend or 0) > 0] + assert priced, f"openai chat was not costed on key ...{key[-6:]}: {rows}" + assert priced[0].status == "success", f"openai chat spend status={priced[0].status!r}" + + @pytest.mark.covers( + "llm.chat_completions.openai.tool_use.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_openai_chat_returns_tool_call( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + require_env("OPENAI_API_KEY") + model = f"e2e-openai-tool-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage(role="user", content="What is the weather in San Francisco? Use the get_weather tool.") + ], + tools=[_WEATHER_TOOL], + tool_choice="required", + max_tokens=128, + ), + ) + ) + _assert_weather_tool_call(response) + + @pytest.mark.covers( + "llm.chat_completions.openai.structured_output.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_openai_chat_structured_output_conforms_to_schema( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + require_env("OPENAI_API_KEY") + model = f"e2e-openai-schema-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content="Extract the person. John Doe is 42 years old.")], + response_format=_PERSON_SCHEMA, + max_tokens=128, + ), + ) + ) + assert response.choices, f"structured output returned no choices: {response}" + content = response.choices[0].message.content if response.choices[0].message else None + assert content, f"structured output returned empty content: {response}" + person = _Person.model_validate_json(content) + assert person.name.strip() and person.age == 42, ( + f"schema-constrained extraction was wrong: {person}" + ) + + @pytest.mark.covers( + "llm.chat_completions.openai.thinking.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_openai_chat_reasoning_reports_reasoning_tokens( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + require_env("OPENAI_API_KEY") + model = f"e2e-openai-reasoning-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content="A train travels 60 miles in 1.5 hours. What is its average speed in mph?", + ) + ], + reasoning_effort="low", + max_tokens=2048, + ), + ) + ) + assert response.choices, f"reasoning call returned no choices: {response}" + message = response.choices[0].message + assert message and message.content and message.content.strip(), f"reasoning call had no answer: {response}" + details = response.usage.completion_tokens_details if response.usage else None + assert details and details.reasoning_tokens and details.reasoning_tokens > 0, ( + f"a reasoning model must report reasoning tokens, got usage={response.usage}" + ) + + @pytest.mark.covers( + "llm.chat_completions.openai.vision.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_openai_chat_vision_describes_image( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + require_env("OPENAI_API_KEY") + model = f"e2e-openai-vision-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model=OPENAI_VISION_BACKEND, api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32))) + _assert_describes_cat(response) + + @pytest.mark.covers( + "llm.chat_completions.openai.prompt_cache_5m.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_openai_chat_prompt_cache_hits_on_repeat( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + require_env("OPENAI_API_KEY") + model = f"e2e-openai-cache-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + body = ChatBody( + model=model, + messages=[ + ChatMessage(role="system", content=CACHE_PREFIX), + ChatMessage(role="user", content="Reply with the single word pong."), + ], + max_tokens=16, + ) + unwrap(client.proxy.chat(key, body)) + second = unwrap(client.proxy.chat(key, body)) + + details = second.usage.prompt_tokens_details if second.usage else None + assert details and details.cached_tokens and details.cached_tokens > 0, ( + f"a repeated large-prefix prompt must report cached prompt tokens, got usage={second.usage}" + ) + + @pytest.mark.covers( + "llm.chat_completions.openai.tool_use.stream.works", + exercised_on=["chat_completions"], + ) + def test_openai_chat_streams_tool_call( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + require_env("OPENAI_API_KEY") + model = f"e2e-openai-tool-stream-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + result = client.proxy.chat_stream( + key, + ChatBody( + model=model, + messages=[ + ChatMessage(role="user", content="What is the weather in San Francisco? Use the get_weather tool.") + ], + tools=[_WEATHER_TOOL], + tool_choice="required", + max_tokens=128, + stream=True, + ), + ) + assert result.ok and result.is_streaming, f"tool stream was not established: {result}" + assert result.stream_error is None, f"tool stream carried an error event: {result.stream_error}" + name, arguments = _streamed_tool_call(result.stream_events) + assert name == "get_weather", f"streamed tool call named {name!r}: {result.stream_events[:5]}" + args = _WeatherArgs.model_validate_json(arguments) + assert args.location.strip(), f"streamed tool call arguments missing location: {arguments!r}" + + +class TestBedrockConverseChatCompletions: + """Bedrock Converse via /chat/completions, the customer's AWS stack. A non-OpenAI + provider must return real content on both the non-streamed and streamed paths. + """ + + def _register(self, client: PassthroughClient, resources: ResourceManager, prefix: str) -> str: + require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") + model = f"{prefix}-{unique_marker()}" + model_id = client.proxy.create_model(model, _bedrock_params()) + resources.defer(lambda: client.proxy.delete_model(model_id)) + return model + + @pytest.mark.covers( + "llm.chat_completions.bedrock_converse.basic.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_bedrock_converse_chat_returns_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-bedrock-chat") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content=f"Reply with the single word pong. {unique_marker()}")], + max_tokens=32, + ), + ) + ) + assert response.choices, f"bedrock converse chat returned no choices: {response}" + content = response.choices[0].message.content if response.choices[0].message else None + assert content and content.strip(), f"bedrock converse returned empty content: {response}" + + @pytest.mark.covers( + "llm.chat_completions.bedrock_converse.basic.stream.works", + exercised_on=["chat_completions"], + ) + def test_bedrock_converse_chat_streams_real_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-bedrock-stream") + key = resources.key() + + result = client.proxy.chat_stream( + key, + ChatBody( + model=model, + messages=[ + ChatMessage(role="user", content=f"Count from 1 to 5, one number per line. {unique_marker()}") + ], + max_tokens=64, + stream=True, + ), + ) + _assert_streamed_completion(result) + + @pytest.mark.covers( + "llm.chat_completions.bedrock_converse.tool_use.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_bedrock_converse_chat_returns_tool_call( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-bedrock-tool") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage(role="user", content="What is the weather in San Francisco? Use the get_weather tool.") + ], + tools=[_WEATHER_TOOL], + tool_choice="required", + max_tokens=128, + ), + ) + ) + _assert_weather_tool_call(response) + + @pytest.mark.covers( + "llm.chat_completions.bedrock_converse.thinking.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_bedrock_converse_chat_returns_reasoning( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-bedrock-thinking") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content="What is 17 times 23? Think it through step by step.")], + thinking=ThinkingParam(type="enabled", budget_tokens=1024), + max_tokens=2048, + ), + ) + ) + assert response.choices, f"bedrock thinking returned no choices: {response}" + message = response.choices[0].message + assert message and message.content and message.content.strip(), ( + f"bedrock thinking returned no answer content: {response}" + ) + assert message.reasoning_content and message.reasoning_content.strip(), ( + "thinking was enabled but no reasoning_content came back on the Bedrock Converse path" + ) + + @pytest.mark.covers( + "llm.chat_completions.bedrock_converse.vision.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_bedrock_converse_chat_vision_describes_image( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-bedrock-vision") + key = resources.key() + + response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32))) + _assert_describes_cat(response) diff --git a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py index 56f2de8bd4f..157caedd561 100644 --- a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py +++ b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py @@ -1,9 +1,9 @@ -"""Live e2e: POST /embeddings returns a real vector. +"""Live e2e: POST /embeddings returns a real vector across OpenAI, Bedrock, Vertex. -Registers an OpenAI embedding deployment at runtime and asserts a non-empty, -non-zero vector came back. Migrated from -litellm-regression-tests/tests/test_inference_endpoints.py; the LIT-3167 guard in -tests/e2e/embeddings/ covers the Gemini embedding path. +Each test registers the deployment it needs at runtime (deleted on teardown) and +asserts a non-empty, non-zero vector came back. The LIT-3167 guard in +tests/e2e/embeddings/ covers the Gemini embedding path; embeddings cost tracking is +covered by tests/e2e/quota_management/spend_tracking/. """ from __future__ import annotations @@ -20,6 +20,7 @@ pytestmark = pytest.mark.e2e class TestEmbeddingsEndpoint: + @pytest.mark.covers("llm.embeddings.openai.basic.nonstream.works") def test_embeddings_returns_vector( self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: @@ -40,3 +41,49 @@ class TestEmbeddingsEndpoint: assert any(component != 0.0 for component in parsed.first_vector), ( f"embedding vector is all zeros: {result.body[:300]}" ) + + @pytest.mark.covers("llm.embeddings.bedrock.basic.nonstream.works") + def test_bedrock_embeddings_returns_vector( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-embeddings-bedrock-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="bedrock/amazon.titan-embed-text-v2:0", aws_region_name="us-west-2" + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.embeddings(key, model, "Say this is a test!") + require_successful_call(result) + parsed = EmbeddingsResult.model_validate_json(result.body) + assert parsed.first_vector, f"/embeddings returned no vector: {result.body[:300]}" + assert any(component != 0.0 for component in parsed.first_vector), ( + f"embedding vector is all zeros: {result.body[:300]}" + ) + + @pytest.mark.covers("llm.embeddings.vertex.basic.nonstream.works") + def test_vertex_embeddings_returns_vector( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-embeddings-vertex-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="vertex_ai/gemini-embedding-2", + vertex_project="os.environ/VERTEXAI_PROJECT", + vertex_location="us-central1", + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.embeddings(key, model, "Say this is a test!") + require_successful_call(result) + parsed = EmbeddingsResult.model_validate_json(result.body) + assert parsed.first_vector, f"/embeddings returned no vector: {result.body[:300]}" + assert any(component != 0.0 for component in parsed.first_vector), ( + f"embedding vector is all zeros: {result.body[:300]}" + ) diff --git a/tests/e2e/llm_translation/test_image_generation_e2e.py b/tests/e2e/llm_translation/test_image_generation_e2e.py index 4d2211f3be4..1ba78a7e083 100644 --- a/tests/e2e/llm_translation/test_image_generation_e2e.py +++ b/tests/e2e/llm_translation/test_image_generation_e2e.py @@ -9,7 +9,7 @@ from __future__ import annotations import pytest -from e2e_config import unique_marker +from e2e_config import require_env, unique_marker from e2e_http import require_successful_call from endpoints_client import EndpointsClient, ImagesResult from lifecycle import ResourceManager @@ -18,7 +18,17 @@ from models import LiteLLMParamsBody pytestmark = pytest.mark.e2e +def _assert_image_returned(body: str) -> None: + parsed = ImagesResult.model_validate_json(body) + assert parsed.data, f"/images/generations returned no data: {body[:300]}" + first = parsed.data[0] + assert first.b64_json or first.url, ( + f"generated image has neither b64_json nor url: {body[:300]}" + ) + + class TestImageGeneration: + @pytest.mark.covers("llm.images_generations.openai.basic.nonstream.works") def test_image_generation_returns_image( self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: @@ -34,9 +44,26 @@ class TestImageGeneration: result = endpoints_client.images(key, model, "Draw a cute cat") require_successful_call(result) - parsed = ImagesResult.model_validate_json(result.body) - assert parsed.data, f"/images/generations returned no data: {result.body[:300]}" - first = parsed.data[0] - assert first.b64_json or first.url, ( - f"generated image has neither b64_json nor url: {result.body[:300]}" + _assert_image_returned(result.body) + + @pytest.mark.covers("llm.images_generations.bedrock.basic.nonstream.works", exercised_on=["images_generations"]) + def test_bedrock_image_generation_returns_image( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") + model = f"e2e-bedrock-image-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="bedrock/amazon.titan-image-generator-v2:0", + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="os.environ/AWS_REGION", + ), ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.images(key, model, "Draw a cute cat") + require_successful_call(result) + _assert_image_returned(result.body) diff --git a/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py b/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py new file mode 100644 index 00000000000..3f2907f202e --- /dev/null +++ b/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py @@ -0,0 +1,155 @@ +"""Live e2e: POST /v1/messages routed to Azure AI Foundry Anthropic deployments. + +Registers `azure_ai/` deployments at runtime and drives the Messages +endpoint through the gateway across the behaviors an Anthropic client relies on: +a basic completion, a streamed completion, and tool use (non-streaming and +streaming). Auth is the Azure API key (`x-api-key`); the deployment reads +`AZURE_AI_API_BASE` / `AZURE_AI_API_KEY` from the proxy env, so no secret is +sent in the request. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import unique_marker +from e2e_http import StreamingResponse, require_successful_call, unwrap +from endpoints_client import EndpointsClient +from lifecycle import ResourceManager +from models import ( + AnthropicCustomTool, + AnthropicMessagesBody, + ChatMessage, + JsonSchemaProperty, + LiteLLMParamsBody, + ToolInputSchema, +) + +pytestmark = pytest.mark.e2e + +AZURE_FOUNDRY_MODEL = "azure_ai/claude-haiku-4-5" + +WEATHER_TOOL = AnthropicCustomTool( + name="get_weather", + description="Get the current weather for a city.", + input_schema=ToolInputSchema( + properties={"city": JsonSchemaProperty(type="string")}, + required=["city"], + ), +) + + +def _assert_streamed_ok(result: StreamingResponse) -> None: + require_successful_call(result) + assert result.is_streaming, f"response was not streamed: {result.headers}" + assert not result.stream_error, f"stream errored: {result.stream_error}" + assert result.stream_events, "stream produced no SSE events" + assert any("content_block_delta" in event for event in result.stream_events), ( + "stream carried no content deltas" + ) + assert any("message_stop" in event for event in result.stream_events), ( + "stream never reached message_stop" + ) + + +class TestAzureFoundryMessages: + def _register( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> tuple[str, str]: + model = f"e2e-azure-foundry-messages-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model=AZURE_FOUNDRY_MODEL, + api_base="os.environ/AZURE_AI_API_BASE", + api_key="os.environ/AZURE_AI_API_KEY", + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + return model, resources.key(models=[model]) + + @pytest.mark.covers("llm.messages.azure_foundry.basic.nonstream.works") + def test_basic_nonstream( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model, key = self._register(endpoints_client, resources) + response = unwrap( + endpoints_client.proxy.messages( + key, + AnthropicMessagesBody( + model=model, + max_tokens=64, + messages=[ChatMessage(role="user", content="Reply with one word.")], + ), + ) + ) + assert response.content, f"no content blocks in response: {response}" + text = "".join(block.text or "" for block in response.content if block.type == "text") + assert text.strip(), f"/v1/messages returned no text: {response}" + + @pytest.mark.covers("llm.messages.azure_foundry.basic.stream.works") + def test_basic_stream( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model, key = self._register(endpoints_client, resources) + result = endpoints_client.proxy.messages_stream( + key, + AnthropicMessagesBody( + model=model, + max_tokens=64, + stream=True, + messages=[ChatMessage(role="user", content="Count from one to three.")], + ), + ) + _assert_streamed_ok(result) + + @pytest.mark.covers("llm.messages.azure_foundry.tool_use.nonstream.works") + def test_tool_use_nonstream( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model, key = self._register(endpoints_client, resources) + response = unwrap( + endpoints_client.proxy.messages( + key, + AnthropicMessagesBody( + model=model, + max_tokens=256, + tools=[WEATHER_TOOL], + messages=[ + ChatMessage(role="user", content="What is the weather in Paris? Use the tool.") + ], + ), + ) + ) + assert response.content, f"no content blocks in response: {response}" + assert any(block.type == "tool_use" for block in response.content), ( + f"model did not call the tool: {response}" + ) + + @pytest.mark.covers("llm.messages.azure_foundry.tool_use.stream.works") + def test_tool_use_stream( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model, key = self._register(endpoints_client, resources) + result = endpoints_client.proxy.messages_stream( + key, + AnthropicMessagesBody( + model=model, + max_tokens=256, + stream=True, + tools=[WEATHER_TOOL], + messages=[ + ChatMessage(role="user", content="What is the weather in Paris? Use the tool.") + ], + ), + ) + require_successful_call(result) + assert result.is_streaming, f"response was not streamed: {result.headers}" + assert not result.stream_error, f"stream errored: {result.stream_error}" + assert result.stream_events, "stream produced no SSE events" + assert any("tool_use" in event for event in result.stream_events), ( + "stream carried no tool_use block" + ) + assert any("message_stop" in event for event in result.stream_events), ( + "stream never reached message_stop" + ) diff --git a/tests/e2e/llm_translation/test_messages_e2e.py b/tests/e2e/llm_translation/test_messages_e2e.py index b0a48f22118..44376218c6b 100644 --- a/tests/e2e/llm_translation/test_messages_e2e.py +++ b/tests/e2e/llm_translation/test_messages_e2e.py @@ -1,7 +1,8 @@ """Live e2e: POST /v1/messages (Anthropic Messages API) returns a real completion. Registers an Anthropic deployment at runtime, drives the Messages endpoint through -the gateway, and asserts an assistant message with text came back. Migrated from +the gateway, and asserts an assistant message with text came back, both +non-streaming and streamed. Migrated from litellm-regression-tests/tests/test_inference_endpoints.py. """ @@ -9,31 +10,163 @@ from __future__ import annotations import pytest -from e2e_config import unique_marker -from e2e_http import require_successful_call +from e2e_config import require_env, unique_marker +from e2e_http import require_successful_call, unwrap from endpoints_client import EndpointsClient, MessagesResult from lifecycle import ResourceManager -from models import LiteLLMParamsBody +from models import ( + AnthropicCustomTool, + AnthropicMessagesBody, + ChatMessage, + JsonSchemaProperty, + LiteLLMParamsBody, + SpendLogRow, + ToolInputSchema, +) pytestmark = pytest.mark.e2e +ANTHROPIC_BACKEND = "anthropic/claude-haiku-4-5" + +WEATHER_TOOL = AnthropicCustomTool( + name="get_weather", + description="Get the current weather for a city.", + input_schema=ToolInputSchema( + properties={"city": JsonSchemaProperty(type="string")}, + required=["city"], + ), +) + + +def _approx_equal(actual: float, expected: float) -> bool: + """Within 1% or 1e-9 absolute - spend math, not exact float identity.""" + return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) + class TestAnthropicMessages: - def test_messages_returns_completion( + def _register( self, endpoints_client: EndpointsClient, resources: ResourceManager - ) -> None: + ) -> tuple[str, str]: model = f"e2e-messages-{unique_marker()}" model_id = endpoints_client.create_model( model, LiteLLMParamsBody( - model="anthropic/claude-haiku-4-5", api_key="os.environ/ANTHROPIC_API_KEY" + model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY" ), ) resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() + return model, resources.key() + + @pytest.mark.covers("llm.messages.anthropic.basic.nonstream.works") + def test_messages_returns_completion( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model, key = self._register(endpoints_client, resources) result = endpoints_client.messages(key, model, "reply with one word") require_successful_call(result) parsed = MessagesResult.model_validate_json(result.body) assert parsed.role == "assistant", f"unexpected role: {result.body[:300]}" assert parsed.text.strip(), f"/v1/messages returned no text: {result.body[:300]}" + + @pytest.mark.covers("llm.messages.anthropic.basic.nonstream.cost_logged") + def test_messages_logs_cost_matching_the_response_header( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + require_env("ANTHROPIC_API_KEY") + model = f"e2e-messages-cost-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY" + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.messages(key, model, f"reply with one word {unique_marker()}") + require_successful_call(result) + parsed = MessagesResult.model_validate_json(result.body) + assert parsed.role == "assistant" and parsed.text.strip(), ( + f"/v1/messages returned no assistant text: {result.body[:300]}" + ) + + # The customer reads per-request cost off the response header (LIT-4076), so + # it must be present and positive on /v1/messages, not only /chat/completions. + header_cost = result.response_cost + assert header_cost is not None and header_cost > 0, ( + "x-litellm-response-cost header missing or non-positive on /v1/messages; " + f"headers={result.headers}" + ) + + # Correlate the spend row by the unique scoped key, not the Anthropic response + # id: on /v1/messages the spend-log request_id is the proxy's own call id, which + # need not equal the message body id, so an id-based poll can miss a correctly + # logged row and time out. The key is fresh per test, so its only priced row is + # this call. + def _priced(rows: list[SpendLogRow]) -> bool: + return any(r.spend is not None and r.spend > 0 for r in rows) + + rows = endpoints_client.proxy.poll_logs_for_key(key, predicate=_priced) + priced = [r for r in rows if r.spend is not None and r.spend > 0] + assert priced, ( + f"no priced /spend/logs row landed for key {key} within the poll window; got {rows}" + ) + row = priced[0] + assert (row.prompt_tokens or 0) > 0 and (row.completion_tokens or 0) > 0, ( + f"messages spend row missing token counts, so the cost is not real usage: {row}" + ) + assert row.spend is not None and _approx_equal(row.spend, header_cost), ( + f"logged spend {row.spend} disagrees with the x-litellm-response-cost header {header_cost}; " + "the customer bills against the header, so the two must match" + ) + + @pytest.mark.covers("llm.messages.anthropic.basic.stream.works") + def test_messages_streams_completion( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model, key = self._register(endpoints_client, resources) + + result = endpoints_client.proxy.messages_stream( + key, + AnthropicMessagesBody( + model=model, + max_tokens=64, + stream=True, + messages=[ChatMessage(role="user", content="Count from one to three.")], + ), + ) + require_successful_call(result) + assert result.is_streaming, f"response was not streamed: {result.headers}" + assert not result.stream_error, f"stream errored: {result.stream_error}" + assert result.stream_events, "stream produced no SSE events" + assert any("content_block_delta" in event for event in result.stream_events), ( + "stream carried no content deltas" + ) + assert any("message_stop" in event for event in result.stream_events), ( + "stream never reached message_stop" + ) + + @pytest.mark.covers("llm.messages.anthropic.tool_use.nonstream.works") + def test_messages_tool_use( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model, key = self._register(endpoints_client, resources) + + response = unwrap( + endpoints_client.proxy.messages( + key, + AnthropicMessagesBody( + model=model, + max_tokens=256, + tools=[WEATHER_TOOL], + messages=[ + ChatMessage(role="user", content="What is the weather in Paris? Use the tool.") + ], + ), + ) + ) + assert response.content, f"no content blocks in response: {response}" + assert any(block.type == "tool_use" for block in response.content), ( + f"model did not call the tool: {response}" + ) diff --git a/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py b/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py index 7a04b044634..97d24e0564b 100644 --- a/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py +++ b/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py @@ -7,7 +7,7 @@ accepted in place on Claude 4.8+/5 (200) but rejected on Claude 4.7 and older ("role 'system' is not supported on this model", 400), and a *leading* system entry is rejected on every model ("messages.0: use the top-level 'system' parameter"). This mirrors Bedrock Invoke (PRs #32578/#32831/#32882); the same -model-gated hoist now runs for these two providers (Kraken Tech RCA gap #3). +model-gated hoist now runs for these two providers (customer RCA gap #3). Flagged models (``supports_mid_conversation_system`` in the cost map: Claude 4.8+ and the 5 family) must keep the reminder in ``messages`` so the top-level @@ -88,9 +88,9 @@ def _system_reminder_turn() -> RichMessage: def _post_messages(client: EndpointsClient, key: str, body: RichMessagesRequest) -> Result[MessagesResult]: - return client.gateway.transport.post( + return client.proxy.transport.post( "/v1/messages", - headers=client.gateway.transport.bearer(key), + headers=client.proxy.transport.bearer(key), json=body, response_type=MessagesResult, ) diff --git a/tests/e2e/llm_translation/test_moderations_e2e.py b/tests/e2e/llm_translation/test_moderations_e2e.py new file mode 100644 index 00000000000..69cf4414a48 --- /dev/null +++ b/tests/e2e/llm_translation/test_moderations_e2e.py @@ -0,0 +1,65 @@ +"""Live e2e: POST /v1/moderations classifies content against the provider policy. + +Registers OpenAI's omni moderation model at runtime and asserts the product +promise on both sides of the decision: clearly violent text comes back flagged +with at least one policy category tripped, and benign text comes back not flagged. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import unique_marker +from e2e_http import unwrap +from endpoints_client import EndpointsClient +from lifecycle import ResourceManager +from models import LiteLLMParamsBody + +pytestmark = pytest.mark.e2e + +VIOLENT_TEXT = "I am going to find you and kill you, and I will hurt everyone you love." +BENIGN_TEXT = "I enjoyed the sunny afternoon and a relaxing walk in the park today." + + +def _register_moderation_model( + endpoints_client: EndpointsClient, resources: ResourceManager +) -> str: + model = f"e2e-moderation-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="openai/omni-moderation-latest", api_key="os.environ/OPENAI_API_KEY" + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + return model + + +class TestModerations: + @pytest.mark.covers("llm.moderations.openai.basic.nonstream.works") + def test_moderations_flags_violent_content( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = _register_moderation_model(endpoints_client, resources) + key = resources.key() + + result = unwrap(endpoints_client.moderations(key, model, VIOLENT_TEXT)) + item = result.first + assert item is not None, f"/moderations returned no results: {result}" + assert item.flagged, f"violent text was not flagged: {item}" + assert item.flagged_categories, ( + f"flagged result reported no true category: {item}" + ) + + def test_moderations_passes_benign_content( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = _register_moderation_model(endpoints_client, resources) + key = resources.key() + + result = unwrap(endpoints_client.moderations(key, model, BENIGN_TEXT)) + item = result.first + assert item is not None, f"/moderations returned no results: {result}" + assert not item.flagged, ( + f"benign text was flagged as {item.flagged_categories}: {item}" + ) diff --git a/tests/e2e/llm_translation/test_passthrough_e2e.py b/tests/e2e/llm_translation/test_passthrough_e2e.py index c8806faf3ea..ed5c657d23e 100644 --- a/tests/e2e/llm_translation/test_passthrough_e2e.py +++ b/tests/e2e/llm_translation/test_passthrough_e2e.py @@ -15,7 +15,8 @@ import pytest from e2e_config import unique_marker from e2e_http import StreamingResponse, require_successful_call -from models import SpendLogRow +from lifecycle import ResourceManager +from models import KeyGenerateBody, SpendLogRow from passthrough_client import ( AnthropicTool, GeminiFunctionDeclaration, @@ -157,3 +158,25 @@ def test_anthropic_passthrough_tool_call_logs_cost( row = _fetch_cost_breakdown(client, result) assert row.custom_llm_provider == "anthropic" + + +class TestPassthroughModelAllowlist: + """A passthrough route must honor the calling key's model allow-list. + + The customer fronts native provider calls through the proxy with custom auth, + so a key scoped to one model must not reach a different model just because the + request goes through the passthrough route rather than /chat/completions. + """ + + @pytest.mark.covers("other.auth.passthrough.model_allowlist_enforced") + def test_passthrough_denies_model_outside_key_allowlist( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + key = client.proxy.generate_key(KeyGenerateBody(models=["gemini-2.5-flash"])) + resources.defer(lambda: client.proxy.delete_key(key)) + + result = client.anthropic_message(key, "claude-haiku-4-5", f"say hi {unique_marker()}") + assert result.status_code == 403, ( + "a key restricted to gemini-2.5-flash must be denied a claude passthrough call, " + f"got {result.status_code}: {result.body[:300]}" + ) diff --git a/tests/e2e/llm_translation/test_passthrough_headers_e2e.py b/tests/e2e/llm_translation/test_passthrough_headers_e2e.py index d9fe37c79ed..045988334d5 100644 --- a/tests/e2e/llm_translation/test_passthrough_headers_e2e.py +++ b/tests/e2e/llm_translation/test_passthrough_headers_e2e.py @@ -1,29 +1,32 @@ """Live e2e: custom pass-through endpoints inject configured headers and honor x-pass-* client headers (prefix stripped) on the way to the upstream. -The upstream is a real public echo service (httpbin.org/anything). Creating the -route via POST /config/pass_through_endpoint, calling it with a virtual key, and -asserting the echo body is the product path operators use; a mock would not -prove the proxy actually rewrote the outbound request. +The upstream is the real Anthropic Messages API rather than an echo service: +Anthropic doesn't echo request headers back, but it does gate real behavior on +two of them, which is enough to prove forwarding without a mock. A static +x-api-key configured on the pass-through endpoint (the caller never supplies +one) must reach upstream, or every call 401s; an invalid x-pass-anthropic-version +sent by the caller must reach upstream with the prefix stripped, and Anthropic +echoes the exact value back in its 400 body, so a unique-per-run marker proves +this specific request's header - not a stale or cached one - got there. """ from __future__ import annotations import pytest -from pydantic import BaseModel, Field, ValidationError +from pydantic import BaseModel, Field from e2e_config import unique_marker -from e2e_http import AuthHeaders, NoBody, StreamingResponse, require_successful_call, unwrap +from e2e_http import AuthHeaders, NoBody, require_successful_call, unwrap +from endpoints_client import MessagesResult from lifecycle import ResourceManager -from models import KeyGenerateBody +from models import ChatMessage, KeyGenerateBody from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e -ECHO_TARGET = "https://httpbin.org/anything" -STATIC_HEADER_NAME = "x-e2e-static-header" -PASS_HEADER_STEM = "e2e-client-marker" -PASS_HEADER_NAME = f"x-pass-{PASS_HEADER_STEM}" +ANTHROPIC_MESSAGES_TARGET = "https://api.anthropic.com/v1/messages" +MODEL = "claude-haiku-4-5-20251001" class PassThroughCreateBody(BaseModel): @@ -48,30 +51,26 @@ class PassThroughDeleteParams(BaseModel): endpoint_id: str -class EchoCallHeaders(AuthHeaders): +class AnthropicPassThroughHeaders(AuthHeaders): content_type: str = Field(default="application/json", serialization_alias="Content-Type") - x_pass_e2e_client_marker: str = Field(serialization_alias="x-pass-e2e-client-marker") + x_pass_anthropic_version: str = Field(serialization_alias="x-pass-anthropic-version") -class EchoBody(BaseModel): - ping: str +class AnthropicMessagesBody(BaseModel): + model: str + max_tokens: int = 8 + messages: list[ChatMessage] -class EchoResponse(BaseModel): - headers: dict[str, str] - - -def _create_passthrough( - client: PassthroughClient, *, path: str, static_value: str -) -> PassThroughEndpoint: +def _create_passthrough(client: PassthroughClient, *, path: str) -> PassThroughEndpoint: created = unwrap( client.proxy.transport.post( "/config/pass_through_endpoint", headers=client.proxy.transport.master, json=PassThroughCreateBody( path=path, - target=ECHO_TARGET, - headers={STATIC_HEADER_NAME: static_value}, + target=ANTHROPIC_MESSAGES_TARGET, + headers={"x-api-key": "os.environ/ANTHROPIC_API_KEY"}, ), response_type=PassThroughCreateResponse, ) @@ -92,12 +91,8 @@ def _delete_passthrough(client: PassthroughClient, endpoint_id: str) -> None: ) -def _echo_headers(resp: StreamingResponse) -> dict[str, str]: - try: - echo = EchoResponse.model_validate_json(resp.body) - except ValidationError as exc: - pytest.fail(f"echo upstream did not return a headers map: {exc}; body={resp.body[:300]}") - return {k.lower(): v for k, v in echo.headers.items()} +def _messages_body() -> AnthropicMessagesBody: + return AnthropicMessagesBody(model=MODEL, messages=[ChatMessage(role="user", content="Say hi.")]) class TestPassthroughHeaders: @@ -110,10 +105,8 @@ class TestPassthroughHeaders: ) -> None: marker = unique_marker() path = f"/e2e-passthrough-headers-{marker}" - static_value = f"static-{marker}" - client_value = f"client-{marker}" - endpoint = _create_passthrough(client, path=path, static_value=static_value) + endpoint = _create_passthrough(client, path=path) assert endpoint.id is not None resources.defer(lambda: _delete_passthrough(client, endpoint.id or "")) @@ -128,23 +121,32 @@ class TestPassthroughHeaders: result = client.proxy.transport.send( path, - headers=EchoCallHeaders( + headers=AnthropicPassThroughHeaders( authorization=f"Bearer {key}", - x_pass_e2e_client_marker=client_value, + x_pass_anthropic_version="2023-06-01", ), - json=EchoBody(ping=marker), + json=_messages_body(), ) require_successful_call(result) + completion = MessagesResult.model_validate_json(result.body) + assert completion.text.strip(), ( + f"static x-api-key must reach Anthropic for the call to succeed at all; got {result.body[:300]}" + ) - upstream = _echo_headers(result) - assert upstream.get(STATIC_HEADER_NAME) == static_value, ( - f"configured pass-through header {STATIC_HEADER_NAME!r} not on upstream " - f"request; got {upstream}" + invalid_version = f"e2e-passhdr-{unique_marker()}" + blocked = client.proxy.transport.send( + path, + headers=AnthropicPassThroughHeaders( + authorization=f"Bearer {key}", + x_pass_anthropic_version=invalid_version, + ), + json=_messages_body(), ) - assert upstream.get(PASS_HEADER_STEM) == client_value, ( - f"x-pass-* header should strip the prefix and forward as {PASS_HEADER_STEM!r}; " - f"got {upstream}" + assert blocked.status_code == 400, ( + f"expected Anthropic to reject the invalid anthropic-version, got " + f"{blocked.status_code}: {blocked.body[:300]}" ) - assert PASS_HEADER_NAME not in upstream, ( - "upstream must not see the x-pass- prefix; proxy should strip it" + assert invalid_version in blocked.body, ( + f"x-pass-anthropic-version must reach upstream with the prefix stripped; " + f"marker missing from Anthropic's error body: {blocked.body[:300]}" ) diff --git a/tests/e2e/llm_translation/test_rerank_e2e.py b/tests/e2e/llm_translation/test_rerank_e2e.py index 4b30ac1ea5c..0857ff65a52 100644 --- a/tests/e2e/llm_translation/test_rerank_e2e.py +++ b/tests/e2e/llm_translation/test_rerank_e2e.py @@ -9,7 +9,7 @@ from __future__ import annotations import pytest -from e2e_config import unique_marker +from e2e_config import require_env, unique_marker from e2e_http import require_successful_call from endpoints_client import EndpointsClient, RerankResult from lifecycle import ResourceManager @@ -23,9 +23,20 @@ DOCUMENTS = [ "Washington, D.C. is the capital of the United States.", "Capital punishment has existed in the United States since before it was a country.", ] +QUERY = "What is the capital of the United States?" + + +def _assert_top_n_scored(body: str) -> None: + parsed = RerankResult.model_validate_json(body) + assert parsed.results, f"/rerank returned no results: {body[:300]}" + assert len(parsed.results) <= 3, f"top_n=3 not honored: {body[:300]}" + assert parsed.results[0].relevance_score is not None, ( + f"top rerank result has no relevance_score: {body[:300]}" + ) class TestRerank: + @pytest.mark.covers("llm.rerank.cohere.basic.nonstream.works") def test_rerank_scores_top_n( self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: @@ -37,13 +48,28 @@ class TestRerank: resources.defer(lambda: endpoints_client.delete_model(model_id)) key = resources.key() - result = endpoints_client.rerank( - key, model, "What is the capital of the United States?", DOCUMENTS, top_n=3 - ) + result = endpoints_client.rerank(key, model, QUERY, DOCUMENTS, top_n=3) require_successful_call(result) - parsed = RerankResult.model_validate_json(result.body) - assert parsed.results, f"/rerank returned no results: {result.body[:300]}" - assert len(parsed.results) <= 3, f"top_n=3 not honored: {result.body[:300]}" - assert parsed.results[0].relevance_score is not None, ( - f"top rerank result has no relevance_score: {result.body[:300]}" + _assert_top_n_scored(result.body) + + @pytest.mark.covers("llm.rerank.bedrock.basic.nonstream.works", exercised_on=["rerank"]) + def test_bedrock_rerank_scores_top_n( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") + model = f"e2e-bedrock-rerank-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="bedrock/amazon.rerank-v1:0", + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="os.environ/AWS_REGION", + ), ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.rerank(key, model, QUERY, DOCUMENTS, top_n=3) + require_successful_call(result) + _assert_top_n_scored(result.body) diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index bd98f11c045..d24d2b53b71 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -13,7 +13,7 @@ from typing import cast import pytest from pydantic import BaseModel, ValidationError -from e2e_config import unique_marker +from e2e_config import require_env, unique_marker from e2e_http import require_successful_call from endpoints_client import ( EndpointsClient, @@ -29,6 +29,26 @@ from models import LiteLLMParamsBody pytestmark = pytest.mark.e2e +BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" + +WEATHER_TOOL = ResponsesFunctionTool( + name="get_weather", + description="Get the weather for a location", + parameters=FunctionParameters( + properties={"location": FunctionParameterProperty(type="string")}, + required=["location"], + ), +) + + +def _bedrock_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=BEDROCK_CONVERSE_BACKEND, + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="os.environ/AWS_REGION", + ) + class WeatherArguments(BaseModel): location: str @@ -190,6 +210,84 @@ class TestResponses: parsed = ResponsesResult.model_validate_json(result.body) assert parsed.text.strip(), f"/responses returned no output text: {result.body[:300]}" + @pytest.mark.covers("llm.responses.anthropic.tool_use.nonstream.works") + def test_responses_anthropic_returns_function_call( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-responses-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="anthropic/claude-haiku-4-5", api_key="os.environ/ANTHROPIC_API_KEY" + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.responses_with_tools( + key, + model, + "What is the weather in San Francisco? Use the get_weather tool.", + [ + ResponsesFunctionTool( + name="get_weather", + description="Get the weather for a location", + parameters=FunctionParameters( + properties={"location": FunctionParameterProperty(type="string")}, + required=["location"], + ), + ) + ], + ) + require_successful_call(result) + parsed = ResponsesResult.model_validate_json(result.body) + function_call = next( + (call for call in parsed.function_calls if call.name == "get_weather"), + None, + ) + assert function_call is not None, f"no get_weather function call: {result.body[:500]}" + assert function_call.arguments is not None + raw_arguments = cast(object, json.loads(function_call.arguments)) + arguments = WeatherArguments.model_validate(raw_arguments) + assert arguments.location, f"function call arguments missing location: {function_call.arguments}" + + @pytest.mark.covers("llm.responses.bedrock_converse.basic.nonstream.works") + def test_responses_bedrock_returns_completion( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") + model = f"e2e-responses-{unique_marker()}" + model_id = endpoints_client.create_model(model, _bedrock_params()) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.responses(key, model, "reply with one word") + require_successful_call(result) + parsed = ResponsesResult.model_validate_json(result.body) + assert parsed.text.strip(), f"/responses over bedrock returned no output text: {result.body[:300]}" + + @pytest.mark.covers("llm.responses.bedrock_converse.tool_use.nonstream.works") + def test_responses_bedrock_returns_function_call( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") + model = f"e2e-responses-{unique_marker()}" + model_id = endpoints_client.create_model(model, _bedrock_params()) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.responses_with_tools( + key, model, "What is the weather in San Francisco? Use the get_weather tool.", [WEATHER_TOOL] + ) + require_successful_call(result) + parsed = ResponsesResult.model_validate_json(result.body) + function_call = next((call for call in parsed.function_calls if call.name == "get_weather"), None) + assert function_call is not None, f"no get_weather function call over bedrock: {result.body[:500]}" + assert function_call.arguments is not None + raw_arguments = cast(object, json.loads(function_call.arguments)) + arguments = WeatherArguments.model_validate(raw_arguments) + assert arguments.location, f"function call arguments missing location: {function_call.arguments}" + def _parse_stream_event( event: str, diff --git a/tests/e2e/load/conftest.py b/tests/e2e/load/conftest.py index e9fba02680d..89a571af83e 100644 --- a/tests/e2e/load/conftest.py +++ b/tests/e2e/load/conftest.py @@ -1,10 +1,12 @@ from __future__ import annotations +import os from collections.abc import Iterator import pytest from requests import RequestException +from e2e_config import WEEKLY_ANOMALY_OPT_IN_ENV from e2e_http import NoBody, Success from load_client import LoadClient, build_client from load_constants import LOAD_MODEL @@ -18,6 +20,22 @@ LOAD_MODEL_PARAMS = LiteLLMParamsBody( ) +def pytest_collection_modifyitems( + config: pytest.Config, items: list[pytest.Item] +) -> None: + if os.environ.get(WEEKLY_ANOMALY_OPT_IN_ENV): + return + deselected = [ + item for item in items if item.get_closest_marker("weekly") is not None + ] + if not deselected: + return + config.hook.pytest_deselected(items=deselected) + items[:] = [ + item for item in items if item.get_closest_marker("weekly") is None + ] + + @pytest.fixture(scope="session") def client(proxy: ProxyClient) -> LoadClient: return build_client(proxy) @@ -33,10 +51,8 @@ def _model_is_servable(proxy: ProxyClient, model_name: str) -> bool: return isinstance(result, Success) and any(entry.id == model_name for entry in result.data.data) -@pytest.fixture(scope="session", autouse=True) -def _ensure_load_model( # pyright: ignore[reportUnusedFunction] # pytest autouse session fixture, wired by name - client: LoadClient, -) -> Iterator[None]: +@pytest.fixture(scope="session") +def ensure_load_model(client: LoadClient) -> Iterator[None]: proxy = client.proxy if _model_is_servable(proxy, LOAD_MODEL): yield @@ -60,7 +76,9 @@ def _ensure_load_model( # pyright: ignore[reportUnusedFunction] # pytest autou @pytest.fixture -def load_key(resources: ResourceManager, client: LoadClient) -> str: +def load_key( + resources: ResourceManager, client: LoadClient, ensure_load_model: None +) -> str: key = client.proxy.generate_key(KeyGenerateBody(models=[LOAD_MODEL], user_id="e2e-load")) resources.defer(lambda: client.proxy.delete_key(key)) return key diff --git a/tests/e2e/load/session_anomaly.py b/tests/e2e/load/session_anomaly.py new file mode 100644 index 00000000000..c29b833635d --- /dev/null +++ b/tests/e2e/load/session_anomaly.py @@ -0,0 +1,299 @@ +from __future__ import annotations + +import time +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass + +from pydantic import BaseModel + +from e2e_config import unique_marker +from e2e_http import Result, Success +from models import CacheControl, RichMessage, TextBlock +from transport import Transport + + +class SessionMessagesRequest(BaseModel): + model: str + max_tokens: int = 128 + system: list[TextBlock] + messages: list[RichMessage] + + +class SessionUsage(BaseModel): + input_tokens: int = 0 + output_tokens: int = 0 + cache_creation_input_tokens: int = 0 + cache_read_input_tokens: int = 0 + + +class SessionContentBlock(BaseModel): + type: str | None = None + text: str | None = None + + +class SessionMessagesResponse(BaseModel): + content: list[SessionContentBlock] = [] + usage: SessionUsage = SessionUsage() + + @property + def text(self) -> str: + return "".join(block.text or "" for block in self.content) + + +@dataclass(frozen=True, slots=True) +class TurnMetric: + turn_index: int + ok: bool + latency_seconds: float + uncached_input_tokens: int + cache_read_tokens: int + cache_creation_tokens: int + failure: str | None + + +@dataclass(frozen=True, slots=True) +class AnomalyReport: + planned_turns: int + attempted_turns: int + failed_turns: int + warm_turns: int + warm_uncached_input_tokens: int + warm_cache_read_tokens: int + warm_cache_creation_tokens: int + p95_turn_seconds: float + + @property + def error_ratio(self) -> float: + return self.failed_turns / self.planned_turns if self.planned_turns else 1.0 + + @property + def warm_cache_read_share(self) -> float: + billed = ( + self.warm_uncached_input_tokens + + self.warm_cache_read_tokens + + self.warm_cache_creation_tokens + ) + return self.warm_cache_read_tokens / billed if billed else 0.0 + + +def _system_prefix_block(marker: str) -> TextBlock: + text = " ".join( + f"Project context paragraph {index} for session {marker}." for index in range(300) + ) + return TextBlock(text=text, cache_control=CacheControl()) + + +def _user_turn_text(marker: str, turn_index: int) -> str: + notes = " ".join( + f"Working note {index} of turn {turn_index} in session {marker}." + for index in range(80) + ) + return f"Reply with one short sentence.\n{notes}" + + +def _reminder_turn() -> RichMessage: + return RichMessage( + role="system", + content=[ + TextBlock( + text="Keep the answer to one short sentence." + ) + ], + ) + + +def _without_cache_control(message: RichMessage) -> RichMessage: + return RichMessage( + role=message.role, + content=[TextBlock(text=block.text) for block in message.content], + ) + + +RETRY_BACKOFF_SECONDS = 2.0 + + +def retried( + call: Callable[[], Result[SessionMessagesResponse]], + attempts: int, + backoff_seconds: float = RETRY_BACKOFF_SECONDS, + sleep: Callable[[float], None] = time.sleep, +) -> Result[SessionMessagesResponse]: + result = call() + if isinstance(result, Success) or attempts <= 1: + return result + sleep(backoff_seconds) + return retried(call, attempts - 1, backoff_seconds, sleep) + + +def _metric( + result: Result[SessionMessagesResponse], turn_index: int, latency_seconds: float +) -> TurnMetric: + if isinstance(result, Success): + usage = result.data.usage + return TurnMetric( + turn_index=turn_index, + ok=True, + latency_seconds=latency_seconds, + uncached_input_tokens=usage.input_tokens, + cache_read_tokens=usage.cache_read_input_tokens, + cache_creation_tokens=usage.cache_creation_input_tokens, + failure=None, + ) + return TurnMetric( + turn_index=turn_index, + ok=False, + latency_seconds=latency_seconds, + uncached_input_tokens=0, + cache_read_tokens=0, + cache_creation_tokens=0, + failure=repr(result), + ) + + +def _drive_turns( + transport: Transport, + key: str, + model: str, + marker: str, + system_block: TextBlock, + history: tuple[RichMessage, ...], + turn_index: int, + remaining_turns: int, + attempts_per_turn: int, +) -> tuple[TurnMetric, ...]: + if remaining_turns == 0: + return () + user_turn = RichMessage( + role="user", + content=[ + TextBlock( + text=_user_turn_text(marker, turn_index), cache_control=CacheControl() + ) + ], + ) + started = time.monotonic() + result = retried( + lambda: transport.post( + "/v1/messages", + headers=transport.bearer(key), + json=SessionMessagesRequest( + model=model, + system=[system_block], + messages=[*history, user_turn], + ), + response_type=SessionMessagesResponse, + ), + attempts_per_turn, + ) + turn = _metric(result, turn_index, time.monotonic() - started) + if not isinstance(result, Success): + return (turn,) + assistant_turn = RichMessage( + role="assistant", content=[TextBlock(text=result.data.text or "Understood.")] + ) + return ( + turn, + *_drive_turns( + transport, + key, + model, + marker, + system_block, + ( + *history, + _without_cache_control(user_turn), + _reminder_turn(), + assistant_turn, + ), + turn_index + 1, + remaining_turns - 1, + attempts_per_turn, + ), + ) + + +def run_session( + transport: Transport, key: str, model: str, turns: int, attempts_per_turn: int +) -> tuple[TurnMetric, ...]: + marker = unique_marker() + return _drive_turns( + transport, + key, + model, + marker, + _system_prefix_block(marker), + (), + 1, + turns, + attempts_per_turn, + ) + + +def run_concurrent_sessions( + transport: Transport, + key: str, + model: str, + sessions: int, + turns_per_session: int, + attempts_per_turn: int, +) -> tuple[TurnMetric, ...]: + with ThreadPoolExecutor(max_workers=sessions) as pool: + futures = [ + pool.submit( + run_session, transport, key, model, turns_per_session, attempts_per_turn + ) + for _ in range(sessions) + ] + return tuple(turn for future in futures for turn in future.result()) + + +def settled_spend( + read_spend: Callable[[], float], + poll_interval: float, + settle_seconds: float, + timeout_seconds: float, + now: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, +) -> float: + deadline = now() + timeout_seconds + settle_seconds + + def settle(previous: float, stable_since: float) -> float: + current = read_spend() + observed = now() + since = stable_since if current == previous else observed + if current > 0 and observed - since >= settle_seconds: + return current + if observed >= deadline: + raise AssertionError( + f"key spend never held a stable non-zero value for {settle_seconds}s " + f"within {timeout_seconds + settle_seconds}s (last read {current}); " + f"spend stopped being recorded, which is itself a spend anomaly" + ) + sleep(poll_interval) + return settle(current, since) + + return settle(-1.0, now()) + + +def _p95(latencies: tuple[float, ...]) -> float: + if not latencies: + return 0.0 + ranked = sorted(latencies) + return ranked[max(0, -(-len(ranked) * 95 // 100) - 1)] + + +def summarize(turns: tuple[TurnMetric, ...], planned_turns: int) -> AnomalyReport: + warm = tuple(turn for turn in turns if turn.ok and turn.turn_index >= 2) + return AnomalyReport( + planned_turns=planned_turns, + attempted_turns=len(turns), + failed_turns=planned_turns - sum(1 for turn in turns if turn.ok), + warm_turns=len(warm), + warm_uncached_input_tokens=sum(turn.uncached_input_tokens for turn in warm), + warm_cache_read_tokens=sum(turn.cache_read_tokens for turn in warm), + warm_cache_creation_tokens=sum(turn.cache_creation_tokens for turn in warm), + p95_turn_seconds=_p95( + tuple(turn.latency_seconds for turn in turns if turn.ok) + ), + ) diff --git a/tests/e2e/load/test_session_anomaly.py b/tests/e2e/load/test_session_anomaly.py new file mode 100644 index 00000000000..80f8aff3ba4 --- /dev/null +++ b/tests/e2e/load/test_session_anomaly.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +from itertools import count, repeat + +import pytest + +from e2e_http import NetworkError, Success +from session_anomaly import ( + SessionMessagesResponse, + TurnMetric, + retried, + settled_spend, + summarize, +) + + +def _ok_turn(turn_index: int) -> TurnMetric: + return TurnMetric( + turn_index=turn_index, + ok=True, + latency_seconds=1.0, + uncached_input_tokens=10, + cache_read_tokens=100, + cache_creation_tokens=5, + failure=None, + ) + + +def _failed_turn(turn_index: int) -> TurnMetric: + return TurnMetric( + turn_index=turn_index, + ok=False, + latency_seconds=1.0, + uncached_input_tokens=0, + cache_read_tokens=0, + cache_creation_tokens=0, + failure="NetworkError()", + ) + + +class TestSummarizePlannedTurns: + def test_session_aborted_on_first_turn_counts_all_its_planned_turns_as_failed( + self, + ) -> None: + completed_session = tuple(_ok_turn(index) for index in range(1, 7)) + aborted_session = (_failed_turn(1),) + + report = summarize((*completed_session, *aborted_session), planned_turns=12) + + assert report.attempted_turns == 7 + assert report.failed_turns == 6 + assert report.error_ratio == 0.5 + + def test_all_planned_turns_completing_reports_zero_failures(self) -> None: + report = summarize( + tuple(_ok_turn(index) for index in range(1, 7)), planned_turns=6 + ) + + assert report.failed_turns == 0 + assert report.error_ratio == 0.0 + + +class TestRetried: + def test_transient_failures_then_success_returns_the_success(self) -> None: + outcome = Success(data=SessionMessagesResponse()) + calls = iter( + (NetworkError(message="overloaded"), NetworkError(message="overloaded"), outcome) + ) + + result = retried(lambda: next(calls), attempts=3, sleep=lambda _: None) + + assert result is outcome + + def test_exhausted_attempts_return_the_last_failure(self) -> None: + last_attempt = NetworkError(message="still overloaded") + never_reached = NetworkError(message="a fourth attempt would break the budget") + calls = iter( + (NetworkError(message="overloaded"), last_attempt, never_reached) + ) + + result = retried(lambda: next(calls), attempts=2, sleep=lambda _: None) + + assert result is last_attempt + assert next(calls) is never_reached + + def test_first_try_success_never_sleeps(self) -> None: + def sleep_means_retry(_: float) -> None: + raise AssertionError("slept after a successful attempt") + + result = retried( + lambda: Success(data=SessionMessagesResponse()), + attempts=3, + sleep=sleep_means_retry, + ) + + assert isinstance(result, Success) + + +class TestSettledSpend: + def test_partial_total_between_batch_flushes_is_not_accepted_as_final(self) -> None: + reads = iter((0.1, 0.1, 0.1, 0.35, 0.35, 0.35, 0.35, 0.35)) + ticks = count(0.0, 2.5) + + spend = settled_spend( + lambda: next(reads), + poll_interval=5.0, + settle_seconds=10.0, + timeout_seconds=100.0, + now=lambda: next(ticks), + sleep=lambda _: None, + ) + + assert spend == 0.35 + + def test_spend_that_never_stabilizes_raises(self) -> None: + reads = (0.1 * step for step in count(1)) + ticks = count(0.0, 2.5) + + with pytest.raises(AssertionError, match="spend anomaly"): + settled_spend( + lambda: next(reads), + poll_interval=5.0, + settle_seconds=5.0, + timeout_seconds=10.0, + now=lambda: next(ticks), + sleep=lambda _: None, + ) + + def test_spend_that_never_becomes_nonzero_raises(self) -> None: + reads = repeat(0.0) + ticks = count(0.0, 2.5) + + with pytest.raises(AssertionError, match="spend anomaly"): + settled_spend( + lambda: next(reads), + poll_interval=5.0, + settle_seconds=5.0, + timeout_seconds=10.0, + now=lambda: next(ticks), + sleep=lambda _: None, + ) diff --git a/tests/e2e/load/test_weekly_session_anomaly_e2e.py b/tests/e2e/load/test_weekly_session_anomaly_e2e.py new file mode 100644 index 00000000000..d4ef883702e --- /dev/null +++ b/tests/e2e/load/test_weekly_session_anomaly_e2e.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import pytest + +from e2e_config import ( + ANOMALY_MAX_ERROR_RATIO, + ANOMALY_MAX_KEY_SPEND_USD, + ANOMALY_MAX_P95_TURN_SECONDS, + ANOMALY_MIN_WARM_CACHE_READ_SHARE, + ANOMALY_SESSIONS, + ANOMALY_SPEND_SETTLE_SECONDS, + ANOMALY_TURN_ATTEMPTS, + ANOMALY_TURNS_PER_SESSION, + unique_marker, +) +from lifecycle import ResourceManager +from load_client import LoadClient +from models import KeyGenerateBody, LiteLLMParamsBody +from proxy_client import ProxyClient +from session_anomaly import run_concurrent_sessions, settled_spend, summarize + +pytestmark = [pytest.mark.e2e, pytest.mark.load, pytest.mark.weekly] + + +@dataclass(frozen=True, slots=True) +class AnomalyRoute: + route_id: str + params: LiteLLMParamsBody + + +ANOMALY_ROUTES = ( + AnomalyRoute( + route_id="anthropic", + params=LiteLLMParamsBody(model="anthropic/claude-sonnet-5"), + ), + AnomalyRoute( + route_id="bedrock_invoke", + params=LiteLLMParamsBody( + model="bedrock/invoke/us.anthropic.claude-sonnet-5", + aws_region_name="us-east-1", + ), + ), +) + + +def _route_id(route: AnomalyRoute) -> str: + return route.route_id + + +def _settled_key_spend(proxy: ProxyClient, key: str) -> float: + return settled_spend( + lambda: proxy.key_info(key).spend or 0.0, + proxy.poll_interval, + ANOMALY_SPEND_SETTLE_SECONDS, + proxy.poll_timeout, + ) + + +class TestWeeklySessionAnomaly: + @pytest.mark.covers("reliability.perf.session_anomaly.under_slo") + @pytest.mark.parametrize("route", ANOMALY_ROUTES, ids=_route_id) + def test_session_load_stays_within_baselines( + self, client: LoadClient, resources: ResourceManager, route: AnomalyRoute + ) -> None: + model_name = f"weekly-anomaly-{route.route_id}-{unique_marker()}" + model_id = client.proxy.create_model(model_name, route.params) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = client.proxy.generate_key( + KeyGenerateBody(models=[model_name], key_alias=model_name) + ) + resources.defer(lambda: client.proxy.delete_key(key)) + + turns = run_concurrent_sessions( + client.proxy.transport, + key, + model_name, + ANOMALY_SESSIONS, + ANOMALY_TURNS_PER_SESSION, + ANOMALY_TURN_ATTEMPTS, + ) + report = summarize(turns, ANOMALY_SESSIONS * ANOMALY_TURNS_PER_SESSION) + failures = tuple(turn.failure for turn in turns if turn.failure) + print(f"{route.route_id} anomaly report: {report}") + + assert report.error_ratio <= ANOMALY_MAX_ERROR_RATIO, ( + f"{route.route_id}: {report.failed_turns}/{report.planned_turns} planned " + f"turns failed or never ran because their session aborted " + f"({report.error_ratio:.1%} > {ANOMALY_MAX_ERROR_RATIO:.1%} allowed); " + f"error rate is anomalously high. Failures: {failures}" + ) + assert report.warm_turns > 0, ( + f"{route.route_id}: no session got past its first turn, so cache and " + f"latency baselines have nothing to read. Failures: {failures}" + ) + assert report.warm_cache_read_share >= ANOMALY_MIN_WARM_CACHE_READ_SHARE, ( + f"{route.route_id}: warm turns read only {report.warm_cache_read_share:.1%} " + f"of billed input tokens from the prompt cache " + f"(read={report.warm_cache_read_tokens}, " + f"creation={report.warm_cache_creation_tokens}, " + f"uncached={report.warm_uncached_input_tokens}), below the " + f"{ANOMALY_MIN_WARM_CACHE_READ_SHARE:.0%} floor; the cached prefix is " + f"being invalidated between turns (the mid-conversation-system cache " + f"collapse signature) or caching stopped working" + ) + assert report.warm_cache_creation_tokens > 0, ( + f"{route.route_id}: warm turns wrote 0 cache-creation tokens across " + f"{report.warm_turns} turns; the moving cache breakpoint stopped writing " + f"new prefix increments" + ) + assert report.p95_turn_seconds <= ANOMALY_MAX_P95_TURN_SECONDS, ( + f"{route.route_id}: p95 turn time {report.p95_turn_seconds:.1f}s exceeds " + f"the {ANOMALY_MAX_P95_TURN_SECONDS:.0f}s ceiling under " + f"{ANOMALY_SESSIONS} concurrent sessions; turn times are anomalously slow" + ) + + spend = _settled_key_spend(client.proxy, key) + assert spend <= ANOMALY_MAX_KEY_SPEND_USD, ( + f"{route.route_id}: gateway recorded ${spend:.4f} for " + f"{report.attempted_turns} turns, above the " + f"${ANOMALY_MAX_KEY_SPEND_USD} ceiling; spend per session is " + f"anomalously high (cache regressions surface here as 2-3x spend)" + ) diff --git a/tests/e2e/load/weekly_anomaly_config.yml b/tests/e2e/load/weekly_anomaly_config.yml new file mode 100644 index 00000000000..08972969cf0 --- /dev/null +++ b/tests/e2e/load/weekly_anomaly_config.yml @@ -0,0 +1,3 @@ +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + store_model_in_db: true diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py index a9dedac8e61..cdc31aeea79 100644 --- a/tests/e2e/management/management_client.py +++ b/tests/e2e/management/management_client.py @@ -14,6 +14,10 @@ from e2e_http import NoBody, ProbeResult, Result, StreamingResponse, Success, Un from models import ( ChatBody, ChatMessage, + CustomerDeleteBody, + CustomerInfoParams, + CustomerNewBody, + CustomerResponse, KeyBlockBody, KeyDeleteBody, KeyGenerateBody, @@ -270,6 +274,35 @@ class ManagementClient: ) ).user_id + def create_customer(self, user_id: str) -> str: + _ = unwrap( + self.proxy.transport.post( + "/customer/new", + headers=self.proxy.transport.master, + json=CustomerNewBody(user_id=user_id), + response_type=CustomerResponse, + ) + ) + return user_id + + def customer_info(self, end_user_id: str) -> CustomerResponse: + return unwrap( + self.proxy.transport.get( + "/customer/info", + headers=self.proxy.transport.master, + params=CustomerInfoParams(end_user_id=end_user_id), + response_type=CustomerResponse, + ) + ) + + def delete_customer(self, user_id: str) -> None: + _ = self.proxy.transport.post( + "/customer/delete", + headers=self.proxy.transport.master, + json=CustomerDeleteBody(user_ids=[user_id]), + response_type=NoBody, + ) + def update_user(self, body: UserUpdateBody) -> None: _ = unwrap( self.proxy.transport.post( diff --git a/tests/e2e/management/test_budget_customer_user_org_e2e.py b/tests/e2e/management/test_budget_customer_user_org_e2e.py new file mode 100644 index 00000000000..54cc18b228b --- /dev/null +++ b/tests/e2e/management/test_budget_customer_user_org_e2e.py @@ -0,0 +1,415 @@ +"""Live e2e coverage for the budget, customer/end-user, user-info and +organization-membership management routes. + +Each test creates its resources under unique ids (deleted on teardown) and +asserts the recorded state the route promises: the budget table reflects a +create/update, a customer round-trips through the info route and disappears after +delete, /user/info echoes what /user/new stored, and an added org member shows up +both in the add response and in /organization/info. The budget/new admin gate is +proven by driving the route under a non-admin key and asserting it is refused. + +Response bodies validate into local pydantic models (only the fields asserted are +modelled) so a shape change fails here instead of passing vacuously. +""" + +from __future__ import annotations + +import math +import time +from collections.abc import Callable + +import pytest +from pydantic import BaseModel, RootModel + +from e2e_config import unique_marker +from e2e_http import NoBody, unwrap +from lifecycle import ResourceManager +from management_client import ManagementClient +from models import KeyGenerateBody, OrgInfoParams, OrgNewBody, UserNewBody + +pytestmark = pytest.mark.e2e + + +def _poll[T](client: ManagementClient, attempt: Callable[[], T | None], failure: str) -> T: + deadline = time.monotonic() + client.proxy.poll_timeout + while time.monotonic() < deadline: + found = attempt() + if found is not None: + return found + time.sleep(client.proxy.poll_interval) + pytest.fail(failure) + + +# ---------- budget ---------- + + +class BudgetNewBody(BaseModel): + max_budget: float + soft_budget: float | None = None + budget_duration: str | None = None + + +class BudgetNewResponse(BaseModel): + budget_id: str + + +class BudgetUpdateBody(BaseModel): + budget_id: str + max_budget: float + + +class BudgetInfoBody(BaseModel): + budgets: list[str] + + +class BudgetRow(BaseModel): + budget_id: str | None = None + max_budget: float | None = None + soft_budget: float | None = None + + +class BudgetInfoResponse(RootModel[list[BudgetRow]]): + pass + + +class BudgetListResponse(RootModel[list[BudgetRow]]): + """GET /budget/list answers with a bare array of budget rows, not an object + wrapping them. Read the rows off .root.""" + + +class BudgetDeleteBody(BaseModel): + id: str + + +def _delete_budget(client: ManagementClient, budget_id: str) -> None: + _ = client.proxy.transport.post( + "/budget/delete", + headers=client.proxy.transport.master, + json=BudgetDeleteBody(id=budget_id), + response_type=NoBody, + ) + + +def _create_budget(client: ManagementClient, resources: ResourceManager, body: BudgetNewBody) -> str: + budget_id = unwrap( + client.proxy.transport.post( + "/budget/new", + headers=client.proxy.transport.master, + json=body, + response_type=BudgetNewResponse, + ) + ).budget_id + resources.defer(lambda: _delete_budget(client, budget_id)) + return budget_id + + +def _budget_rows(client: ManagementClient, budget_id: str) -> tuple[BudgetRow, ...]: + return tuple( + unwrap( + client.proxy.transport.post( + "/budget/info", + headers=client.proxy.transport.master, + json=BudgetInfoBody(budgets=[budget_id]), + response_type=BudgetInfoResponse, + ) + ).root + ) + + +def _budget_list_ids(client: ManagementClient) -> tuple[str, ...]: + return tuple( + row.budget_id + for row in unwrap( + client.proxy.transport.get( + "/budget/list", + headers=client.proxy.transport.master, + params=NoBody(), + response_type=BudgetListResponse, + ) + ).root + if row.budget_id is not None + ) + + +_INITIAL_MAX_BUDGET = 5.5 +_UPDATED_MAX_BUDGET = 91.25 + + +class TestBudgetManagement: + @pytest.mark.covers("mgmt.budget.list.happy_path") + def test_created_budget_appears_in_budget_list( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + budget_id = _create_budget(client, resources, BudgetNewBody(max_budget=_INITIAL_MAX_BUDGET)) + + _ = _poll( + client, + lambda: budget_id if budget_id in _budget_list_ids(client) else None, + f"/budget/list never included the created budget {budget_id}", + ) + + @pytest.mark.covers("mgmt.budget.update.persists") + def test_update_max_budget_persists_to_budget_info( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + budget_id = _create_budget(client, resources, BudgetNewBody(max_budget=_INITIAL_MAX_BUDGET)) + + rows = _budget_rows(client, budget_id) + assert rows, f"/budget/info returned nothing for the freshly created budget {budget_id}" + initial = rows[0].max_budget + assert initial is not None and math.isclose(initial, _INITIAL_MAX_BUDGET, rel_tol=1e-9), ( + f"/budget/info reports max_budget {initial}, created with {_INITIAL_MAX_BUDGET}" + ) + + _ = unwrap( + client.proxy.transport.post( + "/budget/update", + headers=client.proxy.transport.master, + json=BudgetUpdateBody(budget_id=budget_id, max_budget=_UPDATED_MAX_BUDGET), + response_type=NoBody, + ) + ) + + def updated() -> BudgetRow | None: + row = next((r for r in _budget_rows(client, budget_id) if r.budget_id == budget_id), None) + if row is None or row.max_budget is None: + return None + return row if math.isclose(row.max_budget, _UPDATED_MAX_BUDGET, rel_tol=1e-9) else None + + _ = _poll( + client, + updated, + f"/budget/info never reported max_budget {_UPDATED_MAX_BUDGET} for {budget_id} after /budget/update", + ) + + @pytest.mark.covers("mgmt.budget.new.admin_only") + def test_new_is_refused_for_a_non_admin_key( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + key = client.proxy.generate_key(KeyGenerateBody()) + resources.defer(lambda: client.proxy.delete_key(key)) + + outcome = client.proxy.transport.send( + "/budget/new", + headers=client.proxy.transport.bearer(key), + json=BudgetNewBody(max_budget=1.0), + ) + + assert outcome.status_code in (401, 403), ( + f"non-admin key POSTing /budget/new must be refused 401/403, got " + f"{outcome.status_code}: {outcome.body[:300]}" + ) + assert "proxy admin" in outcome.body.lower() or "not allowed" in outcome.body.lower(), ( + f"/budget/new denial body must name the admin-only gate, got: {outcome.body[:300]}" + ) + + +# ---------- customer / end-user ---------- + + +class CustomerNewBody(BaseModel): + user_id: str + max_budget: float | None = None + + +class CustomerNewResponse(BaseModel): + user_id: str + + +class CustomerInfoParams(BaseModel): + end_user_id: str + + +class CustomerInfoResponse(BaseModel): + user_id: str + + +class CustomerDeleteBody(BaseModel): + user_ids: list[str] + + +class CustomerDeleteResponse(BaseModel): + deleted_customers: int + + +def _create_customer( + client: ManagementClient, resources: ResourceManager, route: str, body: CustomerNewBody +) -> str: + user_id = unwrap( + client.proxy.transport.post( + route, + headers=client.proxy.transport.master, + json=body, + response_type=CustomerNewResponse, + ) + ).user_id + resources.defer(lambda: client.proxy.delete_customers([user_id])) + return user_id + + +def _customer_info(client: ManagementClient, route: str, user_id: str) -> CustomerInfoResponse: + return unwrap( + client.proxy.transport.get( + route, + headers=client.proxy.transport.master, + params=CustomerInfoParams(end_user_id=user_id), + response_type=CustomerInfoResponse, + ) + ) + + +class TestCustomerManagement: + @pytest.mark.covers("mgmt.customer.new.happy_path") + def test_new_persists_to_customer_info(self, client: ManagementClient, resources: ResourceManager) -> None: + customer_id = f"e2e-mgmt-cust-{unique_marker()}" + created = _create_customer( + client, resources, "/customer/new", CustomerNewBody(user_id=customer_id, max_budget=7.0) + ) + assert created == customer_id, f"/customer/new echoed user_id {created!r}, created {customer_id!r}" + + info = _customer_info(client, "/customer/info", customer_id) + assert info.user_id == customer_id, ( + f"/customer/info reports user_id {info.user_id!r} for the created customer {customer_id!r}" + ) + + @pytest.mark.covers("mgmt.customer.delete.persists") + def test_delete_removes_the_customer(self, client: ManagementClient, resources: ResourceManager) -> None: + """The teardown's deferred delete fires again on the already-deleted customer + by design: it is the safety net if this test fails before the in-body delete, + and a repeat /customer/delete is absorbed by the warn-only teardown.""" + customer_id = f"e2e-mgmt-cust-{unique_marker()}" + _ = _create_customer(client, resources, "/customer/new", CustomerNewBody(user_id=customer_id, max_budget=3.0)) + + assert _customer_info(client, "/customer/info", customer_id).user_id == customer_id, ( + f"customer {customer_id} was not readable before deletion" + ) + + deleted = unwrap( + client.proxy.transport.post( + "/customer/delete", + headers=client.proxy.transport.master, + json=CustomerDeleteBody(user_ids=[customer_id]), + response_type=CustomerDeleteResponse, + ) + ).deleted_customers + assert deleted == 1, f"/customer/delete reported {deleted} rows removed for one customer" + + def gone() -> bool | None: + return True if client.proxy.transport.probe( + "/customer/info", params=CustomerInfoParams(end_user_id=customer_id) + ).status_code == 404 else None + + _ = _poll(client, gone, f"customer {customer_id} still resolved on /customer/info after /customer/delete") + + @pytest.mark.covers("mgmt.end_user.new.happy_path") + def test_end_user_new_persists_to_end_user_info( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + end_user_id = f"e2e-mgmt-euser-{unique_marker()}" + created = _create_customer(client, resources, "/end_user/new", CustomerNewBody(user_id=end_user_id)) + assert created == end_user_id, f"/end_user/new echoed user_id {created!r}, created {end_user_id!r}" + + info = _customer_info(client, "/end_user/info", end_user_id) + assert info.user_id == end_user_id, ( + f"/end_user/info reports user_id {info.user_id!r} for the created end user {end_user_id!r}" + ) + + +# ---------- user info ---------- + + +class TestUserManagement: + @pytest.mark.covers("mgmt.user.info.happy_path") + def test_new_user_is_readable_via_user_info( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + email = f"e2e-mgmt-{unique_marker()}@example.com" + user_id = client.create_user(UserNewBody(user_email=email, user_role="internal_user")) + resources.defer(lambda: client.delete_user(user_id)) + + info = client.user_info(user_id).user_info + assert info.user_id == user_id, f"/user/info reports user_id {info.user_id!r}, created {user_id!r}" + assert info.user_email == email, f"/user/info reports user_email {info.user_email!r}, configured {email!r}" + assert info.user_role == "internal_user", ( + f"/user/info reports user_role {info.user_role!r}, configured 'internal_user'" + ) + + +# ---------- organization membership ---------- + + +class OrgMemberEntry(BaseModel): + role: str + user_id: str + + +class OrgMemberAddBody(BaseModel): + organization_id: str + member: OrgMemberEntry + + +class OrgMembershipRow(BaseModel): + user_id: str + organization_id: str | None = None + + +class OrgMemberAddResponse(BaseModel): + organization_id: str + updated_organization_memberships: list[OrgMembershipRow] + + +class OrgInfoMembersResponse(BaseModel): + members: list[OrgMembershipRow] = [] + + +class TestOrganizationMembership: + @pytest.mark.covers("mgmt.organization.member_add.happy_path") + def test_member_add_records_membership( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + org_id = client.create_org(OrgNewBody(organization_alias=f"e2e-mgmt-org-{unique_marker()}")) + resources.defer(lambda: client.delete_org(org_id)) + + user_id = client.create_user( + UserNewBody(user_email=f"e2e-mgmt-{unique_marker()}@example.com", user_role="internal_user") + ) + resources.defer(lambda: client.delete_user(user_id)) + + added = unwrap( + client.proxy.transport.post( + "/organization/member_add", + headers=client.proxy.transport.master, + json=OrgMemberAddBody( + organization_id=org_id, + member=OrgMemberEntry(role="internal_user", user_id=user_id), + ), + response_type=OrgMemberAddResponse, + ) + ) + assert added.organization_id == org_id, ( + f"/organization/member_add echoed organization_id {added.organization_id!r}, added to {org_id!r}" + ) + assert any( + row.user_id == user_id and row.organization_id == org_id + for row in added.updated_organization_memberships + ), ( + f"/organization/member_add response does not record {user_id} in org {org_id}: " + f"{added.updated_organization_memberships}" + ) + + def listed() -> bool | None: + members = unwrap( + client.proxy.transport.get( + "/organization/info", + headers=client.proxy.transport.master, + params=OrgInfoParams(organization_id=org_id), + response_type=OrgInfoMembersResponse, + ) + ).members + return True if any(member.user_id == user_id for member in members) else None + + _ = _poll( + client, + listed, + f"/organization/info never listed member {user_id} in org {org_id} after /organization/member_add", + ) diff --git a/tests/e2e/management/test_config_misc_endpoints_e2e.py b/tests/e2e/management/test_config_misc_endpoints_e2e.py new file mode 100644 index 00000000000..6c4de621271 --- /dev/null +++ b/tests/e2e/management/test_config_misc_endpoints_e2e.py @@ -0,0 +1,698 @@ +"""Live e2e: the config and miscellaneous Management/UI routes. + +One method per registry cell, each asserting the real contract against a live +proxy: read-only inventory routes return their documented shape, stateless +validators compute their verdict from the request, and the write routes persist +so a read-back reflects the change. The two routes that mutate global proxy state +(cache settings and router settings, both driven from the admin UI) are exercised +with a benign, self-restoring change so a shared proxy is left as it was found. +""" + +from __future__ import annotations + +import math +import time +from collections.abc import Callable + +import pytest +from pydantic import BaseModel + +from e2e_config import unique_marker +from e2e_http import NoBody, Success, unwrap, unwrap_status +from lifecycle import ResourceManager +from management_client import ManagementClient +from models import KeyGenerateBody, LiteLLMParamsBody, TeamNewBody + +pytestmark = pytest.mark.e2e + + +def _poll[T](client: ManagementClient, attempt: Callable[[], T | None], failure: str) -> T: + deadline = time.monotonic() + client.proxy.poll_timeout + while time.monotonic() < deadline: + found = attempt() + if found is not None: + return found + time.sleep(client.proxy.poll_interval) + pytest.fail(failure) + + +# ---- callbacks ------------------------------------------------------------- + + +class CallbacksListResponse(BaseModel): + success: list[str] + failure: list[str] + success_and_failure: list[str] + + +# ---- cost estimate --------------------------------------------------------- + + +class CostEstimateBody(BaseModel): + model: str + input_tokens: int + output_tokens: int + num_requests_per_day: int | None = None + + +class CostEstimateResponse(BaseModel): + model: str + input_tokens: int + output_tokens: int + cost_per_request: float + input_cost_per_request: float + output_cost_per_request: float + margin_cost_per_request: float + daily_cost: float | None = None + provider: str | None = None + + +# ---- credential migration check -------------------------------------------- + + +class MigrationReport(BaseModel): + residual_legacy: int + total_undecryptable: int + + +class MigrationCheckResponse(BaseModel): + status: str + report: MigrationReport + + +# ---- tool + workflow inventories ------------------------------------------- + + +class ToolListEntry(BaseModel): + name: str | None = None + + +class ToolListResponse(BaseModel): + tools: list[ToolListEntry] + total: int + + +class WorkflowRunEntry(BaseModel): + workflow_id: str | None = None + + +class WorkflowRunsResponse(BaseModel): + runs: list[WorkflowRunEntry] + count: int + + +# ---- compliance ------------------------------------------------------------ + + +class ComplianceGdprBody(BaseModel): + request_id: str + user_id: str + model: str + timestamp: str + + +class ComplianceCheck(BaseModel): + check_name: str + article: str + passed: bool + detail: str + + +class ComplianceResponse(BaseModel): + compliant: bool + regulation: str + checks: list[ComplianceCheck] + + +# ---- cache settings -------------------------------------------------------- + + +class CacheSettingsValue(BaseModel): + type: str + host: str = "" + port: str = "" + + +class CacheSettingsUpdateBody(BaseModel): + cache_settings: CacheSettingsValue + + +class CacheCurrentValues(BaseModel): + type: str | None = None + host: str | None = None + port: str | None = None + + +class CacheGetResponse(BaseModel): + current_values: CacheCurrentValues + + +class CacheUpdateResponse(BaseModel): + status: str + settings: CacheSettingsValue + + +# ---- fallback management --------------------------------------------------- + + +class FallbackShape(BaseModel): + model: str + fallback_models: list[str] + fallback_type: str + + +class FallbackCreateBody(FallbackShape): + pass + + +class FallbackResponse(FallbackShape): + message: str + + +class FallbackGetParams(BaseModel): + fallback_type: str + + +class FallbackGetResponse(FallbackShape): + pass + + +# ---- jwt key mapping ------------------------------------------------------- + + +class JwtKeyMappingNewBody(BaseModel): + jwt_claim_name: str + jwt_claim_value: str + key: str + description: str + + +class JwtInfoParams(BaseModel): + id: str + + +class JwtDeleteBody(BaseModel): + id: str + + +class JwtKeyMappingResponse(BaseModel): + id: str + jwt_claim_name: str + jwt_claim_value: str + is_active: bool + description: str | None = None + + +# ---- router settings via /config/update ------------------------------------ + + +class RouterSettingsPatch(BaseModel): + num_retries: int + + +class ConfigUpdateBody(BaseModel): + router_settings: RouterSettingsPatch + + +class ConfigUpdateResponse(BaseModel): + message: str + + +class RouterCurrentValues(BaseModel): + num_retries: int | None = None + + +class RouterSettingsResponse(BaseModel): + current_values: RouterCurrentValues + + +# ---- mcp server submission ------------------------------------------------- + + +class McpRegisterBody(BaseModel): + server_name: str + url: str + transport: str + description: str + + +class McpServerResponse(BaseModel): + server_id: str + server_name: str | None = None + approval_status: str + transport: str + url: str | None = None + + +class TestInventoryRoutes: + @pytest.mark.covers("mgmt.callback.list.happy_path") + def test_callbacks_list_reports_active_logging_callbacks(self, client: ManagementClient) -> None: + listing = unwrap( + client.proxy.transport.get( + "/callbacks/list", + headers=client.proxy.transport.master, + params=NoBody(), + response_type=CallbacksListResponse, + ) + ) + every = [*listing.success, *listing.failure, *listing.success_and_failure] + assert every, "/callbacks/list reported no active logging callbacks; the proxy always runs the db logger" + assert "_ProxyDBLogger" in every, ( + f"/callbacks/list omitted the always-on _ProxyDBLogger spend logger; got {every}" + ) + + @pytest.mark.covers("mgmt.tool_management.list.happy_path") + def test_tool_list_returns_catalog_with_consistent_total(self, client: ManagementClient) -> None: + listing = unwrap( + client.proxy.transport.get( + "/v1/tool/list", + headers=client.proxy.transport.master, + params=NoBody(), + response_type=ToolListResponse, + ) + ) + assert listing.total == len(listing.tools), ( + f"/v1/tool/list total {listing.total} disagrees with the {len(listing.tools)} tools returned" + ) + + @pytest.mark.covers("mgmt.workflow.list.happy_path") + def test_workflow_runs_list_returns_consistent_count(self, client: ManagementClient) -> None: + listing = unwrap( + client.proxy.transport.get( + "/v1/workflows/runs", + headers=client.proxy.transport.master, + params=NoBody(), + response_type=WorkflowRunsResponse, + ) + ) + assert listing.count == len(listing.runs), ( + f"/v1/workflows/runs count {listing.count} disagrees with the {len(listing.runs)} runs returned" + ) + + @pytest.mark.covers("mgmt.credential_migration.check.happy_path") + def test_credential_migration_check_reports_residual_scan(self, client: ManagementClient) -> None: + report = unwrap( + client.proxy.transport.get( + "/credentials/migrate-encryption/check", + headers=client.proxy.transport.master, + params=NoBody(), + response_type=MigrationCheckResponse, + ) + ) + assert report.status == "success", f"migrate-encryption/check status {report.status!r}, expected 'success'" + assert report.report.residual_legacy >= 0, ( + f"residual_legacy count is negative ({report.report.residual_legacy}); the scan is broken" + ) + assert report.report.total_undecryptable >= 0, ( + f"total_undecryptable count is negative ({report.report.total_undecryptable}); the scan is broken" + ) + + +class TestCostEstimate: + @pytest.mark.covers("mgmt.cost_tracking.estimate.happy_path") + def test_estimate_computes_cost_from_token_counts(self, client: ManagementClient) -> None: + estimate = unwrap( + client.proxy.transport.post( + "/cost/estimate", + headers=client.proxy.transport.master, + json=CostEstimateBody( + model="gpt-4o-mini", input_tokens=1000, output_tokens=500, num_requests_per_day=100 + ), + response_type=CostEstimateResponse, + ) + ) + assert estimate.input_cost_per_request > 0, ( + f"input cost per request is {estimate.input_cost_per_request}; a priced model must cost more than zero" + ) + assert estimate.output_cost_per_request > 0, ( + f"output cost per request is {estimate.output_cost_per_request}; a priced model must cost more than zero" + ) + expected_per_request = ( + estimate.input_cost_per_request + estimate.output_cost_per_request + estimate.margin_cost_per_request + ) + assert math.isclose(estimate.cost_per_request, expected_per_request, rel_tol=1e-9), ( + f"cost_per_request {estimate.cost_per_request} != input+output+margin {expected_per_request}" + ) + assert estimate.daily_cost is not None and math.isclose( + estimate.daily_cost, estimate.cost_per_request * 100, rel_tol=1e-9 + ), f"daily_cost {estimate.daily_cost} != cost_per_request * 100 requests {estimate.cost_per_request * 100}" + + +class TestComplianceRoutes: + @pytest.mark.covers("mgmt.compliance.gdpr.happy_path") + def test_gdpr_check_derives_verdict_from_the_request(self, client: ManagementClient) -> None: + result = unwrap( + client.proxy.transport.post( + "/compliance/gdpr", + headers=client.proxy.transport.master, + json=ComplianceGdprBody( + request_id=f"e2e-gdpr-{unique_marker()}", + user_id=f"e2e-user-{unique_marker()}", + model="gpt-4o-mini", + timestamp="2026-07-21T00:00:00Z", + ), + response_type=ComplianceResponse, + ) + ) + assert result.regulation == "GDPR", ( + f"/compliance/gdpr reported regulation {result.regulation!r}, expected 'GDPR'" + ) + articles = {check.article for check in result.checks} + assert articles == {"Art. 32", "Art. 5(1)(c)", "Art. 30"}, ( + f"/compliance/gdpr returned articles {articles}, expected the three GDPR articles" + ) + assert result.compliant == all(check.passed for check in result.checks), ( + "the overall compliant verdict must be the conjunction of the individual checks" + ) + assert all(check.check_name and check.detail for check in result.checks), ( + "every compliance check must carry a name and a human-readable detail" + ) + + +class TestCacheSettings: + @pytest.mark.covers("mgmt.cache_settings.update.happy_path") + def test_update_persists_cache_backend_to_get( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + """Exercise the update route without changing global state: capture the live + cache backend and write exactly that back, so the config the proxy ends on is + byte-for-byte the one it started with. A teardown restore of the same captured + settings is the safety net if the body fails partway. The update route is only + meaningful against a configured cache, so an unconfigured proxy fails loudly + here rather than being silently switched to redis.""" + before = self._read_settings(client) + assert before.type is not None, ( + "GET /cache/settings reported no cache type; refusing to invent one and mutate the shared proxy" + ) + captured = CacheSettingsValue(type=before.type, host=before.host or "", port=before.port or "") + resources.defer(lambda: self._write_settings(client, captured)) + + updated = unwrap( + client.proxy.transport.post( + "/cache/settings", + headers=client.proxy.transport.master, + json=CacheSettingsUpdateBody(cache_settings=captured), + response_type=CacheUpdateResponse, + ) + ) + assert updated.status == "success", f"/cache/settings update status {updated.status!r}, expected 'success'" + assert updated.settings.type == captured.type, ( + f"/cache/settings echoed type {updated.settings.type!r}, wrote {captured.type!r}" + ) + + def reflected() -> CacheCurrentValues | None: + current = self._read_settings(client) + return current if current.type == captured.type else None + + after = _poll(client, reflected, f"/cache/settings never reported type {captured.type!r} after the update") + assert after.host == captured.host and after.port == captured.port, ( + f"/cache/settings persisted host/port {after.host!r}/{after.port!r}, " + f"wrote {captured.host!r}/{captured.port!r}" + ) + + @staticmethod + def _read_settings(client: ManagementClient) -> CacheCurrentValues: + return unwrap( + client.proxy.transport.get( + "/cache/settings", + headers=client.proxy.transport.master, + params=NoBody(), + response_type=CacheGetResponse, + ) + ).current_values + + @staticmethod + def _write_settings(client: ManagementClient, settings: CacheSettingsValue) -> None: + _ = unwrap( + client.proxy.transport.post( + "/cache/settings", + headers=client.proxy.transport.master, + json=CacheSettingsUpdateBody(cache_settings=settings), + response_type=CacheUpdateResponse, + ) + ) + + +class TestFallbackManagement: + @pytest.mark.covers("mgmt.fallback_management.update.happy_path") + def test_create_persists_and_is_read_back(self, client: ManagementClient, resources: ResourceManager) -> None: + primary = f"e2e-fallback-primary-{unique_marker()}" + secondary = f"e2e-fallback-secondary-{unique_marker()}" + params = LiteLLMParamsBody(model="openai/gpt-5.5", api_key="e2e-dummy-key") + primary_id = client.proxy.create_model(primary, params) + resources.defer(lambda: client.proxy.delete_model(primary_id)) + secondary_id = client.proxy.create_model(secondary, params) + resources.defer(lambda: client.proxy.delete_model(secondary_id)) + resources.defer(lambda: self._delete_fallback(client, primary)) + + created = unwrap( + client.proxy.transport.post( + "/fallback", + headers=client.proxy.transport.master, + json=FallbackCreateBody(model=primary, fallback_models=[secondary], fallback_type="general"), + response_type=FallbackResponse, + ) + ) + assert created.model == primary and created.fallback_models == [secondary], ( + f"/fallback echoed model={created.model!r} fallbacks={created.fallback_models}, " + f"configured {primary!r} -> [{secondary!r}]" + ) + + def read_back() -> FallbackGetResponse | None: + result = client.proxy.transport.get( + f"/fallback/{primary}", + headers=client.proxy.transport.master, + params=FallbackGetParams(fallback_type="general"), + response_type=FallbackGetResponse, + ) + match result: + case Success(data=data) if secondary in data.fallback_models: + return data + case _: + return None + + got = _poll(client, read_back, f"GET /fallback/{primary} never reported {secondary} after /fallback") + assert got.fallback_models == [secondary], ( + f"GET /fallback/{primary} reports fallbacks {got.fallback_models}, configured [{secondary!r}]" + ) + + @staticmethod + def _delete_fallback(client: ManagementClient, model: str) -> None: + _ = client.proxy.transport.delete( + f"/fallback/{model}", + headers=client.proxy.transport.master, + json=NoBody(), + params=FallbackGetParams(fallback_type="general"), + response_type=NoBody, + ) + + +class TestJwtKeyMapping: + @pytest.mark.covers("mgmt.jwt_key_mapping.new.happy_path") + def test_new_persists_mapping_and_is_read_back( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + key = client.proxy.generate_key(KeyGenerateBody()) + resources.defer(lambda: client.proxy.delete_key(key)) + claim_value = f"e2e_jwt_{unique_marker()}" + + created = unwrap( + client.proxy.transport.post( + "/jwt/key/mapping/new", + headers=client.proxy.transport.master, + json=JwtKeyMappingNewBody( + jwt_claim_name="team_id", + jwt_claim_value=claim_value, + key=key, + description="e2e coverage mapping", + ), + response_type=JwtKeyMappingResponse, + ) + ) + resources.defer(lambda: self._delete_mapping(client, created.id)) + assert created.jwt_claim_value == claim_value and created.is_active, ( + f"/jwt/key/mapping/new returned claim_value={created.jwt_claim_value!r} active={created.is_active}, " + f"configured {claim_value!r} active=True" + ) + + info = unwrap( + client.proxy.transport.get( + "/jwt/key/mapping/info", + headers=client.proxy.transport.master, + params=JwtInfoParams(id=created.id), + response_type=JwtKeyMappingResponse, + ) + ) + assert info.id == created.id and info.jwt_claim_name == "team_id" and info.jwt_claim_value == claim_value, ( + f"/jwt/key/mapping/info reports {info.jwt_claim_name!r}={info.jwt_claim_value!r} for id {info.id}, " + f"created team_id={claim_value!r}" + ) + + @staticmethod + def _delete_mapping(client: ManagementClient, mapping_id: str) -> None: + _ = client.proxy.transport.post( + "/jwt/key/mapping/delete", + headers=client.proxy.transport.master, + json=JwtDeleteBody(id=mapping_id), + response_type=NoBody, + ) + + +class TestRouterSettings: + @pytest.mark.covers("mgmt.router_settings.update.happy_path") + def test_config_update_persists_router_setting_to_get( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + """/config/update is the only write path for router_settings (there is no + dedicated router-settings write route). The change is restored on teardown so + the shared proxy keeps its original retry policy.""" + original = self._read_num_retries(client) + assert original is not None, "GET /router/settings did not report num_retries; cannot prove a change" + resources.defer(lambda: self._write_num_retries(client, original)) + + target = original + 5 + response = unwrap( + client.proxy.transport.post( + "/config/update", + headers=client.proxy.transport.master, + json=ConfigUpdateBody(router_settings=RouterSettingsPatch(num_retries=target)), + response_type=ConfigUpdateResponse, + ) + ) + assert "success" in response.message.lower(), ( + f"/config/update reported {response.message!r}, expected a success message" + ) + + _ = _poll( + client, + lambda: True if self._read_num_retries(client) == target else None, + f"GET /router/settings never reported num_retries {target} after /config/update", + ) + + self._write_num_retries(client, original) + restored = _poll( + client, + lambda: original if self._read_num_retries(client) == original else None, + f"GET /router/settings never returned to the original num_retries {original} after the restore", + ) + assert restored == original, f"router num_retries left at {restored}, expected the original {original}" + + @staticmethod + def _read_num_retries(client: ManagementClient) -> int | None: + return unwrap( + client.proxy.transport.get( + "/router/settings", + headers=client.proxy.transport.master, + params=NoBody(), + response_type=RouterSettingsResponse, + ) + ).current_values.num_retries + + @staticmethod + def _write_num_retries(client: ManagementClient, value: int) -> None: + _ = unwrap( + client.proxy.transport.post( + "/config/update", + headers=client.proxy.transport.master, + json=ConfigUpdateBody(router_settings=RouterSettingsPatch(num_retries=value)), + response_type=ConfigUpdateResponse, + ) + ) + + +class TestMcpServerSubmission: + @pytest.mark.covers("mgmt.mcp_server.register.happy_path") + def test_register_submits_pending_server(self, client: ManagementClient, resources: ResourceManager) -> None: + """A non-admin, team-scoped key submits an MCP server for review; the proxy + stores it as pending_review without loading it into the runtime registry.""" + team_id = client.create_team(TeamNewBody(team_alias=f"e2e-mcp-team-{unique_marker()}")) + resources.defer(lambda: client.delete_team(team_id)) + team_key = client.proxy.generate_key(KeyGenerateBody(team_id=team_id)) + resources.defer(lambda: client.proxy.delete_key(team_key)) + + server_name = f"e2e_mcp_{unique_marker()}" + submitted = unwrap_status( + client.proxy.transport.post( + "/v1/mcp/server/register", + headers=client.proxy.transport.bearer(team_key), + json=McpRegisterBody( + server_name=server_name, + url="https://example.com/mcp", + transport="sse", + description="e2e coverage submission", + ), + response_type=McpServerResponse, + ), + 201, + ) + resources.defer(lambda: self._delete_server(client, submitted.server_id)) + assert submitted.approval_status == "pending_review", ( + f"a user submission must be pending_review, got {submitted.approval_status!r}" + ) + assert submitted.server_name == server_name and submitted.transport == "sse", ( + f"/v1/mcp/server/register echoed name={submitted.server_name!r} transport={submitted.transport!r}, " + f"configured {server_name!r}/sse" + ) + + @pytest.mark.covers("mgmt.mcp_server.approve.persists") + def test_approve_activates_submission_and_persists( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + """An admin approving a pending submission flips it to active, and the change + persists to a fresh read of the server.""" + team_id = client.create_team(TeamNewBody(team_alias=f"e2e-mcp-team-{unique_marker()}")) + resources.defer(lambda: client.delete_team(team_id)) + team_key = client.proxy.generate_key(KeyGenerateBody(team_id=team_id)) + resources.defer(lambda: client.proxy.delete_key(team_key)) + + submitted = unwrap( + client.proxy.transport.post( + "/v1/mcp/server/register", + headers=client.proxy.transport.bearer(team_key), + json=McpRegisterBody( + server_name=f"e2e_mcp_{unique_marker()}", + url="https://example.com/mcp", + transport="sse", + description="e2e coverage submission", + ), + response_type=McpServerResponse, + ) + ) + resources.defer(lambda: self._delete_server(client, submitted.server_id)) + assert submitted.approval_status == "pending_review", ( + f"a fresh submission must be pending_review before approval, got {submitted.approval_status!r}" + ) + + approved = unwrap( + client.proxy.transport.put( + f"/v1/mcp/server/{submitted.server_id}/approve", + headers=client.proxy.transport.master, + json=NoBody(), + response_type=McpServerResponse, + ) + ) + assert approved.approval_status == "active", ( + f"approve must flip the submission to active, got {approved.approval_status!r}" + ) + + fetched = unwrap( + client.proxy.transport.get( + f"/v1/mcp/server/{submitted.server_id}", + headers=client.proxy.transport.master, + params=NoBody(), + response_type=McpServerResponse, + ) + ) + assert fetched.server_id == submitted.server_id and fetched.approval_status == "active", ( + f"GET /v1/mcp/server/{submitted.server_id} reports approval_status {fetched.approval_status!r} " + "after approve, expected 'active'" + ) + + @staticmethod + def _delete_server(client: ManagementClient, server_id: str) -> None: + _ = client.proxy.transport.delete( + f"/v1/mcp/server/{server_id}", + headers=client.proxy.transport.master, + json=NoBody(), + response_type=NoBody, + ) diff --git a/tests/e2e/management/test_key_management_e2e.py b/tests/e2e/management/test_key_management_e2e.py new file mode 100644 index 00000000000..711175abb0d --- /dev/null +++ b/tests/e2e/management/test_key_management_e2e.py @@ -0,0 +1,251 @@ +"""Live e2e: the /key management routes' persistence, health, bulk-update, and +admin-only contracts. + +Each test creates its keys under the master key with unique aliases (deleted on +teardown) and asserts the real contract: the info route reflects the write +(persistence), the health route reports the calling key, bulk_update applies to +the target key, and the write routes refuse a non-admin caller. Key writes reach +the auth cache eventually, so the read-backs poll to a deadline instead of +asserting once. +""" + +from __future__ import annotations + +import time +from collections.abc import Callable +from typing import Literal + +import pytest + +from e2e_config import unique_marker +from e2e_http import NoBody, unwrap +from lifecycle import ResourceManager +from management_client import ManagementClient +from models import KeyDeleteBody, KeyGenerateBody, KeyUpdateBody +from pydantic import BaseModel + +pytestmark = pytest.mark.e2e + + +class KeyToggleBlockBody(BaseModel): + key: str + + +class LoggingCallbackStatus(BaseModel): + callbacks: list[str] | None = None + status: str | None = None + details: str | None = None + + +class KeyHealthResponse(BaseModel): + key: Literal["healthy", "unhealthy"] + logging_callbacks: LoggingCallbackStatus | None = None + + +class BulkKeyUpdateItem(BaseModel): + key: str + max_budget: float | None = None + + +class BulkKeyUpdateBody(BaseModel): + keys: list[BulkKeyUpdateItem] + + +class BulkKeyUpdateSuccess(BaseModel): + key: str + + +class BulkKeyUpdateFailure(BaseModel): + key: str + failed_reason: str + + +class BulkKeyUpdateResponse(BaseModel): + total_requested: int + successful_updates: list[BulkKeyUpdateSuccess] + failed_updates: list[BulkKeyUpdateFailure] + + +def _poll[T](client: ManagementClient, attempt: Callable[[], T | None], failure: str) -> T: + deadline = time.monotonic() + client.proxy.poll_timeout + while time.monotonic() < deadline: + found = attempt() + if found is not None: + return found + time.sleep(client.proxy.poll_interval) + pytest.fail(failure) + + +def _generate_key(client: ManagementClient, resources: ResourceManager, body: KeyGenerateBody) -> str: + key = client.proxy.generate_key(body) + resources.defer(lambda: client.proxy.delete_key(key)) + return key + + +def _block(client: ManagementClient, key: str) -> None: + _ = unwrap( + client.proxy.transport.post( + "/key/block", + headers=client.proxy.transport.master, + json=KeyToggleBlockBody(key=key), + response_type=NoBody, + ) + ) + + +def _unblock(client: ManagementClient, key: str) -> None: + _ = unwrap( + client.proxy.transport.post( + "/key/unblock", + headers=client.proxy.transport.master, + json=KeyToggleBlockBody(key=key), + response_type=NoBody, + ) + ) + + +class TestKeyManagementRoutes: + @pytest.mark.covers("mgmt.key.info.persists") + def test_info_reflects_the_fields_the_key_was_created_with( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + alias = f"e2e-mgmt-keyinfo-{unique_marker()}" + key = _generate_key( + client, + resources, + KeyGenerateBody( + models=["gpt-5.5", "gemini-2.5-flash"], + key_alias=alias, + tpm_limit=131313, + rpm_limit=141414, + ), + ) + + info = client.proxy.key_info(key) + assert info.key_alias == alias, f"/key/info reports key_alias {info.key_alias!r}, configured {alias!r}" + assert info.models == ["gpt-5.5", "gemini-2.5-flash"], ( + f"/key/info reports models {info.models}, configured ['gpt-5.5', 'gemini-2.5-flash']" + ) + assert info.tpm_limit == 131313, f"/key/info reports tpm_limit {info.tpm_limit}, configured 131313" + assert info.rpm_limit == 141414, f"/key/info reports rpm_limit {info.rpm_limit}, configured 141414" + + @pytest.mark.covers("mgmt.key.unblock.persists") + def test_unblock_flips_key_info_blocked_back( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + key = _generate_key(client, resources, KeyGenerateBody(models=["gpt-5.5"])) + + _block(client, key) + _ = _poll( + client, + lambda: True if client.proxy.key_info(key).blocked else None, + "/key/info never reported the key blocked after /key/block before the deadline", + ) + + _unblock(client, key) + _ = _poll( + client, + lambda: True if client.proxy.key_info(key).blocked is False else None, + "/key/info never reported the key unblocked after /key/unblock before the deadline", + ) + + @pytest.mark.covers("mgmt.key.health.happy_path") + def test_health_reports_the_calling_key_healthy( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + key = _generate_key(client, resources, KeyGenerateBody(models=["gpt-5.5"])) + + health = unwrap( + client.proxy.transport.post( + "/key/health", + headers=client.proxy.transport.bearer(key), + json=NoBody(), + response_type=KeyHealthResponse, + ) + ) + assert health.key == "healthy", f"/key/health reports {health.key!r} for a key with no logging configured" + assert health.logging_callbacks is None, ( + f"/key/health reports logging_callbacks {health.logging_callbacks!r} for a key with no logging configured" + ) + + @pytest.mark.covers("mgmt.key.bulk_update.happy_path") + def test_bulk_update_applies_max_budget_to_target_key( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + key = _generate_key(client, resources, KeyGenerateBody(models=["gpt-5.5"], max_budget=5.0)) + assert client.proxy.key_info(key).max_budget == 5.0, ( + f"/key/info reports max_budget {client.proxy.key_info(key).max_budget}, configured 5.0" + ) + + result = unwrap( + client.proxy.transport.post( + "/key/bulk_update", + headers=client.proxy.transport.master, + json=BulkKeyUpdateBody(keys=[BulkKeyUpdateItem(key=key, max_budget=42.0)]), + response_type=BulkKeyUpdateResponse, + ) + ) + assert result.total_requested == 1, f"/key/bulk_update reports total_requested {result.total_requested}, sent 1" + assert result.failed_updates == [], f"/key/bulk_update reported failed updates: {result.failed_updates}" + assert [entry.key for entry in result.successful_updates] == [key], ( + f"/key/bulk_update successful_updates {[entry.key for entry in result.successful_updates]} did not target {key}" + ) + + _ = _poll( + client, + lambda: True if client.proxy.key_info(key).max_budget == 42.0 else None, + "/key/info never reported max_budget 42.0 after /key/bulk_update before the deadline", + ) + + @pytest.mark.covers("mgmt.key.generate.admin_only") + def test_generate_forbidden_for_non_admin_key( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + nonadmin = _generate_key(client, resources, KeyGenerateBody(models=["gpt-5.5"])) + + outcome = client.proxy.transport.send( + "/key/generate", + headers=client.proxy.transport.bearer(nonadmin), + json=KeyGenerateBody(models=["gpt-5.5"], key_alias=f"e2e-mgmt-forbidden-{unique_marker()}"), + ) + assert outcome.status_code in (401, 403), ( + f"non-admin key POSTing /key/generate must be denied 401/403, got {outcome.status_code}: {outcome.body[:300]}" + ) + + @pytest.mark.covers("mgmt.key.delete.admin_only") + def test_delete_forbidden_for_non_admin_key( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + nonadmin = _generate_key(client, resources, KeyGenerateBody(models=["gpt-5.5"])) + victim = _generate_key(client, resources, KeyGenerateBody(models=["gpt-5.5"])) + + outcome = client.proxy.transport.send( + "/key/delete", + headers=client.proxy.transport.bearer(nonadmin), + json=KeyDeleteBody(keys=[victim]), + ) + assert outcome.status_code in (401, 403), ( + f"non-admin key POSTing /key/delete must be denied 401/403, got {outcome.status_code}: {outcome.body[:300]}" + ) + assert client.proxy.key_info(victim).blocked in (None, False), ( + "victim key should be unaffected by the denied /key/delete" + ) + + @pytest.mark.covers("mgmt.key.update.admin_only") + def test_update_forbidden_for_non_admin_key( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + nonadmin = _generate_key(client, resources, KeyGenerateBody(models=["gpt-5.5"])) + target = _generate_key(client, resources, KeyGenerateBody(models=["gpt-5.5"])) + + outcome = client.proxy.transport.send( + "/key/update", + headers=client.proxy.transport.bearer(nonadmin), + json=KeyUpdateBody(key=target, models=["gemini-2.5-flash"]), + ) + assert outcome.status_code in (401, 403), ( + f"non-admin key POSTing /key/update must be denied 401/403, got {outcome.status_code}: {outcome.body[:300]}" + ) + assert client.proxy.key_info(target).models == ["gpt-5.5"], ( + f"target key models changed to {client.proxy.key_info(target).models} despite the denied /key/update" + ) diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index 18bc384a879..9b398963ac9 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -610,3 +610,18 @@ class TestManagementRoutePermissions: f"/team/info returned {team_probe.status_code}: {team_probe.body[:300]}" ) assert client.user_count(user_id) == 0, f"user {user_id} was created despite the 403 route denial" + + +class TestCustomer: + @pytest.mark.covers("mgmt.end_user.new.happy_path") + def test_customer_create_persists_to_info( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + customer = f"e2e-customer-{unique_marker()}" + client.create_customer(customer) + resources.defer(lambda: client.delete_customer(customer)) + + info = client.customer_info(customer) + assert info.user_id == customer, ( + f"/customer/info did not report the created end-user; got {info.user_id!r}" + ) diff --git a/tests/e2e/management/test_model_tag_accessgroup_e2e.py b/tests/e2e/management/test_model_tag_accessgroup_e2e.py new file mode 100644 index 00000000000..e6a187ae105 --- /dev/null +++ b/tests/e2e/management/test_model_tag_accessgroup_e2e.py @@ -0,0 +1,385 @@ +"""Live e2e: the model, tag, and model-access-group management routes. + +Each test creates its resources under unique names (deleted on teardown) and +asserts the route's contract against a live proxy: the admin-only guard on +adding a global model, the tag inventory round-trip through /tag/list and +/tag/delete, and creating a model access group then reading it back through +/access_group/{name}/info. Reads that lag a write poll to a deadline instead of +asserting once. + +Request bodies for /model/new are the shared pydantic models; every response +this suite reads is modelled locally so the file is self-contained and no +untyped dict crosses the boundary. +""" + +from __future__ import annotations + +import time +from collections.abc import Callable + +import pytest +from pydantic import BaseModel, ConfigDict, RootModel + +from e2e_config import unique_marker +from e2e_http import NoBody, unwrap +from lifecycle import ResourceManager +from management_client import ManagementClient +from models import KeyGenerateBody, LiteLLMParamsBody, ModelInfoBody, ModelNewBody +from proxy_client import ProxyClient + +pytestmark = pytest.mark.e2e + +_MODEL_PERMISSION_DENIED_MARKER = "does not have permission to make this model call" +_DUMMY_MODEL = "openai/gpt-5.5" +_DUMMY_API_KEY = "e2e-dummy-key" + + +def _poll[T](proxy: ProxyClient, attempt: Callable[[], T | None], failure: str) -> T: + deadline = time.monotonic() + proxy.poll_timeout + while time.monotonic() < deadline: + found = attempt() + if found is not None: + return found + time.sleep(proxy.poll_interval) + pytest.fail(failure) + + +# ---------- tag route models / helpers ---------- + + +class TagCreateBody(BaseModel): + name: str + description: str | None = None + + +class TagDeleteBody(BaseModel): + name: str + + +class TagEntry(BaseModel): + name: str + description: str | None = None + + +class TagCatalog(RootModel[list[TagEntry]]): + """GET /tag/list answers with a bare array of tag configs, not an object + wrapping them; read the rows off .root.""" + + +def _tag_list(client: ManagementClient) -> tuple[TagEntry, ...]: + return tuple( + unwrap( + client.proxy.transport.get( + "/tag/list", + headers=client.proxy.transport.master, + params=NoBody(), + response_type=TagCatalog, + ) + ).root + ) + + +def _create_tag(client: ManagementClient, body: TagCreateBody) -> None: + _ = unwrap( + client.proxy.transport.post( + "/tag/new", + headers=client.proxy.transport.master, + json=body, + response_type=NoBody, + ) + ) + + +def _delete_tag(client: ManagementClient, name: str) -> None: + """Best-effort delete for teardown: a repeat /tag/delete on an already-deleted + tag is a no-op the warn-only teardown absorbs.""" + _ = client.proxy.transport.post( + "/tag/delete", + headers=client.proxy.transport.master, + json=TagDeleteBody(name=name), + response_type=NoBody, + ) + + +def _delete_tag_strict(client: ManagementClient, name: str) -> None: + """Strict delete for the act phase: a failed /tag/delete is a hard failure.""" + _ = unwrap( + client.proxy.transport.post( + "/tag/delete", + headers=client.proxy.transport.master, + json=TagDeleteBody(name=name), + response_type=NoBody, + ) + ) + + +# ---------- access group route models / helpers ---------- + + +class AccessGroupNewBody(BaseModel): + access_group: str + model_names: list[str] + + +class AccessGroupNewResponse(BaseModel): + access_group: str + models_updated: int + + +class AccessGroupInfoResponse(BaseModel): + access_group: str + model_names: list[str] + deployment_count: int + + +def _create_access_group(client: ManagementClient, body: AccessGroupNewBody) -> AccessGroupNewResponse: + return unwrap( + client.proxy.transport.post( + "/access_group/new", + headers=client.proxy.transport.master, + json=body, + response_type=AccessGroupNewResponse, + ) + ) + + +def _access_group_info(client: ManagementClient, access_group: str) -> AccessGroupInfoResponse | None: + result = client.proxy.transport.get( + f"/access_group/{access_group}/info", + headers=client.proxy.transport.master, + params=NoBody(), + response_type=AccessGroupInfoResponse, + ) + return unwrap(result) if result.kind == "success" else None + + +def _delete_access_group(client: ManagementClient, access_group: str) -> None: + """Best-effort delete for teardown; deleting the model behind it removes the + access group too, so a repeat delete is a no-op the teardown absorbs.""" + _ = client.proxy.transport.delete( + f"/access_group/{access_group}/delete", + headers=client.proxy.transport.master, + json=NoBody(), + response_type=NoBody, + ) + + +def _create_db_model(client: ManagementClient, resources: ResourceManager, model_name: str) -> str: + model_id = client.proxy.create_model( + model_name, LiteLLMParamsBody(model=_DUMMY_MODEL, api_key=_DUMMY_API_KEY) + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + return model_id + + +# ---------- model block route models / helpers ---------- + + +class ModelBlockBody(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + model_id: str + + +class ModelInfoBlockDetail(BaseModel): + id: str | None = None + blocked: bool | None = None + + +class ModelInfoBlockEntry(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + model_name: str + model_info: ModelInfoBlockDetail = ModelInfoBlockDetail() + + +class ModelInfoCatalog(BaseModel): + data: list[ModelInfoBlockEntry] = [] + + +def _model_blocked_flag(client: ManagementClient, model_id: str) -> bool | None: + catalog = unwrap( + client.proxy.transport.get( + "/model/info", + headers=client.proxy.transport.master, + params=NoBody(), + response_type=ModelInfoCatalog, + ) + ) + entry = next((row for row in catalog.data if row.model_info.id == model_id), None) + return entry.model_info.blocked if entry is not None else None + + +class TestModelRoutes: + @pytest.mark.covers("mgmt.model.add.admin_only") + def test_non_admin_key_cannot_add_global_model( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + key = client.proxy.generate_key(KeyGenerateBody(models=[])) + resources.defer(lambda: client.proxy.delete_key(key)) + + model_name = f"e2e-mgmt-model-forbidden-{unique_marker()}" + outcome = client.proxy.transport.send( + "/model/new", + headers=client.proxy.transport.bearer(key), + json=ModelNewBody( + model_name=model_name, + litellm_params=LiteLLMParamsBody(model=_DUMMY_MODEL, api_key=_DUMMY_API_KEY), + model_info=ModelInfoBody(), + ), + ) + + assert outcome.status_code == 403, ( + f"non-admin key adding a global model (no team_id) must be denied 403, got " + f"{outcome.status_code}: {outcome.body[:300]}" + ) + assert _MODEL_PERMISSION_DENIED_MARKER in outcome.body, ( + f"403 body must be the model-permission denial, got: {outcome.body[:300]}" + ) + + cataloged = [entry.model_name for entry in client.proxy.model_info()] + assert model_name not in cataloged, ( + f"{model_name!r} was registered in /model/info despite the 403; the admin-only " + f"guard did not block the write" + ) + + @pytest.mark.covers("mgmt.model.block.persists") + def test_block_then_unblock_persists_to_model_info( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + """The blocked flag's persistence is read back from /model/info, not from the + /model/block response: that route currently returns a non-2xx serialization + envelope even though the DB write lands, so the /model/info read-back is the + authoritative persistence contract and keeps this test valid once the + response shape is fixed.""" + model_name = f"e2e-mgmt-model-block-{unique_marker()}" + model_id = _create_db_model(client, resources, model_name) + + assert _model_blocked_flag(client, model_id) is not True, ( + f"{model_name!r} already reports blocked in /model/info before /model/block ran" + ) + + _ = client.proxy.transport.send( + "/model/block", + headers=client.proxy.transport.master, + json=ModelBlockBody(model_id=model_id), + ) + _ = _poll( + client.proxy, + lambda: True if _model_blocked_flag(client, model_id) is True else None, + f"/model/info never reported {model_name!r} blocked after /model/block", + ) + + _ = client.proxy.transport.send( + "/model/unblock", + headers=client.proxy.transport.master, + json=ModelBlockBody(model_id=model_id), + ) + _ = _poll( + client.proxy, + lambda: True if _model_blocked_flag(client, model_id) is not True else None, + f"/model/info never cleared blocked for {model_name!r} after /model/unblock", + ) + + +class TestTagRoutes: + @pytest.mark.covers("mgmt.tag.list.happy_path") + def test_tag_list_reports_created_tag(self, client: ManagementClient, resources: ResourceManager) -> None: + name = f"e2e-mgmt-tag-{unique_marker()}" + description = "coverage: tag inventory" + assert all(entry.name != name for entry in _tag_list(client)), ( + f"tag {name!r} was already listed by /tag/list before /tag/new created it" + ) + + _create_tag(client, TagCreateBody(name=name, description=description)) + resources.defer(lambda: _delete_tag(client, name)) + + entry = _poll( + client.proxy, + lambda: next((entry for entry in _tag_list(client) if entry.name == name), None), + f"/tag/list never listed {name!r} after /tag/new", + ) + assert entry.description == description, ( + f"/tag/list reports description {entry.description!r} for {name!r}, configured {description!r}" + ) + + @pytest.mark.covers("mgmt.tag.delete.persists") + def test_tag_delete_removes_from_list(self, client: ManagementClient, resources: ResourceManager) -> None: + """The teardown's deferred delete fires again on the already-deleted tag by + design: it is the safety net if this test fails before the in-body delete, + and a repeat /tag/delete is a warn-only no-op the teardown absorbs.""" + name = f"e2e-mgmt-tag-{unique_marker()}" + _create_tag(client, TagCreateBody(name=name)) + resources.defer(lambda: _delete_tag(client, name)) + + _ = _poll( + client.proxy, + lambda: True if any(entry.name == name for entry in _tag_list(client)) else None, + f"/tag/list never listed {name!r} after /tag/new; cannot prove deletion removes it", + ) + + _delete_tag_strict(client, name) + + _ = _poll( + client.proxy, + lambda: True if all(entry.name != name for entry in _tag_list(client)) else None, + f"{name!r} still present in /tag/list after /tag/delete at the deadline", + ) + + +class TestModelAccessGroupRoutes: + @pytest.mark.covers("mgmt.access_group.new.happy_path") + def test_new_access_group_tags_the_deployment( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + model_name = f"e2e-mgmt-agmodel-{unique_marker()}" + _ = _create_db_model(client, resources, model_name) + + access_group = f"e2e-mgmt-ag-{unique_marker()}" + created = _create_access_group( + client, AccessGroupNewBody(access_group=access_group, model_names=[model_name]) + ) + resources.defer(lambda: _delete_access_group(client, access_group)) + + assert created.access_group == access_group, ( + f"/access_group/new echoed access_group {created.access_group!r}, requested {access_group!r}" + ) + assert created.models_updated >= 1, ( + f"/access_group/new tagged {created.models_updated} deployments for {model_name!r}, expected >= 1" + ) + + info = _poll( + client.proxy, + lambda: _access_group_info(client, access_group), + f"/access_group/{access_group}/info never resolved the group created by /access_group/new", + ) + assert model_name in info.model_names, ( + f"the group created by /access_group/new does not list {model_name!r} on read-back; " + f"/access_group/info reports members {info.model_names}" + ) + + @pytest.mark.covers("mgmt.access_group.info.happy_path") + def test_access_group_info_reports_membership( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + model_name = f"e2e-mgmt-agmodel-{unique_marker()}" + _ = _create_db_model(client, resources, model_name) + + access_group = f"e2e-mgmt-ag-{unique_marker()}" + _ = _create_access_group( + client, AccessGroupNewBody(access_group=access_group, model_names=[model_name]) + ) + resources.defer(lambda: _delete_access_group(client, access_group)) + + info = _poll( + client.proxy, + lambda: _access_group_info(client, access_group), + f"/access_group/{access_group}/info never resolved the created access group", + ) + assert info.access_group == access_group, ( + f"/access_group/info reports access_group {info.access_group!r}, created {access_group!r}" + ) + assert model_name in info.model_names, ( + f"/access_group/info reports members {info.model_names}, expected to include {model_name!r}" + ) + assert info.deployment_count >= 1, ( + f"/access_group/info reports deployment_count {info.deployment_count}, expected >= 1" + ) diff --git a/tests/e2e/management/test_team_management_e2e.py b/tests/e2e/management/test_team_management_e2e.py new file mode 100644 index 00000000000..108aeaad21b --- /dev/null +++ b/tests/e2e/management/test_team_management_e2e.py @@ -0,0 +1,303 @@ +"""Live e2e: the /team/* management routes' block, membership, and admin-only +contract. + +Each test creates its team/user/key resources under unique names (deleted on +teardown) and asserts both halves of the contract: the recorded state (the info +route reflects the write) and the enforced behavior (a non-admin key is refused). +Team writes reach the read path once their db/cache entry propagates, so the +read-backs poll to a deadline instead of asserting once. + +Everything the shared harness does not already model lives here: the local +request/response models for /team/block, /team/member_update, and the +/team/info fields (blocked flag and per-member budget) these tests assert on. +""" + +from __future__ import annotations + +import time +from collections.abc import Callable +from typing import Literal + +import pytest +from pydantic import BaseModel + +from e2e_config import unique_marker +from e2e_http import NoBody, StreamingResponse, unwrap +from lifecycle import ResourceManager +from management_client import ManagementClient +from models import ( + KeyGenerateBody, + TeamInfoParams, + TeamMemberAddBody, + TeamMemberDeleteBody, + TeamMemberEntry, + TeamNewBody, + UserNewBody, +) + +pytestmark = pytest.mark.e2e + +TeamRole = Literal["admin", "user"] + + +class TeamBlockBody(BaseModel): + team_id: str + + +class MemberUpdateBody(BaseModel): + team_id: str + user_id: str + role: TeamRole | None = None + max_budget_in_team: float | None = None + + +class MemberRoleEntry(BaseModel): + user_id: str | None = None + user_email: str | None = None + role: TeamRole + + +class MemberBudgetTable(BaseModel): + max_budget: float | None = None + + +class TeamMembership(BaseModel): + user_id: str + litellm_budget_table: MemberBudgetTable | None = None + + +class TeamInfoData(BaseModel): + team_alias: str | None = None + models: list[str] = [] + blocked: bool | None = None + members_with_roles: list[MemberRoleEntry] = [] + + +class TeamInfoRead(BaseModel): + team_id: str + team_info: TeamInfoData + team_memberships: list[TeamMembership] = [] + + +def _poll[T](client: ManagementClient, attempt: Callable[[], T | None], failure: str) -> T: + deadline = time.monotonic() + client.proxy.poll_timeout + while time.monotonic() < deadline: + found = attempt() + if found is not None: + return found + time.sleep(client.proxy.poll_interval) + pytest.fail(failure) + + +def _create_team(client: ManagementClient, resources: ResourceManager, alias: str, models: list[str]) -> str: + team_id = client.create_team(TeamNewBody(team_alias=alias, models=models)) + resources.defer(lambda: client.delete_team(team_id)) + return team_id + + +def _create_user(client: ManagementClient, resources: ResourceManager, email: str) -> str: + user_id = client.create_user(UserNewBody(user_email=email, user_role="internal_user")) + resources.defer(lambda: client.delete_user(user_id)) + return user_id + + +def _generate_key(client: ManagementClient, resources: ResourceManager, body: KeyGenerateBody) -> str: + key = client.proxy.generate_key(body) + resources.defer(lambda: client.proxy.delete_key(key)) + return key + + +def _read_team(client: ManagementClient, team_id: str) -> TeamInfoRead: + return unwrap( + client.proxy.transport.get( + "/team/info", + headers=client.proxy.transport.master, + params=TeamInfoParams(team_id=team_id), + response_type=TeamInfoRead, + ) + ) + + +def _set_blocked(client: ManagementClient, team_id: str, *, blocked: bool) -> None: + _ = unwrap( + client.proxy.transport.post( + "/team/unblock" if not blocked else "/team/block", + headers=client.proxy.transport.master, + json=TeamBlockBody(team_id=team_id), + response_type=NoBody, + ) + ) + + +def _member_update(client: ManagementClient, body: MemberUpdateBody) -> None: + _ = unwrap( + client.proxy.transport.post( + "/team/member_update", + headers=client.proxy.transport.master, + json=body, + response_type=NoBody, + ) + ) + + +def _member_role(info: TeamInfoRead, user_id: str) -> TeamRole | None: + return next((m.role for m in info.team_info.members_with_roles if m.user_id == user_id), None) + + +def _member_max_budget(info: TeamInfoRead, user_id: str) -> float | None: + membership = next((tm for tm in info.team_memberships if tm.user_id == user_id), None) + if membership is None or membership.litellm_budget_table is None: + return None + return membership.litellm_budget_table.max_budget + + +def _member_add_status(client: ManagementClient, key: str, team_id: str, user_id: str) -> StreamingResponse: + return client.proxy.transport.send( + "/team/member_add", + headers=client.proxy.transport.bearer(key), + json=TeamMemberAddBody(team_id=team_id, member=TeamMemberEntry(role="user", user_id=user_id)), + ) + + +def _member_delete_status(client: ManagementClient, key: str, team_id: str, user_id: str) -> StreamingResponse: + return client.proxy.transport.send( + "/team/member_delete", + headers=client.proxy.transport.bearer(key), + json=TeamMemberDeleteBody(team_id=team_id, user_id=user_id), + ) + + +class TestTeamManagementRoutes: + @pytest.mark.covers("mgmt.team.info.happy_path") + def test_info_returns_created_team_fields( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + alias = f"e2e-team-info-{unique_marker()}" + team_id = _create_team(client, resources, alias, ["gemini-2.5-flash"]) + + info = _read_team(client, team_id) + assert info.team_id == team_id, f"/team/info echoed team_id {info.team_id!r}, requested {team_id!r}" + assert info.team_info.team_alias == alias, ( + f"/team/info reports team_alias {info.team_info.team_alias!r}, configured {alias!r}" + ) + assert info.team_info.models == ["gemini-2.5-flash"], ( + f"/team/info reports models {info.team_info.models}, configured ['gemini-2.5-flash']" + ) + + @pytest.mark.covers("mgmt.team.block.persists") + def test_block_then_unblock_persists_to_team_info( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + team_id = _create_team(client, resources, f"e2e-team-block-{unique_marker()}", ["gemini-2.5-flash"]) + assert not _read_team(client, team_id).team_info.blocked, "/team/info reports the team blocked before /team/block" + + _set_blocked(client, team_id, blocked=True) + _ = _poll( + client, + lambda: True if _read_team(client, team_id).team_info.blocked else None, + "/team/info never reflected blocked=True after /team/block", + ) + + _set_blocked(client, team_id, blocked=False) + _ = _poll( + client, + lambda: True if _read_team(client, team_id).team_info.blocked is False else None, + "/team/info never reflected blocked=False after /team/unblock", + ) + + @pytest.mark.covers("mgmt.team.member_update.persists") + def test_member_update_persists_role_and_budget( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + user_id = _create_user(client, resources, f"e2e-team-mu-{unique_marker()}@example.com") + team_id = _create_team(client, resources, f"e2e-team-mu-{unique_marker()}", ["gemini-2.5-flash"]) + client.add_team_member(team_id, user_id) + assert _member_role(_read_team(client, team_id), user_id) == "user", ( + f"member {user_id} should start as role 'user' after /team/member_add" + ) + + budget = 4242.0 + _member_update(client, MemberUpdateBody(team_id=team_id, user_id=user_id, role="admin", max_budget_in_team=budget)) + + def updated() -> bool | None: + info = _read_team(client, team_id) + return True if _member_role(info, user_id) == "admin" and _member_max_budget(info, user_id) == budget else None + + _ = _poll( + client, + updated, + f"/team/info never reflected role=admin and max_budget={budget} for {user_id} after /team/member_update", + ) + + @pytest.mark.covers("mgmt.team.member_delete.persists") + def test_member_delete_persists_to_team_info( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + user_id = _create_user(client, resources, f"e2e-team-md-{unique_marker()}@example.com") + team_id = _create_team(client, resources, f"e2e-team-md-{unique_marker()}", ["gemini-2.5-flash"]) + client.add_team_member(team_id, user_id) + assert _member_role(_read_team(client, team_id), user_id) == "user", ( + f"/team/info does not list {user_id} as a member after /team/member_add" + ) + + client.delete_team_member(team_id, user_id) + _ = _poll( + client, + lambda: True if _member_role(_read_team(client, team_id), user_id) is None else None, + f"/team/info still lists {user_id} after /team/member_delete", + ) + + @pytest.mark.covers("mgmt.team.new.admin_only") + def test_new_is_denied_to_non_admin_keys( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + no_role_key = _generate_key(client, resources, KeyGenerateBody(models=[])) + internal_user_id = _create_user(client, resources, f"e2e-team-adm-{unique_marker()}@example.com") + internal_user_key = _generate_key(client, resources, KeyGenerateBody(user_id=internal_user_id)) + + for key, label in ((no_role_key, "role=None"), (internal_user_key, "internal_user")): + outcome = client.team_new_status(key, TeamNewBody(team_alias=f"e2e-team-adm-{unique_marker()}")) + assert outcome.status_code in (401, 403), ( + f"/team/new by a {label} key must be denied 401/403, got {outcome.status_code}: {outcome.body[:300]}" + ) + + @pytest.mark.covers("mgmt.team.member_add.member_forbidden") + def test_member_add_forbidden_to_plain_member( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + _member_id, other_id, member_key, team_id = self._team_with_member_key(client, resources) + + outcome = _member_add_status(client, member_key, team_id, other_id) + assert outcome.status_code == 403, ( + f"/team/member_add by a plain team member must be 403, got {outcome.status_code}: {outcome.body[:300]}" + ) + assert "not allowed" in outcome.body.lower(), ( + f"403 body should say the call is not allowed, got: {outcome.body[:300]}" + ) + + @pytest.mark.covers("mgmt.team.member_delete.member_forbidden") + def test_member_delete_forbidden_to_plain_member( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + member_id, _other_id, member_key, team_id = self._team_with_member_key(client, resources) + + outcome = _member_delete_status(client, member_key, team_id, member_id) + assert outcome.status_code == 403, ( + f"/team/member_delete by a plain team member must be 403, got {outcome.status_code}: {outcome.body[:300]}" + ) + assert "not allowed" in outcome.body.lower(), ( + f"403 body should say the call is not allowed, got: {outcome.body[:300]}" + ) + + @staticmethod + def _team_with_member_key( + client: ManagementClient, resources: ResourceManager + ) -> tuple[str, str, str, str]: + """A team with a plain member (role user) whose key is scoped to that + user + team, plus a second user id the member could try to add.""" + member_id = _create_user(client, resources, f"e2e-team-fb-{unique_marker()}@example.com") + other_id = _create_user(client, resources, f"e2e-team-fb-{unique_marker()}@example.com") + team_id = _create_team(client, resources, f"e2e-team-fb-{unique_marker()}", ["gemini-2.5-flash"]) + client.add_team_member(team_id, member_id) + member_key = _generate_key(client, resources, KeyGenerateBody(user_id=member_id, team_id=team_id)) + return member_id, other_id, member_key, team_id diff --git a/tests/e2e/mcp/mcp_client.py b/tests/e2e/mcp/mcp_client.py index f68fdf63b3f..b0aa4c68e3a 100644 --- a/tests/e2e/mcp/mcp_client.py +++ b/tests/e2e/mcp/mcp_client.py @@ -85,6 +85,37 @@ class McpToolsListResponse(BaseModel): return None +class BlockedWordSpec(BaseModel): + keyword: str + action: str = "BLOCK" + + +class ContentFilterMcpParams(BaseModel): + """litellm_content_filter params scoped to the MCP tool-call hook. mode is + pre_mcp_call because a pre_call config silently no-ops on the tools/call path + (the event type is rewritten to pre_mcp_call for call_mcp_tool), and default_on + is required there because per-key/request guardrail selection is dropped from + the synthetic MCP request the hook sees.""" + + guardrail: str = "litellm_content_filter" + mode: str = "pre_mcp_call" + default_on: bool = True + blocked_words: list[BlockedWordSpec] + + +class GuardrailSpecBody(BaseModel): + guardrail_name: str + litellm_params: ContentFilterMcpParams + + +class GuardrailCreateBody(BaseModel): + guardrail: GuardrailSpecBody + + +class GuardrailCreateResponse(BaseModel): + guardrail_id: str + + class McpCallToolBody(BaseModel): name: str arguments: dict[str, McpToolArg] @@ -186,6 +217,35 @@ class McpClient: response_type=McpToolsListResponse, ) + def register_mcp_content_filter(self, *, name: str, blocked_keyword: str) -> str: + """Register a default-on content-filter guardrail that runs on the MCP + tool-call hook (pre_mcp_call) and blocks a single keyword. The keyword is + unique per test, so default_on only ever intercepts this test's own + banned tool call on the shared proxy.""" + return unwrap( + self.proxy.transport.post( + "/guardrails", + headers=self.proxy.transport.master, + json=GuardrailCreateBody( + guardrail=GuardrailSpecBody( + guardrail_name=name, + litellm_params=ContentFilterMcpParams( + blocked_words=[BlockedWordSpec(keyword=blocked_keyword)], + ), + ) + ), + response_type=GuardrailCreateResponse, + ) + ).guardrail_id + + def delete_guardrail(self, guardrail_id: str) -> None: + _ = self.proxy.transport.delete( + f"/guardrails/{guardrail_id}", + headers=self.proxy.transport.master, + json=NoBody(), + response_type=NoBody, + ) + def call_tool( self, key: str, diff --git a/tests/e2e/mcp/test_mcp_guardrail_e2e.py b/tests/e2e/mcp/test_mcp_guardrail_e2e.py new file mode 100644 index 00000000000..63239444454 --- /dev/null +++ b/tests/e2e/mcp/test_mcp_guardrail_e2e.py @@ -0,0 +1,146 @@ +"""Live e2e: a guardrail on the MCP tool-call path blocks banned content in the +tool arguments before the call reaches the upstream MCP server. + +A general litellm_content_filter guardrail is configured with mode=pre_mcp_call +(the event type the proxy rewrites pre_call to for a call_mcp_tool) and default_on +(per-key/request guardrail selection is dropped from the synthetic MCP request the +hook sees, so default_on is how it attaches to tools/call). The banned keyword is +unique per run, so default_on only ever intercepts this test's own banned call. + +Against the real Datadog MCP server, calling search_datadog_logs with the banned +keyword in the query is blocked with HTTP 400 attributed to the pre_mcp_call hook, +and the tool never runs; the same guardrail lets a clean query through to Datadog. +This is the enforced half (the block) plus the pass-through half in one spec. +""" + +from __future__ import annotations + +import time +from collections.abc import Callable + +import pytest + +from datadog_mcp import SEARCH_LOGS_TOOL, assert_dd_mcp_creds, register_datadog_mcp +from e2e_config import DD_SEARCH_FROM, unique_marker +from e2e_http import Result, Success, UnknownApiError, unwrap +from lifecycle import ResourceManager +from mcp_client import McpCallToolResponse, McpClient, McpToolArguments + +pytestmark = pytest.mark.e2e + +# Stage runs several data-plane pods behind the shared key, and each picks up a +# newly registered guardrail only on its next periodic DB sync (~30s in +# proxy_server.py). Every pod is guaranteed to have refreshed only once a full sync +# interval has elapsed since the create; before then a banned call routed to a +# lagging pod passes through as legitimate in-flight propagation, not a leak. +GUARDRAIL_FULL_SYNC_SECONDS = 40.0 +POST_SYNC_VERIFICATION_CALLS = 4 + + +def _poll_until_blocked( + search: Callable[[str], Result[McpCallToolResponse]], banned_keyword: str, client: McpClient +) -> Result[McpCallToolResponse]: + """Retry a banned tool call until the guardrail blocks it (400) or the deadline + passes, returning the last result. Absorbs the control-plane -> data-plane + guardrail-sync delay so the check waits for enforcement instead of racing it.""" + deadline = time.monotonic() + client.proxy.poll_timeout + last: Result[McpCallToolResponse] = search(f"tell me about {banned_keyword}") + while time.monotonic() < deadline: + if isinstance(last, UnknownApiError) and last.status_code == 400: + return last + time.sleep(client.proxy.poll_interval) + last = search(f"tell me about {banned_keyword}") + return last + + +class TestMcpToolCallGuardrail: + @pytest.mark.covers( + "guardrail.litellm_content_filter.pre_mcp_call.blocks", + exercised_on=["mcp_operations"], + ) + def test_content_filter_blocks_banned_keyword_in_tool_args( + self, client: McpClient, resources: ResourceManager + ) -> None: + assert_dd_mcp_creds() + marker = unique_marker() + banned_keyword = f"e2eblocked{marker}" + + guardrail_id = client.register_mcp_content_filter( + name=f"e2e-mcp-cf-{marker}", blocked_keyword=banned_keyword + ) + guardrail_created_at = time.monotonic() + resources.defer(lambda: client.delete_guardrail(guardrail_id)) + + server_id = register_datadog_mcp(client, resources) + key = client.generate_key(user_id=f"e2e-mcp-guard-{marker}", mcp_servers=[server_id]) + resources.defer(lambda: client.proxy.delete_key(key)) + + tools = unwrap(client.list_tools(key)) + tool_name = tools.tool_name_containing(server_id, SEARCH_LOGS_TOOL) + assert tool_name is not None, ( + f"granted key never saw {SEARCH_LOGS_TOOL} on server {server_id}; " + f"tools={tools.tool_names_for_server(server_id)}" + ) + + def search(query: str) -> Result[McpCallToolResponse]: + arguments: McpToolArguments = { + "query": query, + "from": DD_SEARCH_FROM, + "to": "now", + "max_tokens": 500, + "telemetry": {"intent": "e2e mcp guardrail check"}, + } + return client.call_tool(key, server_id=server_id, name=tool_name, arguments=arguments) + + # Registering the guardrail is a control-plane write; the data-plane worker + # that serves tools/call picks it up on its next guardrail sync, so an + # immediate call can race the propagation and slip through. Poll the banned + # call to the deadline and require a block, so the check proves enforcement + # rather than catching a pre-sync pass-through. The keyword is unique per + # run, so this only ever intercepts this test's own call. + blocked = _poll_until_blocked(search, banned_keyword, client) + match blocked: + case UnknownApiError(status_code=400, body=body): + assert banned_keyword in body or "content blocked" in body.lower(), ( + f"the block must name the content-filter reason, got: {body[:300]}" + ) + assert "pre_mcp_call" in body, ( + f"the block must be attributed to the MCP tool-call hook (pre_mcp_call), got: {body[:300]}" + ) + case _: + pytest.fail( + "content_filter never blocked the banned keyword on the MCP tool call within " + f"{client.proxy.poll_timeout}s (guardrail sync to the data plane never landed); " + f"last result: {blocked}" + ) + + # The block above only proves the one pod that served it has synced; another + # pod could still lack the guardrail and let the banned call reach Datadog. + # Wait out the full sync interval from the create so every pod has refreshed + # from the DB, then require the banned call to stay blocked across several + # attempts. A pass-through now is a genuine partial-propagation leak, not a + # race. Client load balancing still can't guarantee every pod is hit, so this + # samples several worker selections rather than proving all pods synced. + sync_remaining = guardrail_created_at + GUARDRAIL_FULL_SYNC_SECONDS - time.monotonic() + if sync_remaining > 0: + time.sleep(sync_remaining) + for attempt in range(1, POST_SYNC_VERIFICATION_CALLS + 1): + reblocked = search(f"still about {banned_keyword} #{attempt}") + assert isinstance(reblocked, UnknownApiError) and reblocked.status_code == 400, ( + "after the guardrail sync interval every data-plane pod must block the banned " + f"keyword, but attempt {attempt} of {POST_SYNC_VERIFICATION_CALLS} was allowed " + f"through (a pod still lacks the guardrail): {reblocked}" + ) + if attempt < POST_SYNC_VERIFICATION_CALLS: + time.sleep(client.proxy.poll_interval) + + allowed = search(f"e2e-clean-{marker}") + match allowed: + case Success(data=result): + assert result.is_error is not True, ( + f"a clean MCP tool call must reach the server and not error, got: {result}" + ) + case _: + pytest.fail( + f"a clean MCP tool call must pass the guardrail and reach the server; got {allowed}" + ) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 28cc7984598..b3ea9346180 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -115,6 +115,18 @@ class KeyInfoResponse(BaseModel): # ---------- customers ---------- +class CustomerNewBody(BaseModel): + user_id: str + + +class CustomerResponse(BaseModel): + user_id: str | None = None + + +class CustomerInfoParams(BaseModel): + end_user_id: str + + class CustomerDeleteBody(BaseModel): user_ids: list[str] @@ -126,9 +138,26 @@ class ChatMetadata(BaseModel): tags: list[str] | None = None +class ImageUrl(BaseModel): + url: str + + +class TextContentPart(BaseModel): + type: str = "text" + text: str + + +class ImageContentPart(BaseModel): + type: str = "image_url" + image_url: ImageUrl + + +ContentPart = TextContentPart | ImageContentPart + + class ChatMessage(BaseModel): role: str - content: str + content: str | list[ContentPart] class CacheControl(BaseModel): @@ -180,6 +209,7 @@ class ChatBody(BaseModel): tools: list[ChatTool] | None = None tool_choice: str | None = None guardrails: list[str] | None = None + response_format: dict[str, object] | None = None class RouterSettingsOverride(BaseModel): @@ -203,9 +233,19 @@ class ReliabilityChatBody(ChatBody): router_settings_override: RouterSettingsOverride | None = None +class ToolCallFunction(BaseModel): + name: str | None = None + arguments: str | None = None + + +class ToolCall(BaseModel): + function: ToolCallFunction = ToolCallFunction() + + class OutMessage(BaseModel): content: str | None = None reasoning_content: str | None = None + tool_calls: list[ToolCall] | None = None class ChatChoice(BaseModel): @@ -216,6 +256,10 @@ class PromptTokensDetails(BaseModel): cached_tokens: int | None = None +class CompletionTokensDetails(BaseModel): + reasoning_tokens: int | None = None + + class Usage(BaseModel): prompt_tokens: int | None = None completion_tokens: int | None = None @@ -223,6 +267,7 @@ class Usage(BaseModel): cache_read_input_tokens: int | None = None cache_creation_input_tokens: int | None = None prompt_tokens_details: PromptTokensDetails | None = None + completion_tokens_details: CompletionTokensDetails | None = None class ChatResponse(BaseModel): @@ -286,6 +331,7 @@ class CountTokensBody(BaseModel): class AnthropicContentBlock(BaseModel): type: str | None = None + text: str | None = None class AnthropicMessagesResponse(BaseModel): @@ -817,3 +863,23 @@ class TagListResponse(RootModel[list[TagListEntry]]): """GET /tag/list answers with a bare array of tag configs (the stored tags plus any dynamically-seen spend tags), not an object wrapping them. Read the rows off .root.""" + + +# ---------- health / lifecycle ---------- + + +class ReadinessResponse(BaseModel): + """GET /health/readiness (public probe). The low-detail payload a load + balancer sees: `status` plus the resolved DB state (`connected`, + `disconnected`, or `Not connected`).""" + + status: str + db: str | None = None + + +class ReadinessDetailsResponse(ReadinessResponse): + """GET /health/readiness/details (authenticated). Extends the public payload + with the diagnostics only an authenticated caller may read.""" + + litellm_version: str | None = None + success_callbacks: list[str] = [] diff --git a/tests/e2e/logging/otel_client.py b/tests/e2e/otel_client.py similarity index 100% rename from tests/e2e/logging/otel_client.py rename to tests/e2e/otel_client.py diff --git a/tests/e2e/other/conftest.py b/tests/e2e/other/conftest.py new file mode 100644 index 00000000000..9141b6e364e --- /dev/null +++ b/tests/e2e/other/conftest.py @@ -0,0 +1,18 @@ +"""`other` suite's `client` fixture. + +Lifecycle (resources/scoped_key), proxy liveness gate, and the e2e/covers +markers all live in the parent tests/e2e/conftest.py. OtherClient holds the +shared ProxyClient so anything these tests create tears down through it. +""" + +from __future__ import annotations + +import pytest + +from other_client import OtherClient, build_client +from proxy_client import ProxyClient + + +@pytest.fixture(scope="session") +def client(proxy: ProxyClient) -> OtherClient: + return build_client(proxy) diff --git a/tests/e2e/other/other_client.py b/tests/e2e/other/other_client.py new file mode 100644 index 00000000000..1aa83ac42c7 --- /dev/null +++ b/tests/e2e/other/other_client.py @@ -0,0 +1,73 @@ +"""Client for the `other` holding-pen suite: the auth gate (master key vs an +invalid key on an admin route) and the process-lifecycle health probes +(liveness, public readiness, authenticated readiness diagnostics). + +Holds the shared ProxyClient so `resources` / `scoped_key` still clean up, and +adds only the routes these behaviors need. The health probes deliberately send +no auth header (public routes), so they go through the transport with an empty +headers model rather than a bearer. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from e2e_http import NoBody, ProbeResult, Result +from models import ( + ReadinessDetailsResponse, + ReadinessResponse, + UserListParams, + UserListResponse, +) +from proxy_client import ProxyClient + + +@dataclass(frozen=True, slots=True) +class OtherClient: + proxy: ProxyClient + + def liveness(self) -> ProbeResult: + """GET /health/liveliness. Unauthenticated; the probe returns status + + raw body so the test can assert the worker reports itself alive.""" + return self.proxy.transport.probe("/health/liveliness", params=NoBody()) + + def readiness_public(self) -> Result[ReadinessResponse]: + """GET /health/readiness with no credential at all, proving the probe is + safe to expose to an unauthenticated load balancer.""" + return self.proxy.transport.get( + "/health/readiness", + headers=NoBody(), + params=NoBody(), + response_type=ReadinessResponse, + ) + + def readiness_details(self, key: str) -> Result[ReadinessDetailsResponse]: + return self.proxy.transport.get( + "/health/readiness/details", + headers=self.proxy.transport.bearer(key), + params=NoBody(), + response_type=ReadinessDetailsResponse, + ) + + def readiness_details_unauthenticated(self) -> Result[ReadinessDetailsResponse]: + return self.proxy.transport.get( + "/health/readiness/details", + headers=NoBody(), + params=NoBody(), + response_type=ReadinessDetailsResponse, + ) + + def list_users_as(self, key: str) -> Result[UserListResponse]: + """GET /user/list under `key`. Admin-only, so it doubles as the master + key's authorization proof: the master key (proxy admin) reads it, a + non-matching key is rejected before it ever reaches the handler.""" + return self.proxy.transport.get( + "/user/list", + headers=self.proxy.transport.bearer(key), + params=UserListParams(user_ids="e2e-test-user"), + response_type=UserListResponse, + ) + + +def build_client(proxy: ProxyClient) -> OtherClient: + return OtherClient(proxy=proxy) diff --git a/tests/e2e/other/test_health_lifecycle_e2e.py b/tests/e2e/other/test_health_lifecycle_e2e.py new file mode 100644 index 00000000000..2551352e8fa --- /dev/null +++ b/tests/e2e/other/test_health_lifecycle_e2e.py @@ -0,0 +1,65 @@ +"""Live e2e: the process-lifecycle probes Kubernetes and load balancers depend on. + +Liveness and public readiness must answer without a credential (a load balancer +has none), and public readiness must distinguish a healthy worker from one whose +DB is unreachable by reporting the resolved DB state. The detailed readiness +route, by contrast, is authenticated: it exposes diagnostics (version, callbacks, +DB) and must reject an anonymous caller. The suite runs against a proxy configured +with a real database, so a healthy readiness payload reports the DB as connected; +a regression that stopped checking the DB, or dropped the public exposure, fails +here. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import MASTER_KEY +from e2e_http import UnauthorizedError, unwrap +from other_client import OtherClient + +pytestmark = pytest.mark.e2e + + +class TestHealthLifecycle: + @pytest.mark.covers("other.lifecycle.liveness.ping") + def test_liveness_reports_alive_without_auth(self, client: OtherClient) -> None: + probe = client.liveness() + assert probe.status_code == 200, ( + f"liveness must answer 200 for an unauthenticated probe, got " + f"{probe.status_code}: {probe.body[:200]}" + ) + assert "alive" in probe.body.lower(), ( + f"liveness body must confirm the worker is alive, got {probe.body[:200]}" + ) + + @pytest.mark.covers("other.lifecycle.readiness.public_probe") + def test_readiness_is_reachable_without_credentials(self, client: OtherClient) -> None: + readiness = unwrap(client.readiness_public()) + assert readiness.status == "healthy", ( + f"public readiness must report a healthy worker, got status {readiness.status!r}" + ) + + @pytest.mark.covers("other.lifecycle.readiness.reports_db_status") + def test_readiness_reports_connected_db(self, client: OtherClient) -> None: + readiness = unwrap(client.readiness_public()) + assert readiness.db == "connected", ( + "readiness must report the configured database as connected so an " + f"orchestrator can tell a healthy worker from a DB-unreachable one, got {readiness.db!r}" + ) + + @pytest.mark.covers("other.lifecycle.readiness_details.authenticated_diagnostics") + def test_readiness_details_require_auth_and_expose_diagnostics(self, client: OtherClient) -> None: + anonymous = client.readiness_details_unauthenticated() + assert isinstance(anonymous, UnauthorizedError), ( + f"/health/readiness/details must reject an unauthenticated caller, got {anonymous}" + ) + + details = unwrap(client.readiness_details(MASTER_KEY)) + assert details.status == "healthy", f"authenticated readiness status must be healthy, got {details.status!r}" + assert details.litellm_version is not None, ( + "authenticated diagnostics must expose the litellm version" + ) + assert details.db == "connected", ( + f"authenticated diagnostics must report the DB as connected, got {details.db!r}" + ) diff --git a/tests/e2e/other/test_master_key_auth_e2e.py b/tests/e2e/other/test_master_key_auth_e2e.py new file mode 100644 index 00000000000..6ab33c9b62a --- /dev/null +++ b/tests/e2e/other/test_master_key_auth_e2e.py @@ -0,0 +1,37 @@ +"""Live e2e: the master key authenticates and is treated as a proxy admin, and a +key that is not the master key is rejected before reaching the handler. + +/user/list is admin-only, so it proves both halves of the master-key contract in +one route: the master key reads it (authenticated + authorized as admin), while a +freshly minted, never-provisioned token is denied 401 by the auth layer. The +invalid case uses a unique, master-key-shaped token so the check exercises the +credential comparison rather than a value that could collide with a real key. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import MASTER_KEY, unique_marker +from e2e_http import UnauthorizedError, unwrap +from other_client import OtherClient + +pytestmark = pytest.mark.e2e + + +class TestMasterKeyAuth: + @pytest.mark.covers("other.auth.master_key.valid_allows") + def test_master_key_authenticates_and_grants_admin_route(self, client: OtherClient) -> None: + listing = unwrap(client.list_users_as(MASTER_KEY)) + assert listing.total >= 0, ( + "master key reached the admin /user/list handler but the response did not " + f"carry a user count: {listing}" + ) + + @pytest.mark.covers("other.auth.master_key.invalid_denied") + def test_non_matching_master_key_is_denied(self, client: OtherClient) -> None: + bogus = f"sk-{unique_marker()}" + result = client.list_users_as(bogus) + assert isinstance(result, UnauthorizedError), ( + f"a token that is not the master key must be rejected with 401, got {result}" + ) diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index e9611df139b..2998a4b83c6 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -6,3 +6,4 @@ addopts = --strict-markers --strict-config markers = e2e: live test that requires a running proxy and real provider keys load: heavy throughput/load test; collected last so it never perturbs latency-sensitive suites + weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py index d43d8e94898..3bd1b992745 100644 --- a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py @@ -194,6 +194,7 @@ def test_streaming_messages_via_responses_bridge_tracks_spend( @pytest.mark.covers("quota_management.spend_tracking.embeddings.logs_cost") +@pytest.mark.covers("llm.embeddings.openai.basic.nonstream.cost_logged") def test_embedding_writes_nonzero_spend_row( client: SpendClient, scoped_key: str ) -> None: diff --git a/tests/e2e/transport.py b/tests/e2e/transport.py index 005b49272e8..da4252e550e 100644 --- a/tests/e2e/transport.py +++ b/tests/e2e/transport.py @@ -16,7 +16,7 @@ import e2e_http from e2e_http import ( URL, AuthHeaders, - FileUploadForm, + BinaryStream, ProbeResult, Result, StreamingResponse, @@ -32,6 +32,15 @@ class Transport(Protocol): self, path: str, *, headers: BaseModel, json: BaseModel ) -> StreamingResponse: ... + def stream_binary( + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + chunk_size: int = 8192, + ) -> BinaryStream: ... + def send( self, path: str, @@ -65,6 +74,10 @@ class Transport(Protocol): self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] ) -> Result[R]: ... + def put[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: ... + def probe(self, path: str, *, params: BaseModel) -> ProbeResult: ... def upload[R: BaseModel]( @@ -72,9 +85,10 @@ class Transport(Protocol): path: str, *, headers: BaseModel, - form: FileUploadForm, + form: BaseModel, filename: str, content: bytes, + file_content_type: str = "application/jsonl", params: BaseModel | None = None, response_type: type[R], ) -> Result[R]: ... @@ -159,6 +173,17 @@ class HttpTransport: timeout=self.request_timeout, ) + def put[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + return e2e_http.put( + self._url(path), + headers=headers, + json=json, + response_type=response_type, + timeout=self.request_timeout, + ) + def stream( self, path: str, *, headers: BaseModel, json: BaseModel ) -> StreamingResponse: @@ -166,6 +191,22 @@ class HttpTransport: self._url(path), headers=headers, json=json, timeout=self.request_timeout ) + def stream_binary( + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + chunk_size: int = 8192, + ) -> BinaryStream: + return e2e_http.stream_binary( + self._url(path), + headers=headers, + json=json, + chunk_size=chunk_size, + timeout=self.request_timeout, + ) + def send( self, path: str, @@ -197,9 +238,10 @@ class HttpTransport: path: str, *, headers: BaseModel, - form: FileUploadForm, + form: BaseModel, filename: str, content: bytes, + file_content_type: str = "application/jsonl", params: BaseModel | None = None, response_type: type[R], ) -> Result[R]: @@ -209,6 +251,7 @@ class HttpTransport: form=form, filename=filename, content=content, + file_content_type=file_content_type, params=params, response_type=response_type, timeout=self.request_timeout, @@ -234,6 +277,7 @@ CONTROL_PLANE_PREFIXES: tuple[str, ...] = ( "/tag", "/budget", "/model/", + "/access_group", "/spend", "/global", "/config", @@ -318,11 +362,30 @@ class SplitTransport: path, headers=headers, json=json, response_type=response_type ) + def put[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + return self._route(path).put( + path, headers=headers, json=json, response_type=response_type + ) + def stream( self, path: str, *, headers: BaseModel, json: BaseModel ) -> StreamingResponse: return self._route(path).stream(path, headers=headers, json=json) + def stream_binary( + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + chunk_size: int = 8192, + ) -> BinaryStream: + return self._route(path).stream_binary( + path, headers=headers, json=json, chunk_size=chunk_size + ) + def send( self, path: str, @@ -344,9 +407,10 @@ class SplitTransport: path: str, *, headers: BaseModel, - form: FileUploadForm, + form: BaseModel, filename: str, content: bytes, + file_content_type: str = "application/jsonl", params: BaseModel | None = None, response_type: type[R], ) -> Result[R]: @@ -356,6 +420,7 @@ class SplitTransport: form=form, filename=filename, content=content, + file_content_type=file_content_type, params=params, response_type=response_type, ) diff --git a/tests/litellm_utils_tests/test_proxy_budget_reset.py b/tests/litellm_utils_tests/test_proxy_budget_reset.py index 5c96eb619bf..44da3ea06a0 100644 --- a/tests/litellm_utils_tests/test_proxy_budget_reset.py +++ b/tests/litellm_utils_tests/test_proxy_budget_reset.py @@ -30,6 +30,7 @@ def _attrify(d: dict): None)` (et al), which returns None for plain dicts — that would silently skip the row. """ + class _AttrDict(dict): def __getattr__(self, k): try: @@ -120,9 +121,11 @@ async def test_reset_budget_keys_partial_failure(): key1, key2, key3, key4, key5, key6 = ( _attrify(k) for k in [key1, key2, key3, key4, key5, key6] ) - prisma_client.get_data = AsyncMock(return_value=[key1, key2, key3, key4, key5, key6]) + prisma_client.get_data = AsyncMock( + return_value=[key1, key2, key3, key4, key5, key6] + ) - async def fake_reset_key(key, current_time): + async def fake_reset_key(key, current_time, reset_settings=None): if key["id"] == "key1": # Simulate a failure on key1 (for example, this might be due to an invariant check) raise Exception("Simulated failure for key1") @@ -207,9 +210,11 @@ async def test_reset_budget_users_partial_failure(): user1, user2, user3, user4, user5, user6 = ( _attrify(u) for u in [user1, user2, user3, user4, user5, user6] ) - prisma_client.get_data = AsyncMock(return_value=[user1, user2, user3, user4, user5, user6]) + prisma_client.get_data = AsyncMock( + return_value=[user1, user2, user3, user4, user5, user6] + ) - async def fake_reset_user(user, current_time): + async def fake_reset_user(user, current_time, reset_settings=None): if user["id"] == "user1": raise Exception("Simulated failure for user1") else: @@ -397,7 +402,7 @@ async def test_reset_budget_teams_partial_failure(): team1, team2 = _attrify(team1), _attrify(team2) prisma_client.get_data = AsyncMock(return_value=[team1, team2]) - async def fake_reset_team(team, current_time): + async def fake_reset_team(team, current_time, reset_settings=None): if team["id"] == "team1": raise Exception("Simulated failure for team1") else: @@ -513,14 +518,14 @@ async def test_reset_budget_continues_other_categories_on_failure(): job = ResetBudgetJob(proxy_logging_obj, prisma_client) - async def fake_reset_key(key, current_time): + async def fake_reset_key(key, current_time, reset_settings=None): key["spend"] = 0.0 key["budget_reset_at"] = ( current_time + timedelta(seconds=key["budget_duration"]) ).isoformat() return key - async def fake_reset_user(user, current_time): + async def fake_reset_user(user, current_time, reset_settings=None): if user["id"] == "user1": raise Exception("Simulated failure for user1") user["spend"] = 0.0 @@ -529,7 +534,7 @@ async def test_reset_budget_continues_other_categories_on_failure(): ).isoformat() return user - async def fake_reset_team(team, current_time): + async def fake_reset_team(team, current_time, reset_settings=None): team["spend"] = 0.0 team["budget_reset_at"] = ( current_time + timedelta(seconds=team["budget_duration"]) @@ -632,7 +637,7 @@ async def test_service_logger_keys_success(): job = ResetBudgetJob(proxy_logging_obj, prisma_client) - async def fake_reset_key(key, current_time): + async def fake_reset_key(key, current_time, reset_settings=None): key["spend"] = 0.0 key["budget_reset_at"] = ( current_time + timedelta(seconds=key["budget_duration"]) @@ -688,7 +693,7 @@ async def test_service_logger_keys_failure(): job = ResetBudgetJob(proxy_logging_obj, prisma_client) - async def fake_reset_key(key, current_time): + async def fake_reset_key(key, current_time, reset_settings=None): if key["id"] == "key1": raise Exception("Simulated failure for key1") key["spend"] = 0.0 @@ -750,7 +755,7 @@ async def test_service_logger_users_success(): job = ResetBudgetJob(proxy_logging_obj, prisma_client) - async def fake_reset_user(user, current_time): + async def fake_reset_user(user, current_time, reset_settings=None): user["spend"] = 0.0 user["budget_reset_at"] = ( current_time + timedelta(seconds=user["budget_duration"]) @@ -802,7 +807,7 @@ async def test_service_logger_users_failure(): job = ResetBudgetJob(proxy_logging_obj, prisma_client) - async def fake_reset_user(user, current_time): + async def fake_reset_user(user, current_time, reset_settings=None): if user["id"] == "user1": raise Exception("Simulated failure for user1") user["spend"] = 0.0 @@ -863,7 +868,7 @@ async def test_service_logger_teams_success(): job = ResetBudgetJob(proxy_logging_obj, prisma_client) - async def fake_reset_team(team, current_time): + async def fake_reset_team(team, current_time, reset_settings=None): team["spend"] = 0.0 team["budget_reset_at"] = ( current_time + timedelta(seconds=team["budget_duration"]) @@ -915,7 +920,7 @@ async def test_service_logger_teams_failure(): job = ResetBudgetJob(proxy_logging_obj, prisma_client) - async def fake_reset_team(team, current_time): + async def fake_reset_team(team, current_time, reset_settings=None): if team["id"] == "team1": raise Exception("Simulated failure for team1") team["spend"] = 0.0 diff --git a/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json b/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json index a4c50d3c575..41b4c3efb63 100644 --- a/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json +++ b/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json @@ -11,7 +11,7 @@ "user": "", "team_id": "", "organization_id": "", - "metadata": "{\"applied_guardrails\": [], \"batch_models\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"guardrail_information\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}", + "metadata": "{\"applied_guardrails\": [], \"batch_models\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"guardrail_information\": null, \"compression_savings\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}", "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index d22b343d843..ee18c96c393 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -1780,7 +1780,10 @@ def test_update_key_budget_with_temp_budget_increase(): "temp_budget_expiry": expiry_in_isoformat, }, ) - assert _update_key_budget_with_temp_budget_increase(valid_token).max_budget == 200 + result = _update_key_budget_with_temp_budget_increase(valid_token) + assert result.max_budget == 200 + assert result is not valid_token + assert valid_token.max_budget == 100 @pytest.mark.asyncio diff --git a/tests/proxy_unit_tests/test_user_api_key_auth.py b/tests/proxy_unit_tests/test_user_api_key_auth.py index 5471d2668e4..59c4caefa33 100644 --- a/tests/proxy_unit_tests/test_user_api_key_auth.py +++ b/tests/proxy_unit_tests/test_user_api_key_auth.py @@ -1115,6 +1115,7 @@ async def test_jwt_non_admin_team_route_access(monkeypatch): "team_id": None, "team_object": None, "user_id": None, + "user_email": None, "user_object": None, "org_id": None, "org_object": None, diff --git a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py index b745ca8eadf..fbd7e36e298 100644 --- a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py +++ b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py @@ -219,8 +219,7 @@ def _gate(**overrides): kwargs = { "custom_llm_provider": "azure_ai", "litellm_params": GenericLiteLLMParams(api_key="sk-azure", rust=True), - "stream": False, - "rust_stream_eligible": False, + "has_agentic_hook": False, "model": "claude-sonnet-4-5", "api_key": "sk-azure", "api_base": "https://resource.services.ai.azure.com/anthropic", @@ -345,11 +344,11 @@ async def test_gate_skips_rust_for_unsupported_provider(): @pytest.mark.asyncio -async def test_gate_skips_rust_when_streaming_but_not_eligible(): +async def test_gate_skips_rust_for_agentic_hook(): bridge = ExplodingAsyncMessages() litellm.use_litellm_rust(True, amessages=bridge) - response = await _gate(stream=True, rust_stream_eligible=False) + response = await _gate(has_agentic_hook=True) assert response is None assert bridge.calls == 0 @@ -362,8 +361,7 @@ async def test_gate_streams_through_rust_when_eligible_and_strips_stream_flag(): streaming_body = {**REQUEST_BODY, "stream": True} response = await _gate( - stream=True, - rust_stream_eligible=True, + has_agentic_hook=False, request_body=streaming_body, ) diff --git a/tests/test_litellm/interactions/test_openapi_compliance.py b/tests/test_litellm/interactions/test_openapi_compliance.py index 209e99895db..11b08fa45a8 100644 --- a/tests/test_litellm/interactions/test_openapi_compliance.py +++ b/tests/test_litellm/interactions/test_openapi_compliance.py @@ -194,6 +194,7 @@ class TestResponseCompliance: "cancelled", "incomplete", "budget_exceeded", + "queued", ] assert status_prop["enum"] == expected_statuses print(f"✓ Status enum values: {expected_statuses}") diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index b156faf3ea6..9ff67a82f40 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2237,3 +2237,75 @@ def test_token_type_cost_breakdown_applies_regional_uplift(): text_input_cost = 600 * model_info["input_cost_per_token"] * uplift assert text_output_cost + eu.reasoning_cost == pytest.approx(completion_cost) assert text_input_cost + eu.cache_read_cost == pytest.approx(prompt_cost) + + +GEMINI_DAY0_LAUNCH_PRICING = [ + ("gemini-3.6-flash", 1.5e-06, 7.5e-06, 1.5e-07), + ("gemini/gemini-3.6-flash", 1.5e-06, 7.5e-06, 1.5e-07), + ("vertex_ai/gemini-3.6-flash", 1.5e-06, 7.5e-06, 1.5e-07), + ("gemini-3.5-flash-lite", 3e-07, 2.5e-06, 3e-08), + ("gemini/gemini-3.5-flash-lite", 3e-07, 2.5e-06, 3e-08), + ("vertex_ai/gemini-3.5-flash-lite", 3e-07, 2.5e-06, 3e-08), +] + + +@pytest.mark.parametrize("model,input_cost,output_cost,cache_read_cost", GEMINI_DAY0_LAUNCH_PRICING) +def test_gemini_36_flash_and_35_flash_lite_launch_pricing(model, input_cost, output_cost, cache_read_cost): + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model_cost_map = litellm.model_cost[model] + assert model_cost_map["input_cost_per_token"] == input_cost + assert model_cost_map["output_cost_per_token"] == output_cost + assert model_cost_map["output_cost_per_reasoning_token"] == output_cost + assert model_cost_map["cache_read_input_token_cost"] == cache_read_cost + assert model_cost_map["mode"] == "chat" + assert model_cost_map["supports_reasoning"] is True + assert model_cost_map["supports_function_calling"] is True + assert model_cost_map["max_input_tokens"] == 1048576 + + +def test_generic_cost_per_token_gemini_36_flash(): + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + usage = Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500, + completion_tokens_details=CompletionTokensDetailsWrapper( + reasoning_tokens=200, + text_tokens=300, + ), + prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=1000), + ) + prompt_cost, completion_cost = generic_cost_per_token( + model="gemini-3.6-flash", + usage=usage, + custom_llm_provider="gemini", + ) + assert prompt_cost == pytest.approx(0.0015) + assert completion_cost == pytest.approx(0.00375) + + +def test_generic_cost_per_token_gemini_35_flash_lite(): + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + usage = Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500, + completion_tokens_details=CompletionTokensDetailsWrapper( + reasoning_tokens=200, + text_tokens=300, + ), + prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=1000), + ) + prompt_cost, completion_cost = generic_cost_per_token( + model="gemini-3.5-flash-lite", + usage=usage, + custom_llm_provider="gemini", + ) + assert prompt_cost == pytest.approx(0.0003) + assert completion_cost == pytest.approx(0.00125) diff --git a/tests/test_litellm/litellm_core_utils/test_duration_parser.py b/tests/test_litellm/litellm_core_utils/test_duration_parser.py index 3e4446c6672..b6b617610a8 100644 --- a/tests/test_litellm/litellm_core_utils/test_duration_parser.py +++ b/tests/test_litellm/litellm_core_utils/test_duration_parser.py @@ -1,5 +1,5 @@ import unittest -from datetime import datetime, timezone +from datetime import datetime, time, timezone from zoneinfo import ZoneInfo from litellm.litellm_core_utils.duration_parser import get_next_standardized_reset_time @@ -199,5 +199,122 @@ class TestStandardizedResetTime(unittest.TestCase): self.assertEqual(result, expected) +class TestResetTimeOfDay(unittest.TestCase): + """A configurable reset_time_of_day shifts day/week/month resets off midnight.""" + + def test_daily_reset_before_offset_is_today(self): + now = datetime(2023, 5, 15, 8, 0, 0, tzinfo=timezone.utc) + result = get_next_standardized_reset_time( + "1d", now, "UTC", reset_time_of_day=time(12, 0) + ) + self.assertEqual(result, datetime(2023, 5, 15, 12, 0, 0, tzinfo=timezone.utc)) + + def test_daily_reset_after_offset_is_tomorrow(self): + now = datetime(2023, 5, 15, 14, 0, 0, tzinfo=timezone.utc) + result = get_next_standardized_reset_time( + "1d", now, "UTC", reset_time_of_day=time(12, 0) + ) + self.assertEqual(result, datetime(2023, 5, 16, 12, 0, 0, tzinfo=timezone.utc)) + + def test_daily_reset_exactly_at_offset_rolls_forward(self): + now = datetime(2023, 5, 15, 12, 0, 0, tzinfo=timezone.utc) + result = get_next_standardized_reset_time( + "1d", now, "UTC", reset_time_of_day=time(12, 0) + ) + self.assertEqual(result, datetime(2023, 5, 16, 12, 0, 0, tzinfo=timezone.utc)) + + def test_daily_reset_with_seconds_offset(self): + now = datetime(2023, 5, 15, 8, 0, 0, tzinfo=timezone.utc) + result = get_next_standardized_reset_time( + "1d", now, "UTC", reset_time_of_day=time(9, 30, 15) + ) + self.assertEqual(result, datetime(2023, 5, 15, 9, 30, 15, tzinfo=timezone.utc)) + + def test_offset_applies_in_configured_timezone(self): + # 2023-05-15 22:30 UTC == 2023-05-16 01:30 in Jerusalem (IDT, UTC+3), + # so the next noon-Jerusalem reset is 2023-05-16 12:00 IDT. + now = datetime(2023, 5, 15, 22, 30, 0, tzinfo=timezone.utc) + result = get_next_standardized_reset_time( + "1d", now, "Asia/Jerusalem", reset_time_of_day=time(12, 0) + ) + jerusalem = result.astimezone(ZoneInfo("Asia/Jerusalem")) + self.assertEqual( + (jerusalem.year, jerusalem.month, jerusalem.day), (2023, 5, 16) + ) + self.assertEqual(jerusalem.hour, 12) + self.assertEqual(jerusalem.minute, 0) + + def test_weekly_reset_lands_on_monday_at_offset(self): + wednesday = datetime(2023, 5, 17, 15, 45, 0, tzinfo=timezone.utc) + result = get_next_standardized_reset_time( + "7d", wednesday, "UTC", reset_time_of_day=time(12, 0) + ) + self.assertEqual(result, datetime(2023, 5, 22, 12, 0, 0, tzinfo=timezone.utc)) + + def test_weekly_reset_today_is_monday_before_offset_is_today(self): + monday_morning = datetime(2023, 5, 22, 9, 0, 0, tzinfo=timezone.utc) + result = get_next_standardized_reset_time( + "7d", monday_morning, "UTC", reset_time_of_day=time(12, 0) + ) + self.assertEqual(result, datetime(2023, 5, 22, 12, 0, 0, tzinfo=timezone.utc)) + + def test_weekly_reset_today_is_monday_after_offset_is_next_week(self): + monday_afternoon = datetime(2023, 5, 22, 15, 0, 0, tzinfo=timezone.utc) + result = get_next_standardized_reset_time( + "7d", monday_afternoon, "UTC", reset_time_of_day=time(12, 0) + ) + self.assertEqual(result, datetime(2023, 5, 29, 12, 0, 0, tzinfo=timezone.utc)) + + def test_monthly_30d_lands_on_first_at_offset(self): + now = datetime(2023, 5, 15, 10, 30, 0, tzinfo=timezone.utc) + result = get_next_standardized_reset_time( + "30d", now, "UTC", reset_time_of_day=time(12, 0) + ) + self.assertEqual(result, datetime(2023, 6, 1, 12, 0, 0, tzinfo=timezone.utc)) + + def test_monthly_1mo_today_is_first_before_offset_is_today(self): + now = datetime(2023, 5, 1, 9, 0, 0, tzinfo=timezone.utc) + result = get_next_standardized_reset_time( + "1mo", now, "UTC", reset_time_of_day=time(12, 0) + ) + self.assertEqual(result, datetime(2023, 5, 1, 12, 0, 0, tzinfo=timezone.utc)) + + def test_monthly_year_rollover_at_offset(self): + now = datetime(2023, 12, 15, 9, 0, 0, tzinfo=timezone.utc) + result = get_next_standardized_reset_time( + "1mo", now, "UTC", reset_time_of_day=time(12, 0) + ) + self.assertEqual(result, datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc)) + + def test_custom_day_reset_applies_offset(self): + now = datetime(2023, 5, 15, 10, 30, 0, tzinfo=timezone.utc) + result = get_next_standardized_reset_time( + "3d", now, "UTC", reset_time_of_day=time(12, 0) + ) + self.assertEqual(result, datetime(2023, 5, 18, 12, 0, 0, tzinfo=timezone.utc)) + + def test_sub_day_durations_ignore_offset(self): + base = datetime(2023, 5, 15, 15, 20, 30, tzinfo=timezone.utc) + self.assertEqual( + get_next_standardized_reset_time( + "2h", base, "UTC", reset_time_of_day=time(12, 0) + ), + datetime(2023, 5, 15, 16, 0, 0, tzinfo=timezone.utc), + ) + self.assertEqual( + get_next_standardized_reset_time( + "30m", base, "UTC", reset_time_of_day=time(12, 0) + ), + datetime(2023, 5, 15, 15, 30, 0, tzinfo=timezone.utc), + ) + + def test_default_offset_is_midnight(self): + now = datetime(2023, 5, 15, 10, 30, 0, tzinfo=timezone.utc) + self.assertEqual( + get_next_standardized_reset_time("1d", now, "UTC"), + datetime(2023, 5, 16, 0, 0, 0, tzinfo=timezone.utc), + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py index 25d24cfc3ac..1e1b98861b4 100644 --- a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py +++ b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py @@ -412,7 +412,7 @@ class TestAzureAnthropicMidConversationSystem: older Claude, and a *leading* system entry 400s on every model ("messages.0: use the top-level 'system' parameter"). These tests pin the model-aware hoist the config applies so Claude Code sessions neither collapse the prompt cache - on 4.8+ nor hard-fail on 4.7 and older (RCA: Kraken Tech high-spend).""" + on 4.8+ nor hard-fail on 4.7 and older (RCA: customer high-spend).""" def test_supported_model_keeps_mid_conversation_system_in_place(self, local_model_cost_map): messages = [ diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py index ddc2e026e83..ffe21b91ab2 100644 --- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -54,15 +54,24 @@ class FakeBedrockStream: self.input_stream = input_stream if input_stream is not None else FakeInputStream() +class FakeLogging: + def __init__(self, trace_id="trace-nova-sonic"): + self.litellm_trace_id = trace_id + + class DisconnectingClientWS: def __init__(self, messages): self._messages = list(messages) + self.sent_to_client = [] async def receive_text(self): if self._messages: return self._messages.pop(0) raise RuntimeError("client disconnected") + async def send_text(self, message): + self.sent_to_client.append(message) + class ClosableClientWS: def __init__(self): @@ -85,10 +94,14 @@ class EndedBedrockStream: class RealtimeClientWS: def __init__(self): self.closed = False + self.sent_to_client = [] async def receive_text(self): raise RuntimeError("client disconnected") + async def send_text(self, message): + self.sent_to_client.append(message) + async def close(self, code=None, reason=None): self.closed = True @@ -277,6 +290,61 @@ class TestBedrockRealtimeHandler: assert client_ws.closed +class TestBedrockRealtimeSessionLifecycle: + """Server must emit session.created on connect and session.updated on session.update (LIT-4655 regression)""" + + @pytest.mark.asyncio + async def test_session_created_sent_on_connect_before_any_client_input(self, stub_aws_sdk_client): + handler = BedrockRealtime() + websocket = RealtimeClientWS() + + await handler.async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=websocket, + logging_obj=FakeLogging(), + aws_region_name="us-east-1", + aws_access_key_id="k", + aws_secret_access_key="s", + ) + + assert websocket.sent_to_client, "server sent nothing on connect: spec-conformant clients deadlock" + first_event = json.loads(websocket.sent_to_client[0]) + assert first_event["type"] == "session.created" + assert first_event["session"]["id"] == "trace-nova-sonic" + assert first_event["session"]["model"] == "amazon.nova-sonic-v1:0" + + @pytest.mark.asyncio + async def test_session_update_is_acked_with_session_updated(self, stub_aws_models): + handler = BedrockRealtime() + config = BedrockRealtimeConfig() + stream = FakeBedrockStream() + client_ws = DisconnectingClientWS( + [json.dumps({"type": "session.update", "session": {"instructions": "hi", "modalities": ["text"]}})] + ) + + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging() + ) + + acked = [json.loads(message) for message in client_ws.sent_to_client] + updated = [event for event in acked if event["type"] == "session.updated"] + assert updated, "session.update was not acked" + assert updated[0]["session"]["modalities"] == ["text"], "ack must reflect the requested modalities" + + @pytest.mark.asyncio + async def test_no_session_updated_without_logging_obj(self, stub_aws_models): + handler = BedrockRealtime() + config = BedrockRealtimeConfig() + stream = FakeBedrockStream() + client_ws = DisconnectingClientWS( + [json.dumps({"type": "session.update", "session": {"instructions": "hi"}})] + ) + + await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) + + assert client_ws.sent_to_client == [] + + class TestBedrockRealtimeAwsAuth: """AWS auth params passed via litellm_params must reach the Smithy client config (LIT-3923 regression)""" @@ -288,7 +356,7 @@ class TestBedrockRealtimeAwsAuth: await handler.async_realtime( model="amazon.nova-sonic-v1:0", websocket=websocket, - logging_obj=MagicMock(), + logging_obj=FakeLogging(), aws_region_name="us-east-1", aws_access_key_id="litellm-params-access-key", aws_secret_access_key="litellm-params-secret-key", @@ -318,7 +386,7 @@ class TestBedrockRealtimeAwsAuth: await handler.async_realtime( model="amazon.nova-sonic-v1:0", websocket=RealtimeClientWS(), - logging_obj=MagicMock(), + logging_obj=FakeLogging(), aws_region_name="eu-west-1", aws_role_name="arn:aws:iam::123456789012:role/nova-sonic", aws_session_name="realtime-session", diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py index a68aa603b26..aa002b6e302 100644 --- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py +++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py @@ -403,8 +403,9 @@ class TestBedrockRealtimeResponseCreate: class TestBedrockRealtimeResponseTransformation: """Test suite for response transformation""" - def test_transform_session_start_response(self): - """Test sessionStart response transformation""" + def test_bedrock_session_start_does_not_emit_duplicate_session_created(self): + """A Bedrock output sessionStart must not forward a second session.created to the + client; session.created is sent exactly once on connect (LIT-4655)""" config = BedrockRealtimeConfig() logging_obj = MagicMock() logging_obj.litellm_trace_id = "trace_123" @@ -428,10 +429,8 @@ class TestBedrockRealtimeResponseTransformation: }, ) - assert len(result["response"]) == 1 - assert result["response"][0]["type"] == "session.created" - assert result["response"][0]["session"]["id"] == "trace_123" - assert "model" in result["response"][0]["session"] + assert result["response"] == [] + assert result["session_configuration_request"] == json.dumps({"configured": True}) def test_transform_text_output_response(self): """Test textOutput response transformation""" @@ -789,5 +788,47 @@ class TestBedrockRealtimeResponseTransformation: assert len(set(response_ids)) == 1, "Response IDs should be consistent" +class TestBedrockRealtimeSessionEvents: + """session.created / session.updated builders produce spec-shaped events (LIT-4655)""" + + @staticmethod + def _logging(): + from types import SimpleNamespace + + return SimpleNamespace(litellm_trace_id="trace_123") + + def test_session_created_event_shape(self): + event = BedrockRealtimeConfig().session_created_event("amazon.nova-sonic-v1:0", self._logging()) + assert event["type"] == "session.created" + assert event["session"]["id"] == "trace_123" + assert event["session"]["model"] == "amazon.nova-sonic-v1:0" + assert event["session"]["modalities"] == ["text", "audio"] + assert event["event_id"] + + def test_session_updated_event_shape(self): + event = BedrockRealtimeConfig().session_updated_event("amazon.nova-sonic-v1:0", self._logging()) + assert event["type"] == "session.updated" + assert event["session"]["id"] == "trace_123" + assert event["session"]["model"] == "amazon.nova-sonic-v1:0" + assert event["event_id"] + + def test_created_and_updated_have_distinct_event_ids(self): + config = BedrockRealtimeConfig() + logging_obj = self._logging() + created = config.session_created_event("amazon.nova-sonic-v1:0", logging_obj) + updated = config.session_updated_event("amazon.nova-sonic-v1:0", logging_obj) + assert created["event_id"] != updated["event_id"] + + def test_session_updated_reflects_requested_modalities(self): + event = BedrockRealtimeConfig().session_updated_event( + "amazon.nova-sonic-v1:0", self._logging(), modalities=["text"] + ) + assert event["session"]["modalities"] == ["text"] + + def test_session_updated_defaults_modalities_when_unspecified(self): + event = BedrockRealtimeConfig().session_updated_event("amazon.nova-sonic-v1:0", self._logging()) + assert event["session"]["modalities"] == ["text", "audio"] + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/llms/huggingface/embedding/test_huggingface_embedding_handler.py b/tests/test_litellm/llms/huggingface/embedding/test_huggingface_embedding_handler.py index 8a072fa5097..af8321f24a1 100644 --- a/tests/test_litellm/llms/huggingface/embedding/test_huggingface_embedding_handler.py +++ b/tests/test_litellm/llms/huggingface/embedding/test_huggingface_embedding_handler.py @@ -121,6 +121,20 @@ class TestHuggingFaceEmbedding: assert response.usage.prompt_tokens > 0 assert response.usage.total_tokens == response.usage.prompt_tokens + def test_model_name_with_https_substring_uses_api_base(self): + api_base = "https://legit.example/embed" + + litellm.embedding( + model="huggingface/my-https-endpoint", + input=["hello world"], + input_type="embed", + api_base=api_base, + ) + + self.mock_http.assert_called_once() + called_url = self.mock_http.call_args[0][0] + assert called_url == api_base + def test_embedding_with_sentence_similarity_task(self): """Test embedding when task type is sentence-similarity (requires 2+ sentences)""" diff --git a/tests/test_litellm/llms/oobabooga/chat/test_oobabooga.py b/tests/test_litellm/llms/oobabooga/chat/test_oobabooga.py new file mode 100644 index 00000000000..91ebb2bd9d4 --- /dev/null +++ b/tests/test_litellm/llms/oobabooga/chat/test_oobabooga.py @@ -0,0 +1,55 @@ +import os +import sys +from unittest.mock import MagicMock, patch + +sys.path.insert(0, os.path.abspath("../../../../..")) + +import litellm + +MOCK_COMPLETION_RESPONSE = { + "choices": [{"message": {"role": "assistant", "content": "hi there"}}], + "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, +} + + +def _mock_post_response(): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = "ok" + mock_response.json.return_value = MOCK_COMPLETION_RESPONSE + return mock_response + + +def test_model_name_with_https_substring_uses_api_base(): + api_base = "https://legit.example" + + with patch( + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post" + ) as mock_post: + mock_post.return_value = _mock_post_response() + + litellm.completion( + model="oobabooga/my-https-model", + messages=[{"role": "user", "content": "hello"}], + api_base=api_base, + ) + + mock_post.assert_called_once() + called_url = mock_post.call_args[0][0] + assert called_url == f"{api_base}/v1/chat/completions" + + +def test_url_valued_model_still_targets_that_url(): + with patch( + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post" + ) as mock_post: + mock_post.return_value = _mock_post_response() + + litellm.completion( + model="oobabooga/https://sdk-user.example", + messages=[{"role": "user", "content": "hello"}], + ) + + mock_post.assert_called_once() + called_url = mock_post.call_args[0][0] + assert called_url == "https://sdk-user.example/v1/chat/completions" diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 2d09cc0ed32..292bddf1274 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -591,7 +591,7 @@ class TestVertexAnthropicMidConversationSystem: Claude, and a *leading* system entry 400s on every model ("messages.0: use the top-level 'system' parameter"). These tests pin the model-aware hoist so Claude Code sessions neither collapse the prompt cache on 4.8+ nor hard-fail - on 4.7 and older (RCA: Kraken Tech high-spend).""" + on 4.7 and older (RCA: customer high-spend).""" def test_supported_model_keeps_mid_conversation_system_in_place(self, local_model_cost_map): messages = [ diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_sso_assertion_store.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_sso_assertion_store.py new file mode 100644 index 00000000000..a3f46a49ba9 --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_sso_assertion_store.py @@ -0,0 +1,343 @@ +"""Tests for the SSO identity assertion store (EMA subject-token capture). + +Pins the contract of the store that PR 2's ``_id_jag`` subject-sourcing seam will read: +the carrier validates untyped IdP token-response values at the boundary, retention is +gated on an ``oauth2_id_jag`` server being registered, the row is encrypted at rest and +round-trips exactly, a store failure never escapes into the login path, and a salt-key +rotation re-encrypts stored rows like the sibling per-user credential tables. +""" + +import json +import time +from unittest.mock import AsyncMock, MagicMock, patch + +import jwt as pyjwt +import pytest + +from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( + assertion_from_sso_login, + ema_assertion_retention_enabled, + fetch_sso_identity_assertion, + persist_sso_identity_assertion, + retain_sso_identity_assertion_for_ema, + rotate_sso_identity_assertions_master_key, +) +from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper +from litellm.types.mcp import MCPAuth + +SALT_KEY = "test-salt-key-for-sso-assertion-tests-1234" +SIGNING_KEY = "test-idp-signing-key-32-bytes-long-xxxx" +ISSUER = "https://idp.example.com" + + +@pytest.fixture(autouse=True) +def _set_salt_key(monkeypatch): + monkeypatch.setenv("LITELLM_SALT_KEY", SALT_KEY) + + +def _make_id_token(exp_offset: int = 3600, iss: str = ISSUER) -> str: + return pyjwt.encode( + {"iss": iss, "sub": "u1", "exp": int(time.time()) + exp_offset}, + SIGNING_KEY, + algorithm="HS256", + ) + + +def _make_prisma(stored: dict, db_has_id_jag_server: bool = False): + """A fake prisma client whose sso-assertion table reads and writes ``stored`` + (user_id -> assertion_b64), covering upsert, find_unique, find_many, and update. + ``db_has_id_jag_server`` drives the retention gate's authoritative DB fallback; + it is wired explicitly so the gate never reads a truthy bare MagicMock.""" + prisma = MagicMock() + prisma.db.litellm_mcpservertable.find_first = AsyncMock( + return_value=MagicMock() if db_has_id_jag_server else None + ) + + async def _upsert(where, data): + stored[where["user_id"]] = data["update"]["assertion_b64"] + + async def _find_unique(where): + blob = stored.get(where["user_id"]) + if blob is None: + return None + row = MagicMock() + row.user_id = where["user_id"] + row.assertion_b64 = blob + return row + + async def _find_many(): + rows = [] + for user_id, blob in stored.items(): + row = MagicMock() + row.user_id = user_id + row.assertion_b64 = blob + rows.append(row) + return rows + + async def _update(where, data): + stored[where["user_id"]] = data["assertion_b64"] + + prisma.db.litellm_ssoidentityassertion.upsert = AsyncMock(side_effect=_upsert) + prisma.db.litellm_ssoidentityassertion.find_unique = AsyncMock(side_effect=_find_unique) + prisma.db.litellm_ssoidentityassertion.find_many = AsyncMock(side_effect=_find_many) + prisma.db.litellm_ssoidentityassertion.update = AsyncMock(side_effect=_update) + return prisma + + +def _server_with_auth(auth_type): + server = MagicMock() + server.auth_type = auth_type + return server + + +def test_assertion_from_sso_login_happy_path(): + token = _make_id_token() + assertion = assertion_from_sso_login(token, "rt_1") + assert assertion is not None + assert assertion.id_token.get_secret_value() == token + assert assertion.refresh_token is not None + assert assertion.refresh_token.get_secret_value() == "rt_1" + assert assertion.issuer == ISSUER + assert assertion.expires_at is not None + assert assertion.expires_at.timestamp() == pytest.approx(time.time() + 3600, abs=5) + + +def test_assertion_repr_never_leaks_token_material(): + token = _make_id_token() + assertion = assertion_from_sso_login(token, "rt_secret_value") + rendered = repr(assertion) + str(assertion) + assert token not in rendered + assert "rt_secret_value" not in rendered + + +@pytest.mark.parametrize("id_token", [None, "", "not-a-jwt", 12345, ["x"], {"a": 1}]) +def test_assertion_from_sso_login_rejects_unusable_id_token(id_token): + assert assertion_from_sso_login(id_token, "rt") is None + + +@pytest.mark.parametrize("refresh_token", [None, "", 123, ["rt"], {"rt": 1}]) +def test_assertion_from_sso_login_drops_malformed_refresh_token(refresh_token): + assertion = assertion_from_sso_login(_make_id_token(), refresh_token) + assert assertion is not None + assert assertion.refresh_token is None + + +def test_assertion_without_exp_or_iss_still_retained(): + token = pyjwt.encode({"sub": "u1"}, SIGNING_KEY, algorithm="HS256") + assertion = assertion_from_sso_login(token, None) + assert assertion is not None + assert assertion.expires_at is None + assert assertion.issuer is None + + +@pytest.mark.asyncio +async def test_retention_gate_requires_an_id_jag_server(): + with ( + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, + patch("litellm.proxy.proxy_server.prisma_client", _make_prisma({}, db_has_id_jag_server=False)), + ): + manager.config_mcp_servers = { + "s1": _server_with_auth(MCPAuth.oauth2), + "s2": _server_with_auth(None), + } + assert await ema_assertion_retention_enabled() is False + manager.config_mcp_servers = { + "s1": _server_with_auth(MCPAuth.oauth2), + "s2": _server_with_auth(MCPAuth.oauth2_id_jag), + } + assert await ema_assertion_retention_enabled() is True + + +@pytest.mark.asyncio +async def test_retention_gate_reads_the_db_when_config_declares_no_id_jag_server(): + """A DB-backed server added on another pod (or before this pod's DB load) must still enable + retention off the authoritative DB row; False only when neither authority knows one.""" + with patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager: + manager.config_mcp_servers = {"s1": _server_with_auth(MCPAuth.oauth2)} + db_backed = _make_prisma({}, db_has_id_jag_server=True) + with patch("litellm.proxy.proxy_server.prisma_client", db_backed): + assert await ema_assertion_retention_enabled() is True + db_backed.db.litellm_mcpservertable.find_first.assert_awaited_once_with( + where={"auth_type": MCPAuth.oauth2_id_jag.value} + ) + with patch("litellm.proxy.proxy_server.prisma_client", None): + assert await ema_assertion_retention_enabled() is False + + +@pytest.mark.asyncio +async def test_retention_gate_never_consults_the_registry_snapshot(): + """The registry is a per-process snapshot of DB state, stale in either direction: trusting + it positively would keep retaining bearer material after the last EMA server was removed on + another pod, trusting it negatively would drop writes for one added elsewhere. The gate must + judge only the config declaration and the DB row, so a stale snapshot listing an id_jag + server changes nothing.""" + with ( + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, + patch("litellm.proxy.proxy_server.prisma_client", _make_prisma({}, db_has_id_jag_server=False)), + ): + manager.config_mcp_servers = {} + manager.get_registry.return_value = {"stale": _server_with_auth(MCPAuth.oauth2_id_jag)} + assert await ema_assertion_retention_enabled() is False + manager.get_registry.assert_not_called() + + +@pytest.mark.asyncio +async def test_retain_persists_when_only_the_db_knows_the_id_jag_server(): + stored = {} + prisma = _make_prisma(stored, db_has_id_jag_server=True) + with ( + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, + patch("litellm.proxy.proxy_server.prisma_client", prisma), + ): + manager.config_mcp_servers = {} + await retain_sso_identity_assertion_for_ema( + user_id="user-a", assertion=assertion_from_sso_login(_make_id_token(), None) + ) + assert "user-a" in stored + + +@pytest.mark.asyncio +async def test_persist_and_fetch_round_trip_encrypted_at_rest(): + stored = {} + prisma = _make_prisma(stored) + token = _make_id_token() + assertion = assertion_from_sso_login(token, "rt_1") + with patch("litellm.proxy.proxy_server.prisma_client", prisma): + await persist_sso_identity_assertion("user-a", assertion) + fetched = await fetch_sso_identity_assertion("user-a") + assert fetched is not None + assert fetched.id_token.get_secret_value() == token + assert fetched.refresh_token is not None + assert fetched.refresh_token.get_secret_value() == "rt_1" + assert fetched.issuer == assertion.issuer + assert fetched.expires_at == assertion.expires_at + assert token not in stored["user-a"] + assert "rt_1" not in stored["user-a"] + decrypted = decrypt_value_helper(stored["user-a"], "test", exception_type="debug") + assert json.loads(decrypted)["id_token"] == token + + +@pytest.mark.asyncio +async def test_persist_overwrites_previous_login(): + stored = {} + prisma = _make_prisma(stored) + first = _make_id_token(exp_offset=100) + second = _make_id_token(exp_offset=7200) + with patch("litellm.proxy.proxy_server.prisma_client", prisma): + await persist_sso_identity_assertion("user-a", assertion_from_sso_login(first, None)) + await persist_sso_identity_assertion("user-a", assertion_from_sso_login(second, "rt_new")) + fetched = await fetch_sso_identity_assertion("user-a") + assert fetched is not None + assert fetched.id_token.get_secret_value() == second + assert fetched.refresh_token is not None + + +@pytest.mark.asyncio +async def test_fetch_missing_row_returns_none(): + prisma = _make_prisma({}) + with patch("litellm.proxy.proxy_server.prisma_client", prisma): + assert await fetch_sso_identity_assertion("nobody") is None + + +@pytest.mark.asyncio +async def test_fetch_undecryptable_row_returns_none(): + prisma = _make_prisma({"user-a": "not-an-encrypted-blob"}) + with patch("litellm.proxy.proxy_server.prisma_client", prisma): + assert await fetch_sso_identity_assertion("user-a") is None + + +@pytest.mark.asyncio +async def test_fetch_unparseable_payload_returns_none(): + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + + prisma = _make_prisma({"user-a": encrypt_value_helper("]]not json")}) + with patch("litellm.proxy.proxy_server.prisma_client", prisma): + assert await fetch_sso_identity_assertion("user-a") is None + + +@pytest.mark.asyncio +async def test_retain_noop_when_no_id_jag_server(): + stored = {} + prisma = _make_prisma(stored) + with ( + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, + patch("litellm.proxy.proxy_server.prisma_client", prisma), + ): + manager.config_mcp_servers = {"s1": _server_with_auth(MCPAuth.oauth2)} + await retain_sso_identity_assertion_for_ema( + user_id="user-a", assertion=assertion_from_sso_login(_make_id_token(), None) + ) + prisma.db.litellm_ssoidentityassertion.upsert.assert_not_called() + assert stored == {} + + +@pytest.mark.asyncio +async def test_retain_persists_when_id_jag_server_registered(): + stored = {} + prisma = _make_prisma(stored) + with ( + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, + patch("litellm.proxy.proxy_server.prisma_client", prisma), + ): + manager.config_mcp_servers = {"s1": _server_with_auth(MCPAuth.oauth2_id_jag)} + await retain_sso_identity_assertion_for_ema( + user_id="user-a", assertion=assertion_from_sso_login(_make_id_token(), None) + ) + assert "user-a" in stored + + +@pytest.mark.asyncio +async def test_retain_none_assertion_never_consults_gate_or_store(): + gate = MagicMock() + with patch( + "litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store.ema_assertion_retention_enabled", + gate, + ): + await retain_sso_identity_assertion_for_ema(user_id="user-a", assertion=None) + gate.assert_not_called() + + +@pytest.mark.asyncio +async def test_retain_swallows_store_failure(): + prisma = MagicMock() + prisma.db.litellm_ssoidentityassertion.upsert = AsyncMock(side_effect=RuntimeError("db down")) + with ( + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, + patch("litellm.proxy.proxy_server.prisma_client", prisma), + ): + manager.config_mcp_servers = {"s1": _server_with_auth(MCPAuth.oauth2_id_jag)} + await retain_sso_identity_assertion_for_ema( + user_id="user-a", assertion=assertion_from_sso_login(_make_id_token(), None) + ) + + +@pytest.mark.asyncio +async def test_rotation_reencrypts_under_new_key(monkeypatch): + stored = {} + prisma = _make_prisma(stored) + token = _make_id_token() + with patch("litellm.proxy.proxy_server.prisma_client", prisma): + await persist_sso_identity_assertion("user-a", assertion_from_sso_login(token, None)) + original_blob = stored["user-a"] + + new_key = "rotated-sso-assertion-salt-key-5678" + await rotate_sso_identity_assertions_master_key(prisma_client=prisma, new_master_key=new_key) + assert stored["user-a"] != original_blob + + monkeypatch.setenv("LITELLM_SALT_KEY", new_key) + decrypted = decrypt_value_helper(stored["user-a"], "test", exception_type="debug") + assert decrypted is not None + assert json.loads(decrypted)["id_token"] == token + + +@pytest.mark.asyncio +async def test_rotation_skips_unreadable_rows_but_rotates_readable_ones(): + stored = {"good": None, "bad": "garbage-blob"} + prisma = _make_prisma(stored) + token = _make_id_token() + with patch("litellm.proxy.proxy_server.prisma_client", prisma): + await persist_sso_identity_assertion("good", assertion_from_sso_login(token, None)) + good_blob_before = stored["good"] + await rotate_sso_identity_assertions_master_key(prisma_client=prisma, new_master_key="another-new-salt-key-0000") + assert stored["bad"] == "garbage-blob" + assert stored["good"] != good_blob_before diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 0489b197652..636c7fbd3d5 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -8031,3 +8031,120 @@ async def test_bare_origin_discovery_resolves_single_server_not_aggregate(): assert resource_response["authorization_servers"] == ["https://llm.example.com/test_oauth"] finally: global_mcp_server_manager.registry.clear() + + +@pytest.mark.asyncio +async def test_authorize_wall_names_the_fix_for_urlless_servers(): + """LIT-4629: the authorize wall previously said only "authorization url is not set" with no + hint that spec-only servers never discover; the detail must now name both remedies (manual + Authorization URL + Token URL, or an Issuer for RFC 8414 discovery).""" + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + authorize_with_server, + ) + from litellm.types.mcp import MCPAuth, MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="urlless-wall", + name="sheets_wall", + server_name="sheets_wall", + url=None, + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + spec_path="https://example.com/openapi.yaml", + ) + mock_request = MagicMock() + mock_request.base_url = "https://litellm.example.com/" + mock_request.headers = {} + + with pytest.raises(HTTPException) as exc_info: + await authorize_with_server( + request=mock_request, + mcp_server=server, + client_id="client", + redirect_uri="http://localhost/callback", + ) + assert exc_info.value.status_code == 400 + detail_text = str(exc_info.value.detail) + assert "set Authorization URL and Token URL" in detail_text + assert "Issuer" in detail_text + + +@pytest.mark.asyncio +async def test_token_wall_names_the_fix_for_urlless_servers(): + """The /token wall is the second stop on the same misconfiguration (LIT-4629): after an admin + fills only the Authorization URL, the code exchange dies here; the detail must name the + remedies like the authorize wall does.""" + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + exchange_token_with_server, + ) + from litellm.types.mcp import MCPAuth, MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="urlless-token-wall", + name="sheets_token_wall", + server_name="sheets_token_wall", + url=None, + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + spec_path="https://example.com/openapi.yaml", + authorization_url="https://accounts.google.com/o/oauth2/v2/auth", + ) + mock_request = MagicMock() + mock_request.base_url = "https://litellm.example.com/" + mock_request.headers = {} + + with pytest.raises(HTTPException) as exc_info: + await exchange_token_with_server( + request=mock_request, + mcp_server=server, + grant_type="authorization_code", + code="auth-code", + redirect_uri="http://localhost/callback", + client_id="client", + client_secret=None, + code_verifier="verifier", + ) + assert exc_info.value.status_code == 400 + detail_text = str(exc_info.value.detail) + assert "set Token URL manually" in detail_text + assert "Issuer" in detail_text + + +@pytest.mark.asyncio +async def test_register_wall_names_the_fix_for_urlless_servers(): + """The /register wall serves the same missing-authorization-url 400 as authorize; its detail + must carry the same actionable remedies.""" + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + register_client_with_server, + ) + from litellm.types.mcp import MCPAuth, MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="urlless-register-wall", + name="sheets_register_wall", + server_name="sheets_register_wall", + url=None, + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + spec_path="https://example.com/openapi.yaml", + ) + mock_request = MagicMock() + mock_request.base_url = "https://litellm.example.com/" + mock_request.headers = {} + + with pytest.raises(HTTPException) as exc_info: + await register_client_with_server( + request=mock_request, + mcp_server=server, + client_name="client", + grant_types=None, + response_types=None, + token_endpoint_auth_method=None, + ) + assert exc_info.value.status_code == 400 + detail_text = str(exc_info.value.detail) + assert "set Authorization URL and Token URL" in detail_text + assert "Issuer" in detail_text diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py index 73486fe0b6a..b56a12db5b1 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py @@ -1033,3 +1033,166 @@ class TestResolveByokMcpAuthHeader: check_mock.assert_awaited_once_with(server, user_auth) assert result == "caller-header" + + +class TestOpenApiResolvedUpstreamAuth: + """LIT-4629: spec_path servers egress through plain httpx, so the manager's OpenAPI arm must + materialize the v2-resolved credential into the `_request_resolved_auth_headers` ContextVar; + before the fix the resolved token never reached the upstream API.""" + + def _oauth_server(self, **overrides: Any) -> MCPServer: + fields: Dict[str, Any] = dict( + server_id="srv-sheets", + name="google_sheets", + server_name="google_sheets", + url=None, + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + spec_path="https://example.com/sheets-openapi.yaml", + ) + fields.update(overrides) + return MCPServer(**fields) + + @pytest.mark.asyncio + async def test_call_tool_openapi_injects_v2_resolved_token_contextvar(self): + """The managed spec_path arm resolves the v2 credential and sets the ContextVar; kills + the mutant that drops the resolve_openapi_upstream_auth call in call_tool.""" + from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + _request_resolved_auth_headers, + ) + from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + StaticHeaderAuth, + ) + from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + + manager = MCPServerManager() + server = self._oauth_server() + user_auth = UserAPIKeyAuth(user_id="alice", api_key="sk-user") + captured: Dict[str, Any] = {} + + async def fake_openapi_handler(_server, _name, _arguments): + captured["resolved"] = _request_resolved_auth_headers.get() + return MagicMock() + + with patch.object(manager, "_resolve_mcp_server_for_tool_call", return_value=server): + with patch.object( + manager._cred_provider, + "resolve_credentials", + new=AsyncMock(return_value=Ok(StaticHeaderAuth("Bearer stored-user-token"))), + ): + with patch.object(manager, "_call_openapi_tool_handler", side_effect=fake_openapi_handler): + await manager.call_tool( + server_name=server.server_name, + name="get_values", + arguments={}, + user_api_key_auth=user_auth, + ) + + assert captured["resolved"] == {"Authorization": "Bearer stored-user-token"} + assert _request_resolved_auth_headers.get() is None + + @pytest.mark.asyncio + async def test_call_tool_openapi_m2m_missing_token_url_fails_closed(self): + """A url-less M2M spec server with no token_url must fail with a typed error instead of + egressing unauthenticated (the pre-#32259 silent failure this arm previously preserved). + Drives the real adapter/resolver chain: ClientCredentialsConfig with missing grant fields + resolves to a misconfigured CredError, raised as an HTTPException.""" + from fastapi import HTTPException + + manager = MCPServerManager() + server = self._oauth_server( + oauth2_flow="client_credentials", + client_id="m2m-client", + client_secret="m2m-secret", + token_url=None, + ) + called = AsyncMock() + + with patch.object(manager, "_resolve_mcp_server_for_tool_call", return_value=server): + with patch.object(manager, "_call_openapi_tool_handler", new=called): + with pytest.raises(HTTPException): + await manager.call_tool( + server_name=server.server_name, + name="get_values", + arguments={}, + user_api_key_auth=UserAPIKeyAuth(user_id="alice", api_key="sk-user"), + ) + + called.assert_not_awaited() + + @pytest.mark.asyncio + async def test_caller_oauth2_headers_never_become_resolved_for_byok_server(self): + """Greptile P1 regression: BYOK servers defer to v1 (to_server_spec None), and the v1 arm + must never promote caller-supplied oauth2 headers into the resolved-auth slot, where they + would override the per-server BYOK credential and leak the caller's gateway Authorization + upstream.""" + manager = MCPServerManager() + server = MCPServer( + server_id="byok-spec", + name="byok_spec", + server_name="byok_spec", + url=None, + transport=MCPTransport.http, + auth_type=MCPAuth.api_key, + spec_path="https://example.com/openapi.yaml", + is_byok=True, + ) + + resolved, forwarded = await manager.resolve_openapi_upstream_auth( + mcp_server=server, + oauth2_headers={"Authorization": "Bearer sk-litellm-gateway-key"}, + raw_headers=None, + mcp_auth_header="user-byok-key", + user_api_key_auth=UserAPIKeyAuth(user_id="alice", api_key="sk-user"), + forwarded_headers=None, + ) + + assert resolved is None + assert forwarded is None + + @pytest.mark.asyncio + async def test_v1_server_threads_stored_headers_only_without_caller_headers(self): + """The v1 (unmigrated) arm resolves the stored per-user token only when the caller sent no + oauth2 headers of their own; with caller headers present the stored lookup is skipped and + nothing is promoted to resolved.""" + manager = MCPServerManager() + server = MCPServer( + server_id="v1-spec", + name="v1_spec", + server_name="v1_spec", + url=None, + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + spec_path="https://example.com/openapi.yaml", + delegate_auth_to_upstream=True, + ) + stored = {"Authorization": "Bearer stored-v1-token"} + user_auth = UserAPIKeyAuth(user_id="alice", api_key="sk-user") + + with patch.object( + manager, "_resolve_oauth2_headers_for_tool_call", new=AsyncMock(return_value=stored) + ) as lookup: + resolved, _ = await manager.resolve_openapi_upstream_auth( + mcp_server=server, + oauth2_headers=None, + raw_headers=None, + mcp_auth_header=None, + user_api_key_auth=user_auth, + forwarded_headers=None, + ) + assert resolved == stored + lookup.assert_awaited_once_with(server, None, user_auth) + + with patch.object( + manager, "_resolve_oauth2_headers_for_tool_call", new=AsyncMock(return_value=stored) + ) as lookup: + resolved, _ = await manager.resolve_openapi_upstream_auth( + mcp_server=server, + oauth2_headers={"Authorization": "Bearer caller-supplied"}, + raw_headers=None, + mcp_auth_header=None, + user_api_key_auth=user_auth, + forwarded_headers=None, + ) + assert resolved is None + lookup.assert_not_awaited() diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 03f91260955..a5cb16822cf 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -5597,7 +5597,7 @@ class TestMCPServerTimestamps: async def test_build_mcp_server_from_table_persists_discovered_oauth_endpoints(self): """A DB-backed oauth2 server with no configured endpoints discovers them and must write authorization_url, token_url, and scopes back to the row; otherwise the resolved values - live only in memory and one failed re-discovery serves 400 "authorization url is not set" + live only in memory and one failed re-discovery serves the 400 "authorization url is not configured" from /authorize. registration_url must never be persisted because _dcr_bridge_relays_client_registration keys off that column.""" manager = MCPServerManager() @@ -8891,3 +8891,140 @@ async def test_resolve_toolset_tool_permissions_single_db_fetch_across_checks(): assert first == {"server-a": ["lookup_status"]} assert second == first list_toolsets_mock.assert_awaited_once() + + +class TestMaterializeAuthHeaders: + """_materialize_auth_headers drives one step of a resolved httpx.Auth's own flow to turn it + into a header dict for the OpenAPI egress arm, which sends plain headers and cannot carry an + httpx.Auth. Generic across auth shapes via the resolver-arm header_name convention.""" + + @pytest.mark.asyncio + async def test_static_header_auth_materializes_its_header(self): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + _materialize_auth_headers, + ) + from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + StaticHeaderAuth, + ) + + headers = await _materialize_auth_headers(StaticHeaderAuth("Bearer stored-token")) + assert headers == {"Authorization": "Bearer stored-token"} + + @pytest.mark.asyncio + async def test_client_credentials_bearer_auth_materializes_bearer(self): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + _materialize_auth_headers, + ) + from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import ( + ClientCredentialsBearerAuth, + ) + + async def _refetch(_stale: str): + return None + + headers = await _materialize_auth_headers(ClientCredentialsBearerAuth("m2m-token", _refetch)) + assert headers == {"Authorization": "Bearer m2m-token"} + + @pytest.mark.asyncio + async def test_noop_and_none_materialize_to_none(self): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + _materialize_auth_headers, + ) + from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + NoOpAuth, + ) + + assert await _materialize_auth_headers(None) is None + assert await _materialize_auth_headers(NoOpAuth()) is None + + +class TestUrllessIssuerDiscovery: + """LIT-4629: servers with no url (OpenAPI spec_path, stdio) run no resource discovery, so + their OAuth endpoints could only ever come from manual entry; an admin-pinned issuer is a + url-independent trust anchor (RFC 8414 section 3.3) and must unlock discovery for them.""" + + def _urlless_row(self, **overrides): + fields = dict( + server_id="urlless-1", + alias="sheets_urlless", + description="spec-only server", + url=None, + spec_path="https://example.com/sheets-openapi.yaml", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + created_at=datetime.now(), + updated_at=datetime.now(), + ) + fields.update(overrides) + return LiteLLM_MCPServerTable(**fields) + + @pytest.mark.asyncio + async def test_urlless_server_with_issuer_discovers_endpoints(self): + """The gate previously required bool(server_url), so a url-less server with an issuer + configured never ran the issuer-anchored fetch and /authorize 400d. Kills the mutant that + restores the bare bool(server_url) term.""" + manager = MCPServerManager() + row = self._urlless_row(issuer="https://accounts.google.com") + + resolved = MCPOAuthMetadata( + authorization_url="https://accounts.google.com/o/oauth2/v2/auth", + token_url="https://oauth2.googleapis.com/token", + ) + resource_rooted = AsyncMock(return_value=None) + with ( + patch.object(manager, "_fetch_issuer_anchored_oauth_metadata", new=AsyncMock(return_value=resolved)) as anchored, + patch.object(manager, "_descovery_metadata", new=resource_rooted), + ): + built = await manager.build_mcp_server_from_table(row, credentials_are_encrypted=False) + + anchored.assert_awaited_once_with("https://accounts.google.com", None) + resource_rooted.assert_not_awaited() + assert built.issuer_is_anchored is True + assert built.authorization_url == "https://accounts.google.com/o/oauth2/v2/auth" + assert built.token_url == "https://oauth2.googleapis.com/token" + + @pytest.mark.asyncio + async def test_urlless_server_without_issuer_stays_undiscovered(self): + """With neither a url nor an issuer there is no discovery source; the build must not + attempt any fetch and the endpoints stay unset (manual entry remains the only path).""" + manager = MCPServerManager() + row = self._urlless_row() + + anchored = AsyncMock() + resource_rooted = AsyncMock() + with ( + patch.object(manager, "_fetch_issuer_anchored_oauth_metadata", new=anchored), + patch.object(manager, "_descovery_metadata", new=resource_rooted), + ): + built = await manager.build_mcp_server_from_table(row, credentials_are_encrypted=False) + + anchored.assert_not_awaited() + resource_rooted.assert_not_awaited() + assert built.authorization_url is None + assert built.token_url is None + assert built.issuer_is_anchored is False + + @pytest.mark.asyncio + async def test_urlless_obo_with_issuer_discovers_token_url(self): + """oauth2_token_exchange is not a discovery auth type, so the plain gate relax alone + would leave a url-less OBO server undiscovered; with an issuer pinned and no configured + exchange endpoint it must resolve token_url through the issuer-anchored fetch. Kills the + mutant that drops the OBO widening from the anchor computation.""" + manager = MCPServerManager() + row = self._urlless_row( + alias="obo_urlless", + auth_type=MCPAuth.oauth2_token_exchange, + issuer="https://idp.example.com", + ) + + resolved = MCPOAuthMetadata(token_url="https://idp.example.com/token") + resource_rooted = AsyncMock(return_value=None) + with ( + patch.object(manager, "_fetch_issuer_anchored_oauth_metadata", new=AsyncMock(return_value=resolved)) as anchored, + patch.object(manager, "_descovery_metadata", new=resource_rooted), + ): + built = await manager.build_mcp_server_from_table(row, credentials_are_encrypted=False) + + anchored.assert_awaited_once_with("https://idp.example.com", None) + resource_rooted.assert_not_awaited() + assert built.token_url == "https://idp.example.com/token" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py index 39f3c767220..7bcacb3ff4a 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py @@ -17,6 +17,7 @@ import pytest from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( _request_auth_header, _request_extra_headers, + _request_resolved_auth_headers, _resolve_param_list, _resolve_ref, build_input_schema, @@ -1207,3 +1208,61 @@ class TestRequestExtraHeaders: call_args = async_client.get.call_args headers_sent = call_args[1]["headers"] assert "X-TOKEN" not in headers_sent + + @pytest.mark.asyncio + async def test_resolved_auth_headers_win_over_every_other_authorization_source(self): + """The gateway-resolved credential (stored per-user OAuth / minted M2M token) is + authoritative: it must override the BYOK override, static headers, and forwarded caller + headers on the Authorization name, case-insensitively, mirroring _resolve_v2_auth's rule + on the MCPClient path. Without this, a spec_path oauth2 server's completed OAuth flow + stores a token that never reaches the upstream API (LIT-4629).""" + operation = {} + func = create_tool_function( + path="/secure", + method="get", + operation=operation, + base_url="https://api.example.com", + headers={"authorization": "Bearer static-operator"}, + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "secure-data") + mock_client.return_value = async_client + + extra_token = _request_extra_headers.set({"Authorization": "Bearer caller-forwarded"}) + auth_token = _request_auth_header.set("Bearer byok-credential") + resolved_token = _request_resolved_auth_headers.set({"Authorization": "Bearer resolved-oauth"}) + try: + result = await func() + finally: + _request_auth_header.reset(auth_token) + _request_extra_headers.reset(extra_token) + _request_resolved_auth_headers.reset(resolved_token) + + assert result == "secure-data" + headers_sent = async_client.get.call_args[1]["headers"] + authorization_values = [v for k, v in headers_sent.items() if k.lower() == "authorization"] + assert authorization_values == ["Bearer resolved-oauth"] + + @pytest.mark.asyncio + async def test_resolved_auth_headers_not_leaked_between_calls(self): + """After resetting the resolved-auth ContextVar, subsequent calls send no credential.""" + operation = {} + func = create_tool_function( + path="/data", + method="get", + operation=operation, + base_url="https://api.example.com", + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "ok") + mock_client.return_value = async_client + + token = _request_resolved_auth_headers.set({"Authorization": "Bearer resolved-oauth"}) + _request_resolved_auth_headers.reset(token) + + await func() + + headers_sent = async_client.get.call_args[1]["headers"] + assert "Authorization" not in headers_sent diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py index 3ad01e9c3ec..1e4349c3143 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py @@ -218,3 +218,86 @@ async def test_openapi_local_tool_denied_when_server_not_resolvable(): assert exc.value.status_code == 503 pre_call.assert_not_awaited() handle_local.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_openapi_local_tool_injects_resolved_oauth_token(): + """LIT-4629: the local-registry (OpenAPI) dispatch is the primary egress for spec_path + tools, and before the fix it dropped the gateway-resolved OAuth credential entirely, so a + user's completed OAuth flow stored a token that never reached the upstream API. The resolved + credential must land in the `_request_resolved_auth_headers` ContextVar the tool closure + reads. Kills the mutant that deletes the resolve_openapi_upstream_auth call in server.py.""" + from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + _request_resolved_auth_headers, + ) + from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + StaticHeaderAuth, + ) + from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + from litellm.types.mcp import MCPAuth, MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + user = UserAPIKeyAuth( + api_key="sk-user", + user_id="alice", + user_role=LitellmUserRoles.INTERNAL_USER.value, + ) + oauth_server = MCPServer( + server_id="srv-sheets", + name="google_sheets", + server_name="google_sheets", + url=None, + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + spec_path="https://example.com/sheets-openapi.yaml", + ) + + fake_tool = MagicMock() + fake_tool.name = "get_values" + captured: dict = {} + + async def handle_local(_name, _arguments): + captured["resolved"] = _request_resolved_auth_headers.get() + return [] + + with ( + patch.object( + mcp_module.global_mcp_server_manager, + "_get_mcp_server_from_tool_name", + return_value=oauth_server, + ), + patch.object( + mcp_module.global_mcp_server_manager, + "pre_call_tool_check", + new=AsyncMock(return_value={}), + ), + patch.object( + mcp_module.global_mcp_tool_registry, + "get_tool", + return_value=fake_tool, + ), + patch.object( + mcp_module.global_mcp_server_manager._cred_provider, + "resolve_credentials", + new=AsyncMock(return_value=Ok(StaticHeaderAuth("Bearer stored-user-token"))), + ), + patch( + "litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool", + new=handle_local, + ), + patch( + "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed", + return_value=True, + ), + ): + await mcp_module.execute_mcp_tool( + name="get_values", + arguments={}, + allowed_mcp_servers=[oauth_server], + start_time=datetime.now(timezone.utc), + user_api_key_auth=user, + ) + + assert captured["resolved"] == {"Authorization": "Bearer stored-user-token"} + assert _request_resolved_auth_headers.get() is None diff --git a/tests/test_litellm/proxy/a2a/test_agent_card.py b/tests/test_litellm/proxy/a2a/test_agent_card.py index d302bde7895..dfa848e335e 100644 --- a/tests/test_litellm/proxy/a2a/test_agent_card.py +++ b/tests/test_litellm/proxy/a2a/test_agent_card.py @@ -1,10 +1,14 @@ """Unit tests for the pure merge logic in litellm/proxy/a2a/agent_card.py.""" +import pytest + from litellm.proxy.a2a.agent_card import ( LITELLM_A2A_PROTOCOL_VERSION, LITELLM_SECURITY_REQUIREMENTS, LITELLM_SECURITY_SCHEMES, merge_agent_card, + normalize_protocol_version, + resolve_served_protocol_version, ) PROXY_URL = "https://proxy.example/a2a/agent-xyz" @@ -205,3 +209,54 @@ def test_strips_additional_interfaces_to_prevent_backend_url_leak(): ] merged = merge_agent_card(upstream, proxy_url=PROXY_URL, proxy_base_url=PROXY_BASE) assert "additionalInterfaces" not in merged + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("0.3", "0.3"), + ("0.3.0", "0.3"), + ("1.0", "1.0"), + ("1.0.0", "1.0"), + ("1.0.1", "1.0"), + ("0.3.0-rc1", "0.3"), + ("1.0.0-rc.1+build.5", "1.0"), + ("0.2.6", None), + ("2.0", None), + ("0.30", None), + ("0.3.garbage", None), + ("0.3.", None), + ("1.0.not-semver", None), + ("0.3.0.0", None), + ("0.3-rc1", None), + ("garbage", None), + ("", None), + (None, None), + (1.0, None), + ], +) +def test_normalize_protocol_version(raw, expected): + assert normalize_protocol_version(raw) == expected + + +def test_resolve_served_protocol_version_canonicalizes_semver_pins(): + assert resolve_served_protocol_version({"protocolVersion": "0.3.0"}) == "0.3" + assert resolve_served_protocol_version({"protocolVersion": "1.0.0"}) == "1.0" + assert resolve_served_protocol_version({"protocolVersion": "0.3"}) == "0.3" + assert resolve_served_protocol_version({"protocolVersion": "1.0"}) == "1.0" + + +def test_resolve_served_protocol_version_falls_back_for_unsupported(): + assert ( + resolve_served_protocol_version({"protocolVersion": "0.2.6"}) + == LITELLM_A2A_PROTOCOL_VERSION + ) + assert resolve_served_protocol_version(None) == LITELLM_A2A_PROTOCOL_VERSION + + +def test_serves_semver_pinned_protocol_version_as_major_minor(): + card = _full_upstream_card() + card["protocolVersion"] = "0.3.0" + merged = merge_agent_card(card, proxy_url=PROXY_URL, proxy_base_url=PROXY_BASE) + assert merged["protocolVersion"] == "0.3" + assert merged["supportedInterfaces"][0]["protocolVersion"] == "0.3" diff --git a/tests/test_litellm/proxy/a2a/test_version_convert.py b/tests/test_litellm/proxy/a2a/test_version_convert.py index f3c51ca6b72..7eb5debb792 100644 --- a/tests/test_litellm/proxy/a2a/test_version_convert.py +++ b/tests/test_litellm/proxy/a2a/test_version_convert.py @@ -313,3 +313,13 @@ def test_agent_card_with_0_3_pin_and_supported_interfaces_is_lowered(): def test_agent_card_same_version_passthrough(): card = _extended_card_1_0() assert normalize_agent_card(card, "1.0") is card + + +def test_detect_card_version_normalizes_semver_protocol_version(): + from litellm.proxy.a2a.version_convert import _detect_card_version + + assert _detect_card_version({"protocolVersion": "1.0.0"}) == "1.0" + assert ( + _detect_card_version({"protocolVersion": "0.3.0", "supportedInterfaces": []}) + == "0.3" + ) diff --git a/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py b/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py index 3740c01b7fc..bcd3333baf9 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py @@ -540,6 +540,53 @@ class TestAgentRBACProxyAdmin: assert resp.status_code == 200 +class TestAgentProtocolVersionValidation: + """Registration accepts spec-default semver protocolVersion values and still + rejects genuinely unsupported versions.""" + + @pytest.fixture(autouse=True) + def _setup(self, monkeypatch): + self.admin_client = _make_app_with_role(LitellmUserRoles.PROXY_ADMIN) + self.mock_registry = MagicMock() + monkeypatch.setattr(agent_endpoints, "AGENT_REGISTRY", self.mock_registry) + + def _create_agent_with_protocol_version(self, protocol_version: str): + config = _sample_agent_config() + config["agent_card_params"]["protocolVersion"] = protocol_version + with patch("litellm.proxy.proxy_server.prisma_client"): + self.mock_registry.get_agent_by_name = MagicMock(return_value=None) + self.mock_registry.add_agent_to_db = AsyncMock( + return_value=_sample_agent_response() + ) + self.mock_registry.register_agent = MagicMock() + return self.admin_client.post( + "/v1/agents", + json=config, + headers={"Authorization": "Bearer k"}, + ) + + def test_semver_protocol_version_registers_and_stores_major_minor(self): + resp = self._create_agent_with_protocol_version("0.3.0") + assert resp.status_code == 200 + stored_card = self.mock_registry.add_agent_to_db.await_args.kwargs["agent"][ + "agent_card_params" + ] + assert stored_card["protocolVersion"] == "0.3" + assert stored_card["supportedInterfaces"][0]["protocolVersion"] == "0.3" + + def test_unsupported_protocol_version_is_rejected(self): + resp = self._create_agent_with_protocol_version("0.2.6") + assert resp.status_code == 400 + assert "Unsupported protocolVersion '0.2.6'" in resp.json()["detail"] + self.mock_registry.add_agent_to_db.assert_not_awaited() + + def test_malformed_protocol_version_is_rejected(self): + resp = self._create_agent_with_protocol_version("0.3.garbage") + assert resp.status_code == 400 + assert "Unsupported protocolVersion '0.3.garbage'" in resp.json()["detail"] + self.mock_registry.add_agent_to_db.assert_not_awaited() + + class TestCheckAgentManagementPermission: """Unit tests for the _check_agent_management_permission helper.""" diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index 72bd215b9be..9f24c662581 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -1587,7 +1587,7 @@ class TestGetDynamicLitellmParamsClearsAdminConfigOnBaseOverride: } out = get_dynamic_litellm_params( litellm_params=dict(admin_params), - request_kwargs={"base_url": "https://attacker.example"}, + request_kwargs={"base_url": "https://attacker.example", "api_key": "sk-caller"}, ) assert "aws_access_key_id" not in out assert "aws_secret_access_key" not in out @@ -1608,7 +1608,7 @@ class TestGetDynamicLitellmParamsClearsAdminConfigOnBaseOverride: } out = get_dynamic_litellm_params( litellm_params=dict(admin_params), - request_kwargs={"api_base": "self-hosted.example.com:50051"}, + request_kwargs={"api_base": "self-hosted.example.com:50051", "api_key": "sk-caller"}, ) assert out["api_base"] == "self-hosted.example.com:50051" assert "nvcf_function_id" not in out @@ -1626,7 +1626,7 @@ class TestGetDynamicLitellmParamsClearsAdminConfigOnBaseOverride: } out = get_dynamic_litellm_params( litellm_params=dict(admin_params), - request_kwargs={"api_base": "self-hosted.example.com:50051"}, + request_kwargs={"api_base": "self-hosted.example.com:50051", "api_key": "sk-caller"}, ) assert out["api_base"] == "self-hosted.example.com:50051" assert "use_ssl" not in out @@ -1651,6 +1651,7 @@ class TestGetDynamicLitellmParamsClearsAdminConfigOnBaseOverride: }, request_kwargs={ "api_base": "https://attacker.example", + "api_key": "sk-caller", "organization": "org-attacker", "extra_body": {"attacker": "value"}, }, @@ -1674,6 +1675,7 @@ class TestGetDynamicLitellmParamsClearsAdminConfigOnBaseOverride: }, request_kwargs={ "api_base": "https://attacker.example", + "api_key": "sk-caller", "organization": "", "extra_body": "", }, @@ -1701,6 +1703,310 @@ class TestGetDynamicLitellmParamsClearsAdminConfigOnBaseOverride: assert out["api_version"] == "2026-04-01" assert out["api_base"] == "https://admin.upstream/v1" + def test_client_api_key_used_when_supplied_with_base_override(self): + from litellm.router_utils.clientside_credential_handler import ( + get_dynamic_litellm_params, + ) + + out = get_dynamic_litellm_params( + litellm_params={ + "model": "gpt-4", + "api_key": "sk-admin-secret", + "api_base": "https://admin.upstream/v1", + }, + request_kwargs={ + "api_base": "https://attacker.example", + "api_key": "sk-client-byok", + }, + ) + assert out["api_key"] == "sk-client-byok" + assert "sk-admin-secret" not in str(out) + + +_OPENAI_CHAT_RESPONSE = { + "id": "chatcmpl-x", + "object": "chat.completion", + "created": 1, + "model": "gpt-4", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +class TestClientsideBaseOverrideOutboundKey: + """Drive a completion through the router and assert on the outbound request + when the caller overrides ``api_base``.""" + + def _router(self): + from litellm import Router + + return Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "openai/gpt-4", + "api_key": "sk-SERVER-CONFIG", + "api_base": "https://admin.upstream/v1", + }, + } + ] + ) + + @pytest.fixture(autouse=True) + def _ambient_server_key(self, monkeypatch): + import litellm + + monkeypatch.setenv("OPENAI_API_KEY", "sk-SERVER-ENV") + monkeypatch.setattr(litellm, "api_key", None, raising=False) + + def test_caller_key_override_sends_caller_key_never_server_key(self): + import httpx + import respx + + with respx.mock: + route = respx.post("https://caller.example/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_OPENAI_CHAT_RESPONSE) + ) + self._router().completion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + api_base="https://caller.example/v1", + api_key="sk-CALLER", + ) + authorization = route.calls.last.request.headers.get("authorization") + assert authorization == "Bearer sk-CALLER" + assert "SERVER" not in (authorization or "") + + +def _rounds_deep_api_base_payload(rounds, field): + """Build a fallbacks payload with ``api_base`` on a target nested ``rounds`` + fallback-rounds deep, each round wrapped in its own grouping dict.""" + node = {"model": "leaf", "api_base": "https://attacker.example"} + for i in range(rounds): + node = {"model": f"m{i}", field: [{"grp": [node]}]} + return {"model": "gpt-4", field: [{"grp": [node]}]} + + +class TestIsRequestBodySafeBlocksFallbackSmuggle: + """``is_request_body_safe`` runs the banned-param check on every dict target + inside the fallback lists.""" + + @pytest.fixture(autouse=True) + def _disable_url_validation(self, monkeypatch): + import litellm + + monkeypatch.setattr(litellm, "user_url_validation", False, raising=False) + + @pytest.mark.parametrize( + "fallback_key", + ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"], + ) + def test_api_base_smuggled_via_nested_fallback_is_rejected(self, fallback_key): + with pytest.raises(ValueError, match="api_base"): + is_request_body_safe( + request_body={ + "model": "gpt-4", + fallback_key: [ + { + "gpt-4": [ + {"model": "evil", "api_base": "https://attacker.example"}, + ] + } + ], + }, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + + def test_string_only_fallbacks_are_accepted(self): + assert ( + is_request_body_safe( + request_body={ + "model": "gpt-4", + "fallbacks": [{"gpt-4": ["gpt-3.5-turbo", "claude-3-haiku"]}], + }, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + is True + ) + + def test_benign_dict_fallback_entry_is_accepted(self): + assert ( + is_request_body_safe( + request_body={ + "model": "gpt-4", + "fallbacks": [{"gpt-4": [{"model": "gpt-3.5-turbo"}]}], + }, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + is True + ) + + def test_smuggled_fallback_allowed_under_proxy_wide_opt_in(self): + assert ( + is_request_body_safe( + request_body={ + "model": "gpt-4", + "fallbacks": [ + {"gpt-4": [{"model": "byok", "api_base": "https://my-byok.example"}]} + ], + }, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="gpt-4", + ) + is True + ) + + @pytest.mark.parametrize( + "fallback_field", + ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"], + ) + @pytest.mark.parametrize("surface", ["top_level", "router_settings_override"]) + def test_deeply_nested_api_base_smuggle_rejected_on_both_surfaces(self, fallback_field, surface): + nested = [ + { + "always-fail": [ + { + "model": "x", + fallback_field: [ + {"x": [{"model": "deepseek-chat", "api_base": "http://attacker"}]} + ], + } + ] + } + ] + request_body = {"model": "gpt-4"} + if surface == "top_level": + request_body[fallback_field] = nested + else: + request_body["router_settings_override"] = {fallback_field: nested} + with pytest.raises(ValueError, match="api_base"): + is_request_body_safe( + request_body=request_body, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + + def test_router_settings_override_single_level_api_base_rejected(self): + with pytest.raises(ValueError, match="api_base"): + is_request_body_safe( + request_body={ + "model": "gpt-4", + "router_settings_override": { + "fallbacks": [{"gpt-4": [{"model": "x", "api_base": "http://attacker"}]}] + }, + }, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + + def test_model_less_config_dict_api_base_rejected(self): + with pytest.raises(ValueError, match="api_base"): + is_request_body_safe( + request_body={ + "model": "gpt-4", + "fallbacks": [{"gpt-4": [{"api_base": "http://attacker"}]}], + }, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + + def test_nested_api_base_caught_across_router_fallback_rounds(self): + """An ``api_base`` target nested ``ROUTER_MAX_FALLBACKS - 1`` rounds deep + is still reached and rejected.""" + import litellm + + with pytest.raises(ValueError, match="api_base"): + is_request_body_safe( + request_body=_rounds_deep_api_base_payload(litellm.ROUTER_MAX_FALLBACKS - 1, "fallbacks"), + general_settings={}, + llm_router=None, + model="gpt-4", + ) + + def test_grouping_only_deep_chain_is_rejected_at_depth_limit(self): + """A deep grouping-only chain (``{"g": [{"g": [...]}]}``) is rejected at the + validation-depth limit rather than accepted or raising RecursionError.""" + node: object = ["safe-model"] + for _ in range(5000): + node = [{"grp": node}] + with pytest.raises(ValueError, match="depth"): + is_request_body_safe( + request_body={"model": "gpt-4", "fallbacks": node}, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + + def test_pathologically_deep_model_nesting_is_rejected(self): + with pytest.raises(ValueError, match="depth"): + is_request_body_safe( + request_body=_rounds_deep_api_base_payload(5000, "fallbacks"), + general_settings={}, + llm_router=None, + model="gpt-4", + ) + + +class TestIsRequestBodySafeRejectsUrlValuedFallback: + @pytest.mark.parametrize("fallback_field", ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"]) + def test_url_valued_string_fallback_is_rejected(self, fallback_field): + with pytest.raises(ValueError, match="URL-valued fallback"): + is_request_body_safe( + request_body={ + "model": "gpt-4", + fallback_field: [{"gpt-4": ["huggingface/http://attacker.example/path"]}], + }, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + + @pytest.mark.parametrize("fallback_field", ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"]) + def test_url_valued_dict_model_fallback_is_rejected(self, fallback_field): + with pytest.raises(ValueError, match="URL-valued fallback"): + is_request_body_safe( + request_body={ + "model": "gpt-4", + fallback_field: [{"gpt-4": [{"model": "huggingface/http://attacker.example/path"}]}], + }, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + + def test_ordinary_string_fallback_is_allowed(self): + assert ( + is_request_body_safe( + request_body={"model": "gpt-4", "fallbacks": [{"gpt-4": ["gpt-4-backup"]}]}, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + is True + ) + + def test_ordinary_dict_model_fallback_is_allowed(self): + assert ( + is_request_body_safe( + request_body={"model": "gpt-4", "fallbacks": [{"gpt-4": [{"model": "gpt-4-backup"}]}]}, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + is True + ) + class TestIsRequestBodySafeBlocksEndpointTargetingFields: """ @@ -1823,6 +2129,46 @@ class TestIsRequestBodySafeBlocksBedrockProjectOverride: ) +class TestIsRequestBodySafeBlocksVertexCredentialAlias: + @pytest.mark.parametrize("field", ["vertex_ai_credentials"]) + def test_field_in_request_body_is_rejected(self, field): + with pytest.raises(ValueError, match=field): + is_request_body_safe( + request_body={"model": "gpt-4", field: "attacker-supplied"}, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + + @pytest.mark.parametrize("field", ["vertex_ai_credentials"]) + def test_admin_opt_in_proxy_wide_allows(self, field): + assert ( + is_request_body_safe( + request_body={"model": "gpt-4", field: "byok-supplied"}, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="gpt-4", + ) + is True + ) + + def test_legitimate_request_body_param_still_allowed(self): + assert ( + is_request_body_safe( + request_body={ + "model": "gpt-4", + "temperature": 0.7, + "max_tokens": 128, + "user": "end-user-123", + }, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + is True + ) + + class TestIsRequestBodySafeBlocksNVCFFunctionOverride: """``nvcf_function_id`` is rejected as a request-body param unless the admin opted in proxy-wide or per-deployment.""" diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index 13041950f98..ffc5241d027 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -458,6 +458,118 @@ async def test_auth_builder_non_proxy_admin_user_role(): assert result["user_id"] == "test_user_1" +@pytest.mark.asyncio +@pytest.mark.parametrize( + "row_email,expected_email", + [ + ("row@example.com", "row@example.com"), + (None, "claim@example.com"), + ("", "claim@example.com"), + ], +) +async def test_auth_builder_result_includes_user_email(row_email, expected_email): + """LIT-4238: auth_builder must return user_email (user row wins, JWT claim + is the fallback) so the auth object and metrics get the email.""" + api_key = "test_jwt_token" + request_data = {"model": "gpt-4"} + general_settings = {"enforce_rbac": False} + route = "/chat/completions" + + user_object = LiteLLM_UserTable( + user_id="test_user_1", + user_email=row_email, + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + jwt_handler = JWTHandler() + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth() + + with ( + patch.object(jwt_handler, "auth_jwt", new_callable=AsyncMock) as mock_auth_jwt, + patch.object(JWTAuthManager, "check_rbac_role", new_callable=AsyncMock), + patch.object(jwt_handler, "get_rbac_role", return_value=None), + patch.object(jwt_handler, "get_scopes", return_value=[]), + patch.object(jwt_handler, "get_object_id", return_value=None), + patch.object( + JWTAuthManager, + "get_user_info", + new_callable=AsyncMock, + return_value=("test_user_1", "claim@example.com", True), + ), + patch.object(jwt_handler, "get_org_id", return_value=None), + patch.object(jwt_handler, "get_end_user_id", return_value=None), + patch.object( + JWTAuthManager, + "check_admin_access", + new_callable=AsyncMock, + return_value=None, + ) as mock_check_admin, + patch.object( + JWTAuthManager, + "find_and_validate_specific_team_id", + new_callable=AsyncMock, + return_value=(None, None), + ), + patch.object(JWTAuthManager, "get_all_team_ids", return_value=set()), + patch.object( + JWTAuthManager, + "find_team_with_model_access", + new_callable=AsyncMock, + return_value=(None, None), + ), + patch.object( + JWTAuthManager, + "get_objects", + new_callable=AsyncMock, + return_value=(user_object, None, None, None, user_object.user_id), + ), + patch.object(JWTAuthManager, "map_user_to_teams", new_callable=AsyncMock), + patch.object(JWTAuthManager, "validate_object_id", return_value=True), + ): + mock_auth_jwt.return_value = {"sub": "test_user_1", "scope": ""} + + result = await JWTAuthManager.auth_builder( + api_key=api_key, + jwt_handler=jwt_handler, + request_data=request_data, + general_settings=general_settings, + route=route, + prisma_client=None, + user_api_key_cache=None, + parent_otel_span=None, + proxy_logging_obj=None, + ) + + assert result["user_email"] == expected_email + assert mock_check_admin.call_args.kwargs["user_email"] == "claim@example.com" + + +@pytest.mark.asyncio +async def test_check_admin_access_result_includes_user_email(): + """LIT-4238: the scope-based admin path has no user row, so the JWT claim + email must ride the JWTAuthBuilderResult.""" + jwt_handler = JWTHandler() + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( + admin_jwt_scope="litellm_proxy_admin", + admin_allowed_routes=["/chat/completions"], + ) + + result = await JWTAuthManager.check_admin_access( + jwt_handler=jwt_handler, + scopes=["litellm_proxy_admin"], + route="/chat/completions", + user_id="admin-user", + user_email="admin@example.com", + org_id=None, + api_key="test_jwt_token", + jwt_valid_token={"sub": "admin-user"}, + ) + + assert result is not None + assert result["is_proxy_admin"] is True + assert result["user_email"] == "admin@example.com" + + @pytest.mark.asyncio async def test_sync_user_role_and_teams(): from unittest.mock import MagicMock diff --git a/tests/test_litellm/proxy/auth/test_router_override_fallback_auth.py b/tests/test_litellm/proxy/auth/test_router_override_fallback_auth.py index fc0e9aec501..eb1135a240a 100644 --- a/tests/test_litellm/proxy/auth/test_router_override_fallback_auth.py +++ b/tests/test_litellm/proxy/auth/test_router_override_fallback_auth.py @@ -11,12 +11,22 @@ from unittest.mock import AsyncMock, patch import pytest from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.auth_utils import iter_request_fallback_targets from litellm.proxy.auth.user_api_key_auth import ( _enforce_key_and_fallback_model_access, - iter_router_fallback_model_names, + _fallback_target_model_name, ) +def _fallback_model_names(fallbacks): + """Model names the auth check validates for a top-level ``fallbacks`` value.""" + return [ + name + for target in iter_request_fallback_targets({"fallbacks": fallbacks}) + if (name := _fallback_target_model_name(target)) is not None + ] + + def _key_with_models(models: List[str]) -> UserAPIKeyAuth: return UserAPIKeyAuth( api_key="hashed", @@ -26,37 +36,40 @@ def _key_with_models(models: List[str]) -> UserAPIKeyAuth: ) -# ── iter_router_fallback_model_names ───────────────────────────────────────── +# ── fallback model-name extraction ─────────────────────────────────────────── -def testiter_router_fallback_model_names_router_config_shape(): +def test_fallback_model_names_router_config_shape(): """Router-config shape: ``[{primary: [fallback_list]}]``.""" - assert list( - iter_router_fallback_model_names( - [{"gpt-3.5-turbo": ["gpt-4", "claude-3"]}, {"gpt-4o": ["o1"]}] - ) + assert _fallback_model_names( + [{"gpt-3.5-turbo": ["gpt-4", "claude-3"]}, {"gpt-4o": ["o1"]}] ) == ["gpt-4", "claude-3", "o1"] -def testiter_router_fallback_model_names_simple_string_shape(): +def test_fallback_model_names_simple_string_shape(): """Simple top-level shape: list of strings.""" - assert list(iter_router_fallback_model_names(["gpt-4", "claude-3"])) == [ + assert _fallback_model_names(["gpt-4", "claude-3"]) == ["gpt-4", "claude-3"] + + +def test_fallback_model_names_client_side_shape(): + """ClientSideFallbackModel shape: ``[{"model": "..."}]``.""" + assert _fallback_model_names([{"model": "gpt-4"}, {"model": "claude-3"}]) == [ "gpt-4", "claude-3", ] -def testiter_router_fallback_model_names_client_side_shape(): - """ClientSideFallbackModel shape: ``[{"model": "..."}]``.""" - assert list( - iter_router_fallback_model_names([{"model": "gpt-4"}, {"model": "claude-3"}]) - ) == ["gpt-4", "claude-3"] +def test_fallback_model_names_nested_deployment_fallbacks(): + """A deployment target's own nested fallback field is unrolled too.""" + assert _fallback_model_names( + [{"primary": [{"model": "gpt-4", "fallbacks": [{"gpt-4": ["deepseek-chat"]}]}]}] + ) == ["gpt-4", "deepseek-chat"] -def testiter_router_fallback_model_names_empty_or_none(): - assert list(iter_router_fallback_model_names(None)) == [] - assert list(iter_router_fallback_model_names([])) == [] - assert list(iter_router_fallback_model_names("not a list")) == [] +def test_fallback_model_names_empty_or_none(): + assert _fallback_model_names(None) == [] + assert _fallback_model_names([]) == [] + assert _fallback_model_names("not a list") == [] # ── _enforce_key_and_fallback_model_access ──────────────────────────────────── @@ -200,6 +213,98 @@ async def test_top_level_fallback_fields_validated(fallback_field): assert "top-level-smuggled" in seen +@pytest.mark.asyncio +async def test_nested_deployment_fallback_inner_model_validated(): + """A model name nested several fallback rounds deep, inside a deployment + target's own ``fallbacks``, is extracted and passed to can_key_call_model.""" + valid_token = _key_with_models(["gpt-3.5-turbo"]) + request_data = { + "model": "gpt-3.5-turbo", + "fallbacks": [ + { + "gpt-3.5-turbo": [ + { + "model": "gpt-3.5-turbo", + "fallbacks": [{"gpt-3.5-turbo": ["deep-smuggled-model"]}], + } + ] + } + ], + } + + seen: List[str] = [] + + async def fake_can_key_call_model(model, llm_model_list, valid_token, llm_router): + seen.append(model) + + with ( + patch( + "litellm.proxy.auth.user_api_key_auth.can_key_call_model", + side_effect=fake_can_key_call_model, + ), + patch( + "litellm.proxy.auth.user_api_key_auth.is_valid_fallback_model", + new=AsyncMock(), + ), + ): + await _enforce_key_and_fallback_model_access( + valid_token=valid_token, + request_data=request_data, + route="/v1/chat/completions", + request=None, + llm_model_list=None, + llm_router=None, + ) + + assert "deep-smuggled-model" in seen + + +@pytest.mark.asyncio +async def test_model_less_fallback_dict_is_skipped_never_passed_as_none(): + """A fallback target dict without a ``model`` key is skipped, never passed + as ``None`` into can_key_call_model / is_valid_fallback_model.""" + valid_token = _key_with_models(["gpt-3.5-turbo"]) + request_data = { + "model": "gpt-3.5-turbo", + "fallbacks": [ + { + "gpt-3.5-turbo": [ + {"model": "real-fallback"}, + {"api_base": "http://attacker"}, + "string-fallback", + ] + } + ], + } + + seen: List[str] = [] + + async def fake_can_key_call_model(model, llm_model_list, valid_token, llm_router): + seen.append(model) + + with ( + patch( + "litellm.proxy.auth.user_api_key_auth.can_key_call_model", + side_effect=fake_can_key_call_model, + ), + patch( + "litellm.proxy.auth.user_api_key_auth.is_valid_fallback_model", + new=AsyncMock(), + ), + ): + await _enforce_key_and_fallback_model_access( + valid_token=valid_token, + request_data=request_data, + route="/v1/chat/completions", + request=None, + llm_model_list=None, + llm_router=None, + ) + + assert None not in seen + assert seen == ["gpt-3.5-turbo", "real-fallback", "string-fallback"] + + @pytest.mark.asyncio async def test_router_override_without_fallbacks_does_not_break_auth(): """``router_settings_override`` set without any fallback fields is a diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 59b8228530a..2c1948adca1 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -1569,6 +1569,7 @@ class TestJWTOAuth2Coexistence: "token": jwt_token, "team_id": "jwt-team", "user_id": "jwt-human-user", + "user_email": None, "end_user_id": None, "org_id": None, "team_membership": None, @@ -1643,6 +1644,7 @@ class TestJWTOAuth2Coexistence: "token": jwt_token, "team_id": "validated-team", "user_id": "validated-user", + "user_email": "validated@example.com", "end_user_id": "validated-end-user", "org_id": "validated-org", "team_membership": None, @@ -1702,6 +1704,7 @@ class TestJWTOAuth2Coexistence: mock_auto_register.call_args.kwargs["end_user_id"] == "validated-end-user" ) assert result.org_id == "validated-org" + assert result.user_email == "validated@example.com" @pytest.mark.asyncio async def test_routing_override_routes_matching_jwt_to_oauth2(self): @@ -1788,6 +1791,7 @@ class TestJWTOAuth2Coexistence: "token": jwt_token, "team_id": "jwt-team", "user_id": "jwt-user-no-override", + "user_email": None, "end_user_id": None, "org_id": None, "team_membership": None, @@ -1988,6 +1992,7 @@ class TestJWTOAuth2Coexistence: "token": jwt_token, "team_id": "jwt-team", "user_id": "jwt-user-scope-mismatch", + "user_email": None, "end_user_id": None, "org_id": None, "team_membership": None, @@ -2296,6 +2301,7 @@ class TestJWTOAuth2Coexistence: "token": jwt_token, "team_id": None, "user_id": "jwt-admin-user", + "user_email": None, "end_user_id": None, "org_id": None, "team_membership": None, @@ -4255,6 +4261,98 @@ async def test_auth_does_not_rewrite_cached_key_object_back_into_cache(): setattr(_proxy_server_mod, k, v) +class TestJWTAuthUserEmail: + """JWT auth must populate `UserAPIKeyAuth.user_email` (LIT-4238); it feeds + the Prometheus `user_email` label and `user_api_key_user_email` in + StandardLogging/SpendLogs metadata, which were always None for JWT traffic.""" + + def _jwt_request(self, jwt_token): + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.method = "POST" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + return mock_request + + async def _run_jwt_auth(self, mock_jwt_result, jwt_token): + with ( + patch( + "litellm.proxy.proxy_server.general_settings", + {"enable_jwt_auth": True}, + ), + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.proxy_server.master_key", "sk-master"), + patch("litellm.proxy.proxy_server.prisma_client", None), + patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + return_value=mock_jwt_result, + ), + ): + litellm.proxy.proxy_server.jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth(), + ) + return await user_api_key_auth( + request=self._jwt_request(jwt_token), + api_key=f"Bearer {jwt_token}", + ) + + @pytest.mark.asyncio + async def test_jwt_auth_populates_user_email_on_valid_token(self): + jwt_token = "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ1c2VyMSJ9.signature" + mock_jwt_result = { + "is_proxy_admin": False, + "team_object": None, + "user_object": LiteLLM_UserTable( + user_id="jwt-human-user", + user_email="row@example.com", + user_role=LitellmUserRoles.INTERNAL_USER.value, + ), + "end_user_object": None, + "org_object": None, + "token": jwt_token, + "team_id": None, + "user_id": "jwt-human-user", + "user_email": "resolved@example.com", + "end_user_id": None, + "org_id": None, + "team_membership": None, + "jwt_claims": {"sub": "user1"}, + } + + result = await self._run_jwt_auth(mock_jwt_result, jwt_token) + + assert result.user_id == "jwt-human-user" + assert result.user_email == "resolved@example.com" + + @pytest.mark.asyncio + async def test_jwt_auth_populates_user_email_on_proxy_admin(self): + jwt_token = "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ1c2VyMSJ9.signature" + mock_jwt_result = { + "is_proxy_admin": True, + "team_object": None, + "user_object": None, + "end_user_object": None, + "org_object": None, + "token": jwt_token, + "team_id": None, + "user_id": "jwt-admin-user", + "user_email": "admin@example.com", + "end_user_id": None, + "org_id": None, + "team_membership": None, + "jwt_claims": {"sub": "user1"}, + } + + result = await self._run_jwt_auth(mock_jwt_result, jwt_token) + + assert result.user_role == LitellmUserRoles.PROXY_ADMIN + assert result.user_id == "jwt-admin-user" + assert result.user_email == "admin@example.com" + + class TestCheckKeyModelBudgetWithFallback: """`_check_key_model_budget_with_fallback` must reroute a request to the first configured `budget_fallbacks` entry still within its own budget, @@ -4529,6 +4627,9 @@ async def test_temp_budget_increase_applied_for_cached_key(): Seed the auth cache with a key whose spend (5.0) exceeds its original max_budget (2.0) but is under the effective budget (2.0 + 100.0). The cache-hit request must not raise and the resolved token must carry max_budget == 102.0. + + Resolving twice must yield 102.0 both times and leave the cached object at the + original 2.0: the increase is derived per request, never compounded or persisted. """ from datetime import datetime, timedelta @@ -4574,14 +4675,22 @@ async def test_temp_budget_increase_applied_for_cached_key(): new_callable=AsyncMock, ), ): - result = await _user_api_key_auth_builder( - request=mock_request, - api_key=f"Bearer {api_key}", - azure_api_key_header="", - anthropic_api_key_header=None, - google_ai_studio_api_key_header=None, - azure_apim_header=None, - request_data={"model": "gpt-4o-mini"}, + results = tuple( + [ + await _user_api_key_auth_builder( + request=mock_request, + api_key=f"Bearer {api_key}", + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data={"model": "gpt-4o-mini"}, + ) + for _ in range(2) + ] ) - assert result.max_budget == 102.0 + assert all(result.max_budget == 102.0 for result in results) + + cached_after = await user_api_key_cache.async_get_cache(key=hashed_token) + assert cached_after.max_budget == 2.0 diff --git a/tests/test_litellm/proxy/common_utils/test_callback_utils.py b/tests/test_litellm/proxy/common_utils/test_callback_utils.py index 36ff3f3c399..8f390c096d7 100644 --- a/tests/test_litellm/proxy/common_utils/test_callback_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_callback_utils.py @@ -84,6 +84,52 @@ def test_process_callback_with_no_required_env_vars(mock_get_env_vars): assert result["variables"] == {} +@patch( + "litellm.proxy.common_utils.callback_utils.CustomLogger.get_callback_env_vars", + return_value=["LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY"], +) +def test_process_callback_falls_back_to_process_env(mock_get_env_vars, monkeypatch): + """A callback env var set only in the process env must be surfaced. + + The logging integrations read their config from the process environment, so a + callback configured purely via env vars (IaC) is live even with no stored + entry. Reporting it as unset makes a working callback read as unconfigured. + """ + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "env-public-key") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "env-secret-key") + # stored config only carries the public key; the secret is env-only + environment_variables = {"LANGFUSE_PUBLIC_KEY": "db-public-key"} + + result = process_callback( + _callback="langfuse", + callback_type="success", + environment_variables=environment_variables, + ) + + # stored value wins; the env-only var is resolved rather than reported None + assert result["variables"] == { + "LANGFUSE_PUBLIC_KEY": "db-public-key", + "LANGFUSE_SECRET_KEY": "env-secret-key", + } + + +@patch( + "litellm.proxy.common_utils.callback_utils.CustomLogger.get_callback_env_vars", + return_value=["LANGFUSE_SECRET_KEY"], +) +def test_process_callback_reports_none_when_absent_everywhere(mock_get_env_vars, monkeypatch): + """A var set in neither the stored config nor the process env stays None.""" + monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False) + + result = process_callback( + _callback="langfuse", + callback_type="success", + environment_variables={}, + ) + + assert result["variables"] == {"LANGFUSE_SECRET_KEY": None} + + def test_normalize_callback_names_none_returns_empty_list(): assert normalize_callback_names(None) == [] assert normalize_callback_names([]) == [] diff --git a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py index 5e348b1bb7e..be5bc74c385 100644 --- a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py +++ b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py @@ -5,25 +5,23 @@ import sys import time import types from datetime import datetime, timedelta, timezone +from datetime import time as dt_time from typing import Any, Dict, List from unittest.mock import AsyncMock, MagicMock import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path +sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path from litellm._logging import verbose_proxy_logger from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob +from litellm.proxy.common_utils.timezone_utils import BudgetResetSettings from litellm.proxy.utils import ProxyLogging # Mock classes for testing class MockLiteLLMTeamMembership: - async def update_many( - self, where: Dict[str, Any], data: Dict[str, Any] - ) -> Dict[str, Any]: + async def update_many(self, where: Dict[str, Any], data: Dict[str, Any]) -> Dict[str, Any]: # Mock the update_many method for litellm_teammembership return {"count": 1} @@ -32,9 +30,7 @@ class MockLiteLLMVerificationToken: def __init__(self): self.update_many_calls: List[Dict[str, Any]] = [] - async def update_many( - self, where: Dict[str, Any], data: Dict[str, Any] - ) -> Dict[str, Any]: + async def update_many(self, where: Dict[str, Any], data: Dict[str, Any]) -> Dict[str, Any]: self.update_many_calls.append({"where": where, "data": data}) return {"count": 1} @@ -52,9 +48,7 @@ class MockLiteLLMOrganizationTable: self.find_many_calls.append({"where": where}) return self._find_many_results - async def update_many( - self, where: Dict[str, Any], data: Dict[str, Any] - ) -> Dict[str, Any]: + async def update_many(self, where: Dict[str, Any], data: Dict[str, Any]) -> Dict[str, Any]: self.update_many_calls.append({"where": where, "data": data}) return {"count": 1} @@ -72,9 +66,7 @@ class MockLiteLLMTagTable: self.find_many_calls.append({"where": where}) return self._find_many_results - async def update_many( - self, where: Dict[str, Any], data: Dict[str, Any] - ) -> Dict[str, Any]: + async def update_many(self, where: Dict[str, Any], data: Dict[str, Any]) -> Dict[str, Any]: self.update_many_calls.append({"where": where, "data": data}) return {"count": 1} @@ -110,9 +102,7 @@ class MockBatcher: _self._outer = outer def update(_self, where, data): - _self._outer.calls.append( - {"table": _self._table_name, "where": where, "data": data} - ) + _self._outer.calls.append({"table": _self._table_name, "where": where, "data": data}) self.litellm_verificationtoken = _Table("key", self) self.litellm_usertable = _Table("user", self) @@ -172,11 +162,7 @@ class MockPrismaClient: return [item for item in data if hasattr(item, "budget_reset_at")] # Handle specific filtering for enduser table queries - if ( - table_name == "enduser" - and query_type == "find_all" - and "budget_id_list" in kwargs - ): + if table_name == "enduser" and query_type == "find_all" and "budget_id_list" in kwargs: budget_id_list = kwargs["budget_id_list"] # Return endusers that match the budget IDs return [ @@ -188,11 +174,7 @@ class MockPrismaClient: ] # Handle key queries with expires and reset_at - if ( - table_name == "key" - and query_type == "find_all" - and ("expires" in kwargs or "reset_at" in kwargs) - ): + if table_name == "key" and query_type == "find_all" and ("expires" in kwargs or "reset_at" in kwargs): return [item for item in data if hasattr(item, "budget_reset_at")] return data @@ -227,9 +209,7 @@ def mock_proxy_logging(): @pytest.fixture def reset_budget_job(mock_prisma_client, mock_proxy_logging): - return ResetBudgetJob( - proxy_logging_obj=mock_proxy_logging, prisma_client=mock_prisma_client - ) + return ResetBudgetJob(proxy_logging_obj=mock_proxy_logging, prisma_client=mock_prisma_client) # Helper function to run async tests @@ -270,6 +250,40 @@ def test_reset_budget_for_key(reset_budget_job, mock_prisma_client): assert set(write["data"].keys()) == {"spend", "budget_reset_at"} +def test_reset_budget_for_key_honors_injected_reset_time(mock_prisma_client, mock_proxy_logging): + """Injected BudgetResetSettings drives the written reset time end to end (DI, no globals). + + Before the configurable-reset-time change this wrote a midnight reset_at (hour 0); + with noon injected it must write a noon reset_at. + """ + job = ResetBudgetJob( + proxy_logging_obj=mock_proxy_logging, + prisma_client=mock_prisma_client, + reset_settings=BudgetResetSettings(timezone="UTC", reset_time_of_day=dt_time(12, 0)), + ) + now = datetime.now(timezone.utc) + test_key = type( + "LiteLLM_VerificationToken", + (), + { + "spend": 100.0, + "budget_duration": "1d", + "budget_reset_at": now, + "id": "test-key-noon", + "token": "tok-noon", + }, + ) + mock_prisma_client.data["key"] = [test_key] + + asyncio.run(job.reset_budget_for_litellm_keys()) + + key_writes = [c for c in mock_prisma_client.db.batch_calls if c["table"] == "key"] + assert len(key_writes) == 1 + reset_at = key_writes[0]["data"]["budget_reset_at"].astimezone(timezone.utc) + assert reset_at.hour == 12 + assert reset_at.minute == 0 + + def test_reset_budget_for_user(reset_budget_job, mock_prisma_client): # Setup test data with timezone-aware datetime now = datetime.now(timezone.utc) @@ -486,11 +500,7 @@ def test_reset_budget_for_keys_linked_to_budgets(reset_budget_job, mock_prisma_c budgets_to_reset = [test_budget] # Run the method - asyncio.run( - reset_budget_job.reset_budget_for_keys_linked_to_budgets( - budgets_to_reset=budgets_to_reset - ) - ) + asyncio.run(reset_budget_job.reset_budget_for_keys_linked_to_budgets(budgets_to_reset=budgets_to_reset)) # Verify that update_many was called on litellm_verificationtoken calls = mock_prisma_client.db.litellm_verificationtoken.update_many_calls @@ -531,11 +541,7 @@ def test_reset_budget_for_keys_linked_to_budgets_excludes_keys_with_own_budget_d budgets_to_reset = [test_budget] - asyncio.run( - reset_budget_job.reset_budget_for_keys_linked_to_budgets( - budgets_to_reset=budgets_to_reset - ) - ) + asyncio.run(reset_budget_job.reset_budget_for_keys_linked_to_budgets(budgets_to_reset=budgets_to_reset)) calls = mock_prisma_client.db.litellm_verificationtoken.update_many_calls assert len(calls) == 1 @@ -548,17 +554,13 @@ def test_reset_budget_for_keys_linked_to_budgets_excludes_keys_with_own_budget_d assert call["where"]["budget_id"] == {"in": ["7d-budget-tier"]} -def test_reset_budget_for_keys_linked_to_budgets_empty( - reset_budget_job, mock_prisma_client -): +def test_reset_budget_for_keys_linked_to_budgets_empty(reset_budget_job, mock_prisma_client): """ Test that when there are no budgets to reset, no update is performed on the verification token table. """ # Run with empty list - asyncio.run( - reset_budget_job.reset_budget_for_keys_linked_to_budgets(budgets_to_reset=[]) - ) + asyncio.run(reset_budget_job.reset_budget_for_keys_linked_to_budgets(budgets_to_reset=[])) # Verify no update_many calls were made calls = mock_prisma_client.db.litellm_verificationtoken.update_many_calls @@ -584,11 +586,7 @@ def test_reset_budget_for_orgs_linked_to_budgets(reset_budget_job, mock_prisma_c }, ) - asyncio.run( - reset_budget_job.reset_budget_for_orgs_linked_to_budgets( - budgets_to_reset=[test_budget] - ) - ) + asyncio.run(reset_budget_job.reset_budget_for_orgs_linked_to_budgets(budgets_to_reset=[test_budget])) calls = mock_prisma_client.db.litellm_organizationtable.update_many_calls assert len(calls) == 1 @@ -598,16 +596,12 @@ def test_reset_budget_for_orgs_linked_to_budgets(reset_budget_job, mock_prisma_c assert call["data"]["spend"] == 0 -def test_reset_budget_for_orgs_linked_to_budgets_empty( - reset_budget_job, mock_prisma_client -): +def test_reset_budget_for_orgs_linked_to_budgets_empty(reset_budget_job, mock_prisma_client): """ Test that when there are no budgets to reset, no update is performed on the organization table. """ - asyncio.run( - reset_budget_job.reset_budget_for_orgs_linked_to_budgets(budgets_to_reset=[]) - ) + asyncio.run(reset_budget_job.reset_budget_for_orgs_linked_to_budgets(budgets_to_reset=[])) calls = mock_prisma_client.db.litellm_organizationtable.update_many_calls assert len(calls) == 0 @@ -631,11 +625,7 @@ def test_reset_budget_for_tags_linked_to_budgets(reset_budget_job, mock_prisma_c }, ) - asyncio.run( - reset_budget_job.reset_budget_for_tags_linked_to_budgets( - budgets_to_reset=[test_budget] - ) - ) + asyncio.run(reset_budget_job.reset_budget_for_tags_linked_to_budgets(budgets_to_reset=[test_budget])) calls = mock_prisma_client.db.litellm_tagtable.update_many_calls assert len(calls) == 1 @@ -645,16 +635,12 @@ def test_reset_budget_for_tags_linked_to_budgets(reset_budget_job, mock_prisma_c assert call["data"]["spend"] == 0 -def test_reset_budget_for_tags_linked_to_budgets_empty( - reset_budget_job, mock_prisma_client -): +def test_reset_budget_for_tags_linked_to_budgets_empty(reset_budget_job, mock_prisma_client): """ Test that when there are no budgets to reset, no update is performed on the tag table. """ - asyncio.run( - reset_budget_job.reset_budget_for_tags_linked_to_budgets(budgets_to_reset=[]) - ) + asyncio.run(reset_budget_job.reset_budget_for_tags_linked_to_budgets(budgets_to_reset=[])) calls = mock_prisma_client.db.litellm_tagtable.update_many_calls assert len(calls) == 0 @@ -668,9 +654,7 @@ def test_reset_budget_for_tags_linked_to_budgets_empty( ], ids=["30d-calendar-month", "1mo-calendar-month", "1d-next-midnight"], ) -def test_reset_budget_reset_at_date_calendar_aligned( - budget_duration, expected_day, expected_month -): +def test_reset_budget_reset_at_date_calendar_aligned(budget_duration, expected_day, expected_month): """ Verify that _reset_budget_reset_at_date produces calendar-aligned reset times (matching get_budget_reset_time), not sliding-window offsets. @@ -694,7 +678,7 @@ def test_reset_budget_reset_at_date_calendar_aligned( with patch("litellm.proxy.common_utils.timezone_utils.datetime") as mock_dt: mock_dt.now.return_value = fixed_now mock_dt.side_effect = lambda *args, **kwargs: datetime(*args, **kwargs) - asyncio.run(ResetBudgetJob._reset_budget_reset_at_date(test_budget, fixed_now)) + asyncio.run(ResetBudgetJob._reset_budget_reset_at_date(test_budget, fixed_now, BudgetResetSettings())) assert test_budget.budget_reset_at.day == expected_day assert test_budget.budget_reset_at.month == expected_month @@ -724,7 +708,7 @@ def test_reset_budget_reset_at_date_7d_next_monday(): with patch("litellm.proxy.common_utils.timezone_utils.datetime") as mock_dt: mock_dt.now.return_value = fixed_now mock_dt.side_effect = lambda *args, **kwargs: datetime(*args, **kwargs) - asyncio.run(ResetBudgetJob._reset_budget_reset_at_date(test_budget, fixed_now)) + asyncio.run(ResetBudgetJob._reset_budget_reset_at_date(test_budget, fixed_now, BudgetResetSettings())) # Next Monday after Wednesday June 14 is June 19 assert test_budget.budget_reset_at.day == 19 @@ -749,7 +733,7 @@ def test_reset_budget_reset_at_date_none_duration(): }, ) - asyncio.run(ResetBudgetJob._reset_budget_reset_at_date(test_budget, now)) + asyncio.run(ResetBudgetJob._reset_budget_reset_at_date(test_budget, now, BudgetResetSettings())) assert test_budget.budget_reset_at == original_reset_at @@ -773,7 +757,7 @@ def test_reset_budget_reset_at_date_none_reset_at(): with patch("litellm.proxy.common_utils.timezone_utils.datetime") as mock_dt: mock_dt.now.return_value = fixed_now mock_dt.side_effect = lambda *args, **kwargs: datetime(*args, **kwargs) - asyncio.run(ResetBudgetJob._reset_budget_reset_at_date(test_budget, fixed_now)) + asyncio.run(ResetBudgetJob._reset_budget_reset_at_date(test_budget, fixed_now, BudgetResetSettings())) # Should be set to 1st of next month (July 1) assert test_budget.budget_reset_at is not None @@ -781,9 +765,7 @@ def test_reset_budget_reset_at_date_none_reset_at(): assert test_budget.budget_reset_at.month == 7 -def test_budget_table_reset_also_resets_linked_keys( - reset_budget_job, mock_prisma_client -): +def test_budget_table_reset_also_resets_linked_keys(reset_budget_job, mock_prisma_client): """ Integration-style test: when reset_budget_for_litellm_budget_table runs, it should also reset spend for keys linked to the expiring budget tiers @@ -818,9 +800,7 @@ def test_budget_table_reset_also_resets_linked_keys( assert calls[0]["data"]["spend"] == 0 -def test_budget_table_reset_also_resets_linked_orgs( - reset_budget_job, mock_prisma_client -): +def test_budget_table_reset_also_resets_linked_orgs(reset_budget_job, mock_prisma_client): """ Integration-style test: when reset_budget_for_litellm_budget_table runs, it should also reset spend for orgs linked to the expiring budget tiers @@ -853,9 +833,7 @@ def test_budget_table_reset_also_resets_linked_orgs( assert calls[0]["data"]["spend"] == 0 -def test_budget_table_reset_also_resets_linked_tags( - reset_budget_job, mock_prisma_client -): +def test_budget_table_reset_also_resets_linked_tags(reset_budget_job, mock_prisma_client): """ Integration-style test: when reset_budget_for_litellm_budget_table runs, it should also reset spend for tags linked to the expiring budget tiers. @@ -887,9 +865,7 @@ def test_budget_table_reset_also_resets_linked_tags( assert calls[0]["data"]["spend"] == 0 -def test_reset_budget_resets_endusers_with_null_budget_id( - reset_budget_job, mock_prisma_client -): +def test_reset_budget_resets_endusers_with_null_budget_id(reset_budget_job, mock_prisma_client): """ When litellm.max_end_user_budget_id is configured and that budget is being reset, end users with budget_id=NULL should also have their spend @@ -959,17 +935,13 @@ def test_reset_budget_resets_endusers_with_null_budget_id( mock_prisma_client.data["enduser"] = [enduser_with_budget] # Set up the DB mock for NULL-budget-id end users - mock_prisma_client.db.litellm_endusertable.set_find_many_results( - [enduser_no_budget_row] - ) + mock_prisma_client.db.litellm_endusertable.set_find_many_results([enduser_no_budget_row]) asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) # Both end users should have been reset updated = mock_prisma_client.updated_data["enduser"] - assert ( - len(updated) == 2 - ), f"Expected 2 endusers reset (1 explicit + 1 implicit), got {len(updated)}" + assert len(updated) == 2, f"Expected 2 endusers reset (1 explicit + 1 implicit), got {len(updated)}" user_ids = {u.user_id for u in updated} assert "enduser-explicit" in user_ids @@ -986,9 +958,7 @@ def test_reset_budget_resets_endusers_with_null_budget_id( litellm.max_end_user_budget_id = None -def test_reset_budget_skips_null_budget_id_endusers_when_default_not_configured( - reset_budget_job, mock_prisma_client -): +def test_reset_budget_skips_null_budget_id_endusers_when_default_not_configured(reset_budget_job, mock_prisma_client): """ When litellm.max_end_user_budget_id is NOT configured, end users with budget_id=NULL should NOT be fetched or reset. @@ -1073,20 +1043,14 @@ def test_reset_budget_for_team_members_preserves_total_spend(): mock_prisma_client = MagicMock() mock_prisma_client.db.litellm_teammembership.find_many = AsyncMock(return_value=[]) - mock_prisma_client.db.litellm_teammembership.update_many = AsyncMock( - return_value={"count": 1} - ) + mock_prisma_client.db.litellm_teammembership.update_many = AsyncMock(return_value={"count": 1}) - job = ResetBudgetJob( - proxy_logging_obj=MagicMock(), prisma_client=mock_prisma_client - ) + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=mock_prisma_client) asyncio.run(job.reset_budget_for_litellm_team_members([expired_budget])) mock_prisma_client.db.litellm_teammembership.update_many.assert_called_once() - call_kwargs = ( - mock_prisma_client.db.litellm_teammembership.update_many.call_args.kwargs - ) + call_kwargs = mock_prisma_client.db.litellm_teammembership.update_many.call_args.kwargs assert call_kwargs["where"]["budget_id"]["in"] == ["budget-1"] assert call_kwargs["data"] == {"spend": 0} assert "total_spend" not in call_kwargs["data"] @@ -1142,9 +1106,7 @@ def test_reset_budget_windows_uses_is_not_null_filter(monkeypatch): raises `MissingRequiredValueError`. We work around it by using `query_raw` with `IS NOT NULL`. If someone reverts to the ORM filter, this test fails. """ - job, prisma_client, _ = _make_reset_budget_windows_job( - monkeypatch, key_rows=[], team_rows=[] - ) + job, prisma_client, _ = _make_reset_budget_windows_job(monkeypatch, key_rows=[], team_rows=[]) asyncio.run(job.reset_budget_windows()) @@ -1184,15 +1146,11 @@ def test_reset_budget_windows_resets_expired_key_window(monkeypatch): # The `budget_limits` payload is re-serialized JSON with a bumped reset_at. written_windows = json.loads(call_kwargs["data"]["budget_limits"]) assert len(written_windows) == 1 - new_reset_at = datetime.fromisoformat( - written_windows[0]["reset_at"].replace("Z", "+00:00") - ).replace(tzinfo=None) + new_reset_at = datetime.fromisoformat(written_windows[0]["reset_at"].replace("Z", "+00:00")).replace(tzinfo=None) assert new_reset_at > now # The spend counter for this key+window was cleared. - spend_counter_cache.in_memory_cache.set_cache.assert_any_call( - key="spend:key:sk-expired:window:1d", value=0.0 - ) + spend_counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:key:sk-expired:window:1d", value=0.0) def test_reset_budget_windows_skips_unexpired_key_window(monkeypatch): @@ -1206,9 +1164,7 @@ def test_reset_budget_windows_skips_unexpired_key_window(monkeypatch): "budget_limits": [{"budget_duration": "1d", "reset_at": future}], } ] - job, prisma_client, _ = _make_reset_budget_windows_job( - monkeypatch, key_rows=key_rows, team_rows=[] - ) + job, prisma_client, _ = _make_reset_budget_windows_job(monkeypatch, key_rows=key_rows, team_rows=[]) asyncio.run(job.reset_budget_windows()) @@ -1237,9 +1193,7 @@ def test_reset_budget_windows_resets_expired_team_window(monkeypatch): assert call_kwargs["where"] == {"team_id": "team-expired"} assert "budget_limits" in call_kwargs["data"] - spend_counter_cache.in_memory_cache.set_cache.assert_any_call( - key="spend:team:team-expired:window:30d", value=0.0 - ) + spend_counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:team:team-expired:window:30d", value=0.0) def test_reset_budget_windows_handles_string_budget_limits(monkeypatch): @@ -1252,14 +1206,10 @@ def test_reset_budget_windows_handles_string_budget_limits(monkeypatch): key_rows = [ { "token": "sk-string-limits", - "budget_limits": json.dumps( - [{"budget_duration": "1d", "reset_at": expired}] - ), + "budget_limits": json.dumps([{"budget_duration": "1d", "reset_at": expired}]), } ] - job, prisma_client, _ = _make_reset_budget_windows_job( - monkeypatch, key_rows=key_rows, team_rows=[] - ) + job, prisma_client, _ = _make_reset_budget_windows_job(monkeypatch, key_rows=key_rows, team_rows=[]) asyncio.run(job.reset_budget_windows()) @@ -1274,9 +1224,7 @@ def test_reset_budget_windows_skips_row_with_empty_budget_limits(monkeypatch): {"token": "sk-empty-list", "budget_limits": []}, {"token": "sk-empty-str", "budget_limits": ""}, ] - job, prisma_client, _ = _make_reset_budget_windows_job( - monkeypatch, key_rows=key_rows, team_rows=[] - ) + job, prisma_client, _ = _make_reset_budget_windows_job(monkeypatch, key_rows=key_rows, team_rows=[]) asyncio.run(job.reset_budget_windows()) @@ -1361,27 +1309,17 @@ def test_reset_budget_for_team_members_invalidates_redis_counter(monkeypatch): ) prisma_client = MagicMock() - prisma_client.db.litellm_teammembership.find_many = AsyncMock( - return_value=[membership] - ) - prisma_client.db.litellm_teammembership.update_many = AsyncMock( - return_value={"count": 1} - ) + prisma_client.db.litellm_teammembership.find_many = AsyncMock(return_value=[membership]) + prisma_client.db.litellm_teammembership.update_many = AsyncMock(return_value={"count": 1}) job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) asyncio.run(job.reset_budget_for_litellm_team_members([expired_budget])) - counter_cache.in_memory_cache.set_cache.assert_any_call( - key="spend:team_member:alice:team-x", value=0.0, ttl=60 - ) - counter_cache.redis_cache.async_set_cache.assert_any_await( - key="spend:team_member:alice:team-x", value=0.0, ttl=60 - ) + counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:team_member:alice:team-x", value=0.0, ttl=60) + counter_cache.redis_cache.async_set_cache.assert_any_await(key="spend:team_member:alice:team-x", value=0.0, ttl=60) -def test_reset_budget_for_keys_invalidates_redis_counter( - reset_budget_job, mock_prisma_client, monkeypatch -): +def test_reset_budget_for_keys_invalidates_redis_counter(reset_budget_job, mock_prisma_client, monkeypatch): """Key budget reset must clear the Redis spend counter.""" counter_cache = _make_counter_invalidation_job(monkeypatch) @@ -1402,14 +1340,10 @@ def test_reset_budget_for_keys_invalidates_redis_counter( asyncio.run(reset_budget_job.reset_budget_for_litellm_keys()) - counter_cache.in_memory_cache.set_cache.assert_any_call( - key="spend:key:sk-abc", value=0.0, ttl=60 - ) + counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:key:sk-abc", value=0.0, ttl=60) -def test_reset_budget_for_users_invalidates_redis_counter( - reset_budget_job, mock_prisma_client, monkeypatch -): +def test_reset_budget_for_users_invalidates_redis_counter(reset_budget_job, mock_prisma_client, monkeypatch): """User budget reset must clear the Redis spend counter.""" counter_cache = _make_counter_invalidation_job(monkeypatch) @@ -1430,14 +1364,10 @@ def test_reset_budget_for_users_invalidates_redis_counter( asyncio.run(reset_budget_job.reset_budget_for_litellm_users()) - counter_cache.in_memory_cache.set_cache.assert_any_call( - key="spend:user:alice", value=0.0, ttl=60 - ) + counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:user:alice", value=0.0, ttl=60) -def test_reset_budget_for_teams_invalidates_redis_counter( - reset_budget_job, mock_prisma_client, monkeypatch -): +def test_reset_budget_for_teams_invalidates_redis_counter(reset_budget_job, mock_prisma_client, monkeypatch): """Team budget reset must clear the Redis spend counter.""" counter_cache = _make_counter_invalidation_job(monkeypatch) @@ -1458,9 +1388,7 @@ def test_reset_budget_for_teams_invalidates_redis_counter( asyncio.run(reset_budget_job.reset_budget_for_litellm_teams()) - counter_cache.in_memory_cache.set_cache.assert_any_call( - key="spend:team:team-x", value=0.0, ttl=60 - ) + counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:team:team-x", value=0.0, ttl=60) def test_reset_does_not_zero_counter_when_db_write_fails(monkeypatch): @@ -1511,9 +1439,7 @@ def test_reset_does_not_zero_counter_when_db_write_fails(monkeypatch): batcher.commit = failing_commit prisma_client.db.batch_ = MagicMock(return_value=batcher) - job = ResetBudgetJob( - proxy_logging_obj=MockProxyLogging(), prisma_client=prisma_client - ) + job = ResetBudgetJob(proxy_logging_obj=MockProxyLogging(), prisma_client=prisma_client) asyncio.run(job.reset_budget_for_litellm_keys()) @@ -1543,8 +1469,8 @@ def test_reset_budget_for_keys_writes_only_spend_and_reset_at(reset_budget_job, "budget_duration": "30d", "budget_reset_at": now, "token": "sk-problematic", - "object_permission_id": "perm-abc", # would be rejected on update - "budget_limits": [{"max_budget": 5}], # would be rejected on update + "object_permission_id": "perm-abc", # would be rejected on update + "budget_limits": [{"max_budget": 5}], # would be rejected on update "metadata": {"some": "thing"}, }, ) @@ -1570,19 +1496,13 @@ def test_reset_budget_for_keys_linked_to_budgets_invalidates_redis_counter(monke linked_key = type("Key", (), {"token": "sk-linked"}) prisma_client = MagicMock() - prisma_client.db.litellm_verificationtoken.find_many = AsyncMock( - return_value=[linked_key] - ) - prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( - return_value={"count": 1} - ) + prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[linked_key]) + prisma_client.db.litellm_verificationtoken.update_many = AsyncMock(return_value={"count": 1}) job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) asyncio.run(job.reset_budget_for_keys_linked_to_budgets([expired_budget])) - counter_cache.in_memory_cache.set_cache.assert_any_call( - key="spend:key:sk-linked", value=0.0, ttl=60 - ) + counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:key:sk-linked", value=0.0, ttl=60) def test_reset_budget_for_orgs_linked_to_budgets_invalidates_redis_counter(monkeypatch): @@ -1593,22 +1513,14 @@ def test_reset_budget_for_orgs_linked_to_budgets_invalidates_redis_counter(monke linked_org = type("Org", (), {"organization_id": "org-acme"}) prisma_client = MagicMock() - prisma_client.db.litellm_organizationtable.find_many = AsyncMock( - return_value=[linked_org] - ) - prisma_client.db.litellm_organizationtable.update_many = AsyncMock( - return_value={"count": 1} - ) + prisma_client.db.litellm_organizationtable.find_many = AsyncMock(return_value=[linked_org]) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock(return_value={"count": 1}) job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) asyncio.run(job.reset_budget_for_orgs_linked_to_budgets([expired_budget])) - counter_cache.in_memory_cache.set_cache.assert_any_call( - key="spend:org:org-acme", value=0.0, ttl=60 - ) - counter_cache.redis_cache.async_set_cache.assert_any_await( - key="spend:org:org-acme", value=0.0, ttl=60 - ) + counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:org:org-acme", value=0.0, ttl=60) + counter_cache.redis_cache.async_set_cache.assert_any_await(key="spend:org:org-acme", value=0.0, ttl=60) def test_reset_budget_for_tags_linked_to_budgets_invalidates_redis_counter(monkeypatch): @@ -1625,12 +1537,8 @@ def test_reset_budget_for_tags_linked_to_budgets_invalidates_redis_counter(monke job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) asyncio.run(job.reset_budget_for_tags_linked_to_budgets([expired_budget])) - counter_cache.in_memory_cache.set_cache.assert_any_call( - key="spend:tag:tenant-42", value=0.0, ttl=60 - ) - counter_cache.redis_cache.async_set_cache.assert_any_await( - key="spend:tag:tenant-42", value=0.0, ttl=60 - ) + counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:tag:tenant-42", value=0.0, ttl=60) + counter_cache.redis_cache.async_set_cache.assert_any_await(key="spend:tag:tenant-42", value=0.0, ttl=60) def test_reset_budget_for_tags_linked_to_budgets_invalidates_management_cache( @@ -1657,9 +1565,7 @@ def test_reset_budget_for_tags_linked_to_budgets_invalidates_management_cache( job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) asyncio.run(job.reset_budget_for_tags_linked_to_budgets([expired_budget])) - counter_cache.user_api_key_cache.async_delete_cache.assert_any_await( - key="tag:tenant-42" - ) + counter_cache.user_api_key_cache.async_delete_cache.assert_any_await(key="tag:tenant-42") def test_reset_budget_for_tags_linked_to_budgets_invalidates_each_tag_management_cache( @@ -1684,8 +1590,7 @@ def test_reset_budget_for_tags_linked_to_budgets_invalidates_each_tag_management asyncio.run(job.reset_budget_for_tags_linked_to_budgets([expired_budget])) deleted_keys = { - call.kwargs.get("key") - for call in counter_cache.user_api_key_cache.async_delete_cache.await_args_list + call.kwargs.get("key") for call in counter_cache.user_api_key_cache.async_delete_cache.await_args_list } assert deleted_keys == {"tag:tenant-a", "tag:tenant-b", "tag:tenant-c"} @@ -1711,19 +1616,13 @@ def test_reset_budget_for_keys_linked_to_budgets_invalidates_management_cache( linked_key = type("Key", (), {"token": "sk-linked"}) prisma_client = MagicMock() - prisma_client.db.litellm_verificationtoken.find_many = AsyncMock( - return_value=[linked_key] - ) - prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( - return_value={"count": 1} - ) + prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[linked_key]) + prisma_client.db.litellm_verificationtoken.update_many = AsyncMock(return_value={"count": 1}) job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) asyncio.run(job.reset_budget_for_keys_linked_to_budgets([expired_budget])) - counter_cache.user_api_key_cache.async_delete_cache.assert_any_await( - key="sk-linked" - ) + counter_cache.user_api_key_cache.async_delete_cache.assert_any_await(key="sk-linked") def test_reset_budget_for_orgs_linked_to_budgets_invalidates_management_cache( @@ -1736,19 +1635,14 @@ def test_reset_budget_for_orgs_linked_to_budgets_invalidates_management_cache( linked_org = type("Org", (), {"organization_id": "org-acme"}) prisma_client = MagicMock() - prisma_client.db.litellm_organizationtable.find_many = AsyncMock( - return_value=[linked_org] - ) - prisma_client.db.litellm_organizationtable.update_many = AsyncMock( - return_value={"count": 1} - ) + prisma_client.db.litellm_organizationtable.find_many = AsyncMock(return_value=[linked_org]) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock(return_value={"count": 1}) job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) asyncio.run(job.reset_budget_for_orgs_linked_to_budgets([expired_budget])) deleted_keys = { - call.kwargs.get("key") - for call in counter_cache.user_api_key_cache.async_delete_cache.await_args_list + call.kwargs.get("key") for call in counter_cache.user_api_key_cache.async_delete_cache.await_args_list } assert deleted_keys == { "org_id:org-acme", @@ -1768,19 +1662,13 @@ def test_reset_budget_for_team_members_invalidates_management_cache(monkeypatch) ) prisma_client = MagicMock() - prisma_client.db.litellm_teammembership.find_many = AsyncMock( - return_value=[membership] - ) - prisma_client.db.litellm_teammembership.update_many = AsyncMock( - return_value={"count": 1} - ) + prisma_client.db.litellm_teammembership.find_many = AsyncMock(return_value=[membership]) + prisma_client.db.litellm_teammembership.update_many = AsyncMock(return_value={"count": 1}) job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) asyncio.run(job.reset_budget_for_litellm_team_members([expired_budget])) - counter_cache.user_api_key_cache.async_delete_cache.assert_any_await( - key="team-x_alice" - ) + counter_cache.user_api_key_cache.async_delete_cache.assert_any_await(key="team-x_alice") def test_reset_budget_for_tags_linked_to_budgets_management_cache_delete_failure_still_resets( @@ -1788,9 +1676,7 @@ def test_reset_budget_for_tags_linked_to_budgets_management_cache_delete_failure ): """If ``async_delete_cache`` raises, the DB cascade must still complete.""" counter_cache = _make_counter_invalidation_job(monkeypatch) - counter_cache.user_api_key_cache.async_delete_cache = AsyncMock( - side_effect=RuntimeError("cache unavailable") - ) + counter_cache.user_api_key_cache.async_delete_cache = AsyncMock(side_effect=RuntimeError("cache unavailable")) expired_budget = type("B", (), {"budget_id": "budget-1"}) linked_tag = type("Tag", (), {"tag_name": "tenant-42"}) diff --git a/tests/test_litellm/proxy/common_utils/test_timezone_utils.py b/tests/test_litellm/proxy/common_utils/test_timezone_utils.py index 80b813226df..7f686c53c95 100644 --- a/tests/test_litellm/proxy/common_utils/test_timezone_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_timezone_utils.py @@ -1,19 +1,33 @@ import os import sys -from datetime import datetime, timezone +from datetime import datetime, time, timezone from zoneinfo import ZoneInfo +import pytest + sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system path import litellm from litellm.proxy.common_utils.timezone_utils import ( + BudgetResetSettings, + compute_budget_reset_at, + get_budget_reset_settings, get_budget_reset_time, get_budget_reset_timezone, + parse_budget_reset_time, ) +def _restore_attr(obj, name, original): + if original is None: + if hasattr(obj, name): + delattr(obj, name) + else: + setattr(obj, name, original) + + def test_get_budget_reset_time(): """ Test that the budget reset time is set to the first of the next month @@ -100,3 +114,69 @@ def test_get_budget_reset_time_respects_timezone(): delattr(litellm, "timezone") else: litellm.timezone = original + + +def test_parse_budget_reset_time_hh_mm(): + assert parse_budget_reset_time("12:00") == time(12, 0) + + +def test_parse_budget_reset_time_hh_mm_ss(): + assert parse_budget_reset_time("09:30:15") == time(9, 30, 15) + + +def test_parse_budget_reset_time_unset_defaults_to_midnight(): + assert parse_budget_reset_time(None) == time(0, 0) + assert parse_budget_reset_time("") == time(0, 0) + + +def test_parse_budget_reset_time_invalid_string_raises(): + with pytest.raises(ValueError): + parse_budget_reset_time("25:00") + with pytest.raises(ValueError): + parse_budget_reset_time("noon") + + +def test_parse_budget_reset_time_non_string_raises(): + # Unquoted "12:00" in YAML parses to the int 720; it must fail loudly, + # not silently fall back to midnight. + with pytest.raises(ValueError): + parse_budget_reset_time(720) + + +def test_get_budget_reset_settings_reads_globals(): + orig_tz = getattr(litellm, "timezone", None) + orig_rt = getattr(litellm, "budget_reset_time", None) + try: + litellm.timezone = "Asia/Jerusalem" + litellm.budget_reset_time = "12:00" + settings = get_budget_reset_settings() + assert settings.timezone == "Asia/Jerusalem" + assert settings.reset_time_of_day == time(12, 0) + finally: + _restore_attr(litellm, "timezone", orig_tz) + _restore_attr(litellm, "budget_reset_time", orig_rt) + + +def test_compute_budget_reset_at_applies_offset(): + settings = BudgetResetSettings( + timezone="Asia/Jerusalem", reset_time_of_day=time(12, 0) + ) + reset_at = compute_budget_reset_at("1d", settings) + jerusalem = reset_at.astimezone(ZoneInfo("Asia/Jerusalem")) + assert jerusalem.hour == 12 + assert jerusalem.minute == 0 + assert reset_at > datetime.now(timezone.utc) + + +def test_get_budget_reset_time_honors_global_budget_reset_time(): + orig_tz = getattr(litellm, "timezone", None) + orig_rt = getattr(litellm, "budget_reset_time", None) + try: + litellm.timezone = "UTC" + litellm.budget_reset_time = "12:00" + reset_at = get_budget_reset_time(budget_duration="1d") + assert reset_at.astimezone(timezone.utc).hour == 12 + assert reset_at.astimezone(timezone.utc).minute == 0 + finally: + _restore_attr(litellm, "timezone", orig_tz) + _restore_attr(litellm, "budget_reset_time", orig_rt) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py index 07c40aa763d..4021f922877 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py @@ -10,14 +10,19 @@ import pytest sys.path.insert(0, os.path.abspath("../../../../..")) +import httpx from fastapi import HTTPException import litellm import litellm.types.utils from litellm._logging import verbose_proxy_logger from litellm.caching import DualCache +from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.model_armor import ModelArmorGuardrail +from litellm.proxy.guardrails.guardrail_hooks.model_armor.model_armor import ( + ModelArmorAPIError, +) from litellm.types.guardrails import GuardrailEventHooks @@ -403,8 +408,9 @@ async def test_model_armor_api_error_handling(): "metadata": {"guardrails": ["model-armor-test"]}, } - # Should raise HTTPException for API error - with pytest.raises(HTTPException) as exc_info: + # An API failure propagates as ModelArmorAPIError, not a content-block + # HTTPException, so guardrail trace status stays guardrail_failed_to_respond + with pytest.raises(ModelArmorAPIError) as exc_info: await guardrail.async_pre_call_hook( user_api_key_dict=mock_user_api_key_dict, cache=mock_cache, @@ -412,9 +418,8 @@ async def test_model_armor_api_error_handling(): call_type="completion", ) - assert exc_info.value.status_code == 400 - assert "Model Armor API error" in str(exc_info.value.detail) - assert "upstream 500" in str(exc_info.value.detail) + assert exc_info.value.detail == "Model Armor API error (upstream 500)" + assert "Internal Server Error" not in str(exc_info.value.detail) @pytest.mark.asyncio @@ -622,7 +627,7 @@ async def test_model_armor_streaming_block_yields_sse_error(): @pytest.mark.asyncio -async def test_model_armor_api_failure_returns_400(): +async def test_model_armor_api_failure_raises_sanitized_error(): """Test that Model Armor API failures raise HTTP 400, not the upstream status code.""" guardrail = ModelArmorGuardrail( template_id="test-template", @@ -643,15 +648,544 @@ async def test_model_armor_api_failure_returns_400(): with patch.object( guardrail.async_handler, "post", AsyncMock(return_value=mock_response) ): - with pytest.raises(HTTPException) as exc_info: + with pytest.raises(ModelArmorAPIError) as exc_info: await guardrail.make_model_armor_request( content="test content", source="user_prompt", ) - # Should be 400, NOT the upstream 500 - assert exc_info.value.status_code == 400 - assert "upstream 500" in str(exc_info.value.detail) + assert exc_info.value.detail == "Model Armor API error (upstream 500)" + assert "Internal Server Error" not in str(exc_info.value.detail) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sanitize", [True, False]) +async def test_model_armor_error_output_sanitization(sanitize: bool): + marker = "SYNTHETIC_MODEL_ARMOR_MARKER" + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + guardrail_name="model-armor-test", + sanitize_error_detail=sanitize, + ) + guardrail._ensure_access_token_async = AsyncMock( + return_value=("test-token", "test-project") + ) + + error_response = AsyncMock(status_code=500, text=marker) + with patch.object( + guardrail.async_handler, "post", AsyncMock(return_value=error_response) + ), patch.object(verbose_proxy_logger, "debug") as debug_log, patch.object( + verbose_proxy_logger, "error" + ) as error_log, pytest.raises(ModelArmorAPIError) as exc_info: + await guardrail.make_model_armor_request(content=marker) + + direct_log = f"{debug_log.call_args_list} {error_log.call_args_list}" + if sanitize: + assert marker not in str(exc_info.value.detail) + assert marker not in direct_log + else: + assert marker in str(exc_info.value.detail) + assert marker in direct_log + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fail_on_error", [True, False]) +async def test_model_armor_api_error_honors_fail_open(fail_on_error: bool): + """An upstream API failure (raised by the real handler as MaskedHTTPStatusError) + must block with a sanitized 400 when fail_on_error is true and let the request + proceed when the operator configured fail-open.""" + marker = "SYNTHETIC_FAIL_OPEN_MARKER" + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + guardrail_name="model-armor-test", + fail_on_error=fail_on_error, + ) + guardrail._ensure_access_token_async = AsyncMock( + return_value=("test-token", "test-project") + ) + guardrail.should_run_guardrail = Mock(return_value=True) + + request = httpx.Request("POST", "https://modelarmor.example.test/v1") + upstream = httpx.Response(503, content=marker.encode(), request=request) + original = httpx.HTTPStatusError("Service Unavailable", request=request, response=upstream) + masked = MaskedHTTPStatusError(original, message=marker, text=marker) + + request_data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "synthetic input"}], + "metadata": {}, + } + + with patch.object(guardrail.async_handler, "post", AsyncMock(side_effect=masked)): + if fail_on_error: + with pytest.raises(ModelArmorAPIError) as exc_info: + await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=MagicMock(spec=DualCache), + data=request_data, + call_type="completion", + ) + assert exc_info.value.detail == "Model Armor API error (upstream 503)" + assert marker not in str(exc_info.value.detail) + else: + result = await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=MagicMock(spec=DualCache), + data=request_data, + call_type="completion", + ) + assert result is request_data + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fail_on_error", [True, False]) +async def test_model_armor_api_error_fail_open_moderation_and_post_call(fail_on_error: bool): + """The during-call and post-call hooks route API failures through fail_on_error + exactly like pre-call: sanitized 400 when failing closed, pass-through when open.""" + api_error = ModelArmorAPIError("Model Armor API error (upstream 503)") + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + guardrail_name="model-armor-test", + fail_on_error=fail_on_error, + ) + guardrail.make_model_armor_request = AsyncMock(side_effect=api_error) + guardrail.should_run_guardrail = Mock(return_value=True) + + request_data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "synthetic input"}], + "metadata": {}, + } + mock_llm_response = litellm.ModelResponse() + mock_llm_response.choices = [ + litellm.Choices(message=litellm.Message(content="model output")) + ] + + if fail_on_error: + with pytest.raises(ModelArmorAPIError) as mod_exc: + await guardrail.async_moderation_hook( + data=dict(request_data), + user_api_key_dict=UserAPIKeyAuth(), + call_type="completion", + ) + assert mod_exc.value.detail == "Model Armor API error (upstream 503)" + + with pytest.raises(ModelArmorAPIError) as post_exc: + await guardrail.async_post_call_success_hook( + data=dict(request_data), + user_api_key_dict=UserAPIKeyAuth(), + response=mock_llm_response, + ) + assert post_exc.value.detail == "Model Armor API error (upstream 503)" + else: + moderated = await guardrail.async_moderation_hook( + data=dict(request_data), + user_api_key_dict=UserAPIKeyAuth(), + call_type="completion", + ) + assert moderated is not None + + result = await guardrail.async_post_call_success_hook( + data=dict(request_data), + user_api_key_dict=UserAPIKeyAuth(), + response=mock_llm_response, + ) + assert result is mock_llm_response + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fail_on_error", [True, False]) +async def test_model_armor_api_error_fail_open_streaming(fail_on_error: bool): + """A streaming-path API failure yields a sanitized SSE error frame when failing + closed and passes the original chunks through when the operator opted into fail-open.""" + api_error = ModelArmorAPIError("Model Armor API error (upstream 503)") + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + guardrail_name="model-armor-test", + fail_on_error=fail_on_error, + ) + guardrail.make_model_armor_request = AsyncMock(side_effect=api_error) + guardrail.should_run_guardrail = Mock(return_value=True) + + async def mock_stream(): + yield litellm.ModelResponseStream( + choices=[ + litellm.types.utils.StreamingChoices( + delta=litellm.types.utils.Delta(content="streamed output") + ) + ] + ) + + chunks = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=mock_stream(), + request_data={ + "model": "gpt-4", + "messages": [{"role": "user", "content": "synthetic input"}], + "metadata": {}, + }, + ): + chunks.append(chunk) + + if fail_on_error: + assert len(chunks) == 1 + assert isinstance(chunks[0], str) + assert "Model Armor API error (upstream 503)" in chunks[0] + assert '"code": "500"' in chunks[0] + else: + assert len(chunks) == 1 + assert isinstance(chunks[0], litellm.ModelResponseStream) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fail_on_error", [True, False]) +async def test_model_armor_api_error_fail_open_file_scan(fail_on_error: bool): + """A file-scan API failure blocks with the sanitized detail when failing closed + and skips the attachment when the operator opted into fail-open.""" + api_error = ModelArmorAPIError("Model Armor API error (upstream 503)") + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + guardrail_name="model-armor-test", + fail_on_error=fail_on_error, + ) + guardrail.make_model_armor_request = AsyncMock(side_effect=api_error) + + pdf_b64 = base64.b64encode(b"%PDF-1.4 synthetic").decode() + messages = [ + { + "role": "user", + "content": [ + { + "type": "file", + "file": { + "file_data": f"data:application/pdf;base64,{pdf_b64}", + "filename": "synthetic.pdf", + "format": "application/pdf", + }, + } + ], + } + ] + data = {"metadata": {}} + + if fail_on_error: + with pytest.raises(ModelArmorAPIError) as exc_info: + await guardrail._scan_request_files(messages=messages, data=data) + assert exc_info.value.detail == "Model Armor API error (upstream 503)" + else: + assert await guardrail._scan_request_files(messages=messages, data=data) is None + + +def test_model_armor_hot_reload_null_stays_sanitized(): + """update_in_memory_litellm_params assigns raw fields; an explicit null in a + hot-reloaded config must not disable sanitization.""" + from litellm.types.guardrails import LitellmParams + + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + guardrail_name="model-armor-test", + ) + guardrail.update_in_memory_litellm_params( + LitellmParams(guardrail="model_armor", mode="pre_call", sanitize_error_detail=None) + ) + assert guardrail.sanitize_error_detail is True + + guardrail.update_in_memory_litellm_params( + LitellmParams(guardrail="model_armor", mode="pre_call", sanitize_error_detail=False) + ) + assert guardrail.sanitize_error_detail is False + + +def test_model_armor_redactor_depth_cap_fails_closed(): + """Past the recursion cap the redactor must return the redaction sentinel, + never raw content, and must not raise RecursionError.""" + from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH + from litellm.proxy.guardrails.guardrail_hooks.model_armor.model_armor import ( + _redact_scanned_content, + ) + + marker = "SYNTHETIC_DEEP_MARKER" + payload: dict = {"safe_key": marker, "items": [{"safe_key": marker}]} + for _ in range(DEFAULT_MAX_RECURSE_DEPTH + 5): + payload = {"nested": payload} + + redacted = _redact_scanned_content(payload) + assert marker not in str(redacted) + + shallow = _redact_scanned_content({"filterResults": [{"text": marker, "matchState": "MATCH_FOUND"}]}) + assert shallow == {"filterResults": [{"text": "[REDACTED]", "matchState": "MATCH_FOUND"}]} + + uri_payload = _redact_scanned_content( + { + "maliciousUriFilterResult": { + "matchState": "MATCH_FOUND", + "maliciousUriMatchedItems": [{"uri": f"https://evil.example/{marker}"}], + } + } + ) + assert uri_payload == { + "maliciousUriFilterResult": { + "matchState": "MATCH_FOUND", + "maliciousUriMatchedItems": "[REDACTED]", + } + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sanitize", [True, False]) +async def test_model_armor_handler_raised_http_error_sanitized(sanitize: bool): + """The real AsyncHTTPHandler raises on non-2xx via raise_for_status, so a non-200 + never returns a response object. The raised MaskedHTTPStatusError carries the raw + upstream body in its message; the guardrail must convert it to a sanitized + HTTPException instead of letting it bubble raw to callers and logs.""" + marker = "SYNTHETIC_MODEL_ARMOR_MARKER" + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + guardrail_name="model-armor-test", + sanitize_error_detail=sanitize, + ) + guardrail._ensure_access_token_async = AsyncMock( + return_value=("test-token", "test-project") + ) + + request = httpx.Request("POST", "https://modelarmor.example.test/v1") + upstream = httpx.Response(403, content=marker.encode(), request=request) + original = httpx.HTTPStatusError("Forbidden", request=request, response=upstream) + masked = MaskedHTTPStatusError(original, message=marker, text=marker) + + with patch.object( + guardrail.async_handler, "post", AsyncMock(side_effect=masked) + ), patch.object(verbose_proxy_logger, "debug") as debug_log, patch.object( + verbose_proxy_logger, "error" + ) as error_log, pytest.raises(ModelArmorAPIError) as exc_info: + await guardrail.make_model_armor_request(content=marker) + + direct_log = f"{debug_log.call_args_list} {error_log.call_args_list}" + assert "403" in str(exc_info.value.detail) + if sanitize: + assert marker not in str(exc_info.value.detail) + assert marker not in direct_log + else: + assert marker in str(exc_info.value.detail) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sanitize", [True, False]) +async def test_model_armor_post_call_logging_redacts_scanned_content(sanitize: bool): + marker = "SYNTHETIC_POST_CALL_MARKER" + armor_response = { + "sanitizationResult": { + "filterMatchState": "NO_MATCH_FOUND", + "filterResults": { + "sdp": { + "sdpFilterResult": { + "deidentifyResult": { + "matchState": "MATCH_FOUND", + "data": {"text": marker}, + } + } + } + }, + } + } + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + guardrail_name="model-armor-test", + mask_response_content=True, + sanitize_error_detail=sanitize, + ) + guardrail.make_model_armor_request = AsyncMock(return_value=armor_response) + guardrail.should_run_guardrail = Mock(return_value=True) + + mock_llm_response = litellm.ModelResponse() + mock_llm_response.choices = [ + litellm.Choices(message=litellm.Message(content="model output")) + ] + request_data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "synthetic input"}], + "metadata": {}, + "litellm_logging_obj": MagicMock(), + } + + with patch( + "litellm.proxy.common_utils.callback_utils.add_guardrail_response_to_standard_logging_object" + ) as add_logging: + await guardrail.async_post_call_success_hook( + data=request_data, + user_api_key_dict=UserAPIKeyAuth(), + response=mock_llm_response, + ) + + logged = add_logging.call_args.kwargs["guardrail_response"] + assert logged["guardrail_status"] == "success" + logged_armor_response = logged["guardrail_response"]["model_armor_response"] + if sanitize: + assert marker not in str(logged_armor_response) + assert ( + logged_armor_response["sanitizationResult"]["filterResults"]["sdp"][ + "sdpFilterResult" + ]["deidentifyResult"]["matchState"] + == "MATCH_FOUND" + ) + else: + assert logged_armor_response == armor_response + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sanitize", [True, False]) +async def test_model_armor_streaming_logging_redacts_scanned_content(sanitize: bool): + marker = "SYNTHETIC_STREAMING_MARKER" + armor_response = { + "sanitizationResult": { + "filterMatchState": "NO_MATCH_FOUND", + "sanitizedText": marker, + } + } + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + guardrail_name="model-armor-test", + sanitize_error_detail=sanitize, + ) + guardrail.make_model_armor_request = AsyncMock(return_value=armor_response) + guardrail.should_run_guardrail = Mock(return_value=True) + + async def mock_stream(): + yield litellm.ModelResponseStream( + choices=[ + litellm.types.utils.StreamingChoices( + delta=litellm.types.utils.Delta(content="streamed output") + ) + ] + ) + + request_data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "synthetic input"}], + "metadata": {}, + } + + async for _ in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=mock_stream(), + request_data=request_data, + ): + pass + + logged_response = request_data["metadata"]["_model_armor_response"] + if sanitize: + assert logged_response == { + "sanitizationResult": { + "filterMatchState": "NO_MATCH_FOUND", + "sanitizedText": "[REDACTED]", + } + } + assert marker not in str(logged_response) + else: + assert logged_response == armor_response + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sanitize", [True, False]) +async def test_model_armor_match_found_sanitizes_caller_and_logging(sanitize: bool): + marker = "SYNTHETIC_MATCH_FOUND_MARKER" + armor_response = { + "sanitizationResult": { + "filterResults": { + "sdp": { + "sdpFilterResult": { + "inspectResult": { + "matchState": "MATCH_FOUND", + "findings": [{"marker": marker}], + } + } + } + } + } + } + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + guardrail_name="model-armor-test", + event_hook=[GuardrailEventHooks.pre_mcp_call], + sanitize_error_detail=sanitize, + ) + guardrail.make_model_armor_request = AsyncMock(return_value=armor_response) + guardrail.should_run_guardrail = Mock(return_value=True) + request_data = { + "messages": [{"role": "user", "content": "synthetic input"}], + "metadata": {}, + } + + with pytest.raises(HTTPException) as exc_info: + await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=MagicMock(spec=DualCache), + data=request_data, + call_type=litellm.types.utils.CallTypes.call_mcp_tool.value, + ) + + detail = exc_info.value.detail + logged_response = request_data["metadata"]["_model_armor_response"] + if sanitize: + assert detail == {"error": "Content blocked by Model Armor"} + assert logged_response == { + "sanitizationResult": { + "filterResults": { + "sdp": { + "sdpFilterResult": { + "inspectResult": { + "matchState": "MATCH_FOUND", + "findings": "[REDACTED]", + } + } + } + } + } + } + assert marker not in str(detail) + assert marker not in str(logged_response) + else: + assert detail["model_armor_response"] == armor_response + assert logged_response == armor_response + assert marker in str(detail) + assert marker in str(logged_response) + + +def test_model_armor_sanitize_error_detail_config_wiring(): + from litellm.proxy.guardrails.guardrail_hooks.model_armor import ( + initialize_guardrail, + ) + from litellm.types.guardrails import LitellmParams + + config = {"guardrail_name": "model-armor-test"} + params = { + "guardrail": "model_armor", + "mode": "pre_mcp_call", + "template_id": "test-template", + "project_id": "test-project", + } + opted_out = initialize_guardrail( + LitellmParams(**params, sanitize_error_detail=False), config + ) + explicit_null = initialize_guardrail( + LitellmParams(**params, sanitize_error_detail=None), config + ) + default = initialize_guardrail(LitellmParams(**params), config) + + assert opted_out.sanitize_error_detail is False + assert explicit_null.sanitize_error_detail is True + assert default.sanitize_error_detail is True def test_model_armor_ui_friendly_name(): @@ -1394,7 +1928,10 @@ async def test_model_armor_guardrail_status_intervened_vs_failed(): ) info = request_data["metadata"]["standard_logging_guardrail_information"] + assert info[0]["guardrail_name"] == guardrail.guardrail_name assert info[0]["guardrail_status"] == "guardrail_intervened" + assert "model_armor_response" not in info[0]["guardrail_response"] + assert "sanitizationResult" not in info[0]["guardrail_response"] # 2: if an API error - guardrail status should be guardrail_failed_to_respond" guardrail2 = ModelArmorGuardrail( diff --git a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py index f27f1197090..3ff8e2a6886 100644 --- a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py @@ -347,8 +347,8 @@ async def test_scim_create_user_respects_default_role_set_via_ui(mocker, monkeyp # Step 3: Create a user via SCIM scim_user = SCIMUser( schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], - userName="idontexist@krakentest.tech", - emails=[SCIMUserEmail(value="idontexist@krakentest.tech")], + userName="idontexist@example.com", + emails=[SCIMUserEmail(value="idontexist@example.com")], ) mock_prisma_client = mocker.MagicMock() @@ -364,7 +364,7 @@ async def test_scim_create_user_respects_default_role_set_via_ui(mocker, monkeyp new_user_mock = mocker.patch( "litellm.proxy.management_endpoints.scim.scim_v2.new_user", - AsyncMock(return_value=NewUserRequest(user_id="idontexist@krakentest.tech")), + AsyncMock(return_value=NewUserRequest(user_id="idontexist@example.com")), ) mocker.patch( diff --git a/tests/test_litellm/proxy/management_endpoints/test_cache_settings_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_cache_settings_endpoints.py index f4c6d4f8d15..2504b5744fc 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_cache_settings_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_cache_settings_endpoints.py @@ -17,9 +17,14 @@ from litellm.proxy._types import LitellmTableNames, LitellmUserRoles from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.proxy.management_endpoints.cache_settings_endpoints import ( _CACHE_SENSITIVE_FIELDS, + _REDACTED_VALUE, CacheSettingsManager, CacheSettingsUpdateRequest, CacheTestRequest, + _merge_over_saved, + _overlay_environment, + _parse_stored_settings, + _redact_credentials, _resolve_cache_url_precedence, get_cache_settings, test_cache_connection, @@ -610,3 +615,510 @@ async def test_update_cache_settings_no_audit_when_disabled(monkeypatch): ) assert audit_calls == [] + + +class TestParseStoredSettings: + """The stored blob arrives as a JSON string or a parsed dict; both must + normalize to a dict so the secret-preservation read never silently drops it.""" + + def test_parses_a_json_string(self): + assert _parse_stored_settings('{"host": "h", "password": "pw"}') == {"host": "h", "password": "pw"} + + def test_passes_a_dict_through(self): + assert _parse_stored_settings({"host": "h", "password": "pw"}) == {"host": "h", "password": "pw"} + + def test_non_mapping_becomes_empty(self): + assert _parse_stored_settings(None) == {} + assert _parse_stored_settings("[1, 2]") == {} + + +class TestMergeOverSaved: + """The secret-preservation contract behind the redacted-resubmit fix.""" + + def test_redacted_secret_restores_stored_value(self): + # same connection target, an unrelated field edited: the stored secret + # is restored behind the redacted resubmit + merged = _merge_over_saved( + incoming={"type": "redis", "host": "samehost", "namespace": "new", "password": _REDACTED_VALUE}, + saved={"type": "redis", "host": "samehost", "password": "realpw"}, + ) + assert merged["namespace"] == "new" + assert merged["password"] == "realpw" + + def test_stored_secret_not_replayed_to_a_different_target(self): + # credential replay guard: omitting the password while pointing at a new + # host must NOT resurrect the stored secret (it would be sent elsewhere) + merged = _merge_over_saved( + incoming={"type": "redis", "host": "attacker.example.com", "password": _REDACTED_VALUE}, + saved={"type": "redis", "host": "real-redis", "password": "realpw"}, + ) + assert "password" not in merged + + def test_omitted_secret_restores_stored_value(self): + # same host (target unchanged), password field omitted entirely + merged = _merge_over_saved( + incoming={"type": "redis", "host": "samehost", "namespace": "n"}, + saved={"type": "redis", "host": "samehost", "password": "realpw"}, + ) + assert merged["password"] == "realpw" + + def test_sentinel_password_not_replayed_to_different_sentinel_nodes(self): + # sentinel target change with an omitted sentinel_password must not + # resurrect the stored one and send it to the caller's sentinels + merged = _merge_over_saved( + incoming={"type": "redis", "sentinel_nodes": [["attacker", 26379]], "service_name": "mymaster"}, + saved={ + "type": "redis", + "sentinel_nodes": [["real", 26379]], + "service_name": "mymaster", + "sentinel_password": "realsp", + }, + ) + assert "sentinel_password" not in merged + + def test_sentinel_password_preserved_when_sentinel_target_unchanged(self): + merged = _merge_over_saved( + incoming={"type": "redis", "sentinel_nodes": [["real", 26379]], "service_name": "mymaster"}, + saved={ + "type": "redis", + "sentinel_nodes": [["real", 26379]], + "service_name": "mymaster", + "sentinel_password": "realsp", + }, + ) + assert merged["sentinel_password"] == "realsp" + + def test_password_not_replayed_to_different_cluster_nodes(self): + merged = _merge_over_saved( + incoming={"type": "redis", "redis_startup_nodes": [{"host": "attacker", "port": "7001"}]}, + saved={ + "type": "redis", + "redis_startup_nodes": [{"host": "real", "port": "7001"}], + "password": "realpw", + }, + ) + assert "password" not in merged + + def test_equivalent_target_representations_still_preserve_secret(self): + # the client sends port as a string, storage holds it as an int: the + # target is unchanged, so the untouched password must not be dropped + merged = _merge_over_saved( + incoming={"type": "redis", "host": "h", "port": "6379", "password": _REDACTED_VALUE}, + saved={"type": "redis", "host": "h", "port": 6379, "password": "realpw"}, + ) + assert merged["password"] == "realpw" + + def test_explicit_empty_string_clears_the_secret(self): + merged = _merge_over_saved( + incoming={"type": "redis", "host": "h", "password": ""}, + saved={"type": "redis", "host": "h", "password": "realpw"}, + ) + assert merged.get("password") == "" + + def test_explicit_null_clears_the_secret(self): + # an explicit null is a clear, not an omission, so it must not restore + merged = _merge_over_saved( + incoming={"type": "redis", "host": "h", "password": None}, + saved={"type": "redis", "host": "h", "password": "realpw"}, + ) + assert merged.get("password") is None + + def test_secret_not_reused_when_a_pinned_target_field_is_omitted(self): + # omitting the host (a pinned target) means the request does not describe + # the stored target, so the stored secret must not be restored (and thus + # cannot be sent to whatever host the incomplete request resolves to) + merged = _merge_over_saved( + incoming={"type": "redis", "port": "6379"}, + saved={"type": "redis", "host": "real", "port": 6379, "password": "realpw"}, + ) + assert "password" not in merged + + def test_redacted_secret_with_no_stored_value_is_dropped(self): + # env-sourced secret: nothing stored to restore, so the marker must not + # be persisted; the environment stays the source at runtime + merged = _merge_over_saved( + incoming={"type": "redis", "host": "h", "password": _REDACTED_VALUE}, + saved={}, + ) + assert "password" not in merged + + def test_new_secret_value_wins(self): + merged = _merge_over_saved( + incoming={"password": "brandnewpw"}, + saved={"password": "realpw"}, + ) + assert merged["password"] == "brandnewpw" + + def test_switching_from_url_to_host_port_drops_stored_url(self): + # admin migrates a url-mode cache to discrete host/port: the stored url + # must not be resurrected (url precedence would then discard host/port) + merged = _merge_over_saved( + incoming={"type": "redis", "host": "newhost", "port": "6379"}, + saved={"type": "redis", "url": "redis://:pw@oldhost:6379/0"}, + ) + assert "url" not in merged + assert merged["host"] == "newhost" + assert merged["port"] == "6379" + + def test_untouched_url_is_preserved_without_a_discrete_target(self): + # a url-mode save that touches nothing keeps the stored url + merged = _merge_over_saved( + incoming={"type": "redis", "namespace": "ns"}, + saved={"type": "redis", "url": "redis://:pw@host:6379/0"}, + ) + assert merged["url"] == "redis://:pw@host:6379/0" + + +def test_overlay_environment_fills_unset_connection_fields(monkeypatch): + """A cache with no stored connection resolves REDIS_* env for the UI.""" + for var in ("REDIS_URL", "REDIS_HOST", "REDIS_PORT", "REDIS_PASSWORD", "REDIS_USERNAME"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("REDIS_HOST", "redis.internal") + monkeypatch.setenv("REDIS_PORT", "6380") + monkeypatch.setenv("REDIS_PASSWORD", "env-password") + + effective = _overlay_environment({}) + + assert effective["host"] == "redis.internal" + assert effective["port"] == "6380" + assert effective["password"] == "env-password" + assert effective["type"] == "redis" + + +def test_overlay_environment_stored_value_wins(monkeypatch): + monkeypatch.setenv("REDIS_HOST", "env-host") + effective = _overlay_environment({"type": "redis", "host": "stored-host"}) + assert effective["host"] == "stored-host" + + +@pytest.mark.asyncio +async def test_get_cache_settings_falls_back_to_redis_env(monkeypatch): + """A cache configured purely through REDIS_* env vars shows its effective + connection instead of a blank page, with the password redacted.""" + for var in ("REDIS_URL", "REDIS_HOST", "REDIS_PORT", "REDIS_PASSWORD", "REDIS_USERNAME"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("REDIS_HOST", "redis.internal") + monkeypatch.setenv("REDIS_PORT", "6380") + monkeypatch.setenv("REDIS_PASSWORD", "env-password") + + mock_prisma = MagicMock() + mock_prisma.db.litellm_cacheconfig.find_unique = AsyncMock(return_value=None) + + proxy_config = MagicMock() + proxy_config._decrypt_db_variables = MagicMock(side_effect=lambda variables_dict: dict(variables_dict)) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", proxy_config), + ): + response = await get_cache_settings(user_api_key_dict=_admin_auth()) + + values = response.current_values + assert values["host"] == "redis.internal" + assert values["port"] == "6380" + assert values["type"] == "redis" + # the env password is surfaced as configured, not leaked in plaintext + assert values["password"] == _REDACTED_VALUE + + +@pytest.mark.asyncio +async def test_get_cache_settings_redacts_password_with_marker(monkeypatch): + for var in ("REDIS_URL", "REDIS_HOST", "REDIS_PORT", "REDIS_PASSWORD", "REDIS_USERNAME"): + monkeypatch.delenv(var, raising=False) + cache_row = MagicMock() + cache_row.cache_settings = json.dumps( + {"type": "redis", "host": "h", "password": "supersecret", "namespace": "ns"} + ) + mock_prisma = MagicMock() + mock_prisma.db.litellm_cacheconfig.find_unique = AsyncMock(return_value=cache_row) + proxy_config = MagicMock() + proxy_config._decrypt_db_variables = MagicMock(side_effect=lambda variables_dict: dict(variables_dict)) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", proxy_config), + ): + response = await get_cache_settings(user_api_key_dict=_admin_auth()) + + assert response.current_values["password"] == _REDACTED_VALUE + assert response.current_values["namespace"] == "ns" + + +@pytest.mark.asyncio +async def test_get_cache_settings_url_mode_hides_env_discrete_fields(monkeypatch): + """A url-mode stored config must not surface env-overlaid host/port. + + Otherwise a no-op save would submit the env host and, via url precedence, + silently switch the cache off its configured url. + """ + for var in ("REDIS_URL", "REDIS_HOST", "REDIS_PORT", "REDIS_PASSWORD", "REDIS_USERNAME"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("REDIS_HOST", "env-host") + monkeypatch.setenv("REDIS_PORT", "6380") + + cache_row = MagicMock() + cache_row.cache_settings = {"type": "redis", "url": "redis://:pw@stored-host:6379/0"} + mock_prisma = MagicMock() + mock_prisma.db.litellm_cacheconfig.find_unique = AsyncMock(return_value=cache_row) + proxy_config = MagicMock() + proxy_config._decrypt_db_variables = MagicMock(side_effect=lambda variables_dict: dict(variables_dict)) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", proxy_config), + ): + response = await get_cache_settings(user_api_key_dict=_admin_auth()) + + values = response.current_values + assert values["url"] == _REDACTED_VALUE + # the env host/port must not leak in and shadow the url + assert "host" not in values + assert "port" not in values + + +def _mock_proxy_config_identity_crypto(): + proxy_config = MagicMock() + proxy_config._encrypt_env_variables = MagicMock( + side_effect=lambda environment_variables: dict(environment_variables) + ) + proxy_config._decrypt_db_variables = MagicMock(side_effect=lambda variables_dict: dict(variables_dict)) + proxy_config._init_cache = MagicMock() + proxy_config.switch_on_llm_response_caching = MagicMock() + return proxy_config + + +@pytest.mark.asyncio +async def test_update_preserves_stored_password_on_redacted_resubmit(monkeypatch): + """Editing an unrelated field and re-submitting the redacted password must + keep the stored secret, not persist the marker over a working password.""" + monkeypatch.setattr(litellm, "store_audit_logs", False) + + existing = MagicMock() + # prisma returns the Json column as an already-parsed dict, not a JSON + # string; a reader that json.loads unconditionally would drop the whole row + existing.cache_settings = {"type": "redis", "host": "oldhost", "password": "realpw"} + mock_prisma = MagicMock() + mock_prisma.db.litellm_cacheconfig.find_unique = AsyncMock(return_value=existing) + mock_prisma.db.litellm_cacheconfig.upsert = AsyncMock() + proxy_config = _mock_proxy_config_identity_crypto() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + ): + result = await update_cache_settings( + request=CacheSettingsUpdateRequest( + # same host (the target is unchanged), an unrelated field edited + cache_settings={"type": "redis", "host": "oldhost", "namespace": "edited", "password": _REDACTED_VALUE} + ), + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + persisted = proxy_config._encrypt_env_variables.call_args.kwargs["environment_variables"] + assert persisted["host"] == "oldhost" + assert persisted["namespace"] == "edited" + assert persisted["password"] == "realpw" + # the response never echoes the plaintext secret back either + assert result["settings"]["password"] == _REDACTED_VALUE + + +@pytest.mark.asyncio +async def test_update_drops_env_sourced_redacted_secret(monkeypatch): + """With no stored row, a re-submitted redacted secret is env-sourced; the + marker must not be persisted so the environment stays the source.""" + monkeypatch.setattr(litellm, "store_audit_logs", False) + + mock_prisma = MagicMock() + mock_prisma.db.litellm_cacheconfig.find_unique = AsyncMock(return_value=None) + mock_prisma.db.litellm_cacheconfig.upsert = AsyncMock() + proxy_config = _mock_proxy_config_identity_crypto() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + ): + await update_cache_settings( + request=CacheSettingsUpdateRequest( + cache_settings={"type": "redis", "host": "h", "password": _REDACTED_VALUE} + ), + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + persisted = proxy_config._encrypt_env_variables.call_args.kwargs["environment_variables"] + assert "password" not in persisted + + +@pytest.mark.asyncio +async def test_update_applies_new_password(monkeypatch): + """A real new secret value replaces the stored one.""" + monkeypatch.setattr(litellm, "store_audit_logs", False) + + existing = MagicMock() + existing.cache_settings = json.dumps({"type": "redis", "host": "h", "password": "oldpw"}) + mock_prisma = MagicMock() + mock_prisma.db.litellm_cacheconfig.find_unique = AsyncMock(return_value=existing) + mock_prisma.db.litellm_cacheconfig.upsert = AsyncMock() + proxy_config = _mock_proxy_config_identity_crypto() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + ): + await update_cache_settings( + request=CacheSettingsUpdateRequest( + cache_settings={"type": "redis", "host": "h", "password": "brandnewpw"} + ), + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + persisted = proxy_config._encrypt_env_variables.call_args.kwargs["environment_variables"] + assert persisted["password"] == "brandnewpw" + + +@pytest.mark.asyncio +async def test_test_cache_connection_survives_saved_lookup_failure(monkeypatch): + """A failed saved-settings lookup must not block the connection test. + + The test endpoint reads the stored row to resolve a redacted credential, but + that read can raise (a misconfigured or unavailable client), and it must fall + back to the submitted settings rather than abort — otherwise a shared client + left in an odd state by another test would break every connection test. + """ + monkeypatch.setattr(litellm, "store_audit_logs", False) + + # a client whose find_unique is not awaitable, so the saved read raises + bad_prisma = MagicMock() + proxy_config = MagicMock() + proxy_config._decrypt_db_variables = MagicMock(side_effect=lambda variables_dict: dict(variables_dict)) + + cache_instance = MagicMock() + cache_instance.cache = MagicMock() + cache_instance.cache.test_connection = AsyncMock(return_value={"status": "success", "message": "ok"}) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", bad_prisma), + patch("litellm.proxy.proxy_server.proxy_config", proxy_config), + patch("litellm.Cache") as mock_cache_class, + ): + mock_cache_class.return_value = cache_instance + result = await test_cache_connection( + request=CacheTestRequest(cache_settings={"type": "redis", "host": "h", "port": "6379", "password": "pw"}), + user_api_key_dict=_admin_auth(), + ) + + mock_cache_class.assert_called_once() + assert result.status == "success" + + +@pytest.mark.asyncio +async def test_get_cache_settings_does_not_surface_non_display_env_credentials(monkeypatch): + """The env overlay must not leak credential kwargs the UI does not manage. + + _redis_kwargs_from_environment resolves every redis.Redis kwarg, including + secrets like azure_client_secret; only cache display fields may be surfaced, + so a non-admin reading /cache/settings never retrieves such a credential. + """ + for var in ("REDIS_URL", "REDIS_HOST", "REDIS_PORT", "REDIS_PASSWORD", "REDIS_AZURE_CLIENT_SECRET"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("REDIS_HOST", "redis.internal") + monkeypatch.setenv("REDIS_AZURE_CLIENT_SECRET", "super-azure-secret") + + mock_prisma = MagicMock() + mock_prisma.db.litellm_cacheconfig.find_unique = AsyncMock(return_value=None) + proxy_config = MagicMock() + proxy_config._decrypt_db_variables = MagicMock(side_effect=lambda variables_dict: dict(variables_dict)) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", proxy_config), + ): + response = await get_cache_settings(user_api_key_dict=_admin_auth()) + + values = response.current_values + assert values.get("host") == "redis.internal" + # the non-display credential must not appear in the response at all + assert "azure_client_secret" not in values + assert "super-azure-secret" not in values.values() + + +@pytest.mark.asyncio +async def test_test_cache_connection_does_not_log_plaintext_credentials(monkeypatch, caplog): + """The connection test must not write the resolved plaintext secret to logs. + + _merge_over_saved substitutes the stored password for a redacted resubmit, so + the settings dict carries the real secret; the debug log must redact it. + """ + import logging + + existing = MagicMock() + existing.cache_settings = {"type": "redis", "host": "h", "port": "6379", "password": "realredispw"} + mock_prisma = MagicMock() + mock_prisma.db.litellm_cacheconfig.find_unique = AsyncMock(return_value=existing) + proxy_config = MagicMock() + proxy_config._decrypt_db_variables = MagicMock(side_effect=lambda variables_dict: dict(variables_dict)) + + cache_instance = MagicMock() + cache_instance.cache = MagicMock() + cache_instance.cache.test_connection = AsyncMock(return_value={"status": "success", "message": "ok"}) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", proxy_config), + patch("litellm.Cache") as mock_cache_class, + caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"), + ): + mock_cache_class.return_value = cache_instance + # resubmit the redacted marker; the merge resolves it to the stored secret + await test_cache_connection( + request=CacheTestRequest( + cache_settings={"type": "redis", "host": "h", "port": "6379", "password": _REDACTED_VALUE} + ), + user_api_key_dict=_admin_auth(), + ) + + # the real password was used to build the client but never written to the log + assert mock_cache_class.call_args.kwargs["password"] == "realredispw" + assert "realredispw" not in caplog.text + + +@pytest.mark.asyncio +async def test_test_cache_connection_does_not_replay_saved_password_to_new_host(monkeypatch): + """Credential-replay guard on the connection test. + + A caller that submits a different host while omitting the password must not + have the stored password restored and sent to the caller-chosen host. + """ + existing = MagicMock() + existing.cache_settings = {"type": "redis", "host": "real-redis", "port": "6379", "password": "realredispw"} + mock_prisma = MagicMock() + mock_prisma.db.litellm_cacheconfig.find_unique = AsyncMock(return_value=existing) + proxy_config = MagicMock() + proxy_config._decrypt_db_variables = MagicMock(side_effect=lambda variables_dict: dict(variables_dict)) + + cache_instance = MagicMock() + cache_instance.cache = MagicMock() + cache_instance.cache.test_connection = AsyncMock(return_value={"status": "success", "message": "ok"}) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", proxy_config), + patch("litellm.Cache") as mock_cache_class, + ): + mock_cache_class.return_value = cache_instance + await test_cache_connection( + request=CacheTestRequest( + cache_settings={"type": "redis", "host": "attacker.example.com", "port": "6379"} + ), + user_api_key_dict=_admin_auth(), + ) + + called_kwargs = mock_cache_class.call_args.kwargs + # the stored password is NOT sent to the attacker-chosen host + assert called_kwargs.get("password") != "realredispw" + assert "password" not in called_kwargs diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index dffca3093fa..51f72f91dc3 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -14994,3 +14994,76 @@ async def test_list_keys_without_expires_param_forwards_none(): mock_helper.assert_called_once() assert mock_helper.call_args.kwargs["expires_filter"] is None + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.management_endpoints.key_management_endpoints.rotate_sso_identity_assertions_master_key" +) +@patch( + "litellm.proxy.management_endpoints.key_management_endpoints.rotate_mcp_user_env_vars_master_key" +) +@patch( + "litellm.proxy.management_endpoints.key_management_endpoints.rotate_mcp_user_credentials_master_key" +) +@patch( + "litellm.proxy.management_endpoints.key_management_endpoints.rotate_mcp_server_credentials_master_key" +) +async def test_rotate_master_key_rotates_sso_identity_assertions( + mock_rotate_mcp_server, + mock_rotate_mcp_user, + mock_rotate_env_vars, + mock_rotate_sso, +): + """Master-key rotation must re-encrypt the SSO identity assertion store alongside + the sibling per-user encrypted tables, or a salt rotation orphans every stored + assertion (step 4d).""" + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _rotate_master_key, + ) + + mock_prisma_client = AsyncMock() + mock_prisma_client.db = MagicMock() + mock_prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + mock_tx = AsyncMock() + mock_tx.litellm_proxymodeltable = MagicMock() + mock_tx.litellm_proxymodeltable.delete_many = AsyncMock() + mock_tx.litellm_proxymodeltable.create_many = AsyncMock() + mock_prisma_client.db.tx = MagicMock( + return_value=AsyncMock( + __aenter__=AsyncMock(return_value=mock_tx), + __aexit__=AsyncMock(return_value=False), + ) + ) + mock_prisma_client.db.litellm_config.find_many = AsyncMock(return_value=[]) + mock_prisma_client.db.litellm_credentialstable.find_many = AsyncMock( + return_value=[] + ) + + mock_proxy_config = MagicMock() + mock_proxy_config.decrypt_model_list_from_db.return_value = [] + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="test-user", + ) + + with patch( + "litellm.proxy.proxy_server.proxy_config", + mock_proxy_config, + ): + await _rotate_master_key( + prisma_client=mock_prisma_client, + user_api_key_dict=user_api_key_dict, + current_master_key="sk-old-master-key", + new_master_key="sk-new-master-key", + ) + + mock_rotate_sso.assert_awaited_once_with( + prisma_client=mock_prisma_client, + new_master_key="sk-new-master-key", + ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 5631aa69102..e1856860c8a 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -1458,7 +1458,7 @@ async def test_get_generic_sso_response_with_additional_headers(): "fastapi_sso.sso.generic.create_provider", return_value=mock_sso_class ): # Act - result, received_response, _ = await get_generic_sso_response( + result, received_response, _, _ = await get_generic_sso_response( request=mock_request, jwt_handler=mock_jwt_handler, generic_client_id=generic_client_id, @@ -1522,7 +1522,7 @@ async def test_get_generic_sso_response_with_empty_headers(): "fastapi_sso.sso.generic.create_provider", return_value=mock_sso_class ): # Act - result, received_response, _ = await get_generic_sso_response( + result, received_response, _, _ = await get_generic_sso_response( request=mock_request, jwt_handler=mock_jwt_handler, generic_client_id=generic_client_id, @@ -2893,6 +2893,7 @@ class TestCLIKeyRegenerationFlow: prefill_user_code=None, result=mock_result, received_response=None, + sso_assertion=None, ) @pytest.mark.asyncio @@ -2933,6 +2934,7 @@ class TestCLIKeyRegenerationFlow: prefill_user_code="WXYZ-2345", result=mock_result, received_response=None, + sso_assertion=None, ) def test_get_redirect_url_does_not_include_existing_key_in_url(self): @@ -7019,7 +7021,7 @@ class TestPKCEStateCookieBinding: ): jwt_handler = MagicMock(spec=JWTHandler) jwt_handler.get_team_ids_from_jwt.return_value = [] - result, _, _ = await get_generic_sso_response( + result, _, _, _ = await get_generic_sso_response( request=mock_request, jwt_handler=jwt_handler, generic_client_id="cid", @@ -7078,7 +7080,7 @@ async def test_debug_sso_callback_renders_full_jwt_claims(): } async def fake_get_generic_sso_response(**kwargs): - return parsed_openid, raw_userinfo_with_leaked_token, access_token_payload + return parsed_openid, raw_userinfo_with_leaked_token, access_token_payload, None with ( patch.dict( @@ -7374,3 +7376,266 @@ async def test_auth_callback_without_oauth_error_proceeds_to_normal_flow(): assert exc_info.value.status_code == 500 assert "DB not connected" in str(exc_info.value.detail) + + +# ── SSO identity assertion capture + persist wiring (EMA) ───────────────────── + + +def _ema_id_token(sub: str = "u1") -> str: + import time as _time + + import jwt as _pyjwt + + return _pyjwt.encode( + {"iss": "https://idp.example.com", "sub": sub, "exp": int(_time.time()) + 3600}, + "test-idp-signing-key-32-bytes-long-xxxx", + algorithm="HS256", + ) + + +@pytest.mark.asyncio +async def test_pkce_arm_captures_sso_assertion(): + """The PKCE token exchange strips bearer fields from received_response for safety; + the typed assertion carrier must still capture id_token + refresh_token.""" + from litellm.proxy.management_endpoints.ui_sso import ( + SSOAuthenticationHandler, + get_generic_sso_response, + ) + + id_token = _ema_id_token() + mock_request = MagicMock(spec=Request) + mock_request.query_params = {"state": "matched-state", "code": "auth-code"} + mock_request.cookies = {"litellm_oauth_state": "matched-state"} + + with ( + patch.object( + SSOAuthenticationHandler, + "prepare_token_exchange_parameters", + AsyncMock( + return_value={ + "code_verifier": "verifier", + "_pkce_cache_key": "pkce_verifier:matched-state", + } + ), + ), + patch.object( + SSOAuthenticationHandler, + "_pkce_token_exchange", + AsyncMock( + return_value={ + "access_token": "tok", + "id_token": id_token, + "refresh_token": "rt_from_idp", + "sub": "user@example.com", + "email": "user@example.com", + } + ), + ), + patch.object(SSOAuthenticationHandler, "_delete_pkce_verifier", AsyncMock()), + patch("fastapi_sso.sso.base.DiscoveryDocument"), + patch("fastapi_sso.sso.generic.create_provider", return_value=MagicMock()), + patch.dict( + os.environ, + { + "GENERIC_CLIENT_SECRET": "x", + "GENERIC_AUTHORIZATION_ENDPOINT": "https://idp.example.com/auth", + "GENERIC_TOKEN_ENDPOINT": "https://idp.example.com/token", + "GENERIC_USERINFO_ENDPOINT": "https://idp.example.com/userinfo", + "GENERIC_CLIENT_USE_PKCE": "true", + }, + ), + ): + jwt_handler = MagicMock(spec=JWTHandler) + jwt_handler.get_team_ids_from_jwt.return_value = [] + result, received_response, _, sso_assertion = await get_generic_sso_response( + request=mock_request, + jwt_handler=jwt_handler, + generic_client_id="cid", + redirect_url="https://proxy.example.com/sso/callback", + sso_jwt_handler=None, + ) + + assert sso_assertion is not None + assert sso_assertion.id_token.get_secret_value() == id_token + assert sso_assertion.refresh_token is not None + assert sso_assertion.refresh_token.get_secret_value() == "rt_from_idp" + # The sanitized received_response must still not carry bearer material. + assert "id_token" not in (received_response or {}) + assert "refresh_token" not in (received_response or {}) + + +@pytest.mark.asyncio +async def test_verify_and_process_arm_captures_sso_assertion(): + """The non-PKCE generic arm reads the raw bearer fields off the fastapi-sso client.""" + from litellm.proxy.management_endpoints.ui_sso import get_generic_sso_response + + id_token = _ema_id_token() + mock_request = MagicMock(spec=Request) + mock_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler.get_team_ids_from_jwt.return_value = [] + + mock_sso_instance = MagicMock() + mock_sso_instance.verify_and_process = AsyncMock( + return_value={"sub": "u1", "email": "u@example.com"} + ) + mock_sso_instance.access_token = None + mock_sso_instance.id_token = id_token + mock_sso_instance.refresh_token = "rt_from_idp" + mock_sso_class = MagicMock(return_value=mock_sso_instance) + + with patch.dict( + os.environ, + { + "GENERIC_CLIENT_SECRET": "test_secret", + "GENERIC_AUTHORIZATION_ENDPOINT": "https://auth.example.com/auth", + "GENERIC_TOKEN_ENDPOINT": "https://auth.example.com/token", + "GENERIC_USERINFO_ENDPOINT": "https://auth.example.com/userinfo", + }, + ): + with patch("fastapi_sso.sso.base.DiscoveryDocument"): + with patch( + "fastapi_sso.sso.generic.create_provider", return_value=mock_sso_class + ): + _, _, _, sso_assertion = await get_generic_sso_response( + request=mock_request, + jwt_handler=mock_jwt_handler, + generic_client_id="test_client_id", + redirect_url="http://test.com/callback", + sso_jwt_handler=None, + ) + + assert sso_assertion is not None + assert sso_assertion.id_token.get_secret_value() == id_token + assert sso_assertion.refresh_token is not None + assert sso_assertion.refresh_token.get_secret_value() == "rt_from_idp" + + +@pytest.mark.asyncio +async def test_redirect_from_openid_persists_assertion_under_canonical_user_id(): + """The browser funnel persists the captured assertion AFTER canonical user + resolution, keyed by the user_id admission will later resolve (the key-generation + response user_id), not the raw IdP subject.""" + from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( + assertion_from_sso_login, + ) + + assertion = assertion_from_sso_login(_ema_id_token(), "rt_1") + assert assertion is not None + mock_request = MagicMock(spec=Request) + mock_request.base_url = "http://localhost:4000/" + mock_request.cookies = {} + + retain_mock = AsyncMock() + with ( + patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), + patch("litellm.proxy.proxy_server.master_key", "sk-master"), + patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.premium_user", False), + patch("litellm.proxy.proxy_server.user_custom_sso", None), + patch("litellm.proxy.proxy_server.proxy_logging_obj", MagicMock()), + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), + patch( + "litellm.proxy.proxy_server.generate_key_helper_fn", + AsyncMock( + return_value={"token": "sk-ui-key", "user_id": "canonical-user-id"} + ), + ), + patch( + "litellm.proxy.management_endpoints.ui_sso.get_user_info_from_db", + AsyncMock(return_value=None), + ), + patch( + "litellm.proxy.management_endpoints.ui_sso.check_and_update_if_proxy_admin_id", + AsyncMock(return_value="internal_user"), + ), + patch( + "litellm.proxy.management_endpoints.ui_sso.retain_sso_identity_assertion_for_ema", + retain_mock, + ), + ): + response = await SSOAuthenticationHandler.get_redirect_response_from_openid( + result=CustomOpenID( + id="raw-idp-subject", + email="u@example.com", + first_name="U", + last_name="Ser", + display_name="U Ser", + provider="generic", + team_ids=[], + user_role=None, + ), + request=mock_request, + received_response=None, + generic_client_id="cid", + ui_access_mode=None, + access_token_payload=None, + jwt_handler=None, + sso_assertion=assertion, + ) + + retain_mock.assert_awaited_once_with( + user_id="canonical-user-id", assertion=assertion + ) + assert response is not None + + +@pytest.mark.asyncio +async def test_cli_completion_persists_assertion_under_db_user_id(): + """The CLI funnel persists the captured assertion under the DB-resolved user_id.""" + from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( + assertion_from_sso_login, + ) + from litellm.proxy.management_endpoints.ui_sso import ( + _complete_cli_sso_callback_session, + ) + + assertion = assertion_from_sso_login(_ema_id_token(), None) + assert assertion is not None + mock_request = MagicMock(spec=Request) + mock_request.base_url = "http://localhost:4000/" + + user_info = MagicMock() + user_info.user_id = "cli-user-id" + user_info.user_role = "internal_user" + user_info.models = [] + user_info.teams = [] + + retain_mock = AsyncMock() + with ( + patch( + "litellm.proxy.management_endpoints.ui_sso.get_user_info_from_db", + AsyncMock(return_value=user_info), + ), + patch( + "litellm.proxy.management_endpoints.ui_sso._fetch_cli_sso_team_details", + AsyncMock(return_value=[]), + ), + patch( + "litellm.proxy.management_endpoints.ui_sso.build_cli_sso_attribution_metadata", + return_value={}, + ), + patch( + "litellm.proxy.management_endpoints.ui_sso.retain_sso_identity_assertion_for_ema", + retain_mock, + ), + ): + response = await _complete_cli_sso_callback_session( + request=mock_request, + key="cli-login-id", + flow={}, + result={"sub": "raw-idp-subject"}, + parsed_openid_result={ + "user_id": "raw-idp-subject", + "user_email": "u@example.com", + "user_role": None, + }, + user_defined_values=None, + prisma_client=MagicMock(), + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + sso_assertion=assertion, + ) + + retain_mock.assert_awaited_once_with(user_id="cli-user-id", assertion=assertion) + assert response.status_code == 200 diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 93b21c9d3c1..8d1d8185e4d 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -8,6 +8,7 @@ Pins covered: from __future__ import annotations +import json import os from types import SimpleNamespace from typing import Any, Dict @@ -407,6 +408,124 @@ async def test_ProxyConfig_save_config_invalid_path_raises(monkeypatch): await pc.save_config({"x": 1}) +@pytest.mark.asyncio +async def test_ProxyConfig_save_config_db_omits_environment_variables_by_default(monkeypatch): + """A save_config after get_config() (which resolves os.environ/ placeholders + to plaintext and merges the environment_variables section) must not snapshot + those env vars into the DB config row. Persisting them would make a stale DB + row shadow YAML/container env on every subsequent restart.""" + mock_prisma = MagicMock() + mock_prisma.insert_data = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + # a valid salt so the env-var encryption path (reached only if the pop + # regresses) runs cleanly, making this fail on the assertion below rather + # than on an incidental encryption crash + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-test-salt-key") + + pc = ProxyConfig() + cfg = { + "model_list": [{"model_name": "gpt-4o"}], + "litellm_settings": {"success_callback": ["langfuse"]}, + "environment_variables": {"OPENAI_API_KEY": "sk-from-yaml"}, + } + await pc.save_config(cfg) + + mock_prisma.insert_data.assert_awaited_once() + written = mock_prisma.insert_data.await_args.kwargs["data"] + assert "environment_variables" not in written + # unrelated sections are still persisted; model_list is stripped as before + assert written["litellm_settings"] == {"success_callback": ["langfuse"]} + assert "model_list" not in written + # the caller's dict is not mutated (save_config works on a copy) + assert cfg["environment_variables"] == {"OPENAI_API_KEY": "sk-from-yaml"} + + +@pytest.mark.asyncio +async def test_ProxyConfig_save_config_db_persists_environment_variables_when_opted_in(monkeypatch): + """The explicit opt-in path (include_env_vars=True) still persists env vars, + encrypted, so the dedicated config-update flow can write them.""" + mock_prisma = MagicMock() + mock_prisma.insert_data = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-test-salt-key") + + pc = ProxyConfig() + cfg = {"litellm_settings": {}, "environment_variables": {"OPENAI_API_KEY": "sk-explicit"}} + await pc.save_config(cfg, include_env_vars=True) + + mock_prisma.insert_data.assert_awaited_once() + written = mock_prisma.insert_data.await_args.kwargs["data"] + assert set(written["environment_variables"].keys()) == {"OPENAI_API_KEY"} + # value is encrypted at rest, not the plaintext it came in as + assert written["environment_variables"]["OPENAI_API_KEY"] != "sk-explicit" + + +def _install_fake_config_repo(monkeypatch, existing_row): + """Route ProxyConfig's ConfigRepository through an in-memory fake that + records the value written to the environment_variables row.""" + captured: dict = {} + + class _FakeTable: + async def find_first(self, where): + return SimpleNamespace(param_value=existing_row) if existing_row is not None else None + + async def upsert(self, where, data): + captured["value"] = json.loads(data["update"]["param_value"]) + + class _FakeRepo: + def __init__(self, client): + self.table = _FakeTable() + + monkeypatch.setattr("litellm.proxy.proxy_server.ConfigRepository", _FakeRepo) + monkeypatch.setattr("litellm.proxy.proxy_server.invalidate_config_param", AsyncMock()) + return captured + + +@pytest.mark.asyncio +async def test_ProxyConfig_save_environment_variables_merges_sets_and_deletes(monkeypatch): + """The per-key env-var write updates/deletes only the named keys and leaves + every other stored key untouched, so an unrelated env var is never lost or + snapshotted.""" + captured = _install_fake_config_repo( + monkeypatch, + existing_row={"EXISTING_KEY": "ciphertext-existing", "UI_LOGO_PATH": "old-logo", "LITELLM_FAVICON_URL": "old"}, + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MagicMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-test-salt-key") + + pc = ProxyConfig() + await pc.save_environment_variables({"UI_LOGO_PATH": "new-logo", "LITELLM_FAVICON_URL": None}) + + written = captured["value"] + # unrelated key preserved byte-for-byte + assert written["EXISTING_KEY"] == "ciphertext-existing" + # set key updated and encrypted (not the plaintext) + assert "UI_LOGO_PATH" in written and written["UI_LOGO_PATH"] != "new-logo" + # None-valued key deleted + assert "LITELLM_FAVICON_URL" not in written + + +@pytest.mark.asyncio +async def test_ProxyConfig_save_environment_variables_noop_without_db(monkeypatch): + """With no DB configured the per-key write must do nothing (never touch the + config repository).""" + captured = _install_fake_config_repo(monkeypatch, existing_row={}) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + + pc = ProxyConfig() + await pc.save_environment_variables({"UI_LOGO_PATH": "x"}) + + assert "value" not in captured + + # --------------------------------------------------------------------------- # ProxyConfig._check_for_os_environ_vars # --------------------------------------------------------------------------- @@ -950,6 +1069,32 @@ async def test_ProxyConfig_load_config_wires_general_settings_url_validation(tmp litellm.provider_url_destination_allowed_hosts = original_provider_hosts +@pytest.mark.asyncio +async def test_ProxyConfig_load_config_wires_config_reload_interval(tmp_path, monkeypatch): + """general_settings.proxy_config_reload_interval_seconds must reach the proxy_server + module global that schedules the DB config-reload jobs, so operators can tune multi-pod + convergence from config.yaml.""" + import litellm.proxy.proxy_server as proxy_server + + f = tmp_path / "c.yaml" + f.write_text( + "model_list: []\n" + "general_settings:\n" + " proxy_config_reload_interval_seconds: 47\n" + "litellm_settings: {}\n" + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + + original = proxy_server.proxy_config_reload_interval_seconds + try: + await ProxyConfig().load_config(router=None, config_file_path=str(f)) + assert proxy_server.proxy_config_reload_interval_seconds == 47 + finally: + proxy_server.proxy_config_reload_interval_seconds = original + + @pytest.mark.asyncio async def test_ProxyConfig_load_config_missing_file_raises(monkeypatch): monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py index 4ac6fc46a61..ad3c470acf3 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py @@ -13,6 +13,7 @@ Routes covered: from __future__ import annotations +import json from unittest.mock import AsyncMock, MagicMock from .conftest import VOLATILE_KEYS, normalize @@ -473,6 +474,83 @@ def test_config_list_happy_admin(client, auth_as, mock_prisma, monkeypatch): } +def test_config_list_exposes_config_reload_interval(client, auth_as, mock_prisma, monkeypatch): + """proxy_config_reload_interval_seconds must surface in the admin UI general-settings + list as an Integer field defaulting to 30, so operators can tune multi-pod convergence + from the dashboard.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + row = MagicMock() + row.param_value = {} + table.find_first = AsyncMock(return_value=row) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get("/config/list", params={"config_type": "general_settings"}) + assert response.status_code == 200 + by_name = {entry["field_name"]: entry for entry in response.json()} + assert "proxy_config_reload_interval_seconds" in by_name + entry = by_name["proxy_config_reload_interval_seconds"] + assert entry["field_type"] == "Integer" + assert entry["field_default_value"] == 30 + + +def test_config_field_update_accepts_config_reload_interval(client, auth_as, mock_prisma, monkeypatch): + """POST /config/field/update accepts proxy_config_reload_interval_seconds and persists + it to the DB general_settings row for all pods to pick up.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + table.find_first = AsyncMock(return_value=None) + upsert_row = { + "param_name": "general_settings", + "param_value": {"proxy_config_reload_interval_seconds": 45}, + "id": "row-1", + } + table.upsert = AsyncMock(return_value=upsert_row) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/field/update", + json={ + "field_name": "proxy_config_reload_interval_seconds", + "field_value": 45, + "config_type": "general_settings", + }, + ) + assert response.status_code == 200 + upserted = table.upsert.call_args.kwargs["data"]["create"]["param_value"] + assert json.loads(upserted)["proxy_config_reload_interval_seconds"] == 45 + + +def test_config_field_update_rejects_non_positive_config_reload_interval(client, auth_as, mock_prisma, monkeypatch): + """A non-positive proxy_config_reload_interval_seconds from the UI is rejected with a 400 + and never persisted, since APScheduler requires a positive interval.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + table.find_first = AsyncMock(return_value=None) + table.upsert = AsyncMock() + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/field/update", + json={ + "field_name": "proxy_config_reload_interval_seconds", + "field_value": 0, + "config_type": "general_settings", + }, + ) + assert response.status_code == 400 + table.upsert.assert_not_called() + + def test_config_list_non_admin_rejected(client, auth_as, mock_prisma, monkeypatch): """Non-admin gets a 400 with the role embedded in the error message.""" from litellm.proxy import proxy_server as ps diff --git a/tests/test_litellm/proxy/test_provider_url_destination_guard.py b/tests/test_litellm/proxy/test_provider_url_destination_guard.py index 51cd76105d0..c8771abbc8e 100644 --- a/tests/test_litellm/proxy/test_provider_url_destination_guard.py +++ b/tests/test_litellm/proxy/test_provider_url_destination_guard.py @@ -39,6 +39,46 @@ class TestRejectUrlValuedDestinations: assert exc_info.value.status_code == 400 assert exc_info.value.detail["param"] == "model" + def test_provider_prefixed_url_rejected(self): + with pytest.raises(HTTPException) as exc_info: + _reject_url_valued_destinations( + {"model": "huggingface/https://attacker.example/v1"} + ) + assert exc_info.value.status_code == 400 + assert exc_info.value.detail["param"] == "model" + + def test_comma_batch_smuggled_url_rejected(self): + with pytest.raises(HTTPException) as exc_info: + _reject_url_valued_destinations( + {"model": "gpt-4,huggingface/https://attacker.example/v1"} + ) + assert exc_info.value.status_code == 400 + assert exc_info.value.detail["param"] == "model" + + def test_provider_prefixed_uppercase_scheme_url_rejected(self): + with pytest.raises(HTTPException) as exc_info: + _reject_url_valued_destinations( + {"model": "huggingface/HTTPS://evil.example/v1"} + ) + assert exc_info.value.status_code == 400 + assert exc_info.value.detail["param"] == "model" + + def test_provider_prefixed_plain_model_passes(self): + _reject_url_valued_destinations({"model": "huggingface/BAAI/bge-small-en"}) + + def test_comma_batch_plain_models_pass(self): + _reject_url_valued_destinations({"model": "gpt-4,huggingface/BAAI/bge-small-en"}) + + def test_provider_prefixed_url_respects_allowlist(self, monkeypatch): + monkeypatch.setattr( + litellm, + "provider_url_destination_allowed_hosts", + ["trusted.example"], + ) + _reject_url_valued_destinations( + {"model": "huggingface/https://trusted.example/v1"} + ) + def test_url_valued_file_id_rejected(self): with pytest.raises(HTTPException) as exc_info: _reject_url_valued_destinations( diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index a100e7837f4..8a19c6b4406 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -751,6 +751,97 @@ async def test_initialize_scheduled_jobs_credentials(monkeypatch): assert len(mock_scheduler_calls) > 0 +@pytest.mark.asyncio +async def test_initialize_scheduled_jobs_uses_configured_config_reload_interval(monkeypatch): + """ + The DB config-reload jobs (add_deployment, get_credentials) that keep multi-pod + deployments in sync must be scheduled at the configured + proxy_config_reload_interval_seconds, not a hardcoded value. + """ + monkeypatch.delenv("DISABLE_PRISMA_SCHEMA_UPDATE", raising=False) + monkeypatch.delenv("STORE_MODEL_IN_DB", raising=False) + from litellm.proxy.proxy_server import ProxyStartupEvent + from litellm.proxy.utils import ProxyLogging + + mock_prisma_client = MagicMock() + mock_proxy_logging = MagicMock(spec=ProxyLogging) + mock_proxy_logging.slack_alerting_instance = MagicMock() + mock_proxy_config = AsyncMock() + mock_scheduler = MagicMock() + + configured_interval = 47 + + with ( + patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch("litellm.proxy.proxy_server.get_secret_bool", return_value=True), + patch( + "litellm.proxy.proxy_server.proxy_config_reload_interval_seconds", + configured_interval, + ), + patch("litellm.proxy.proxy_server.AsyncIOScheduler", return_value=mock_scheduler), + ): + await ProxyStartupEvent.initialize_scheduled_background_jobs( + general_settings={}, + prisma_client=mock_prisma_client, + proxy_budget_rescheduler_min_time=1, + proxy_budget_rescheduler_max_time=2, + proxy_batch_write_at=5, + proxy_logging_obj=mock_proxy_logging, + ) + + scheduled_seconds = { + job_call.kwargs["id"]: job_call.kwargs.get("seconds") + for job_call in mock_scheduler.add_job.call_args_list + if "id" in job_call.kwargs + } + assert scheduled_seconds["add_deployment_job"] == configured_interval + assert scheduled_seconds["get_credentials_job"] == configured_interval + + +@pytest.mark.asyncio +async def test_initialize_scheduled_jobs_rejects_non_positive_config_reload_interval(monkeypatch): + """ + A non-positive proxy_config_reload_interval_seconds (misconfig via env/config/DB) would + make APScheduler reject the job and crash startup, so the scheduler must fall back to the + 30s default instead of forwarding the bad value. + """ + monkeypatch.delenv("DISABLE_PRISMA_SCHEMA_UPDATE", raising=False) + monkeypatch.delenv("STORE_MODEL_IN_DB", raising=False) + from litellm.proxy.proxy_server import ProxyStartupEvent + from litellm.proxy.utils import ProxyLogging + + mock_prisma_client = MagicMock() + mock_proxy_logging = MagicMock(spec=ProxyLogging) + mock_proxy_logging.slack_alerting_instance = MagicMock() + mock_proxy_config = AsyncMock() + mock_scheduler = MagicMock() + + with ( + patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch("litellm.proxy.proxy_server.get_secret_bool", return_value=True), + patch("litellm.proxy.proxy_server.proxy_config_reload_interval_seconds", 0), + patch("litellm.proxy.proxy_server.AsyncIOScheduler", return_value=mock_scheduler), + ): + await ProxyStartupEvent.initialize_scheduled_background_jobs( + general_settings={}, + prisma_client=mock_prisma_client, + proxy_budget_rescheduler_min_time=1, + proxy_budget_rescheduler_max_time=2, + proxy_batch_write_at=5, + proxy_logging_obj=mock_proxy_logging, + ) + + scheduled_seconds = { + job_call.kwargs["id"]: job_call.kwargs.get("seconds") + for job_call in mock_scheduler.add_job.call_args_list + if "id" in job_call.kwargs + } + assert scheduled_seconds["add_deployment_job"] == 30 + assert scheduled_seconds["get_credentials_job"] == 30 + + @pytest.mark.asyncio async def test_initialize_scheduled_jobs_hydrates_mcp_when_store_model_in_db_false(monkeypatch): """ @@ -924,6 +1015,102 @@ def test_get_config_custom_callback_api_env_vars(monkeypatch): } +@patch( + "litellm.proxy.common_utils.callback_utils.CustomLogger.get_callback_env_vars", + return_value=["LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_HOST"], +) +def test_get_config_callbacks_fall_back_to_process_env(mock_env_vars, monkeypatch): + """A callback configured purely via process env vars is surfaced. + + An IaC deployment sets LANGFUSE_* on the gateway and never touches the UI, + so nothing is stored in the config environment_variables overlay. The read + endpoint must still report the live values instead of blanks. + """ + from litellm.proxy.proxy_server import app, proxy_config, user_api_key_auth + + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-env-only") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-env-only") + monkeypatch.setenv("LANGFUSE_HOST", "https://cloud.langfuse.com") + + config_data = { + "litellm_settings": {"success_callback": ["langfuse"]}, + "general_settings": {}, + "environment_variables": {}, + } + mock_router = MagicMock() + mock_router.get_settings.return_value = {} + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", mock_router) + monkeypatch.setattr(proxy_config, "get_config", AsyncMock(return_value=config_data)) + + original_overrides = app.dependency_overrides.copy() + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234" + ) + + client = TestClient(app) + try: + response = client.get("/get/config/callbacks") + finally: + app.dependency_overrides = original_overrides + + assert response.status_code == 200 + langfuse_cb = next( + (cb for cb in response.json()["callbacks"] if cb["name"] == "langfuse"), None + ) + assert langfuse_cb is not None + assert langfuse_cb["variables"] == { + "LANGFUSE_PUBLIC_KEY": "pk-env-only", + "LANGFUSE_SECRET_KEY": "sk-env-only", + "LANGFUSE_HOST": "https://cloud.langfuse.com", + } + + +@patch( + "litellm.proxy.common_utils.callback_utils.CustomLogger.get_callback_env_vars", + return_value=["LANGFUSE_SECRET_KEY", "LANGFUSE_HOST"], +) +def test_get_config_callback_env_secrets_redacted_for_non_admin(mock_env_vars, monkeypatch): + """Surfacing env vars must not widen who can read secret values. + + The callback role gate redacts sensitive keys for anyone below full admin, + and that must hold whether the value came from the stored config or the + process env. A non-secret var (LANGFUSE_HOST) still resolves for context. + """ + from litellm.proxy.proxy_server import app, proxy_config, user_api_key_auth + + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-env-only-secret") + monkeypatch.setenv("LANGFUSE_HOST", "https://cloud.langfuse.com") + + config_data = { + "litellm_settings": {"success_callback": ["langfuse"]}, + "general_settings": {}, + "environment_variables": {}, + } + mock_router = MagicMock() + mock_router.get_settings.return_value = {} + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", mock_router) + monkeypatch.setattr(proxy_config, "get_config", AsyncMock(return_value=config_data)) + + original_overrides = app.dependency_overrides.copy() + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-user" + ) + + client = TestClient(app) + try: + response = client.get("/get/config/callbacks") + finally: + app.dependency_overrides = original_overrides + + assert response.status_code == 200 + langfuse_cb = next( + (cb for cb in response.json()["callbacks"] if cb["name"] == "langfuse"), None + ) + assert langfuse_cb is not None + assert langfuse_cb["variables"]["LANGFUSE_SECRET_KEY"] == "REDACTED" + assert langfuse_cb["variables"]["LANGFUSE_HOST"] == "https://cloud.langfuse.com" + + def test_get_config_returns_email_settings(monkeypatch): """ Regression for https://github.com/BerriAI/litellm/issues/19221 diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index 69845ec59c2..805baed9e1e 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -53,6 +53,7 @@ def mock_proxy_config(monkeypatch): # Add a counter to track save_config calls save_config_call_count = 0 + saved_env_updates: list = [] async def mock_save_config(new_config=None): nonlocal mock_config, save_config_call_count @@ -61,13 +62,22 @@ def mock_proxy_config(monkeypatch): mock_config = new_config return mock_config + async def mock_save_environment_variables(updates): + saved_env_updates.append(updates) + from litellm.proxy.proxy_server import proxy_config monkeypatch.setattr(proxy_config, "get_config", mock_get_config) monkeypatch.setattr(proxy_config, "save_config", mock_save_config) + monkeypatch.setattr(proxy_config, "save_environment_variables", mock_save_environment_variables) - # Return both the config and the call counter - return {"config": mock_config, "save_call_count": lambda: save_config_call_count} + # Return the config, the save_config call counter, and any env-var updates + # the endpoint routed through the dedicated save_environment_variables path + return { + "config": mock_config, + "save_call_count": lambda: save_config_call_count, + "env_updates": lambda: saved_env_updates, + } @pytest.fixture @@ -840,11 +850,18 @@ class TestProxySettingEndpoints: assert data["status"] == "success" assert data["theme_config"]["logo_url"] == "https://example.com/new-logo.png" - # Verify config was updated - updated_config = mock_proxy_config["config"] - assert "UI_LOGO_PATH" in updated_config["environment_variables"] + # The logo path is applied to the live process immediately + assert os.environ["UI_LOGO_PATH"] == "https://example.com/new-logo.png" assert mock_proxy_config["save_call_count"]() == 1 + # env vars are persisted through the dedicated per-key path, and ONLY + # the two keys this endpoint owns are touched. The unrelated SSO env + # vars in the merged config are never snapshotted. + env_updates = mock_proxy_config["env_updates"]() + assert env_updates == [ + {"UI_LOGO_PATH": "https://example.com/new-logo.png", "LITELLM_FAVICON_URL": None} + ] + def test_update_ui_theme_settings_with_favicon( self, mock_proxy_config, mock_auth, monkeypatch ): @@ -869,13 +886,15 @@ class TestProxySettingEndpoints: == "https://example.com/custom-favicon.ico" ) - updated_config = mock_proxy_config["config"] - assert "UI_LOGO_PATH" in updated_config["environment_variables"] - assert "LITELLM_FAVICON_URL" in updated_config["environment_variables"] - assert ( - updated_config["environment_variables"]["LITELLM_FAVICON_URL"] - == "https://example.com/custom-favicon.ico" - ) + assert os.environ["UI_LOGO_PATH"] == "https://example.com/new-logo.png" + assert os.environ["LITELLM_FAVICON_URL"] == "https://example.com/custom-favicon.ico" + # Only the two owned keys are persisted, both with their new values + assert mock_proxy_config["env_updates"]() == [ + { + "UI_LOGO_PATH": "https://example.com/new-logo.png", + "LITELLM_FAVICON_URL": "https://example.com/custom-favicon.ico", + } + ] def test_update_ui_theme_settings_clear_favicon( self, mock_proxy_config, mock_auth, monkeypatch @@ -925,6 +944,88 @@ class TestProxySettingEndpoints: assert data["values"]["logo_url"] == "https://example.com/logo.png" assert data["values"]["favicon_url"] == "https://example.com/favicon.ico" + def test_get_ui_theme_settings_falls_back_to_process_env( + self, mock_proxy_config, monkeypatch + ): + """Branding supplied only as process env vars must surface in the read. + + A deployment that sets UI_LOGO_PATH / LITELLM_FAVICON_URL via IaC and + never touches the UI has no stored ui_theme_config, yet the branding is + live, so the settings page must reflect it rather than reading blank. + """ + monkeypatch.delenv("UI_LOGO_PATH", raising=False) + monkeypatch.delenv("LITELLM_FAVICON_URL", raising=False) + monkeypatch.setenv("UI_LOGO_PATH", "https://cdn.example.com/logo.png") + monkeypatch.setenv("LITELLM_FAVICON_URL", "https://cdn.example.com/favicon.ico") + + response = client.get("/get/ui_theme_settings") + + assert response.status_code == 200 + values = response.json()["values"] + assert values["logo_url"] == "https://cdn.example.com/logo.png" + assert values["favicon_url"] == "https://cdn.example.com/favicon.ico" + + def test_get_ui_theme_settings_stored_value_wins_over_env( + self, mock_auth, monkeypatch + ): + """A stored ui_theme_config field outranks the env var for that field. + + The env fallback only fills fields the stored config leaves blank, so the + UI-driven flow is unchanged while an unstored field still resolves. + """ + from litellm.proxy.proxy_server import proxy_config + + stored_config = { + "litellm_settings": { + "ui_theme_config": {"logo_url": "https://db.example.com/logo.png"} + } + } + + async def mock_get_config(): + return stored_config + + monkeypatch.setattr(proxy_config, "get_config", mock_get_config) + monkeypatch.setenv("UI_LOGO_PATH", "https://env.example.com/logo.png") + monkeypatch.setenv("LITELLM_FAVICON_URL", "https://env.example.com/favicon.ico") + + response = client.get("/get/ui_theme_settings") + + assert response.status_code == 200 + values = response.json()["values"] + assert values["logo_url"] == "https://db.example.com/logo.png" + assert values["favicon_url"] == "https://env.example.com/favicon.ico" + + def test_get_ui_theme_settings_reports_unset_when_absent_everywhere( + self, mock_proxy_config, monkeypatch + ): + """A field set in neither the stored config nor the env stays null.""" + monkeypatch.delenv("UI_LOGO_PATH", raising=False) + monkeypatch.delenv("LITELLM_FAVICON_URL", raising=False) + + response = client.get("/get/ui_theme_settings") + + assert response.status_code == 200 + values = response.json()["values"] + assert values["logo_url"] is None + assert values["favicon_url"] is None + + def test_get_ui_theme_settings_does_not_disclose_local_path_env_value( + self, mock_proxy_config, monkeypatch + ): + """This endpoint is public, so an env-configured local filesystem branding + path must never be surfaced to anonymous callers; only public http(s) URLs. + """ + monkeypatch.setenv("UI_LOGO_PATH", "/mnt/secret/internal/logo.png") + monkeypatch.setenv("LITELLM_FAVICON_URL", "file:///etc/favicon.ico") + + response = client.get("/get/ui_theme_settings") + + assert response.status_code == 200 + values = response.json()["values"] + # the local path / file scheme is withheld rather than disclosed + assert values["logo_url"] is None + assert values["favicon_url"] is None + def test_get_ui_settings(self, mock_auth, monkeypatch): """Test retrieving UI settings with allowlist sanitization""" from unittest.mock import AsyncMock, MagicMock diff --git a/tests/test_litellm/test_router_per_deployment_num_retries.py b/tests/test_litellm/test_router_per_deployment_num_retries.py index af2372616a6..25574fcb268 100644 --- a/tests/test_litellm/test_router_per_deployment_num_retries.py +++ b/tests/test_litellm/test_router_per_deployment_num_retries.py @@ -3,11 +3,15 @@ Unit tests for per-deployment num_retries in litellm_params GitHub Issue: #18968 - Per-deployment max_retries/num_retries in litellm_params is not used in retry logic """ +import httpx import pytest +import pytest_asyncio from unittest.mock import patch import litellm from litellm import Router +from litellm.types.router import RetryPolicy +from litellm.integrations.custom_logger import CustomLogger class TestPerDeploymentNumRetries: @@ -319,3 +323,255 @@ class TestNumRetriesNoneGuard: # 1 initial attempt + at least 1 retry -> proves None fell back to a positive int assert calls["n"] >= 2 + + +class TestNoProviderRetryAmplification: + """ + A routed request must reach the upstream provider exactly ``1 + `` + times. The Router is the sole retry owner for routed calls, so the provider SDK + must never retry on top of it. Otherwise a per-deployment ``num_retries`` set in + ``litellm_params`` is applied twice - once by the Router loop and once as the + provider client's ``max_retries`` - turning one request into ``(1 + num_retries) ** 2`` + upstream requests. + + These tests count actual upstream HTTP requests through the full Router completion + path by injecting a counting transport via ``litellm.aclient_session`` (the + documented seam the OpenAI client builder reads), so both Router-level and any + provider-SDK-level retries are observed. + """ + + @staticmethod + def _install_counting_upstream() -> dict: + """Route every upstream POST to a 500 and count it. ``retry-after: 0`` keeps + provider-SDK backoff at zero so a mutated (double-retrying) build stays fast.""" + counter = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + counter["n"] += 1 + return httpx.Response( + 500, + headers={"retry-after": "0"}, + json={"error": {"message": "boom", "type": "server_error"}}, + ) + + litellm.aclient_session = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + return counter + + @pytest_asyncio.fixture(autouse=True) + async def _isolate_clients(self): + litellm.in_memory_llm_clients_cache.flush_cache() + yield + session = litellm.aclient_session + litellm.aclient_session = None + litellm.in_memory_llm_clients_cache.flush_cache() + if session is not None: + await session.aclose() + + @staticmethod + def _router(api_base: str, litellm_params: dict, **router_kwargs) -> Router: + params = {"model": "openai/gpt-4o-mini", "api_base": api_base, "api_key": "sk-fake"} + params.update(litellm_params) + return Router(model_list=[{"model_name": "mock", "litellm_params": params}], **router_kwargs) + + async def _call_and_count(self, router: Router, **call_kwargs) -> int: + counter = self._install_counting_upstream() + with patch("asyncio.sleep", return_value=None): + with pytest.raises(litellm.InternalServerError): + await router.acompletion( + model="mock", messages=[{"role": "user", "content": "hi"}], **call_kwargs + ) + return counter["n"] + + @pytest.mark.asyncio + @pytest.mark.parametrize("num_retries", [2, 5]) + async def test_deployment_num_retries_sends_no_extra_provider_requests(self, num_retries): + """ + Deployment ``num_retries=N`` (every attempt failing) must send exactly ``N + 1`` + upstream requests, not ``(N + 1) ** 2``. This is the amplification regression: + an unfixed build sends 9 (N=2) or 36 (N=5). + """ + counter = self._install_counting_upstream() + router = self._router( + f"https://amp-{num_retries}.local/v1", {"num_retries": num_retries}, num_retries=1 + ) + with patch("asyncio.sleep", return_value=None): + with pytest.raises(litellm.InternalServerError): + await router.acompletion(model="mock", messages=[{"role": "user", "content": "hi"}]) + assert counter["n"] == num_retries + 1 + + @pytest.mark.asyncio + async def test_request_max_retries_does_not_nest_with_router_retries(self): + """ + A request-body ``max_retries`` must not make the provider SDK retry on top of the + Router. With deployment ``num_retries=5`` and request ``max_retries=3`` the count + stays ``6``; a build that lets either value reach the provider SDK sends 24 or 36. + """ + router = self._router("https://nest-req.local/v1", {"num_retries": 5}, num_retries=1) + assert await self._call_and_count(router, max_retries=3) == 6 + + @pytest.mark.asyncio + async def test_deployment_max_retries_does_not_nest_with_router_retries(self): + """ + A deployment-level ``max_retries`` is likewise never applied on top of the Router's + retries for a routed call: deployment ``num_retries=5`` plus ``max_retries=3`` still + sends exactly ``6`` upstream requests. + """ + router = self._router( + "https://nest-dep.local/v1", {"num_retries": 5, "max_retries": 3}, num_retries=1 + ) + assert await self._call_and_count(router) == 6 + + @pytest.mark.asyncio + async def test_retry_policy_configured_does_not_reintroduce_amplification(self): + """ + With a retry policy configured alongside a per-deployment ``num_retries=5``, the + provider SDK still must not retry: exactly ``6`` upstream requests, not 36. + """ + router = self._router( + "https://policy.local/v1", + {"num_retries": 5}, + num_retries=1, + retry_policy=RetryPolicy(InternalServerErrorRetries=2), + ) + assert await self._call_and_count(router) == 6 + + @pytest.mark.asyncio + async def test_global_num_retries_not_amplified(self): + """ + Global ``num_retries`` (no per-deployment setting) already behaves correctly and + must stay that way: ``num_retries=3`` sends ``4`` upstream requests. + """ + router = self._router("https://global.local/v1", {}, num_retries=3) + assert await self._call_and_count(router) == 4 + + @pytest.mark.asyncio + async def test_direct_completion_still_forwards_num_retries_to_provider(self): + """ + For a NON-routed direct ``litellm.acompletion`` call, ``num_retries`` remains an + alias for the provider client's ``max_retries`` (the instructor use case). The + provider SDK therefore retries in addition to litellm's own retry wrapper, so the + upstream count exceeds ``num_retries + 1`` - proving the routed-call fix did not + change direct-call behaviour. + """ + counter = self._install_counting_upstream() + num_retries = 2 + with patch("asyncio.sleep", return_value=None): + with pytest.raises(litellm.InternalServerError): + await litellm.acompletion( + model="openai/gpt-4o-mini", + api_base="https://direct.local/v1", + api_key="sk-fake", + messages=[{"role": "user", "content": "hi"}], + num_retries=num_retries, + ) + assert counter["n"] > num_retries + 1 + + +class _AttemptCounter(CustomLogger): + """Counts upstream call attempts via the pre-call hook (one per attempt).""" + + def __init__(self): + self.attempts = 0 + + def log_pre_api_call(self, model, messages, kwargs): + self.attempts += 1 + + +class TestRequestNumRetriesBeatsGlobal: + """ + A per-request num_retries (request body or the x-litellm-num-retries header, both of + which arrive as the num_retries kwarg) must take precedence over the global + litellm.num_retries (litellm_settings.num_retries on the proxy) during retry handling. + + The regression: the @client wrapper stamped the global litellm.num_retries onto the + raised exception, and async_function_with_retries then adopted that stamped value, + overwriting the request-level num_retries it had already resolved. This exercises the + real retry loop end to end (the failing call flows through the wrapped litellm.acompletion), + which the kwargs-merge-only test above does not. + """ + + @pytest.fixture(autouse=True) + def _restore_litellm_globals(self): + prev_num_retries = litellm.num_retries + prev_callbacks = litellm.callbacks + yield + litellm.num_retries = prev_num_retries + litellm.callbacks = prev_callbacks + + @staticmethod + def _router(global_num_retries): + return Router( + model_list=[ + { + "model_name": "mock", + "litellm_params": { + "model": "openai/mock", + "api_key": "sk-fake", + "mock_response": "litellm.InternalServerError", + }, + } + ], + num_retries=global_num_retries, + ) + + async def _count_attempts(self, *, global_num_retries, request_num_retries): + counter = _AttemptCounter() + litellm.callbacks = [counter] + litellm.num_retries = global_num_retries + router = self._router(global_num_retries) + kwargs = {"model": "mock", "messages": [{"role": "user", "content": "hi"}]} + if request_num_retries is not None: + kwargs["num_retries"] = request_num_retries + with patch("asyncio.sleep", return_value=None): + with pytest.raises(litellm.InternalServerError): + await router.acompletion(**kwargs) + return counter.attempts + + @pytest.mark.asyncio + async def test_request_num_retries_overrides_global(self): + """global=3 + request=1 -> 2 attempts (1 initial + 1 retry), not 4 (1 + global 3).""" + attempts = await self._count_attempts(global_num_retries=3, request_num_retries=1) + assert attempts == 2 + + @pytest.mark.asyncio + async def test_request_num_retries_zero_disables_retries_despite_global(self): + """global=3 + request=0 -> a single attempt (retries disabled by the request).""" + attempts = await self._count_attempts(global_num_retries=3, request_num_retries=0) + assert attempts == 1 + + @pytest.mark.asyncio + async def test_global_num_retries_applies_when_request_omits_it(self): + """No request num_retries -> the global still applies: 1 initial + 3 retries = 4.""" + attempts = await self._count_attempts(global_num_retries=3, request_num_retries=None) + assert attempts == 4 + + @pytest.mark.asyncio + async def test_deployment_num_retries_reaches_wrapper_when_no_request_value(self): + """ + With no request value and the router default at 0, a deployment's + litellm_params.num_retries reaches the wrapped call, is carried on the raised + exception, and is applied: deployment 2 -> 1 initial + 2 retries = 3 (not 1). + """ + counter = _AttemptCounter() + litellm.callbacks = [counter] + litellm.num_retries = None + router = Router( + model_list=[ + { + "model_name": "mock", + "litellm_params": { + "model": "openai/mock", + "api_key": "sk-fake", + "mock_response": "litellm.InternalServerError", + "num_retries": 2, + }, + } + ], + num_retries=0, + ) + with patch("asyncio.sleep", return_value=None): + with pytest.raises(litellm.InternalServerError): + await router.acompletion( + model="mock", messages=[{"role": "user", "content": "hi"}] + ) + assert counter.attempts == 3 diff --git a/ui/litellm-dashboard/e2e_tests/fixtures/menuMappings.ts b/ui/litellm-dashboard/e2e_tests/fixtures/menuMappings.ts index 4a4bb64c8ed..d6e7ea86982 100644 --- a/ui/litellm-dashboard/e2e_tests/fixtures/menuMappings.ts +++ b/ui/litellm-dashboard/e2e_tests/fixtures/menuMappings.ts @@ -26,7 +26,8 @@ export const menuLabelToPage: Record = { "Cost Tracking": Page.CostTracking, "UI Theme": Page.UiTheme, // Experimental submenu items - Caching: Page.Caching, + "Response Cache": Page.Caching, + Caching: Page.Caching, // Legacy label support Prompts: Page.Prompts, Budgets: Page.Budgets, "API Playground": Page.TransformRequest, diff --git a/ui/litellm-dashboard/e2e_tests/run_e2e.sh b/ui/litellm-dashboard/e2e_tests/run_e2e.sh index ed0641d04e6..ea95f18890c 100755 --- a/ui/litellm-dashboard/e2e_tests/run_e2e.sh +++ b/ui/litellm-dashboard/e2e_tests/run_e2e.sh @@ -26,6 +26,7 @@ IS_CI="${CI:-false}" CONTAINER_NAME="litellm-e2e-postgres-$$" MOCK_PID="" PROXY_PID="" +PROXY_LOG="" # --- Ensure common tool paths are available (local dev only) --- if [ "$IS_CI" = "false" ]; then @@ -40,6 +41,7 @@ cleanup() { echo "Cleaning up..." [ -n "$MOCK_PID" ] && kill "$MOCK_PID" 2>/dev/null || true [ -n "$PROXY_PID" ] && kill "$PROXY_PID" 2>/dev/null || true + [ -n "$PROXY_LOG" ] && rm -f "$PROXY_LOG" || true if [ "$IS_CI" = "false" ]; then docker stop "$CONTAINER_NAME" 2>/dev/null || true fi @@ -124,6 +126,7 @@ echo "UI build copied and restructured" # --- Python environment --- echo "=== Setting up Python environment ===" cd "$REPO_ROOT" +export UV_PYTHON="${UV_PYTHON:-3.13}" uv sync --group dev --group proxy-dev --extra proxy --frozen --quiet uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma @@ -143,16 +146,18 @@ done # --- LiteLLM proxy --- echo "=== Starting LiteLLM proxy ===" cd "$REPO_ROOT" +PROXY_LOG="${TMPDIR:-/tmp}/litellm-e2e-proxy-$$.log" uv run --no-sync python -m litellm.proxy.proxy_cli \ --config "$SCRIPT_DIR/fixtures/config.yml" \ - --port 4000 & + --port 4000 >"$PROXY_LOG" 2>&1 & PROXY_PID=$! -echo "Waiting for proxy..." +echo "Waiting for proxy (logs: $PROXY_LOG)..." PROXY_READY=0 for i in $(seq 1 180); do if ! kill -0 "$PROXY_PID" 2>/dev/null; then - echo "Error: proxy process exited unexpectedly" + echo "Error: proxy process exited unexpectedly. Proxy output:" + tail -n 100 "$PROXY_LOG" exit 1 fi HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true) @@ -163,7 +168,8 @@ for i in $(seq 1 180); do sleep 1 done if [ "$PROXY_READY" -ne 1 ]; then - echo "Error: proxy did not become healthy within 180 seconds" + echo "Error: proxy did not become healthy within 180 seconds. Proxy output:" + tail -n 100 "$PROXY_LOG" exit 1 fi echo "Proxy is ready." diff --git a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/credentials.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/credentials.spec.ts index 8b7824813a4..7c836068567 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/credentials.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/credentials.spec.ts @@ -38,7 +38,8 @@ test.describe("Edit LLM credential", () => { const row = page.locator("tr", { hasText: credentialName }); await expect(row).toBeVisible({ timeout: 15_000 }); - await row.getByRole("button").first().click(); + await row.getByTestId(`credential-actions-${credentialName}`).click(); + await page.getByTestId("credential-action-edit").click(); const modal = page.locator(".ant-modal-content").filter({ hasText: "Edit Credential" }); await expect(modal).toBeVisible({ timeout: 10_000 }); diff --git a/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts index 7e42d07ae7c..b220dc09ae2 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts @@ -8,7 +8,16 @@ import { MIGRATED_E2E_PAGES } from "../../fixtures/migratedPages"; import type { Page as PlaywrightPage } from "@playwright/test"; const sidebarButtons = { - [Role.ProxyAdmin]: ["Virtual Keys", "Playground", "Models", "Usage", "Teams", "Internal Users", "AI Hub"], + [Role.ProxyAdmin]: [ + "Virtual Keys", + "Playground", + "Models", + "Usage", + "Teams", + "Internal Users", + "AI Hub", + "Response Cache", + ], }; /** Migrated pages live at a path route; legacy pages keep the ?page= query param. */ diff --git a/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/keys.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/keys.spec.ts index a55c19a53de..c44957ea737 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/keys.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/proxy-admin/keys.spec.ts @@ -103,7 +103,8 @@ test.describe("Proxy Admin - Keys", () => { await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 }); - await page.getByRole("button", { name: "Delete Key" }).click(); + await page.getByRole("button", { name: "More key actions" }).click(); + await page.getByRole("menuitem", { name: "Delete Key" }).click(); const modal = page.locator(".ant-modal:visible"); await expect(modal).toBeVisible({ timeout: 5_000 }); diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index f935af8907d..6072c8c725b 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -12,14 +12,6 @@ "count": 1 } }, - "src/app/(dashboard)/agents/_components/AgentsPanel.tsx": { - "no-restricted-imports": { - "count": 1 - }, - "react-hooks/set-state-in-effect": { - "count": 1 - } - }, "src/app/(dashboard)/agents/_components/add_agent_form.tsx": { "no-nested-ternary": { "count": 3 @@ -185,11 +177,6 @@ "count": 1 } }, - "src/app/(dashboard)/cost-tracking/_components/provider_display_helpers.test.ts": { - "unused-imports/no-unused-imports": { - "count": 1 - } - }, "src/app/(dashboard)/cost-tracking/_components/provider_margin_table.tsx": { "no-restricted-imports": { "count": 1 @@ -639,11 +626,6 @@ "count": 2 } }, - "src/app/(dashboard)/memory/_components/MemoryView.tsx": { - "react-hooks/set-state-in-effect": { - "count": 1 - } - }, "src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx": { "no-restricted-imports": { "count": 1 @@ -697,11 +679,6 @@ "count": 1 } }, - "src/app/(dashboard)/organizations/_components/organizations.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.tsx": { "no-restricted-imports": { "count": 1 @@ -1220,9 +1197,6 @@ } }, "src/app/(dashboard)/users/_components/view_users.tsx": { - "no-nested-ternary": { - "count": 1 - }, "no-restricted-imports": { "count": 1 }, @@ -1230,22 +1204,6 @@ "count": 1 } }, - "src/app/(dashboard)/users/_components/view_users/columns.tsx": { - "max-params": { - "count": 1 - }, - "no-restricted-imports": { - "count": 1 - } - }, - "src/app/(dashboard)/users/_components/view_users/table.tsx": { - "no-nested-ternary": { - "count": 1 - }, - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/users/_components/view_users/user_info_view.tsx": { "no-restricted-imports": { "count": 1 @@ -1539,23 +1497,6 @@ "count": 2 } }, - "src/components/ToolPolicies.tsx": { - "no-nested-ternary": { - "count": 1 - }, - "no-restricted-imports": { - "count": 1 - }, - "react-hooks/set-state-in-effect": { - "count": 1 - }, - "react-hooks/static-components": { - "count": 7 - }, - "unused-imports/no-unused-imports": { - "count": 1 - } - }, "src/components/UIAccessControlForm.tsx": { "no-restricted-imports": { "count": 1 @@ -1882,17 +1823,6 @@ "count": 1 } }, - "src/components/model_dashboard/HealthCheckComponent.tsx": { - "no-nested-ternary": { - "count": 3 - }, - "no-restricted-imports": { - "count": 1 - }, - "react-hooks/immutability": { - "count": 1 - } - }, "src/components/model_dashboard/all_models_table.tsx": { "no-nested-ternary": { "count": 1 @@ -1901,25 +1831,6 @@ "count": 1 } }, - "src/components/model_dashboard/health_check_columns.tsx": { - "max-params": { - "count": 1 - }, - "no-nested-ternary": { - "count": 2 - }, - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/model_dashboard/table.tsx": { - "no-nested-ternary": { - "count": 1 - }, - "no-restricted-imports": { - "count": 1 - } - }, "src/components/model_filters.tsx": { "no-restricted-imports": { "count": 1 @@ -2225,7 +2136,7 @@ }, "src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx": { "no-nested-ternary": { - "count": 4 + "count": 3 } }, "src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx": { diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 7a65b63b33c..742c1e4a63f 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -14,6 +14,7 @@ "@base-ui/react": "^1.6.0", "@headlessui/tailwindcss": "0.2.2", "@heroicons/react": "1.0.6", + "@hookform/resolvers": "5.4.0", "@tanstack/react-pacer": "0.22.1", "@tanstack/react-query": "5.100.7", "@tanstack/react-table": "8.21.3", @@ -34,13 +35,15 @@ "react": "18.3.1", "react-copy-to-clipboard": "5.1.1", "react-dom": "18.3.1", + "react-hook-form": "7.82.0", "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", "react-syntax-highlighter": "15.6.6", "recharts": "3.9.2", "remark-gfm": "4.0.1", "tailwind-merge": "3.4.0", - "uuid": "14.0.0" + "uuid": "14.0.0", + "zod": "3.25.76" }, "devDependencies": { "@eslint/js": "9.39.2", @@ -799,9 +802,9 @@ } }, "node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", + "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", "license": "MIT", "optional": true, "dependencies": { @@ -1556,6 +1559,18 @@ "react": ">= 16" } }, + "node_modules/@hookform/resolvers": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/@hookform/resolvers/-/resolvers-5.4.0.tgz", + "integrity": "sha512-EIsqr/t/qbinPIhGjMdtvutIN1Kk4uwbROE9/UQ93CAVGR7GkA7Y92+fX80OzXi/OB67jVFYwKGO1WzkxmkFZw==", + "license": "MIT", + "dependencies": { + "@standard-schema/utils": "^0.3.0" + }, + "peerDependencies": { + "react-hook-form": "^7.55.0" + } + }, "node_modules/@humanfs/core": { "version": "0.19.2", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", @@ -1633,9 +1648,9 @@ } }, "node_modules/@img/sharp-darwin-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", - "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.3.tgz", + "integrity": "sha512-RMnFX7YQsMoh7lWfcM4NEHHymBX/rLuKNPVM84XE9ONPcaSCDgE7CHIHpSgPcO2xcRthgBy1HfNO319mwhIAkg==", "cpu": [ "arm64" ], @@ -1645,19 +1660,19 @@ "darwin" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.2.4" + "@img/sharp-libvips-darwin-arm64": "1.3.2" } }, "node_modules/@img/sharp-darwin-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", - "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.3.tgz", + "integrity": "sha512-Xo+5uFBtLN0BKqieTxiFzFPQAUlBbbH5iBKyRX/z1JrbnYsHTfKJnUfL8+p2TPXr1pXqao4eeL4Rl144uDpK9w==", "cpu": [ "x64" ], @@ -1667,19 +1682,38 @@ "darwin" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.2.4" + "@img/sharp-libvips-darwin-x64": "1.3.2" + } + }, + "node_modules/@img/sharp-freebsd-wasm32": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.3.tgz", + "integrity": "sha512-lUxcqWIj2wMQ9BrwNjngcr1gWUr5xgaGThBRqPPalIC2n67Cqj1uPh8NnA/ZhAg8hUbKl+kVHKwgUIwe6ZYPrg==", + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "dependencies": { + "@img/sharp-wasm32": "0.35.3" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", - "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.2.tgz", + "integrity": "sha512-9J6ypZFpQBj4YnePGoq/S38w6nz+vqg5WZLrLGY4YuSemdMq47GMLBPO42MzwdGwpg/agZ7xzZcFHa48xlywfg==", "cpu": [ "arm64" ], @@ -1693,9 +1727,9 @@ } }, "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", - "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.2.tgz", + "integrity": "sha512-m2pW1n6cns9VaubNwsZ+c3CRYjxNQWgJ5gPlnL1nbBcpkBvFm6SCFN5o0psFHI8w9n11NKhFkeEDns98tiqbEw==", "cpu": [ "x64" ], @@ -1709,9 +1743,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", - "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.2.tgz", + "integrity": "sha512-1eMLzy92I4J6rmi4mAT8yC3HxOtniyGELlzGbNMLLeqe052ahFQ0h6LFq+lh5DsDIdYViIDst08abvSbcEdLXQ==", "cpu": [ "arm" ], @@ -1725,9 +1759,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", - "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.2.tgz", + "integrity": "sha512-dqVSFynCox4C/J8kT16V7SIFAns0IjgLwkvYT7p8LQVmJ5OS5b6tI9IGflxTeuBS//zXeFIUbwt5dwxyZ17cnA==", "cpu": [ "arm64" ], @@ -1741,9 +1775,9 @@ } }, "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", - "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.2.tgz", + "integrity": "sha512-3z0NHDxD6n5I9gc05U1eW1AyRm+Gznzq3naMrthPNqE6oYykcogW0l/jfpJdjYnuNl8R7yI9pNbE1XiUeyq0Aw==", "cpu": [ "ppc64" ], @@ -1757,9 +1791,9 @@ } }, "node_modules/@img/sharp-libvips-linux-riscv64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", - "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.2.tgz", + "integrity": "sha512-bsb4rI+NldGOsXuej2r8OdSS8+zXDVaCWxyWrcv6kneTOlgAHtZABRzBBCwdsPiD90J4myNJuHpg6kA20ImW/w==", "cpu": [ "riscv64" ], @@ -1773,9 +1807,9 @@ } }, "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", - "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.2.tgz", + "integrity": "sha512-/ABshyj8gCpyIrNXnHn4LorDJ0HHm1VhXPBlxZ8zAtfVPAaSafXPGn+sUSIRiwaSBy0mmFjSjiXI5mkcwdChKQ==", "cpu": [ "s390x" ], @@ -1789,9 +1823,9 @@ } }, "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", - "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.2.tgz", + "integrity": "sha512-ITPEtgffGJ0S6G9dRyw/366tJQqFRcHWPHhC+Stpg3Z8AEMrDrTr2lhdz4f/Y/HMbRh//7Z5mBzEpVdi62Oc3w==", "cpu": [ "x64" ], @@ -1805,9 +1839,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", - "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.2.tgz", + "integrity": "sha512-zE9EdiUzUmg5mDT5a1rk5fYJ6GWPloTwWBYDS14naqHsL+EaMpDj1AWnpLgh3u0YCORv2Tt50wrcrpYqkP97Kw==", "cpu": [ "arm64" ], @@ -1821,9 +1855,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", - "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.2.tgz", + "integrity": "sha512-m0lrLiUt+lBYnCFr8qV/65yMR4E/c7/wf78I5eKTdkEakFAlZ9QlzEM3QIhhAwVeUhLAHLcCq7a7Vszq/oFNZQ==", "cpu": [ "x64" ], @@ -1837,9 +1871,9 @@ } }, "node_modules/@img/sharp-linux-arm": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", - "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.3.tgz", + "integrity": "sha512-affVWCTLooy8TSxbDx2qkzuDeaWLNVBA+P//FNBirHsXpP2fuBhk5AuboYUnrDnzoXes8GFjpTx0SBFOCRg+FA==", "cpu": [ "arm" ], @@ -1849,19 +1883,19 @@ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.2.4" + "@img/sharp-libvips-linux-arm": "1.3.2" } }, "node_modules/@img/sharp-linux-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", - "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.3.tgz", + "integrity": "sha512-QgKDspHPnrU+GQ55XPhGwyhC8acLVOOSyAvo1oVfFmrIXLkDNmGWzAfDZ4xK8oSA1qBQrALcHX0G5UZni/SuFQ==", "cpu": [ "arm64" ], @@ -1871,19 +1905,19 @@ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.2.4" + "@img/sharp-libvips-linux-arm64": "1.3.2" } }, "node_modules/@img/sharp-linux-ppc64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", - "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.3.tgz", + "integrity": "sha512-sMd8rDxmpLOwv/7N44klFjOD5DUO7FLdjiXDI0hoxYaf7Ar262dQIEkosE98bps+5HPLtp/EvNqeqQtOycP/IA==", "cpu": [ "ppc64" ], @@ -1893,19 +1927,19 @@ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-ppc64": "1.2.4" + "@img/sharp-libvips-linux-ppc64": "1.3.2" } }, "node_modules/@img/sharp-linux-riscv64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", - "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.3.tgz", + "integrity": "sha512-0Eob78yjlYPfL5vMNWAW55l3R9Y6BQS/gOfe0ZcP9mEz9ohhKSt4im1hayiknXgf8AWrFqMvJcKIdmLmEe7yeQ==", "cpu": [ "riscv64" ], @@ -1915,19 +1949,19 @@ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-riscv64": "1.2.4" + "@img/sharp-libvips-linux-riscv64": "1.3.2" } }, "node_modules/@img/sharp-linux-s390x": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", - "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.3.tgz", + "integrity": "sha512-KgAxQ0DxpNOq1rG2t5cgTgShJFGSuU7XO45cqC+1NVOuZnP6tlgZRuSYOfNupGkHID0o3cJOsw4DVeJpMovcGw==", "cpu": [ "s390x" ], @@ -1937,19 +1971,19 @@ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.2.4" + "@img/sharp-libvips-linux-s390x": "1.3.2" } }, "node_modules/@img/sharp-linux-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", - "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.3.tgz", + "integrity": "sha512-8pqvxubL2PGdhlPy6GLqzDYMUjyRmKAwKHYKixpdJYBUK7PJ0C029XdsnpFIdgRZG68fZiGdHVWcKPvtiPB4cA==", "cpu": [ "x64" ], @@ -1959,19 +1993,19 @@ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.2.4" + "@img/sharp-libvips-linux-x64": "1.3.2" } }, "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", - "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.3.tgz", + "integrity": "sha512-Vz0iQjzzcSX3HCbfwFfCSG/9SCIqyO0mH2sXyiHaAYfBk0cRsCWXRyQYX0ovCK/PAQBbTzQ0dsPQHh5MAFL59w==", "cpu": [ "arm64" ], @@ -1981,19 +2015,19 @@ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + "@img/sharp-libvips-linuxmusl-arm64": "1.3.2" } }, "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", - "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.3.tgz", + "integrity": "sha512-6O1NPKcDVj9QEdg7Hx549EX8U0rp6yXQERqru6yRN7fGBn32UvIRJUlWnk+8xDCiG76hXVBbX82NZ/ZKr0euIg==", "cpu": [ "x64" ], @@ -2003,38 +2037,54 @@ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + "@img/sharp-libvips-linuxmusl-x64": "1.3.2" } }, "node_modules/@img/sharp-wasm32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", - "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", - "cpu": [ - "wasm32" - ], + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.3.tgz", + "integrity": "sha512-cZ0XkcYGpHZkqW6iCkqTcmUC0CD9DhD5d/qeZlZkfRBn6GnHniZXLUo5+9xw8Iv76YE6LQFN9YNBlKREcCG76w==", "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", "optional": true, "dependencies": { - "@emnapi/runtime": "^1.7.0" + "@emnapi/runtime": "^1.11.1" }, "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-webcontainers-wasm32": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.3.tgz", + "integrity": "sha512-2rnq7bX3NzeR2T4YWgz8qiG4h3TSdMe+vN1iQXpJleSJ3SM5zQ8Fy2SyyXAWlbxpEZ2Y+Z4u1BePgJEYbSy80Q==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/sharp-wasm32": "0.35.3" + }, + "engines": { + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-win32-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", - "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.3.tgz", + "integrity": "sha512-4bPwFdMbeC4JQ8L8LOyWp6nsHcboP5fxkp6iPOXz2Vg49R42TuMs2whkJ5OAP4/Ul035qOzy0AecOF9VOscn4w==", "cpu": [ "arm64" ], @@ -2044,16 +2094,16 @@ "win32" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-win32-ia32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", - "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.3.tgz", + "integrity": "sha512-r53mXsBN6lFUDiST764SvgwUdHAqM4rPAiDzAmf4fLoB6X/rkfyTrLCg6+g17wJJiCmB3JYgHuUldCWUIRFSXw==", "cpu": [ "ia32" ], @@ -2063,16 +2113,16 @@ "win32" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": "^20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-win32-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", - "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.3.tgz", + "integrity": "sha512-D4y1vNeZrIIJCN+uHaWVtH86B+aCrdMYYjicy9pXHvbGZeGYLLSd3wdVuC37FxVXlU1ARsk84eKWfWMXGYEqvA==", "cpu": [ "x64" ], @@ -2082,7 +2132,7 @@ "win32" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" @@ -11780,6 +11830,22 @@ "react": "^18.3.1" } }, + "node_modules/react-hook-form": { + "version": "7.82.0", + "resolved": "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.82.0.tgz", + "integrity": "sha512-Zw/uFZ2dO+02GHlBn7JFGn8kZJ7LdM33B/0BXOovzFay+CMhf94JMw5BVu+F1tVkUKjNvBuaE3fz5BJhga10Tg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/react-hook-form" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17 || ^18 || ^19" + } + }, "node_modules/react-is": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", @@ -12503,48 +12569,53 @@ } }, "node_modules/sharp": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", - "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", - "hasInstallScript": true, + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.3.tgz", + "integrity": "sha512-ej0zVHuZGHCiABXcNxeYhpRnPNPAcvbG8RMdBAhDAxLKkCRVSpK3Iyu7qbqw3JMzoj0REeM6f3tJLtVwl0023Q==", "license": "Apache-2.0", "optional": true, "dependencies": { - "@img/colour": "^1.0.0", + "@img/colour": "^1.1.0", "detect-libc": "^2.1.2", - "semver": "^7.7.3" + "semver": "^7.8.5" }, "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.34.5", - "@img/sharp-darwin-x64": "0.34.5", - "@img/sharp-libvips-darwin-arm64": "1.2.4", - "@img/sharp-libvips-darwin-x64": "1.2.4", - "@img/sharp-libvips-linux-arm": "1.2.4", - "@img/sharp-libvips-linux-arm64": "1.2.4", - "@img/sharp-libvips-linux-ppc64": "1.2.4", - "@img/sharp-libvips-linux-riscv64": "1.2.4", - "@img/sharp-libvips-linux-s390x": "1.2.4", - "@img/sharp-libvips-linux-x64": "1.2.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", - "@img/sharp-libvips-linuxmusl-x64": "1.2.4", - "@img/sharp-linux-arm": "0.34.5", - "@img/sharp-linux-arm64": "0.34.5", - "@img/sharp-linux-ppc64": "0.34.5", - "@img/sharp-linux-riscv64": "0.34.5", - "@img/sharp-linux-s390x": "0.34.5", - "@img/sharp-linux-x64": "0.34.5", - "@img/sharp-linuxmusl-arm64": "0.34.5", - "@img/sharp-linuxmusl-x64": "0.34.5", - "@img/sharp-wasm32": "0.34.5", - "@img/sharp-win32-arm64": "0.34.5", - "@img/sharp-win32-ia32": "0.34.5", - "@img/sharp-win32-x64": "0.34.5" + "@img/sharp-darwin-arm64": "0.35.3", + "@img/sharp-darwin-x64": "0.35.3", + "@img/sharp-freebsd-wasm32": "0.35.3", + "@img/sharp-libvips-darwin-arm64": "1.3.2", + "@img/sharp-libvips-darwin-x64": "1.3.2", + "@img/sharp-libvips-linux-arm": "1.3.2", + "@img/sharp-libvips-linux-arm64": "1.3.2", + "@img/sharp-libvips-linux-ppc64": "1.3.2", + "@img/sharp-libvips-linux-riscv64": "1.3.2", + "@img/sharp-libvips-linux-s390x": "1.3.2", + "@img/sharp-libvips-linux-x64": "1.3.2", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.2", + "@img/sharp-libvips-linuxmusl-x64": "1.3.2", + "@img/sharp-linux-arm": "0.35.3", + "@img/sharp-linux-arm64": "0.35.3", + "@img/sharp-linux-ppc64": "0.35.3", + "@img/sharp-linux-riscv64": "0.35.3", + "@img/sharp-linux-s390x": "0.35.3", + "@img/sharp-linux-x64": "0.35.3", + "@img/sharp-linuxmusl-arm64": "0.35.3", + "@img/sharp-linuxmusl-x64": "0.35.3", + "@img/sharp-webcontainers-wasm32": "0.35.3", + "@img/sharp-win32-arm64": "0.35.3", + "@img/sharp-win32-ia32": "0.35.3", + "@img/sharp-win32-x64": "0.35.3" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/shebang-command": { @@ -14156,7 +14227,6 @@ "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "devOptional": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index b5e93d175bf..0f54c536297 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -30,6 +30,7 @@ "@base-ui/react": "^1.6.0", "@headlessui/tailwindcss": "0.2.2", "@heroicons/react": "1.0.6", + "@hookform/resolvers": "5.4.0", "@tanstack/react-pacer": "0.22.1", "@tanstack/react-query": "5.100.7", "@tanstack/react-table": "8.21.3", @@ -50,13 +51,15 @@ "react": "18.3.1", "react-copy-to-clipboard": "5.1.1", "react-dom": "18.3.1", + "react-hook-form": "7.82.0", "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", "react-syntax-highlighter": "15.6.6", "recharts": "3.9.2", "remark-gfm": "4.0.1", "tailwind-merge": "3.4.0", - "uuid": "14.0.0" + "uuid": "14.0.0", + "zod": "3.25.76" }, "devDependencies": { "@eslint/js": "9.39.2", @@ -100,7 +103,8 @@ "axios": "1.13.6", "postcss": "8.5.13", "esbuild": "0.28.1", - "date-fns": "^4.4.0" + "date-fns": "^4.4.0", + "sharp": "^0.35.0" }, "engines": { "node": ">=20.9.0", diff --git a/ui/litellm-dashboard/public/assets/logos/ai21.svg b/ui/litellm-dashboard/public/assets/logos/ai21.svg index 7e62a9517af..3c8c75e6d6f 100644 --- a/ui/litellm-dashboard/public/assets/logos/ai21.svg +++ b/ui/litellm-dashboard/public/assets/logos/ai21.svg @@ -1 +1 @@ -AI21 \ No newline at end of file +AI21 \ No newline at end of file diff --git a/ui/litellm-dashboard/public/assets/logos/promptguard.svg b/ui/litellm-dashboard/public/assets/logos/promptguard.svg index 44cdd52eae3..4b2fd3c386e 100644 --- a/ui/litellm-dashboard/public/assets/logos/promptguard.svg +++ b/ui/litellm-dashboard/public/assets/logos/promptguard.svg @@ -1,5 +1,5 @@ + viewBox="0 0 1024 1024" enable-background="new 0 0 1024 1024" xml:space="preserve"> Soniox +Soniox diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.test.tsx index 7c8aaa2b785..a1484ffb5c5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.test.tsx @@ -38,6 +38,7 @@ const mockAccessGroups: AccessGroupResponse[] = [ const mockUseAccessGroups = vi.fn(); const mockUseDeleteAccessGroup = vi.fn(); const mockMutate = vi.fn(); +const mockUseAuthorized = vi.fn(); vi.mock("@/app/(dashboard)/hooks/accessGroups/useAccessGroups", () => ({ useAccessGroups: () => mockUseAccessGroups(), @@ -47,6 +48,10 @@ vi.mock("@/app/(dashboard)/hooks/accessGroups/useDeleteAccessGroup", () => ({ useDeleteAccessGroup: () => mockUseDeleteAccessGroup(), })); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => mockUseAuthorized(), +})); + vi.mock("./AccessGroupsDetailsPage", () => ({ AccessGroupDetail: ({ accessGroupId, onBack }: { accessGroupId: string; onBack: () => void }) => (
@@ -65,49 +70,42 @@ vi.mock("./AccessGroupsModal/AccessGroupCreateModal", () => ({ ) : null, })); -vi.mock("@/components/common_components/IconActionButton/TableIconActionButtons/TableIconActionButton", () => ({ - default: ({ variant, tooltipText, onClick }: { variant: string; tooltipText: string; onClick: () => void }) => ( - - ), -})); +const makeGroups = (count: number): AccessGroupResponse[] => + Array.from({ length: count }, (_, index) => { + const suffix = String(index + 1).padStart(2, "0"); + return { + ...mockAccessGroups[0], + access_group_id: `ag-${suffix}`, + access_group_name: `Group ${suffix}`, + description: `Group ${suffix} description`, + }; + }); + +const openRowMenu = async (user: ReturnType, groupId: string) => { + await user.click(screen.getByTestId(`access-group-actions-${groupId}`)); + return screen.findByTestId("access-group-action-delete"); +}; describe("AccessGroupsPage", () => { beforeEach(() => { vi.clearAllMocks(); - mockUseAccessGroups.mockReturnValue({ - data: mockAccessGroups, - isLoading: false, - }); - mockUseDeleteAccessGroup.mockReturnValue({ - mutate: mockMutate, - isPending: false, - }); + mockUseAccessGroups.mockReturnValue({ data: mockAccessGroups, isLoading: false }); + mockUseDeleteAccessGroup.mockReturnValue({ mutate: mockMutate, isPending: false }); + mockUseAuthorized.mockReturnValue({ userRole: "Admin", accessToken: "sk-test" }); }); - it("should render", () => { - renderWithProviders(); - expect(screen.getByRole("heading", { name: "Access Groups" })).toBeInTheDocument(); - }); - - it("should display page title and subtitle", () => { + it("renders the page title and subtitle", () => { renderWithProviders(); expect(screen.getByRole("heading", { name: "Access Groups" })).toBeInTheDocument(); expect(screen.getByText("Manage resource permissions for your organization")).toBeInTheDocument(); }); - it("should display Create Access Group button", () => { + it("shows the Create Access Group button for an admin", () => { renderWithProviders(); expect(screen.getByRole("button", { name: /create access group/i })).toBeInTheDocument(); }); - it("should display search input with placeholder", () => { - renderWithProviders(); - expect(screen.getByPlaceholderText("Search groups by name, ID, or description...")).toBeInTheDocument(); - }); - - it("should display access groups in table", () => { + it("renders every access group row", () => { renderWithProviders(); expect(screen.getByText("ag-1")).toBeInTheDocument(); expect(screen.getByText("Admin Group")).toBeInTheDocument(); @@ -115,57 +113,70 @@ describe("AccessGroupsPage", () => { expect(screen.getByText("Read Only")).toBeInTheDocument(); }); - it("should display resource counts for each group", () => { + it("renders resource counts for each group", () => { renderWithProviders(); - const table = screen.getByRole("table"); - expect(table).toHaveTextContent("2"); - expect(table).toHaveTextContent("1"); + // ag-1 has 2 models, 1 mcp server, 1 agent. + const adminRow = screen.getByText("ag-1").closest("tr") as HTMLElement; + expect(within(adminRow).getByTitle("2 Models")).toHaveTextContent("2"); + expect(within(adminRow).getByTitle("1 MCP Servers")).toHaveTextContent("1"); + expect(within(adminRow).getByTitle("1 Agents")).toHaveTextContent("1"); }); - it("should filter groups by search text matching name", async () => { + it("shows the expected column headers", () => { + renderWithProviders(); + expect(screen.getByRole("columnheader", { name: /^ID$/i })).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: /Name/i })).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: /Resources/i })).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: /Created/i })).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: /Updated/i })).toBeInTheDocument(); + }); + + it("filters by name", async () => { const user = userEvent.setup(); renderWithProviders(); - const searchInput = screen.getByPlaceholderText("Search groups by name, ID, or description..."); - await user.type(searchInput, "Admin"); + await user.type(screen.getByPlaceholderText("Search groups by name, ID, or description..."), "Admin"); expect(screen.getByText("Admin Group")).toBeInTheDocument(); expect(screen.queryByText("Read Only")).not.toBeInTheDocument(); }); - it("should filter groups by search text matching ID", async () => { + it("filters by ID", async () => { const user = userEvent.setup(); renderWithProviders(); - const searchInput = screen.getByPlaceholderText("Search groups by name, ID, or description..."); - await user.type(searchInput, "ag-2"); + await user.type(screen.getByPlaceholderText("Search groups by name, ID, or description..."), "ag-2"); expect(screen.getByText("Read Only")).toBeInTheDocument(); expect(screen.queryByText("Admin Group")).not.toBeInTheDocument(); }); - it("should filter groups by search text matching description", async () => { + it("filters by description", async () => { const user = userEvent.setup(); renderWithProviders(); - const searchInput = screen.getByPlaceholderText("Search groups by name, ID, or description..."); - await user.type(searchInput, "read-only"); + await user.type(screen.getByPlaceholderText("Search groups by name, ID, or description..."), "read-only"); expect(screen.getByText("Read Only")).toBeInTheDocument(); expect(screen.queryByText("Admin Group")).not.toBeInTheDocument(); }); - it("should reset to first page when search text changes", async () => { + it("shows the filtered empty state when nothing matches", async () => { const user = userEvent.setup(); renderWithProviders(); - const searchInput = screen.getByPlaceholderText("Search groups by name, ID, or description..."); - await user.type(searchInput, "Admin"); - const pagination = screen.getByText(/groups/); - expect(pagination).toHaveTextContent("1 groups"); + await user.type(screen.getByPlaceholderText("Search groups by name, ID, or description..."), "no-such-group"); + expect(screen.getByText("No matching access groups")).toBeInTheDocument(); + expect(screen.queryByText("Admin Group")).not.toBeInTheDocument(); }); - it("should open create modal when Create Access Group button is clicked", async () => { - const user = userEvent.setup(); + it("shows the empty state when there are no groups", () => { + mockUseAccessGroups.mockReturnValue({ data: [], isLoading: false }); renderWithProviders(); - await user.click(screen.getByRole("button", { name: /create access group/i })); - expect(screen.getByTestId("create-access-group-modal")).toBeInTheDocument(); + expect(screen.getByText("No access groups yet")).toBeInTheDocument(); }); - it("should close create modal when cancel is clicked", async () => { + it("renders loading skeletons on the initial load", () => { + mockUseAccessGroups.mockReturnValue({ data: undefined, isLoading: true }); + renderWithProviders(); + expect(screen.getAllByTestId("skeleton-row").length).toBeGreaterThan(0); + expect(screen.queryByText("Admin Group")).not.toBeInTheDocument(); + }); + + it("opens and closes the create modal", async () => { const user = userEvent.setup(); renderWithProviders(); await user.click(screen.getByRole("button", { name: /create access group/i })); @@ -174,33 +185,22 @@ describe("AccessGroupsPage", () => { expect(screen.queryByTestId("create-access-group-modal")).not.toBeInTheDocument(); }); - it("should navigate to detail view when group ID is clicked", async () => { + it("opens the detail view when the ID cell is clicked and returns via Back", async () => { const user = userEvent.setup(); renderWithProviders(); await user.click(screen.getByText("ag-1")); expect(screen.getByTestId("access-group-detail")).toBeInTheDocument(); expect(screen.getByText("Detail for ag-1")).toBeInTheDocument(); - }); - - it("should return to list view when Back is clicked from detail", async () => { - const user = userEvent.setup(); - renderWithProviders(); - await user.click(screen.getByText("ag-1")); - expect(screen.getByTestId("access-group-detail")).toBeInTheDocument(); await user.click(screen.getByRole("button", { name: "Back" })); expect(screen.queryByTestId("access-group-detail")).not.toBeInTheDocument(); expect(screen.getByText("Admin Group")).toBeInTheDocument(); }); - it("should open delete modal when delete action is clicked", async () => { + it("opens the delete modal from the row actions menu", async () => { const user = userEvent.setup(); renderWithProviders(); - const deleteButtons = screen.getAllByRole("button", { - name: "Delete access group", - }); - await user.click(deleteButtons[0]); + await user.click(await openRowMenu(user, "ag-1")); const dialog = screen.getByRole("dialog", { name: "Delete Access Group" }); - expect(dialog).toBeInTheDocument(); expect( within(dialog).getByText("Are you sure you want to delete this access group? This action cannot be undone."), ).toBeInTheDocument(); @@ -209,71 +209,49 @@ describe("AccessGroupsPage", () => { expect(within(dialog).getByText("Admin Group")).toBeInTheDocument(); }); - it("should close delete modal when cancel is clicked", async () => { + it("closes the delete modal on cancel without deleting", async () => { const user = userEvent.setup(); renderWithProviders(); - const deleteButtons = screen.getAllByRole("button", { - name: "Delete access group", - }); - await user.click(deleteButtons[0]); + await user.click(await openRowMenu(user, "ag-1")); const dialog = screen.getByRole("dialog", { name: "Delete Access Group" }); await user.click(within(dialog).getByRole("button", { name: "Cancel" })); expect(screen.queryByRole("dialog", { name: "Delete Access Group" })).not.toBeInTheDocument(); + expect(mockMutate).not.toHaveBeenCalled(); }); - it("should call delete mutation when delete is confirmed", async () => { + it("calls the delete mutation with the group ID when confirmed", async () => { const user = userEvent.setup(); mockMutate.mockImplementation((_id: string, opts?: { onSuccess?: () => void }) => { opts?.onSuccess?.(); }); renderWithProviders(); - const deleteButtons = screen.getAllByRole("button", { - name: "Delete access group", - }); - await user.click(deleteButtons[0]); + await user.click(await openRowMenu(user, "ag-1")); const dialog = screen.getByRole("dialog", { name: "Delete Access Group" }); - const deleteConfirmButton = within(dialog).getByRole("button", { name: /delete/i }); - await user.click(deleteConfirmButton); + await user.click(within(dialog).getByRole("button", { name: /delete/i })); expect(mockMutate).toHaveBeenCalledWith("ag-1", expect.any(Object)); }); - it("should display pagination with total count", () => { - renderWithProviders(); - expect(screen.getByText("2 groups")).toBeInTheDocument(); - }); - - it("should show table headers for ID, Name, Resources, and Actions", () => { - renderWithProviders(); - expect(screen.getByRole("columnheader", { name: /ID/i })).toBeInTheDocument(); - expect(screen.getByRole("columnheader", { name: /Name/i })).toBeInTheDocument(); - expect(screen.getByRole("columnheader", { name: /Resources/i })).toBeInTheDocument(); - expect(screen.getByRole("columnheader", { name: /Actions/i })).toBeInTheDocument(); - }); - - it("should display loading state when data is loading", () => { - mockUseAccessGroups.mockReturnValue({ - data: undefined, - isLoading: true, - }); - renderWithProviders(); - const table = screen.getByRole("table"); - expect(table).toBeInTheDocument(); - }); - - it("should display empty state when no groups match search", async () => { + it("still shows matches when searching from a later page", async () => { const user = userEvent.setup(); + mockUseAccessGroups.mockReturnValue({ data: makeGroups(25), isLoading: false }); renderWithProviders(); - const searchInput = screen.getByPlaceholderText("Search groups by name, ID, or description..."); - await user.type(searchInput, "nonexistent-group-xyz"); - expect(screen.getByRole("table")).toBeInTheDocument(); + + await user.click(screen.getByTestId("pagination-next")); + expect(screen.getByText("ag-11")).toBeInTheDocument(); + expect(screen.queryByText("ag-01")).not.toBeInTheDocument(); + + // The only match lives on page 1, so the page index must reset or the table reads as empty. + await user.type(screen.getByPlaceholderText("Search groups by name, ID, or description..."), "ag-01"); + expect(await screen.findByText("ag-01")).toBeInTheDocument(); + expect(screen.queryByText("No matching access groups")).not.toBeInTheDocument(); }); - it("should display empty data when useAccessGroups returns empty array", () => { - mockUseAccessGroups.mockReturnValue({ - data: [], - isLoading: false, - }); + it("hides the Create button and row actions for a non-admin", () => { + mockUseAuthorized.mockReturnValue({ userRole: "Admin Viewer", accessToken: "sk-test" }); renderWithProviders(); - expect(screen.getByRole("table")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /create access group/i })).not.toBeInTheDocument(); + expect(screen.queryByTestId("access-group-actions-ag-1")).not.toBeInTheDocument(); + // The read-only view still lists the groups. + expect(screen.getByText("Admin Group")).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx index dbbf4e35900..0de6596f57c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx @@ -1,38 +1,17 @@ import { AccessGroupResponse, useAccessGroups } from "@/app/(dashboard)/hooks/accessGroups/useAccessGroups"; import { useDeleteAccessGroup } from "@/app/(dashboard)/hooks/accessGroups/useDeleteAccessGroup"; import { PlusOutlined } from "@ant-design/icons"; -import { - ColumnDef, - flexRender, - getCoreRowModel, - getSortedRowModel, - Row, - SortingState, - useReactTable, -} from "@tanstack/react-table"; -import { Button, Card, Flex, Input, Layout, Pagination, Space, Table, Tag, theme, Tooltip, Typography } from "antd"; -import { BotIcon, LayersIcon, SearchIcon, ServerIcon } from "lucide-react"; -import { useEffect, useMemo, useState } from "react"; +import { Button, Flex, Input, Layout, Space, theme, Typography } from "antd"; +import { SearchIcon } from "lucide-react"; +import { useMemo, useState } from "react"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; -import TableIconActionButton from "@/components/common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; -import { - SortState, - TableHeaderSortDropdown, -} from "@/components/common_components/TableHeaderSortDropdown/TableHeaderSortDropdown"; -import { DateCell, IdCell } from "@/components/shared/table_cells"; import { AccessGroupDetail } from "./AccessGroupsDetailsPage"; import { AccessGroupCreateModal } from "./AccessGroupsModal/AccessGroupCreateModal"; +import { AccessGroupsTable } from "./AccessGroupsTable"; import { AccessGroup } from "./types"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { isProxyAdminRole } from "@/utils/roles"; -declare module "@tanstack/react-table" { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - interface ColumnMeta { - responsive?: string[]; - } -} - const { Title, Text } = Typography; const { Content } = Layout; @@ -52,55 +31,6 @@ function mapResponseToAccessGroup(r: AccessGroupResponse): AccessGroup { updatedBy: r.updated_by ?? "", }; } -function buildAntdColumns( - table: ReturnType>, - rowLookup: Map>, - onSortingChange: (s: SortingState) => void, -) { - const headers = table.getHeaderGroups()[0]?.headers ?? []; - - return headers.map((header) => { - const canSort = header.column.getCanSort(); - const isSorted = header.column.getIsSorted(); - const meta = header.column.columnDef.meta as { responsive?: string[] } | undefined; - - const col: Record = { - title: ( -
- {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} - {canSort && ( - { - if (newState === false) { - onSortingChange([]); - } else { - onSortingChange([{ id: header.column.id, desc: newState === "desc" }]); - } - }} - columnId={header.column.id} - /> - )} -
- ), - key: header.id, - width: header.column.columnDef.size, - render: (_: unknown, record: AccessGroup) => { - const row = rowLookup.get(record.id); - if (!row) return null; - const cell = row.getVisibleCells().find((c) => c.column.id === header.id); - if (!cell) return null; - return flexRender(cell.column.columnDef.cell, cell.getContext()); - }, - }; - - if (meta?.responsive) { - col.responsive = meta.responsive; - } - - return col; - }); -} export function AccessGroupsPage() { const { token } = theme.useToken(); @@ -113,151 +43,19 @@ export function AccessGroupsPage() { const [selectedGroupId, setSelectedGroupId] = useState(null); const [isCreateModalVisible, setIsCreateModalVisible] = useState(false); const [searchText, setSearchText] = useState(""); - const [currentPage, setCurrentPage] = useState(1); - const [sorting, setSorting] = useState([]); const [groupToDelete, setGroupToDelete] = useState(null); const deleteMutation = useDeleteAccessGroup(); - const pageSize = 10; - useEffect(() => { - setCurrentPage(1); - }, [searchText]); - - // ---------- filtered data ---------- - const filteredGroups = useMemo( - () => - groups.filter( - (group) => - group.name.toLowerCase().includes(searchText.toLowerCase()) || - group.id.toLowerCase().includes(searchText.toLowerCase()) || - group.description.toLowerCase().includes(searchText.toLowerCase()), - ), - [groups, searchText], - ); - - // ---------- TanStack column definitions ---------- - const columnDefs = useMemo[]>( - () => [ - { - id: "id", - accessorKey: "id", - header: () => ID, - enableSorting: false, - size: 170, - cell: ({ row }) => , - }, - { - id: "name", - accessorKey: "name", - header: () => Name, - enableSorting: true, - cell: ({ getValue }) => getValue() as string, - }, - { - id: "resources", - header: () => Resources, - enableSorting: false, - cell: ({ row }) => { - const record = row.original; - const modelIds = record.modelIds ?? []; - const mcpServerIds = record.mcpServerIds ?? []; - const agentIds = record.agentIds ?? []; - return ( - - - - - - {modelIds?.length} - - - - - - - - {mcpServerIds?.length} - - - - - - - - {agentIds?.length} - - - - - ); - }, - }, - { - id: "createdAt", - accessorKey: "createdAt", - header: () => Created, - enableSorting: true, - sortingFn: "datetime", - cell: ({ getValue }) => , - meta: { responsive: ["lg"] }, - }, - { - id: "updatedAt", - accessorKey: "updatedAt", - header: () => Updated, - enableSorting: false, - cell: ({ getValue }) => , - meta: { responsive: ["xl"] }, - }, - ...(canModify - ? [ - { - id: "actions", - header: () => Actions, - enableSorting: false, - cell: ({ row }: { row: Row }) => ( - - setGroupToDelete(row.original)} - /> - - ), - }, - ] - : []), - ], - // setSelectedGroup is stable (useState setter) - // eslint-disable-next-line react-hooks/exhaustive-deps - [canModify], - ); - - // ---------- TanStack table instance ---------- - const table = useReactTable({ - data: filteredGroups, - columns: columnDefs, - state: { sorting }, - onSortingChange: setSorting, - getCoreRowModel: getCoreRowModel(), - getSortedRowModel: getSortedRowModel(), - getRowId: (row) => row.id, - }); - - // All sorted rows from TanStack - const sortedRows = table.getRowModel().rows; - - // Paginated slice - const paginatedRows = sortedRows.slice((currentPage - 1) * pageSize, currentPage * pageSize); - - // Map for O(1) lookup by record id in antd render() - const rowLookup = useMemo(() => new Map(paginatedRows.map((row) => [row.original.id, row])), [paginatedRows]); - - // Convert TanStack headers → antd columns - const antdColumns = buildAntdColumns(table, rowLookup, setSorting); - - // antd dataSource (just the originals for the current page) - const dataSource = paginatedRows.map((row) => row.original); + const filteredGroups = useMemo(() => { + const query = searchText.trim().toLowerCase(); + if (!query) return groups; + return groups.filter( + (group) => + group.name.toLowerCase().includes(query) || + group.id.toLowerCase().includes(query) || + group.description.toLowerCase().includes(query), + ); + }, [groups, searchText]); if (selectedGroupId) { return setSelectedGroupId(null)} />; @@ -279,34 +77,25 @@ export function AccessGroupsPage() { )} - - - } - placeholder="Search groups by name, ID, or description..." - style={{ maxWidth: 400 }} - value={searchText} - onChange={(e) => setSearchText(e.target.value)} - allowClear - /> - setCurrentPage(page)} - size="small" - showTotal={(total) => `${total} groups`} - showSizeChanger={false} - /> - - - + + } + placeholder="Search groups by name, ID, or description..." + style={{ maxWidth: 400 }} + value={searchText} + onChange={(e) => setSearchText(e.target.value)} + allowClear + /> + + + 0} + canModify={canModify} + onGroupClick={setSelectedGroupId} + onDeleteClick={setGroupToDelete} + /> setIsCreateModalVisible(false)} /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsTable.tsx new file mode 100644 index 00000000000..10d1735d3e7 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsTable.tsx @@ -0,0 +1,72 @@ +"use client"; + +import { SortingState } from "@tanstack/react-table"; +import { Layers } from "lucide-react"; +import { useMemo, useState } from "react"; + +import { DataTable } from "@/components/shared/DataTable"; + +import { getAccessGroupsTableColumns } from "./AccessGroupsTableColumns"; +import { AccessGroup } from "./types"; + +interface AccessGroupsTableProps { + groups: AccessGroup[]; + isLoading: boolean; + isFiltered: boolean; + canModify: boolean; + onGroupClick: (id: string) => void; + onDeleteClick: (group: AccessGroup) => void; +} + +const PAGE_SIZE_OPTIONS = [10, 25, 50]; + +function EmptyState({ isFiltered }: { isFiltered: boolean }) { + return ( +
+
+ +
+
+ {isFiltered ? "No matching access groups" : "No access groups yet"} +
+
+ {isFiltered + ? "Try a different search term." + : "Create an access group to manage resource permissions for your organization."} +
+
+ ); +} + +export function AccessGroupsTable({ + groups, + isLoading, + isFiltered, + canModify, + onGroupClick, + onDeleteClick, +}: AccessGroupsTableProps) { + const [sorting, setSorting] = useState([]); + + const columns = useMemo(() => { + const deps = { canModify, onGroupClick, onDeleteClick }; + return getAccessGroupsTableColumns(deps); + }, [canModify, onGroupClick, onDeleteClick]); + + return ( + group.id || String(index)} + sortingMode="client" + sorting={sorting} + onSortingChange={setSorting} + paginationMode="client" + pageSizeOptions={PAGE_SIZE_OPTIONS} + isLoading={isLoading} + loadingMessage="Loading access groups…" + noDataMessage={} + size="compact" + /> + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsTableColumns.tsx new file mode 100644 index 00000000000..ae65f161b1e --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsTableColumns.tsx @@ -0,0 +1,182 @@ +"use client"; + +import { ColumnDef } from "@tanstack/react-table"; +import { Bot, Layers, MoreHorizontal, Server, Trash2 } from "lucide-react"; + +import { DataTableSortHeader } from "@/components/shared/DataTable"; +import { DateCell, IdentityCell } from "@/components/shared/table_cells"; +import { buttonVariants } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { cn } from "@/lib/cva.config"; + +import { AccessGroup } from "./types"; + +interface ResourceTone { + icon: typeof Layers; + className: string; +} + +const RESOURCE_TONES: Record<"models" | "mcpServers" | "agents", ResourceTone> = { + models: { icon: Layers, className: "bg-blue-50 text-blue-700 ring-blue-600/20" }, + mcpServers: { icon: Server, className: "bg-cyan-50 text-cyan-700 ring-cyan-600/20" }, + agents: { icon: Bot, className: "bg-purple-50 text-purple-700 ring-purple-600/20" }, +}; + +function ResourcesCell({ group }: { group: AccessGroup }) { + const items = [ + { key: "models" as const, label: "Models", count: group.modelIds.length }, + { key: "mcpServers" as const, label: "MCP Servers", count: group.mcpServerIds.length }, + { key: "agents" as const, label: "Agents", count: group.agentIds.length }, + ]; + + return ( +
+ {items.map((item) => { + const tone = RESOURCE_TONES[item.key]; + const Icon = tone.icon; + return ( + + + {item.count} + + ); + })} +
+ ); +} + +function AccessGroupRowActions({ + group, + onDeleteClick, +}: { + group: AccessGroup; + onDeleteClick: (group: AccessGroup) => void; +}) { + return ( + + + + + + onDeleteClick(group)} + > + + Delete access group + + + + ); +} + +interface AccessGroupsTableColumnsDeps { + canModify: boolean; + onGroupClick: (id: string) => void; + onDeleteClick: (group: AccessGroup) => void; +} + +export const getAccessGroupsTableColumns = ({ + canModify, + onGroupClick, + onDeleteClick, +}: AccessGroupsTableColumnsDeps): ColumnDef[] => { + const columns: ColumnDef[] = [ + { + id: "id", + accessorKey: "id", + meta: { title: "ID" }, + header: "ID", + size: 200, + enableSorting: false, + cell: ({ row }) => ( + onGroupClick(row.original.id)} + /> + ), + }, + { + id: "name", + accessorKey: "name", + meta: { title: "Name" }, + header: ({ column }) => , + size: 220, + enableSorting: true, + cell: ({ row }) => { + const name = row.original.name; + return ( + + {name || "-"} + + ); + }, + }, + { + id: "resources", + meta: { title: "Resources" }, + header: "Resources", + size: 220, + enableSorting: false, + cell: ({ row }) => , + }, + { + id: "createdAt", + accessorKey: "createdAt", + meta: { title: "Created" }, + header: ({ column }) => , + size: 150, + enableSorting: true, + sortingFn: "datetime", + cell: ({ row }) => , + }, + { + id: "updatedAt", + accessorKey: "updatedAt", + meta: { title: "Updated" }, + header: "Updated", + size: 150, + enableSorting: false, + cell: ({ row }) => , + }, + ]; + + if (!canModify) { + return columns; + } + + return [ + ...columns, + { + id: "actions", + meta: { className: "text-right", headerClassName: "text-right" }, + header: () => Actions, + size: 64, + enableSorting: false, + enableHiding: false, + cell: ({ row }) => ( +
+ +
+ ), + }, + ]; +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.test.tsx index 48674f21883..441d300436a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.test.tsx @@ -1,12 +1,13 @@ import React from "react"; -import { render, screen, waitFor, act, fireEvent, within } from "@testing-library/react"; +import { act, render, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { describe, it, expect, vi, beforeEach } from "vitest"; import AgentsPanel from "./AgentsPanel"; import * as networking from "@/components/networking"; vi.mock("@/components/networking", () => ({ getAgentsList: vi.fn().mockResolvedValue({ agents: [] }), - deleteAgentCall: vi.fn(), + deleteAgentCall: vi.fn().mockResolvedValue({}), })); vi.mock("./add_agent_form", () => ({ @@ -19,56 +20,54 @@ vi.mock("./agent_info", () => ({ describe("AgentsPanel", () => { beforeEach(() => { - vi.clearAllMocks(); + // mockReset (not mockClear) so an unconsumed *Once queue cannot leak into the next test + vi.mocked(networking.getAgentsList).mockReset().mockResolvedValue({ agents: [] }); + vi.mocked(networking.deleteAgentCall).mockReset().mockResolvedValue({}); }); - it("should render the Agents panel title", async () => { + it("should render the Agents panel title", () => { render(); expect(screen.getByText("Agents")).toBeInTheDocument(); }); - it("should show Add New Agent button for admin users", async () => { + it("should show Add New Agent button for admin users", () => { render(); - expect(screen.getByText("+ Add New Agent")).toBeInTheDocument(); + expect(screen.getByText("Add New Agent")).toBeInTheDocument(); }); - it("should show Add New Agent button for proxy_admin users", async () => { + it("should show Add New Agent button for proxy_admin users", () => { render(); - expect(screen.getByText("+ Add New Agent")).toBeInTheDocument(); + expect(screen.getByText("Add New Agent")).toBeInTheDocument(); }); - it("should not show Add New Agent button for internal_user role", async () => { + it("should not show Add New Agent button for internal_user role", () => { render(); - expect(screen.queryByText("+ Add New Agent")).not.toBeInTheDocument(); + expect(screen.queryByText("Add New Agent")).not.toBeInTheDocument(); }); - it("should not show Add New Agent button for internal_user_viewer role", async () => { + it("should not show Add New Agent button for internal_user_viewer role", () => { render(); - expect(screen.queryByText("+ Add New Agent")).not.toBeInTheDocument(); + expect(screen.queryByText("Add New Agent")).not.toBeInTheDocument(); }); - it("should show Actions column header for admin role", async () => { + it("should show the Actions column for admin role", async () => { render(); - await waitFor(() => { - expect(screen.getByRole("columnheader", { name: /actions/i })).toBeInTheDocument(); - }); + expect(await screen.findByRole("columnheader", { name: /actions/i })).toBeInTheDocument(); }); - it("should not show Actions column header for internal user role", async () => { + it("should not show the Actions column for internal user role", async () => { render(); await waitFor(() => { expect(screen.queryByRole("columnheader", { name: /actions/i })).not.toBeInTheDocument(); - // confirm table is rendered (not still loading) expect(screen.getByRole("table")).toBeInTheDocument(); }); }); - it("should render the Health Check toggle", async () => { - render(); + it("should render the Health Check toggle for admins and non-admins", () => { + const { unmount } = render(); expect(screen.getByText("Health Check")).toBeInTheDocument(); - }); + unmount(); - it("should render the Health Check toggle for non-admin users too", async () => { render(); expect(screen.getByText("Health Check")).toBeInTheDocument(); }); @@ -108,19 +107,187 @@ describe("AgentsPanel", () => { expect(within(keylessRow).getByText("Needs Setup")).toBeInTheDocument(); }); - it("should call getAgentsList with health_check=true when toggle is enabled", async () => { + it("should refetch with health_check=true when the toggle is enabled", async () => { + const user = userEvent.setup(); render(); await waitFor(() => { expect(networking.getAgentsList).toHaveBeenCalledWith("test-token", false); }); - const toggle = screen.getByRole("switch"); - await act(async () => { - fireEvent.click(toggle); - }); + await user.click(screen.getByRole("switch")); await waitFor(() => { expect(networking.getAgentsList).toHaveBeenCalledWith("test-token", true); }); }); + + it("should delete an agent through the ⋯ menu and confirm modal, then refetch", async () => { + const user = userEvent.setup(); + vi.mocked(networking.getAgentsList).mockResolvedValue({ + agents: [ + { + agent_id: "agent-9", + agent_name: "Doomed Agent", + litellm_params: { model: "gpt-4" }, + spend: 0, + keys: [], + }, + ], + }); + + render(); + + await user.click(await screen.findByTestId("agent-actions-agent-9")); + await user.click(await screen.findByTestId("agent-action-delete")); + + const modal = await screen.findByRole("dialog"); + await user.click(within(modal).getByRole("button", { name: /^delete$/i })); + + await waitFor(() => { + expect(networking.deleteAgentCall).toHaveBeenCalledWith("test-token", "agent-9"); + }); + // one initial load + one post-delete refetch + await waitFor(() => { + expect(vi.mocked(networking.getAgentsList).mock.calls.length).toBeGreaterThanOrEqual(2); + }); + }); + + it("should show a loading skeleton on initial load and clear it once agents arrive", async () => { + render(); + expect(screen.getAllByTestId("skeleton-row").length).toBeGreaterThan(0); + await waitFor(() => { + expect(screen.queryByTestId("skeleton-row")).not.toBeInTheDocument(); + }); + }); + + it("should clear the loading state when there is no access token rather than skeleton forever", async () => { + render(); + await waitFor(() => { + expect(screen.queryByTestId("skeleton-row")).not.toBeInTheDocument(); + }); + expect(screen.getByText("No agents yet")).toBeInTheDocument(); + expect(networking.getAgentsList).not.toHaveBeenCalled(); + }); + + it("should not show rows fetched with a previous access token after the token changes", async () => { + const agentFor = (name: string) => ({ + agent_id: `id-${name}`, + agent_name: name, + litellm_params: { model: "gpt-4" }, + spend: 0, + keys: [], + }); + let resolveSecond: (value: { agents: ReturnType[] }) => void = () => {}; + vi.mocked(networking.getAgentsList) + .mockResolvedValueOnce({ agents: [agentFor("first-token-agent")] }) + .mockImplementationOnce( + () => + new Promise((resolve) => { + resolveSecond = resolve; + }), + ); + + const { rerender } = render(); + expect(await screen.findByText("first-token-agent")).toBeInTheDocument(); + + rerender(); + + // the previous token's rows must not linger while the new token loads + expect(screen.queryByText("first-token-agent")).not.toBeInTheDocument(); + expect(screen.getAllByTestId("skeleton-row").length).toBeGreaterThan(0); + + await act(async () => { + resolveSecond({ agents: [agentFor("second-token-agent")] }); + }); + expect(await screen.findByText("second-token-agent")).toBeInTheDocument(); + }); + + it("should drop previous rows when the fetch for a new token fails", async () => { + vi.mocked(networking.getAgentsList) + .mockResolvedValueOnce({ + agents: [ + { agent_id: "stale", agent_name: "Stale Agent", litellm_params: { model: "gpt-4" }, spend: 0, keys: [] }, + ], + }) + .mockRejectedValueOnce(new Error("unauthorized")); + + const { rerender } = render(); + expect(await screen.findByText("Stale Agent")).toBeInTheDocument(); + + rerender(); + + await waitFor(() => { + expect(screen.getByText("No agents yet")).toBeInTheDocument(); + }); + expect(screen.queryByText("Stale Agent")).not.toBeInTheDocument(); + }); + + it("should ignore a superseded response so it cannot overwrite the current token's rows", async () => { + let resolveFirst: (value: { + agents: { agent_id: string; agent_name: string; litellm_params: { model: string }; spend: number; keys: [] }[]; + }) => void = () => {}; + vi.mocked(networking.getAgentsList) + .mockImplementationOnce( + () => + new Promise((resolve) => { + resolveFirst = resolve; + }), + ) + .mockResolvedValueOnce({ + agents: [ + { agent_id: "current", agent_name: "Current Agent", litellm_params: { model: "gpt-4" }, spend: 0, keys: [] }, + ], + }); + + const { rerender } = render(); + rerender(); + + expect(await screen.findByText("Current Agent")).toBeInTheDocument(); + + // the slow token-a response lands last and must be discarded + await act(async () => { + resolveFirst({ + agents: [ + { agent_id: "stale", agent_name: "Superseded Agent", litellm_params: { model: "gpt-4" }, spend: 0, keys: [] }, + ], + }); + }); + + expect(screen.queryByText("Superseded Agent")).not.toBeInTheDocument(); + expect(screen.getByText("Current Agent")).toBeInTheDocument(); + }); + + it("should keep rows visible during a health-check refetch instead of re-showing the skeleton", async () => { + const user = userEvent.setup(); + const agents = [ + { + agent_id: "agent-1", + agent_name: "Stable Agent", + litellm_params: { model: "gpt-4" }, + spend: 0, + keys: [], + }, + ]; + let resolveRefetch: (value: { agents: typeof agents }) => void = () => {}; + vi.mocked(networking.getAgentsList) + .mockResolvedValueOnce({ agents }) + .mockImplementationOnce( + () => + new Promise((resolve) => { + resolveRefetch = resolve; + }), + ); + + render(); + expect(await screen.findByText("Stable Agent")).toBeInTheDocument(); + + await user.click(screen.getByRole("switch")); + + expect(screen.getByText("Stable Agent")).toBeInTheDocument(); + expect(screen.queryByTestId("skeleton-row")).not.toBeInTheDocument(); + + await act(async () => { + resolveRefetch({ agents }); + }); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.tsx index 84634620426..a4a71530c84 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.tsx @@ -1,27 +1,15 @@ import React, { useState, useEffect } from "react"; -import { - Button, - Card, - Table, - TableBody, - TableCell, - TableHead, - TableHeaderCell, - TableRow, - Badge, - Text, -} from "@tremor/react"; -import { Modal, Alert, Tooltip, Skeleton, Switch } from "antd"; -import { CheckCircleOutlined } from "@ant-design/icons"; +import { Modal, Alert } from "antd"; +import { Plus } from "lucide-react"; import { getAgentsList, deleteAgentCall } from "@/components/networking"; import AddAgentForm from "./add_agent_form"; import { isAdminRole } from "@/utils/roles"; import AgentInfoView from "./agent_info"; +import AgentsTable from "./AgentsTable"; import NotificationsManager from "@/components/molecules/notifications_manager"; import { Agent } from "@/components/agents/types"; import { Team } from "@/components/key_team_helpers/key_list"; -import { DateCell, IdCell, MoneyCell, StatusBadge } from "@/components/shared/table_cells"; -import TableIconActionButton from "@/components/common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; +import { Button } from "@/components/ui/button"; interface AgentsPanelProps { accessToken: string | null; @@ -36,37 +24,66 @@ interface AgentsResponse { const AgentsPanel: React.FC = ({ accessToken, userRole, teams }) => { const [agentsList, setAgentsList] = useState([]); const [isAddModalVisible, setIsAddModalVisible] = useState(false); - const [isLoading, setIsLoading] = useState(false); + const [isLoading, setIsLoading] = useState(true); const [isDeleting, setIsDeleting] = useState(false); + const [isHealthCheckLoading, setIsHealthCheckLoading] = useState(false); const [agentToDelete, setAgentToDelete] = useState<{ id: string; name: string } | null>(null); const [selectedAgentId, setSelectedAgentId] = useState(null); const [healthCheckEnabled, setHealthCheckEnabled] = useState(false); const isAdmin = userRole ? isAdminRole(userRole) : false; - const fetchAgents = async (healthCheck?: boolean) => { + useEffect(() => { + let cancelled = false; + const loadForToken = async () => { + if (!accessToken) { + setAgentsList([]); + setIsLoading(false); + return; + } + setIsLoading(true); + try { + const response: AgentsResponse = await getAgentsList(accessToken, false); + if (!cancelled) { + setAgentsList(response.agents || []); + } + } catch (error) { + console.error("Error fetching agents:", error); + if (!cancelled) { + setAgentsList([]); + } + } finally { + if (!cancelled) { + setIsLoading(false); + } + } + }; + loadForToken(); + return () => { + cancelled = true; + }; + }, [accessToken]); + + const refetchAgents = async (healthCheck: boolean) => { if (!accessToken) { return; } - - setIsLoading(true); try { - const response: AgentsResponse = await getAgentsList(accessToken, healthCheck ?? healthCheckEnabled); + const response: AgentsResponse = await getAgentsList(accessToken, healthCheck); setAgentsList(response.agents || []); } catch (error) { console.error("Error fetching agents:", error); - } finally { - setIsLoading(false); } }; - useEffect(() => { - fetchAgents(); - }, [accessToken]); - - const handleHealthCheckToggle = (checked: boolean) => { + const handleHealthCheckToggle = async (checked: boolean) => { setHealthCheckEnabled(checked); - fetchAgents(checked); + setIsHealthCheckLoading(true); + try { + await refetchAgents(checked); + } finally { + setIsHealthCheckLoading(false); + } }; const handleAddAgent = () => { @@ -81,7 +98,7 @@ const AgentsPanel: React.FC = ({ accessToken, userRole, teams }; const handleSuccess = () => { - fetchAgents(); + refetchAgents(healthCheckEnabled); }; const handleDeleteClick = (agentId: string, agentName: string) => { @@ -95,7 +112,7 @@ const AgentsPanel: React.FC = ({ accessToken, userRole, teams try { await deleteAgentCall(accessToken, agentToDelete.id); NotificationsManager.success(`Agent "${agentToDelete.name}" deleted successfully`); - fetchAgents(); + await refetchAgents(healthCheckEnabled); } catch (error) { console.error("Error deleting agent:", error); NotificationsManager.fromBackend("Failed to delete agent"); @@ -109,14 +126,6 @@ const AgentsPanel: React.FC = ({ accessToken, userRole, teams setAgentToDelete(null); }; - const sortedAgents = [...agentsList].sort((a, b) => { - const dateA = a.created_at ? new Date(a.created_at).getTime() : 0; - const dateB = b.created_at ? new Date(b.created_at).getTime() : 0; - return dateB - dateA; - }); - - const columnCount = isAdmin ? 7 : 6; - return (
@@ -132,25 +141,14 @@ const AgentsPanel: React.FC = ({ accessToken, userRole, teams showIcon className="mb-3" /> -
- {isAdmin && ( + {isAdmin && ( +
- )} - -
- - Health Check - -
-
-
+
+ )}
{selectedAgentId ? ( @@ -161,73 +159,16 @@ const AgentsPanel: React.FC = ({ accessToken, userRole, teams isAdmin={isAdmin} /> ) : ( - - {isLoading ? ( - - ) : ( -
- - - Agent Name - Agent ID - Spend (USD) - Model - Created - Status - {isAdmin && Actions} - - - - {sortedAgents.length === 0 ? ( - - - - No agents found. Click "+ Add New Agent" to create one. - - - - ) : ( - sortedAgents.map((agent) => ( - - - {agent.agent_name} - - - setSelectedAgentId(id)} /> - - - - - - - {agent.litellm_params?.model || "N/A"} - - - - - - - {(agent.keys?.length ?? 0) > 0 ? ( - - ) : ( - - )} - - {isAdmin && ( - - handleDeleteClick(agent.agent_id, agent.agent_name)} - /> - - )} - - )) - )} - -
- )} -
+ setSelectedAgentId(id)} + onDeleteClick={handleDeleteClick} + /> )} = {}): Agent => ({ + agent_id: "agent-1", + agent_name: "Test Agent", + litellm_params: { model: "gpt-4" }, + spend: 0, + keys: [{ token: "hash-1", key_alias: "primary", key_name: "sk-...1" }], + created_at: "2023-01-01T00:00:00Z", + ...overrides, +}); + +describe("AgentsTable", () => { + it("renders every column header", () => { + render(); + for (const header of ["Agent Name", "Agent ID", "Spend (USD)", "Model", "Created", "Status"]) { + expect(screen.getByText(header)).toBeInTheDocument(); + } + }); + + it("renders the agent's model and opens the detail view when the ID cell is clicked", async () => { + const user = userEvent.setup(); + const onAgentClick = vi.fn(); + const agent = makeAgent({ agent_id: "agent-xyz", agent_name: "Router", litellm_params: { model: "claude-3-5" } }); + render(); + + expect(screen.getByText("claude-3-5")).toBeInTheDocument(); + + await user.click(screen.getByText("agent-xyz")); + expect(onAgentClick).toHaveBeenCalledWith("agent-xyz"); + }); + + it("marks agents Active when they have keys and Needs Setup when they have none", () => { + render( + , + ); + + const keyedRow = screen.getByText("Keyed Agent").closest("tr")!; + const keylessRow = screen.getByText("Keyless Agent").closest("tr")!; + expect(within(keyedRow).getByText("Active")).toBeInTheDocument(); + expect(within(keylessRow).getByText("Needs Setup")).toBeInTheDocument(); + }); + + it("deletes an agent through the ⋯ actions menu", async () => { + const user = userEvent.setup(); + const onDeleteClick = vi.fn(); + const agent = makeAgent({ agent_id: "agent-9", agent_name: "Doomed Agent" }); + render(); + + await user.click(screen.getByTestId("agent-actions-agent-9")); + await user.click(await screen.findByTestId("agent-action-delete")); + + expect(onDeleteClick).toHaveBeenCalledWith("agent-9", "Doomed Agent"); + }); + + it("hides the actions column entirely for non-admins", () => { + const agent = makeAgent({ agent_id: "agent-2" }); + render(); + + expect(screen.queryByTestId("agent-actions-agent-2")).not.toBeInTheDocument(); + expect(screen.queryByRole("columnheader", { name: /actions/i })).not.toBeInTheDocument(); + expect(screen.getByRole("table")).toBeInTheDocument(); + }); + + it("shows the actions column for admins", () => { + render(); + expect(screen.getByRole("columnheader", { name: /actions/i })).toBeInTheDocument(); + expect(screen.getByTestId("agent-actions-agent-3")).toBeInTheDocument(); + }); + + it("defaults to sorting by created_at descending (newest first)", () => { + render( + , + ); + + const bodyRows = screen.getAllByRole("row").slice(1); + expect(bodyRows[0].textContent).toContain("Beta Agent"); + expect(bodyRows[1].textContent).toContain("Alpha Agent"); + }); + + it("sorts agents with no created_at last, never ahead of dated ones", () => { + render( + , + ); + + const bodyRows = screen.getAllByRole("row").slice(1); + expect(bodyRows[0].textContent).toContain("Beta Agent"); + expect(bodyRows[1].textContent).toContain("Alpha Agent"); + expect(bodyRows[2].textContent).toContain("Undated Agent"); + }); + + it("shows a rich empty state when there are no agents", () => { + render(); + expect(screen.getByText("No agents yet")).toBeInTheDocument(); + expect(screen.queryByTestId("skeleton-row")).not.toBeInTheDocument(); + }); + + it("renders loading skeleton rows on initial load instead of the empty state", () => { + render(); + expect(screen.getAllByTestId("skeleton-row").length).toBeGreaterThan(0); + expect(screen.queryByText("No agents yet")).not.toBeInTheDocument(); + }); + + it("invokes the health-check toggle from the toolbar", async () => { + const user = userEvent.setup(); + const onHealthCheckToggle = vi.fn(); + render(); + + expect(screen.getByText("Health Check")).toBeInTheDocument(); + await user.click(screen.getByRole("switch")); + expect(onHealthCheckToggle).toHaveBeenCalledWith(true, expect.anything()); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTable.tsx new file mode 100644 index 00000000000..824ae47f3e6 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTable.tsx @@ -0,0 +1,88 @@ +"use client"; + +import { SortingState } from "@tanstack/react-table"; +import { Tooltip, Switch } from "antd"; +import { CheckCircleOutlined } from "@ant-design/icons"; +import { Bot } from "lucide-react"; +import React, { useMemo, useState } from "react"; + +import { Agent } from "@/components/agents/types"; +import { DataTable } from "@/components/shared/DataTable"; + +import { getAgentsTableColumns } from "./AgentsTableColumns"; + +interface AgentsTableProps { + agents: Agent[]; + isLoading: boolean; + isAdmin: boolean; + healthCheckEnabled: boolean; + isHealthCheckLoading: boolean; + onHealthCheckToggle: (checked: boolean) => void; + onAgentClick: (agentId: string) => void; + onDeleteClick: (agentId: string, agentName: string) => void; +} + +const DEFAULT_SORTING: SortingState = [{ id: "created_at", desc: true }]; + +function EmptyState() { + return ( +
+
+ +
+
No agents yet
+
Add an agent to make it available in your organization.
+
+ ); +} + +const AgentsTable: React.FC = ({ + agents, + isLoading, + isAdmin, + healthCheckEnabled, + isHealthCheckLoading, + onHealthCheckToggle, + onAgentClick, + onDeleteClick, +}) => { + const [sorting, setSorting] = useState(DEFAULT_SORTING); + + const columns = useMemo( + () => getAgentsTableColumns({ isAdmin, onAgentClick, onDeleteClick }), + [isAdmin, onAgentClick, onDeleteClick], + ); + + return ( + agent.agent_id || String(index)} + sortingMode="client" + sorting={sorting} + onSortingChange={setSorting} + isLoading={isLoading} + loadingMessage="Loading agents…" + noDataMessage={} + size="compact" + toolbar={() => ( +
+ +
+ + Health Check + +
+
+
+ )} + /> + ); +}; + +export default AgentsTable; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTableColumns.tsx new file mode 100644 index 00000000000..a8fe3973a42 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsTableColumns.tsx @@ -0,0 +1,163 @@ +"use client"; + +import { ColumnDef } from "@tanstack/react-table"; +import { MoreHorizontal, Trash2 } from "lucide-react"; + +import { Agent } from "@/components/agents/types"; +import { DataTableSortHeader } from "@/components/shared/DataTable"; +import { DateCell, IdentityCell, MoneyCell, StatusBadge } from "@/components/shared/table_cells"; +import { Badge } from "@/components/ui/badge"; +import { buttonVariants } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { cn } from "@/lib/cva.config"; + +interface AgentRowActionsProps { + agent: Agent; + onDeleteClick: (agentId: string, agentName: string) => void; +} + +function AgentRowActions({ agent, onDeleteClick }: AgentRowActionsProps) { + return ( + + + + + + onDeleteClick(agent.agent_id, agent.agent_name)} + > + + Delete + + + + ); +} + +interface AgentsTableColumnsDeps { + isAdmin: boolean; + onAgentClick: (agentId: string) => void; + onDeleteClick: (agentId: string, agentName: string) => void; +} + +export const getAgentsTableColumns = ({ + isAdmin, + onAgentClick, + onDeleteClick, +}: AgentsTableColumnsDeps): ColumnDef[] => [ + { + id: "agent_name", + accessorKey: "agent_name", + meta: { title: "Agent Name" }, + header: ({ column }) => , + size: 200, + enableSorting: true, + cell: ({ row }) => { + const name = row.original.agent_name; + return ( + + {name || "-"} + + ); + }, + }, + { + id: "agent_id", + accessorKey: "agent_id", + meta: { title: "Agent ID" }, + header: ({ column }) => , + size: 200, + enableSorting: true, + cell: ({ row }) => ( + onAgentClick(row.original.agent_id)} + /> + ), + }, + { + id: "spend", + accessorKey: "spend", + meta: { title: "Spend (USD)" }, + header: ({ column }) => , + size: 130, + enableSorting: true, + cell: ({ row }) => , + }, + { + id: "model", + meta: { title: "Model" }, + header: "Model", + size: 170, + enableSorting: false, + cell: ({ row }) => { + const model = row.original.litellm_params?.model; + if (!model) { + return N/A; + } + return ( + + + {model} + + + ); + }, + }, + { + id: "created_at", + accessorFn: (agent) => { + const timestamp = agent.created_at ? new Date(agent.created_at).getTime() : 0; + return Number.isNaN(timestamp) ? 0 : timestamp; + }, + meta: { title: "Created" }, + header: ({ column }) => , + size: 150, + enableSorting: true, + cell: ({ row }) => , + }, + { + id: "status", + meta: { title: "Status" }, + header: "Status", + size: 130, + enableSorting: false, + cell: ({ row }) => { + const hasKeys = (row.original.keys?.length ?? 0) > 0; + return hasKeys ? ( + + ) : ( + + ); + }, + }, + ...(isAdmin + ? [ + { + id: "actions", + meta: { className: "text-right", headerClassName: "text-right" }, + header: () => Actions, + size: 64, + enableSorting: false, + enableHiding: false, + cell: ({ row }) => ( +
+ +
+ ), + } satisfies ColumnDef, + ] + : []), +]; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.test.tsx new file mode 100644 index 00000000000..767e7c2ae5f --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.test.tsx @@ -0,0 +1,89 @@ +import React from "react"; +import { render, screen, fireEvent, within } from "@testing-library/react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import AddAgentForm from "./add_agent_form"; +import * as networking from "@/components/networking"; +import type { AgentCreateInfo } from "@/components/networking"; + +vi.mock("@/components/networking", () => ({ + createAgentCall: vi.fn(), + getAgentCreateMetadata: vi.fn(), + getAgentsList: vi.fn(), + keyCreateForAgentCall: vi.fn(), + keyListCall: vi.fn(), + keyUpdateCall: vi.fn(), + modelAvailableCall: vi.fn(), +})); + +vi.mock("./agent_card_discovery", () => ({ + default: () =>
, +})); + +vi.mock("./agent_form_fields", () => ({ + default: () =>
, +})); + +const a2aInfo: AgentCreateInfo = { + agent_type: "a2a", + agent_type_display_name: "A2A Agent", + description: "Agent-to-agent protocol", + logo_url: "/ui/assets/logos/a2a_agent.png", + credential_fields: [], + use_a2a_form_fields: true, +}; + +const renderForm = () => + render(); + +describe("AddAgentForm logos", () => { + beforeEach(() => { + vi.mocked(networking.getAgentCreateMetadata).mockReset().mockResolvedValue([a2aInfo]); + vi.mocked(networking.getAgentsList).mockReset().mockResolvedValue({ agents: [] }); + vi.mocked(networking.keyListCall).mockReset().mockResolvedValue({ keys: [] }); + vi.mocked(networking.modelAvailableCall).mockReset().mockResolvedValue({ data: [] }); + }); + + it("renders the modal title and agent type selection logos as images from logo_url", async () => { + renderForm(); + + const titleLogo = await screen.findByAltText("Agent logo"); + expect(titleLogo).toBeInstanceOf(HTMLImageElement); + expect(titleLogo).toHaveAttribute("src", expect.stringContaining("assets/logos/a2a_agent.png")); + + const selectionLogo = await screen.findByAltText("A2A Agent logo"); + expect(selectionLogo).toBeInstanceOf(HTMLImageElement); + expect(selectionLogo).toHaveAttribute("src", expect.stringContaining("assets/logos/a2a_agent.png")); + }); + + it("renders the option logo when the agent type dropdown is opened", async () => { + renderForm(); + + await screen.findByAltText("A2A Agent logo"); + fireEvent.mouseDown(screen.getByRole("combobox")); + + const optionLogos = await screen.findAllByAltText("A2A Agent logo"); + expect(optionLogos.length).toBeGreaterThanOrEqual(2); + optionLogos.forEach((img) => { + expect(img).toHaveAttribute("src", expect.stringContaining("assets/logos/a2a_agent.png")); + }); + }); + + it("swaps a failing logo for a letter avatar and warns with the url", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + renderForm(); + + const titleLogo = await screen.findByAltText("Agent logo"); + const header = screen.getByText("Add New Agent").parentElement!; + fireEvent.error(titleLogo); + + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining("assets/logos/a2a_agent.png")); + expect(screen.queryByAltText("Agent logo")).not.toBeInTheDocument(); + expect(within(header).getByText("A")).toBeInTheDocument(); + + const selectionLogo = screen.getByAltText("A2A Agent logo"); + fireEvent.error(selectionLogo); + expect(screen.queryByAltText("A2A Agent logo")).not.toBeInTheDocument(); + expect(warnSpy).toHaveBeenCalledTimes(2); + warnSpy.mockRestore(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx index 8ca2b5afe16..e35388b78da 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx @@ -1,7 +1,7 @@ import React, { useState, useEffect } from "react"; import { Modal, Form, Select, Input, Steps, Radio, Tag, Divider, Switch, InputNumber, Collapse } from "antd"; import MessageManager from "@/components/molecules/message_manager"; -import { resolveLogoSrc } from "@/lib/assetPaths"; +import { Logo } from "@/components/molecules/logo/Logo"; import { Button } from "@tremor/react"; import { CheckCircleFilled, KeyOutlined, RobotOutlined, AppstoreOutlined, InfoCircleOutlined } from "@ant-design/icons"; import CreatedKeyDisplay from "@/components/shared/CreatedKeyDisplay"; @@ -712,17 +712,13 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok value={info.agent_type} label={
- + {info.agent_type_display_name}
} >
- {info.agent_type_display_name} +
{info.agent_type_display_name}
{info.description &&
{info.description}
} @@ -948,7 +944,7 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok title={
{selectedLogo && currentStep < 1 && ( - Agent + )}

Add New Agent

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx index 17d14cd7fac..13472a3d1df 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx @@ -76,6 +76,22 @@ describe("CacheDashboard cache analytics charts", () => { expect(screen.getByText("Cached Completion Tokens vs Generated Completion Tokens")).toBeInTheDocument(); }); + it("scopes the analytics tab to the response cache, not provider prompt caching", async () => { + renderDashboard(); + + expect(await screen.findByText(/is not shown here/)).toBeInTheDocument(); + expect(screen.getByRole("link", { name: "response cache" })).toHaveAttribute( + "href", + "https://docs.litellm.ai/docs/proxy/caching", + ); + expect(screen.getByRole("link", { name: "prompt caching" })).toHaveAttribute( + "href", + "https://docs.litellm.ai/docs/completion/prompt_caching", + ); + expect(screen.queryByText("Cached Tokens")).not.toBeInTheDocument(); + expect(screen.getAllByText("Cached Completion Tokens").length).toBeGreaterThan(0); + }); + it("renders the requests chart with each category legend-bound to its fill and stacked in order", async () => { renderDashboard(); const { requestsCard } = await findChartCards(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx index b8e8dc8adb1..51c0b85cedb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx @@ -282,6 +282,28 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole + + Analytics for LiteLLM's{" "} + + response cache + {" "} + (e.g. Redis / in-memory): requests answered from cache without calling the LLM provider. Provider-side{" "} + + prompt caching + {" "} + (cached input tokens from Anthropic, OpenAI, etc.) is not shown here; see "Prompt Caching + Metrics" on the Usage page or individual requests in the Logs page. + = ({ accessToken, token, userRole

- Cached Tokens + Cached Completion Tokens

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFieldSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFieldSection.tsx index ced822cd796..96106869009 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFieldSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFieldSection.tsx @@ -10,6 +10,7 @@ interface CacheFieldSectionProps { embeddingModels: EmbeddingModelOption[]; gridCols?: string; headingLevel?: "h4" | "h5"; + configuredSecrets?: ReadonlySet; } const CacheFieldSection: React.FC = ({ @@ -19,6 +20,7 @@ const CacheFieldSection: React.FC = ({ embeddingModels, gridCols = "grid-cols-1 gap-6 sm:grid-cols-2", headingLevel = "h4", + configuredSecrets, }) => { const fields = fieldsForSection(section, redisType); if (fields.length === 0) { @@ -32,7 +34,12 @@ const CacheFieldSection: React.FC = ({ {title}

{fields.map((field) => ( - + ))}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFormField.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFormField.tsx index d92ca302901..dbd8c32d18d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFormField.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFormField.tsx @@ -7,22 +7,29 @@ export interface EmbeddingModelOption { label: string; } +export const SECRET_ALREADY_SET_PLACEHOLDER = "Already set. Enter a new value to replace it."; + interface CacheFormFieldProps { field: CacheField; embeddingModels: EmbeddingModelOption[]; + isSecretConfigured?: boolean; } -const renderControl = (field: CacheField, embeddingModels: EmbeddingModelOption[]): React.ReactNode => { +const renderControl = ( + field: CacheField, + embeddingModels: EmbeddingModelOption[], + placeholder: string, +): React.ReactNode => { switch (field.type) { case "boolean": return ; case "password": - return ; + return ; case "integer": case "float": - return ; + return ; case "list": - return ; + return ; case "model-select": return ( ; + return ; } }; -const CacheFormField: React.FC = ({ field, embeddingModels }) => ( +const CacheFormField: React.FC = ({ field, embeddingModels, isSecretConfigured = false }) => ( = ({ field, embeddingModels rules={field.rules} valuePropName={field.type === "boolean" ? "checked" : "value"} > - {renderControl(field, embeddingModels)} + {renderControl(field, embeddingModels, isSecretConfigured ? SECRET_ALREADY_SET_PLACEHOLDER : field.helpText)} ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts index 1f5b566fc5f..e33b525c3ef 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts @@ -8,6 +8,10 @@ export type CacheSection = "connection" | "cluster" | "sentinel" | "semantic" | export type CacheFieldRule = NonNullable[number]; +// Marker the backend returns for a configured credential and maps back to the +// stored secret on save, so the plaintext never round-trips through the form. +export const REDACTED_VALUE = "***REDACTED***"; + export interface CacheField { readonly name: string; readonly label: string; @@ -17,6 +21,9 @@ export interface CacheField { readonly redisType: RedisType | null; readonly defaultValue?: string | number | boolean; readonly rules?: CacheFieldRule[]; + // Credential field: never prefilled into the form, and dropped from the save + // payload when left untouched so the redacted marker is never persisted. + readonly secret?: boolean; } export const REDIS_TYPES: readonly RedisType[] = ["node", "cluster", "sentinel", "semantic"]; @@ -93,6 +100,7 @@ export const CACHE_FIELDS: readonly CacheField[] = [ helpText: "Full Redis/Valkey connection URL (e.g. redis://:password@host:6379/1). When set, it takes precedence over Host, Port, Password, and Database Index.", redisType: null, + secret: true, }, { name: "host", @@ -128,6 +136,7 @@ export const CACHE_FIELDS: readonly CacheField[] = [ section: "connection", helpText: "Redis server password", redisType: null, + secret: true, }, { name: "username", @@ -170,6 +179,7 @@ export const CACHE_FIELDS: readonly CacheField[] = [ section: "sentinel", helpText: "Password for Redis Sentinel authentication", redisType: "sentinel", + secret: true, }, { name: "similarity_threshold", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts index 79f28a97842..c530519ee06 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect } from "vitest"; -import { buildCachePayload, buildInitialValues, fieldsForSection } from "./cacheSettingsUtils"; +import { buildCachePayload, buildInitialValues, configuredSecretFields, fieldsForSection } from "./cacheSettingsUtils"; +import { REDACTED_VALUE } from "./cacheSettingsFields"; describe("fieldsForSection", () => { it("should only include a redis-type-specific field when that type is selected", () => { @@ -83,4 +84,49 @@ describe("buildCachePayload", () => { const payload = buildCachePayload("node", { sentinel_nodes: '[["localhost",26379]]' }, { forTesting: false }); expect(payload).not.toHaveProperty("sentinel_nodes"); }); + + it("should drop a secret whose value is the redacted marker so it is never persisted", () => { + const payload = buildCachePayload( + "node", + { host: "localhost", password: REDACTED_VALUE, url: REDACTED_VALUE }, + { forTesting: false }, + ); + expect(payload).not.toHaveProperty("password"); + expect(payload).not.toHaveProperty("url"); + expect(payload.host).toBe("localhost"); + }); + + it("should send a real new secret value the admin typed", () => { + const payload = buildCachePayload("node", { password: "brandnewpw" }, { forTesting: false }); + expect(payload.password).toBe("brandnewpw"); + }); +}); + +describe("secret handling", () => { + it("buildInitialValues never prefills a credential, even when the server reports it configured", () => { + const serverValues = { + host: "localhost", + password: REDACTED_VALUE, + url: REDACTED_VALUE, + sentinel_password: REDACTED_VALUE, + }; + const values = buildInitialValues(serverValues); + expect(values.password).toBe(""); + expect(values.url).toBe(""); + expect(values.sentinel_password).toBe(""); + // non-secret fields are still prefilled + expect(values.host).toBe("localhost"); + }); + + it("configuredSecretFields reports which credentials the server marked as set", () => { + const configured = configuredSecretFields({ + password: REDACTED_VALUE, + url: "", + host: "localhost", + }); + expect(configured.has("password")).toBe(true); + expect(configured.has("url")).toBe(false); + // a non-secret field is never reported as a configured secret + expect(configured.has("host")).toBe(false); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts index 088da21961c..7b9454a37c3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts @@ -1,4 +1,4 @@ -import { CACHE_FIELDS, CacheField, CacheSection, RedisType } from "./cacheSettingsFields"; +import { CACHE_FIELDS, CacheField, CacheSection, REDACTED_VALUE, RedisType } from "./cacheSettingsFields"; export type CacheFormValue = string | number | boolean | undefined; export type CacheFormValues = Record; @@ -11,7 +11,20 @@ export const isFieldVisible = (field: CacheField, redisType: RedisType): boolean export const fieldsForSection = (section: CacheSection, redisType: RedisType): CacheField[] => CACHE_FIELDS.filter((field) => field.section === section && isFieldVisible(field, redisType)); +const hasValue = (raw: unknown): boolean => raw !== undefined && raw !== null && raw !== ""; + +// Credential fields the server reports as configured (returned as the redacted +// marker). Used to show an "already set" hint without ever holding the secret. +export const configuredSecretFields = (currentValues: Record): ReadonlySet => + new Set(CACHE_FIELDS.filter((field) => field.secret && hasValue(currentValues[field.name])).map((f) => f.name)); + const initialValueForField = (field: CacheField, raw: unknown): CacheFormValue => { + // Never prefill a credential: the server sends the redacted marker for a + // configured secret, and echoing it back would persist the marker. + if (field.secret) { + return ""; + } + const source = raw ?? field.defaultValue; if (field.type === "boolean") { @@ -35,6 +48,11 @@ export const buildInitialValues = (currentValues: Record): Cach Object.fromEntries(CACHE_FIELDS.map((field) => [field.name, initialValueForField(field, currentValues[field.name])])); const saveValueForField = (field: CacheField, raw: CacheFormValue): CacheSavePayloadValue | undefined => { + // A redacted secret echoed back untouched must never be persisted as a value. + if (field.secret && raw === REDACTED_VALUE) { + return undefined; + } + if (field.type === "boolean") { return Boolean(raw); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx index 4382769ae9c..fea2c04015b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx @@ -8,7 +8,7 @@ import RedisTypeSelector from "./RedisTypeSelector"; import CacheFieldSection from "./CacheFieldSection"; import { EmbeddingModelOption } from "./CacheFormField"; import { REDIS_TYPES, REDIS_TYPE_DESCRIPTIONS, RedisType } from "./cacheSettingsFields"; -import { buildCachePayload, buildInitialValues, CacheFormValues } from "./cacheSettingsUtils"; +import { buildCachePayload, buildInitialValues, CacheFormValues, configuredSecretFields } from "./cacheSettingsUtils"; interface CacheSettingsProps { accessToken: string | null; @@ -25,6 +25,7 @@ const CacheSettings: React.FC = ({ accessToken }) => { const [embeddingModels, setEmbeddingModels] = useState([]); const [isTesting, setIsTesting] = useState(false); const [isSaving, setIsSaving] = useState(false); + const [configuredSecrets, setConfiguredSecrets] = useState>(new Set()); const loadCacheSettings = useCallback(async () => { if (!accessToken) { @@ -34,6 +35,7 @@ const CacheSettings: React.FC = ({ accessToken }) => { const data = (await getCacheSettingsCall(accessToken)) as { current_values?: Record }; const currentValues = data.current_values ?? {}; form.setFieldsValue(buildInitialValues(currentValues)); + setConfiguredSecrets(configuredSecretFields(currentValues)); setRedisType(toRedisType(currentValues.redis_type)); } catch (error) { console.error("Failed to load cache settings:", error); @@ -144,6 +146,7 @@ const CacheSettings: React.FC = ({ accessToken }) => { section="connection" redisType={redisType} embeddingModels={embeddingModels} + configuredSecrets={configuredSecrets} />
@@ -166,6 +169,7 @@ const CacheSettings: React.FC = ({ accessToken }) => { section="sentinel" redisType={redisType} embeddingModels={embeddingModels} + configuredSecrets={configuredSecrets} />
)} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_margin_form.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_margin_form.test.tsx index 21ee41936c1..1ededd9e4b1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_margin_form.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_margin_form.test.tsx @@ -6,25 +6,6 @@ import { renderWithProviders } from "../../../../../tests/test-utils"; import AddMarginForm from "./add_margin_form"; import { MarginConfig } from "./types"; -vi.mock("@/components/provider_info_helpers", () => ({ - Providers: { - OpenAI: "OpenAI", - Anthropic: "Anthropic", - }, - provider_map: { - OpenAI: "openai", - Anthropic: "anthropic", - }, - providerLogoMap: { - OpenAI: "https://example.com/openai.png", - Anthropic: "https://example.com/anthropic.png", - }, -})); - -vi.mock("./provider_display_helpers", () => ({ - handleImageError: vi.fn(), -})); - const DEFAULT_PROPS = { marginConfig: {} as MarginConfig, selectedProvider: undefined, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_margin_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_margin_form.tsx index f2c06387301..a17b7fc4ac3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_margin_form.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_margin_form.tsx @@ -2,10 +2,9 @@ import React from "react"; import { TextInput, Button } from "@tremor/react"; import { Select as AntdSelect, Form, Tooltip, Radio } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; -import { Providers, provider_map, providerLogoMap } from "@/components/provider_info_helpers"; -import { resolveLogoSrc } from "@/lib/assetPaths"; +import { Providers, provider_map } from "@/components/provider_info_helpers"; +import { Logo } from "@/components/molecules/logo/Logo"; import { MarginConfig } from "./types"; -import { handleImageError } from "./provider_display_helpers"; interface AddMarginFormProps { marginConfig: MarginConfig; @@ -73,12 +72,7 @@ const AddMarginForm: React.FC = ({ return (
- {`${providerEnum} handleImageError(e, providerDisplayName)} - /> + {providerDisplayName}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_provider_form.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_provider_form.test.tsx index 48d23d4645d..08fb63c32b9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_provider_form.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_provider_form.test.tsx @@ -5,25 +5,7 @@ import userEvent from "@testing-library/user-event"; import { renderWithProviders } from "../../../../../tests/test-utils"; import AddProviderForm from "./add_provider_form"; import { DiscountConfig } from "./types"; - -vi.mock("@/components/provider_info_helpers", () => ({ - Providers: { - OpenAI: "OpenAI", - Anthropic: "Anthropic", - }, - provider_map: { - OpenAI: "openai", - Anthropic: "anthropic", - }, - providerLogoMap: { - OpenAI: "https://example.com/openai.png", - Anthropic: "https://example.com/anthropic.png", - }, -})); - -vi.mock("./provider_display_helpers", () => ({ - handleImageError: vi.fn(), -})); +import { Providers, providerLogoMap } from "@/components/provider_info_helpers"; const DEFAULT_PROPS = { discountConfig: {} as DiscountConfig, @@ -84,4 +66,18 @@ describe("AddProviderForm", () => { renderWithProviders(); expect(screen.getByText("%")).toBeInTheDocument(); }); + + it("renders the selected provider's bundled logo via the shared Logo component", async () => { + renderWithProviders(); + + const logo = await screen.findByRole("img", { name: `${Providers.OpenAI} logo` }); + expect(logo.getAttribute("src")).toBe(providerLogoMap[Providers.OpenAI]); + }); + + it("falls back to a letter avatar for a selected provider that has no bundled logo", () => { + renderWithProviders(); + + expect(screen.queryByRole("img", { name: `${Providers.PG_VECTOR} logo` })).not.toBeInTheDocument(); + expect(screen.getByText(Providers.PG_VECTOR.charAt(0))).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_provider_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_provider_form.tsx index c4961263533..0fdaed8814b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_provider_form.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_provider_form.tsx @@ -2,10 +2,9 @@ import React from "react"; import { TextInput, Button } from "@tremor/react"; import { Select as AntdSelect, Form, Tooltip } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; -import { Providers, provider_map, providerLogoMap } from "@/components/provider_info_helpers"; -import { resolveLogoSrc } from "@/lib/assetPaths"; +import { Providers, provider_map } from "@/components/provider_info_helpers"; +import { Logo } from "@/components/molecules/logo/Logo"; import { DiscountConfig } from "./types"; -import { handleImageError } from "./provider_display_helpers"; interface AddProviderFormProps { discountConfig: DiscountConfig; @@ -60,12 +59,7 @@ const AddProviderForm: React.FC = ({ return (
- {`${providerEnum} handleImageError(e, providerDisplayName)} - /> + {providerDisplayName}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.test.tsx index 0e1c7da92ba..0dae83ba808 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.test.tsx @@ -49,11 +49,7 @@ vi.mock("@/components/provider_info_helpers", () => ({ Providers: { OpenAI: "OpenAI" }, provider_map: { OpenAI: "openai" }, providerLogoMap: {}, -})); - -vi.mock("./provider_display_helpers", () => ({ - getProviderDisplayInfo: vi.fn(() => ({ displayName: "OpenAI", logo: "", enumKey: "OpenAI" })), - handleImageError: vi.fn(), + getProviderLogoAndName: (providerValue: string) => ({ logo: "", displayName: providerValue }), })); const ADMIN_PROPS = { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/index.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/index.ts index 8de7fdd7271..90701dd8f1f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/index.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/index.ts @@ -11,7 +11,6 @@ export type { MarginConfig, CostMarginResponse, } from "./types"; -export type { ProviderDisplayInfo } from "./provider_display_helpers"; export * from "./provider_display_helpers"; export { useDiscountConfig } from "./use_discount_config"; export { useMarginConfig } from "./use_margin_config"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_discount_table.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_discount_table.test.tsx index c1c43ebdb4f..2e8dbb429f0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_discount_table.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_discount_table.test.tsx @@ -43,15 +43,6 @@ vi.mock("@tremor/react", () => ({ }, })); -vi.mock("./provider_display_helpers", () => ({ - getProviderDisplayInfo: vi.fn((providerValue: string) => ({ - displayName: providerValue === "openai" ? "OpenAI" : providerValue, - logo: providerValue === "openai" ? "https://example.com/openai.png" : "", - enumKey: providerValue === "openai" ? "OpenAI" : null, - })), - handleImageError: vi.fn(), -})); - const DEFAULT_DISCOUNT_CONFIG = { openai: 0.05, anthropic: 0.1, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_discount_table.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_discount_table.tsx index d802f6d83dd..8727d6cb33c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_discount_table.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_discount_table.tsx @@ -3,7 +3,8 @@ import { TextInput, Icon, Text } from "@tremor/react"; import { TrashIcon, PencilAltIcon, CheckIcon, XIcon } from "@heroicons/react/outline"; import { SimpleTable } from "@/components/common_components/simple_table"; import { DiscountConfig } from "./types"; -import { getProviderDisplayInfo, handleImageError } from "./provider_display_helpers"; +import { getProviderLogoAndName } from "@/components/provider_info_helpers"; +import { Logo } from "@/components/molecules/logo/Logo"; interface ProviderDiscountTableProps { discountConfig: DiscountConfig; @@ -55,8 +56,8 @@ const ProviderDiscountTable: React.FC = ({ const data: ProviderDiscountRow[] = Object.entries(discountConfig) .map(([provider, discount]) => ({ provider, discount })) .sort((a, b) => { - const displayA = getProviderDisplayInfo(a.provider).displayName; - const displayB = getProviderDisplayInfo(b.provider).displayName; + const displayA = getProviderLogoAndName(a.provider).displayName; + const displayB = getProviderLogoAndName(b.provider).displayName; return displayA.localeCompare(displayB); }); @@ -67,17 +68,10 @@ const ProviderDiscountTable: React.FC = ({ { header: "Provider", cell: (row) => { - const { displayName, logo } = getProviderDisplayInfo(row.provider); + const { displayName } = getProviderLogoAndName(row.provider); return (
- {logo && ( - {`${displayName} handleImageError(e, displayName)} - /> - )} + {displayName}
); @@ -129,7 +123,7 @@ const ProviderDiscountTable: React.FC = ({ { header: "Actions", cell: (row) => { - const { displayName } = getProviderDisplayInfo(row.provider); + const { displayName } = getProviderLogoAndName(row.provider); return ( ({ - Providers: { - OpenAI: "OpenAI", - Anthropic: "Anthropic", - Azure: "Azure", - }, provider_map: { OpenAI: "openai", Anthropic: "anthropic", Azure: "azure", }, - providerLogoMap: { - OpenAI: "https://example.com/openai.png", - Anthropic: "https://example.com/anthropic.png", - Azure: "https://example.com/azure.png", - }, })); -describe("getProviderDisplayInfo", () => { - it("should return display name and logo for a known backend provider value", () => { - const info = getProviderDisplayInfo("openai"); - expect(info.displayName).toBe("OpenAI"); - expect(info.logo).toBe("https://example.com/openai.png"); - expect(info.enumKey).toBe("OpenAI"); - }); - - it("should return the raw value as display name for an unknown provider", () => { - const info = getProviderDisplayInfo("my-custom-provider"); - expect(info.displayName).toBe("my-custom-provider"); - expect(info.logo).toBe(""); - expect(info.enumKey).toBeNull(); - }); - - it("should match a provider by its backend value regardless of casing", () => { - const info = getProviderDisplayInfo("anthropic"); - expect(info.displayName).toBe("Anthropic"); - expect(info.enumKey).toBe("Anthropic"); - }); -}); - describe("getProviderBackendValue", () => { it("should return the backend value for a known provider enum key", () => { expect(getProviderBackendValue("OpenAI")).toBe("openai"); @@ -54,38 +22,3 @@ describe("getProviderBackendValue", () => { expect(getProviderBackendValue("UnknownProvider")).toBeNull(); }); }); - -describe("handleImageError", () => { - it("should replace the img element with a fallback div showing the first letter", () => { - const img = document.createElement("img"); - const parent = document.createElement("div"); - parent.appendChild(img); - - const event = { target: img } as any; - handleImageError(event, "OpenAI"); - - expect(parent.querySelector("img")).toBeNull(); - const fallback = parent.firstChild as HTMLElement; - expect(fallback.tagName).toBe("DIV"); - expect(fallback.textContent).toBe("O"); - }); - - it("should use the first character of the fallback text as the label", () => { - const img = document.createElement("img"); - const parent = document.createElement("div"); - parent.appendChild(img); - - const event = { target: img } as any; - handleImageError(event, "Anthropic"); - - const fallback = parent.firstChild as HTMLElement; - expect(fallback.textContent).toBe("A"); - }); - - it("should do nothing if the image has no parent element", () => { - const img = document.createElement("img"); - const event = { target: img } as any; - // Should not throw - expect(() => handleImageError(event, "OpenAI")).not.toThrow(); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_display_helpers.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_display_helpers.ts index 5489eb12487..ed98ba3586b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_display_helpers.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_display_helpers.ts @@ -1,28 +1,4 @@ -import { Providers, provider_map, providerLogoMap } from "@/components/provider_info_helpers"; -import { resolveLogoSrc } from "@/lib/assetPaths"; - -export interface ProviderDisplayInfo { - displayName: string; - logo: string; - enumKey: string | null; -} - -/** - * Convert backend provider value (e.g., "openai") to display info - */ -export const getProviderDisplayInfo = (providerValue: string): ProviderDisplayInfo => { - const enumKey = Object.keys(provider_map).find( - (key) => provider_map[key as keyof typeof provider_map] === providerValue, - ); - - if (enumKey) { - const displayName = Providers[enumKey as keyof typeof Providers]; - const logo = resolveLogoSrc(providerLogoMap[displayName]) ?? ""; - return { displayName, logo, enumKey }; - } - - return { displayName: providerValue, logo: "", enumKey: null }; -}; +import { provider_map } from "@/components/provider_info_helpers"; /** * Convert provider enum key (e.g., "OpenAI") to backend value (e.g., "openai") @@ -30,17 +6,3 @@ export const getProviderDisplayInfo = (providerValue: string): ProviderDisplayIn export const getProviderBackendValue = (providerEnum: string): string | null => { return provider_map[providerEnum as keyof typeof provider_map] || null; }; - -/** - * Handle image error by replacing with fallback div - */ -export const handleImageError = (e: React.SyntheticEvent, fallbackText: string) => { - const target = e.target as HTMLImageElement; - const parent = target.parentElement; - if (parent) { - const fallbackDiv = document.createElement("div"); - fallbackDiv.className = "w-5 h-5 rounded-full bg-gray-200 flex items-center justify-center text-xs"; - fallbackDiv.textContent = fallbackText.charAt(0); - parent.replaceChild(fallbackDiv, target); - } -}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_margin_table.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_margin_table.test.tsx index e1b17dea23d..170e61141b6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_margin_table.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/provider_margin_table.test.tsx @@ -4,6 +4,7 @@ import { screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { renderWithProviders } from "../../../../../tests/test-utils"; import ProviderMarginTable from "./provider_margin_table"; +import { Providers, providerLogoMap } from "@/components/provider_info_helpers"; vi.mock("@heroicons/react/outline", () => ({ TrashIcon: function TrashIcon() { @@ -43,15 +44,6 @@ vi.mock("@tremor/react", () => ({ }, })); -vi.mock("./provider_display_helpers", () => ({ - getProviderDisplayInfo: vi.fn((providerValue: string) => { - if (providerValue === "openai") return { displayName: "OpenAI", logo: "", enumKey: "OpenAI" }; - if (providerValue === "anthropic") return { displayName: "Anthropic", logo: "", enumKey: "Anthropic" }; - return { displayName: providerValue, logo: "", enumKey: null }; - }), - handleImageError: vi.fn(), -})); - describe("ProviderMarginTable", () => { const onMarginChange = vi.fn(); const onRemoveProvider = vi.fn(); @@ -95,6 +87,30 @@ describe("ProviderMarginTable", () => { expect(screen.getByText("OpenAI")).toBeInTheDocument(); }); + it("should render the provider's bundled logo via the shared Logo component", () => { + renderWithProviders( + , + ); + const logo = screen.getByRole("img", { name: `${Providers.OpenAI} logo` }); + expect(logo.getAttribute("src")).toBe(providerLogoMap[Providers.OpenAI]); + }); + + it("should fall back to a letter avatar for a provider with no bundled logo", () => { + renderWithProviders( + , + ); + expect(screen.queryByRole("img")).not.toBeInTheDocument(); + expect(screen.getByText("m")).toBeInTheDocument(); + }); + it("should display the global provider as 'Global (All Providers)'", () => { renderWithProviders( = ({ .sort((a, b) => { if (a.provider === "global") return -1; if (b.provider === "global") return 1; - const displayA = getProviderDisplayInfo(a.provider).displayName; - const displayB = getProviderDisplayInfo(b.provider).displayName; + const displayA = getProviderLogoAndName(a.provider).displayName; + const displayB = getProviderLogoAndName(b.provider).displayName; return displayA.localeCompare(displayB); }); @@ -115,17 +116,10 @@ const ProviderMarginTable: React.FC = ({
); } - const { displayName, logo } = getProviderDisplayInfo(row.provider); + const { displayName } = getProviderLogoAndName(row.provider); return (
- {logo && ( - {`${displayName} handleImageError(e, displayName)} - /> - )} + {displayName}
); @@ -186,7 +180,7 @@ const ProviderMarginTable: React.FC = ({ { header: "Actions", cell: (row) => { - const displayName = row.provider === "global" ? "Global" : getProviderDisplayInfo(row.provider).displayName; + const displayName = row.provider === "global" ? "Global" : getProviderLogoAndName(row.provider).displayName; return ( ({ getGuardrailsList: vi.fn(), @@ -48,7 +48,8 @@ vi.mock("@/utils/roles", () => ({ isAdminRole: vi.fn((role: string) => role === "admin"), })); -vi.mock("./guardrail_info_helpers", () => ({ +vi.mock("./guardrail_info_helpers", async (importOriginal) => ({ + ...(await importOriginal()), getGuardrailLogoAndName: vi.fn(() => ({ logo: null, displayName: "Test Provider", @@ -78,6 +79,7 @@ describe("GuardrailsPanel", () => { }; const mockGetGuardrailsList = vi.mocked(getGuardrailsList); + const mockDeleteGuardrailCall = vi.mocked(deleteGuardrailCall); beforeEach(() => { vi.clearAllMocks(); @@ -107,4 +109,35 @@ describe("GuardrailsPanel", () => { fireEvent.click(screen.getByText("Guardrails")); expect(screen.getByText("Add New Guardrail")).toBeInTheDocument(); }); + + it("should delete the clicked guardrail after confirming in the modal", async () => { + render(); + fireEvent.click(screen.getByText("Guardrails")); + + fireEvent.click(await screen.findByTestId("delete-button")); + + const modal = within(await screen.findByRole("dialog")); + expect(modal.getByText("Delete Guardrail")).toBeInTheDocument(); + expect(modal.getByText("test-guardrail-1")).toBeInTheDocument(); + expect(modal.getByText("Test Provider")).toBeInTheDocument(); + + fireEvent.click(modal.getByRole("button", { name: "Delete" })); + + await waitFor(() => { + expect(mockDeleteGuardrailCall).toHaveBeenCalledWith("test-token", "test-guardrail-1"); + }); + expect(mockGetGuardrailsList).toHaveBeenCalledTimes(2); + }); + + it("should not delete anything when the modal is cancelled", async () => { + render(); + fireEvent.click(screen.getByText("Guardrails")); + + fireEvent.click(await screen.findByTestId("delete-button")); + const modal = within(await screen.findByRole("dialog")); + + fireEvent.click(modal.getByRole("button", { name: "Cancel" })); + + expect(mockDeleteGuardrailCall).not.toHaveBeenCalled(); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.test.tsx index 8fc0d36c2b4..91d95155e82 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.test.tsx @@ -41,3 +41,17 @@ describe("AddGuardrailForm close behavior", () => { expect(onClose).toHaveBeenCalledTimes(1); }); }); + +describe("AddGuardrailForm provider options", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("renders provider options with logos from the bundled guardrail logo map", async () => { + renderForm(); + fireEvent.mouseDown(screen.getByLabelText("Guardrail Provider")); + + const logo = await screen.findByAltText("Presidio PII logo"); + expect(logo.getAttribute("src")).toContain("microsoft_azure.svg"); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx index 202568b478d..17331014c57 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx @@ -12,10 +12,10 @@ import { type CompetitorIntentConfig } from "./content_filter/CompetitorIntentCo import { choiceToSkipSystemForCreate, choiceToSkipToolForCreate, + getGuardrailLogo, getGuardrailProviders, getSupportedModesForProvider, guardrail_provider_map, - guardrailLogoMap, populateGuardrailProviderMap, populateGuardrailProviders, shouldRenderContentFilterConfigSettings, @@ -23,7 +23,7 @@ import { shouldRenderPIIConfigSettings, toModeArray, } from "./guardrail_info_helpers"; -import { resolveLogoSrc } from "@/lib/assetPaths"; +import { Logo } from "@/components/molecules/logo/Logo"; import GuardrailOptionalParams from "./guardrail_optional_params"; import GuardrailProviderFields from "./guardrail_provider_fields"; import LLMJudgeFields from "./llm_judge/LLMJudgeFields"; @@ -725,53 +725,19 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a dropdownRender={(menu) => menu} showSearch={true} > - {Object.entries(getGuardrailProviders()).map(([key, value]) => ( -
- } - > + {Object.entries(getGuardrailProviders()).map(([key, value]) => { + const optionContent = (
- {guardrailLogoMap[value] && ( - { - // Hide broken image icon if image fails to load - e.currentTarget.style.display = "none"; - }} - /> - )} + {value}
- - ))} + ); + return ( + + ); + })} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrailTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrailTableColumns.tsx index 9ceb6ba244b..ec3d05a6907 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrailTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrailTableColumns.tsx @@ -16,6 +16,7 @@ import { import { cn } from "@/lib/cva.config"; import { getGuardrailLogoAndName } from "./guardrail_info_helpers"; +import { Logo } from "@/components/molecules/logo/Logo"; const CONFIG_DELETE_HINT = "Config guardrails are defined in the config file and cannot be deleted from the dashboard."; @@ -23,16 +24,7 @@ function GuardrailProviderCell({ provider }: { provider: string }) { const { logo, displayName } = getGuardrailLogoAndName(provider); return (
- {logo ? ( - { - (event.currentTarget as HTMLImageElement).style.display = "none"; - }} - /> - ) : null} + {displayName}
); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_card.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_card.test.tsx index 0fa5d2ffcd2..2d1f35e456c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_card.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_card.test.tsx @@ -53,15 +53,28 @@ describe("GuardrailCard", () => { expect(screen.queryByText(/F1:/)).not.toBeInTheDocument(); }); + it("should render the logo through the shared Logo component with the card src", () => { + render(); + const img = screen.getByAltText("Test Guardrail logo"); + expect(img.getAttribute("src")).toContain("/logos/test.svg"); + }); + + it("should pass a bundled static-import src through unchanged", () => { + const bundledCard: GuardrailCardInfo = { ...baseCard, logo: "/_next/static/media/akto.svg" }; + render(); + expect(screen.getByAltText("Test Guardrail logo")).toHaveAttribute("src", "/_next/static/media/akto.svg"); + }); + it("should show fallback initial when logo fails to load", () => { render(); - const img = screen.getByRole("presentation"); + const img = screen.getByAltText("Test Guardrail logo"); act(() => { fireEvent.error(img); }); expect(screen.getByText("T")).toBeInTheDocument(); + expect(screen.queryByAltText("Test Guardrail logo")).not.toBeInTheDocument(); }); it("should show fallback initial when logo src is empty", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_card.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_card.tsx index 8e9fcc21dfe..53abf3eb81c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_card.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_card.tsx @@ -1,42 +1,7 @@ import React, { useState } from "react"; import { CheckCircleFilled } from "@ant-design/icons"; import { GuardrailCardInfo } from "./guardrail_garden_data"; -import { resolveLogoSrc } from "@/lib/assetPaths"; - -const LogoWithFallback: React.FC<{ src: string; name: string }> = ({ src, name }) => { - const [hasError, setHasError] = useState(false); - - if (hasError || !src) { - return ( -
- {name?.charAt(0) || "?"} -
- ); - } - - return ( - setHasError(true)} - /> - ); -}; +import { Logo } from "@/components/molecules/logo/Logo"; const GuardrailCard: React.FC<{ card: GuardrailCardInfo; onClick: () => void }> = ({ card, onClick }) => { const [hovered, setHovered] = useState(false); @@ -61,7 +26,7 @@ const GuardrailCard: React.FC<{ card: GuardrailCardInfo; onClick: () => void }> > {/* Icon + Name row */}
- + {card.name}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.test.ts new file mode 100644 index 00000000000..13909e48185 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest"; +import { ALL_CARDS, LITELLM_CONTENT_FILTER_CARDS, PARTNER_GUARDRAIL_CARDS } from "./guardrail_garden_data"; + +const EXPECTED_PARTNER_LOGO_FILES: Record = { + presidio: "microsoft_azure.svg", + bedrock: "bedrock.svg", + lakera: "lakeraai.jpeg", + openai_moderation: "openai_small.svg", + google_model_armor: "google.svg", + guardrails_ai: "guardrails_ai.jpeg", + zscaler: "zscaler.svg", + panw: "palo_alto_networks.jpeg", + cisco_ai_defense: "cisco.png", + noma: "noma_security.png", + aporia: "aporia.png", + aim: "aim_security.jpeg", + cato_networks: "cato_networks.svg", + prompt_security: "prompt_security.png", + lasso: "lasso.png", + pangea: "pangea.png", + enkryptai: "enkrypt_ai.avif", + javelin: "javelin.png", + pillar: "pillar.jpeg", + akto: "akto.svg", + promptguard: "promptguard.svg", + xecguard: "xecguard.svg", + deepkeep: "deepkeep.svg", + repelloai: "repelloai.png", + straiker: "straiker.svg", +}; + +describe("guardrail_garden_data logos", () => { + it("points every partner card at its own provider's bundled logo file", () => { + expect(new Set(PARTNER_GUARDRAIL_CARDS.map((card) => card.id))).toEqual( + new Set(Object.keys(EXPECTED_PARTNER_LOGO_FILES)), + ); + for (const card of PARTNER_GUARDRAIL_CARDS) { + expect(card.logo, `card ${card.id}`).toContain(EXPECTED_PARTNER_LOGO_FILES[card.id]); + } + }); + + it("uses the LiteLLM logo for every content filter card", () => { + for (const card of LITELLM_CONTENT_FILTER_CARDS) { + expect(card.logo, `card ${card.id}`).toContain("litellm_logo.jpg"); + } + }); + + it("bundles every card logo instead of referencing runtime /ui asset paths", () => { + for (const card of ALL_CARDS) { + expect(card.logo, `card ${card.id}`).not.toBe(""); + expect(card.logo, `card ${card.id}`).not.toContain("/ui/assets/logos/"); + } + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.ts b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.ts index a29d12f53f9..744af89a357 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.ts @@ -1,3 +1,5 @@ +import { guardrailLogoMap } from "./guardrail_info_helpers"; + export interface GuardrailCardInfo { id: string; name: string; @@ -16,7 +18,7 @@ export interface GuardrailCardInfo { providerKey?: string; } -const ASSET_PREFIX = "/ui/assets/logos/"; +const litellmContentFilterLogo = guardrailLogoMap["LiteLLM Content Filter"]; export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ { @@ -26,7 +28,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ "Detects requests for personalized financial advice, investment recommendations, or financial planning.", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Topic Blocker"], eval: { f1: 100.0, @@ -42,7 +44,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ description: "Detects insults, name-calling, and personal attacks directed at the chatbot, staff, or other people.", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Topic Blocker"], eval: { f1: 100.0, @@ -58,7 +60,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ description: "Detects requests for unauthorized legal advice, case analysis, or legal recommendations.", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Topic Blocker"], }, { @@ -67,7 +69,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ description: "Detects requests for medical diagnosis, treatment recommendations, or health advice.", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Topic Blocker"], }, { @@ -76,7 +78,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ description: "Detects content related to violence, criminal planning, attacks, and violent threats.", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Safety"], }, { @@ -85,7 +87,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ description: "Detects content related to self-harm, suicide, and dangerous self-destructive behavior.", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Safety"], }, { @@ -94,7 +96,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ description: "Detects content that could endanger child safety or exploit minors.", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Safety"], }, { @@ -103,7 +105,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ description: "Detects content related to illegal weapons manufacturing, distribution, or acquisition.", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Safety"], }, { @@ -112,7 +114,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ description: "Detects gender-based discrimination, stereotypes, and biased language.", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Bias"], }, { @@ -121,7 +123,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ description: "Detects racial discrimination, stereotypes, and racially biased content.", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Bias"], }, { @@ -130,7 +132,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ description: "Detects religious discrimination, intolerance, and religiously biased content.", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Bias"], }, { @@ -139,7 +141,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ description: "Detects discrimination based on sexual orientation and related biased content.", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Bias"], }, { @@ -148,7 +150,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ description: "Detects jailbreak attempts designed to bypass AI safety guidelines and restrictions.", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Prompt Injection"], }, { @@ -157,7 +159,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ description: "Detects attempts to extract sensitive data through prompt manipulation.", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Prompt Injection"], }, { @@ -166,7 +168,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ description: "Detects SQL injection attempts embedded in prompts.", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Prompt Injection"], }, { @@ -175,7 +177,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ description: "Detects attempts to inject malicious code through prompts.", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Prompt Injection"], }, { @@ -184,7 +186,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ description: "Detects attempts to extract or override system prompts.", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Prompt Injection"], }, { @@ -193,7 +195,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ description: "Detects toxic, abusive, and hateful language across multiple languages (EN, AU, DE, ES, FR).", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Toxicity"], }, { @@ -203,7 +205,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ "Detect and block sensitive data patterns like SSNs, credit card numbers, API keys, and custom regex patterns.", category: "litellm", subcategory: "Patterns", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["PII", "Regex", "Data Protection"], }, { @@ -213,7 +215,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ "Block or mask content containing specific keywords or phrases. Upload custom word lists or add individual terms.", category: "litellm", subcategory: "Keywords", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Keywords", "Blocklist"], }, { @@ -223,7 +225,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ "Detects markdown fenced code blocks in requests and responses. Block or mask executable code (e.g. Python, JavaScript, Bash) by language with configurable confidence.", category: "litellm", subcategory: "Code Safety", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Code", "Safety", "Prompt Injection"], }, { @@ -233,7 +235,7 @@ export const LITELLM_CONTENT_FILTER_CARDS: GuardrailCardInfo[] = [ "Block or reframe competitor comparison and ranking intent. Detect when users ask to compare or recommend competitors (airline or generic competitor lists).", category: "litellm", subcategory: "Content Category", - logo: `${ASSET_PREFIX}litellm_logo.jpg`, + logo: litellmContentFilterLogo, tags: ["Content Category", "Competitor", "Topic Blocker"], }, ]; @@ -245,7 +247,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ description: "Microsoft Presidio for PII detection and anonymization. Supports 30+ entity types with configurable actions.", category: "partner", - logo: `${ASSET_PREFIX}microsoft_azure.svg`, + logo: guardrailLogoMap["Presidio PII"], tags: ["PII", "Microsoft"], providerKey: "PresidioPII", }, @@ -254,7 +256,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "Bedrock Guardrail", description: "AWS Bedrock Guardrails for content filtering, topic avoidance, and sensitive information detection.", category: "partner", - logo: `${ASSET_PREFIX}bedrock.svg`, + logo: guardrailLogoMap["Bedrock Guardrail"], tags: ["AWS", "Content Safety"], providerKey: "Bedrock", }, @@ -263,7 +265,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "Lakera", description: "AI security platform protecting against prompt injections, data leakage, and harmful content.", category: "partner", - logo: `${ASSET_PREFIX}lakeraai.jpeg`, + logo: guardrailLogoMap["Lakera"], tags: ["Security", "Prompt Injection"], providerKey: "Lakera", }, @@ -272,7 +274,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "OpenAI Moderation", description: "OpenAI's content moderation API for detecting harmful content across multiple categories.", category: "partner", - logo: `${ASSET_PREFIX}openai_small.svg`, + logo: guardrailLogoMap["OpenAI Moderation"], tags: ["Content Moderation", "OpenAI"], }, { @@ -280,7 +282,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "Google Cloud Model Armor", description: "Google Cloud's model protection service for safe and responsible AI deployments.", category: "partner", - logo: `${ASSET_PREFIX}google.svg`, + logo: guardrailLogoMap["Google Cloud Model Armor"], tags: ["Google Cloud", "Safety"], }, { @@ -288,7 +290,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "Guardrails AI", description: "Open-source framework for adding structural, type, and quality guarantees to LLM outputs.", category: "partner", - logo: `${ASSET_PREFIX}guardrails_ai.jpeg`, + logo: guardrailLogoMap["Guardrails AI"], tags: ["Open Source", "Validation"], }, { @@ -296,7 +298,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "Zscaler AI Guard", description: "Enterprise AI security from Zscaler for monitoring and protecting AI/ML workloads.", category: "partner", - logo: `${ASSET_PREFIX}zscaler.svg`, + logo: guardrailLogoMap["Zscaler AI Guard"], tags: ["Enterprise", "Security"], }, { @@ -304,7 +306,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "PANW Prisma AIRS", description: "Palo Alto Networks Prisma AI Runtime Security for securing AI applications in production.", category: "partner", - logo: `${ASSET_PREFIX}palo_alto_networks.jpeg`, + logo: guardrailLogoMap["PANW Prisma AIRS"], tags: ["Enterprise", "Security"], }, { @@ -313,7 +315,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ description: "Cisco AI Defense Inspection API for runtime protection: prompt injection, PII/PCI/PHI, harassment, hate speech, profanity, violence, and code detection.", category: "partner", - logo: `${ASSET_PREFIX}cisco.png`, + logo: guardrailLogoMap["Cisco AI Defense"], tags: ["Enterprise", "Security", "Prompt Injection", "PII"], providerKey: "CiscoAiDefense", }, @@ -322,7 +324,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "Noma Security", description: "AI security platform for detecting and preventing AI-specific threats and vulnerabilities.", category: "partner", - logo: `${ASSET_PREFIX}noma_security.png`, + logo: guardrailLogoMap["Noma Security"], tags: ["Security", "Threat Detection"], }, { @@ -330,7 +332,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "Aporia AI", description: "Real-time AI guardrails for hallucination detection, topic control, and policy enforcement.", category: "partner", - logo: `${ASSET_PREFIX}aporia.png`, + logo: guardrailLogoMap["Aporia AI"], tags: ["Hallucination", "Policy"], }, { @@ -338,7 +340,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "AIM Guardrail", description: "AIM Security guardrails for comprehensive AI threat detection and mitigation.", category: "partner", - logo: `${ASSET_PREFIX}aim_security.jpeg`, + logo: guardrailLogoMap["AIM Guardrail"], tags: ["Security", "Threat Detection"], }, { @@ -346,7 +348,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "Cato Networks Guardrail", description: "Cato Networks guardrails for comprehensive AI threat detection and mitigation.", category: "partner", - logo: `${ASSET_PREFIX}cato_networks.svg`, + logo: guardrailLogoMap["Cato Networks Guardrail"], tags: ["Security", "Threat Detection"], }, { @@ -354,7 +356,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "Prompt Security", description: "Protect against prompt injection attacks, data leakage, and other LLM security threats.", category: "partner", - logo: `${ASSET_PREFIX}prompt_security.png`, + logo: guardrailLogoMap["Prompt Security"], tags: ["Prompt Injection", "Security"], }, { @@ -362,7 +364,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "Lasso Guardrail", description: "Content moderation and safety guardrails for responsible AI deployments.", category: "partner", - logo: `${ASSET_PREFIX}lasso.png`, + logo: guardrailLogoMap["Lasso Guardrail"], tags: ["Content Moderation"], }, { @@ -370,7 +372,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "Pangea Guardrail", description: "Pangea's AI guardrails for secure, compliant, and trustworthy AI applications.", category: "partner", - logo: `${ASSET_PREFIX}pangea.png`, + logo: guardrailLogoMap["Pangea Guardrail"], tags: ["Compliance", "Security"], }, { @@ -378,7 +380,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "EnkryptAI", description: "AI security and governance platform for enterprise AI safety and compliance.", category: "partner", - logo: `${ASSET_PREFIX}enkrypt_ai.avif`, + logo: guardrailLogoMap["EnkryptAI"], tags: ["Enterprise", "Governance"], }, { @@ -386,7 +388,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "Javelin Guardrails", description: "AI gateway with built-in guardrails for secure and compliant AI operations.", category: "partner", - logo: `${ASSET_PREFIX}javelin.png`, + logo: guardrailLogoMap["Javelin Guardrails"], tags: ["Gateway", "Security"], }, { @@ -394,7 +396,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "Pillar Guardrail", description: "AI safety platform for monitoring, testing, and securing AI systems.", category: "partner", - logo: `${ASSET_PREFIX}pillar.jpeg`, + logo: guardrailLogoMap["Pillar Guardrail"], tags: ["Monitoring", "Safety"], }, { @@ -402,7 +404,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ name: "Akto Guardrail", description: "AI security platform from Akto.io with automatic monitoring and guardrails for AI/ML applications.", category: "partner", - logo: `${ASSET_PREFIX}akto.svg`, + logo: guardrailLogoMap["Akto"], tags: ["Security", "Safety", "Monitoring"], }, { @@ -411,7 +413,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ description: "AI security gateway with prompt injection detection, PII redaction, topic filtering, entity blocklists, and hallucination detection. Self-hostable with drop-in proxy integration.", category: "partner", - logo: `${ASSET_PREFIX}promptguard.svg`, + logo: guardrailLogoMap["PromptGuard"], tags: ["Security", "Prompt Injection", "PII"], providerKey: "Promptguard", eval: { @@ -428,7 +430,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ description: "CyCraft XecGuard AI security gateway. Multi-policy scanning (prompt injection, harmful content, PII, system-prompt enforcement) plus RAG context grounding.", category: "partner", - logo: `${ASSET_PREFIX}xecguard.svg`, + logo: guardrailLogoMap["XecGuard"], tags: ["Security", "Policy", "Grounding", "RAG"], providerKey: "Xecguard", }, @@ -438,7 +440,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ description: "DeepKeep AI Firewall for comprehensive LLM security — prompt injection detection, PII protection, content moderation, and policy enforcement with configurable guardrail pipelines.", category: "partner", - logo: `${ASSET_PREFIX}deepkeep.svg`, + logo: guardrailLogoMap["DeepKeep AI Firewall"], tags: ["Security", "Prompt Injection", "PII", "Firewall"], providerKey: "Deepkeep", }, @@ -448,7 +450,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ description: "RepelloAI Argus scans prompts and responses against policies configured per asset in the Repello dashboard.", category: "partner", - logo: `${ASSET_PREFIX}repelloai.png`, + logo: guardrailLogoMap["RepelloAI Argus"], tags: ["Security", "Policy", "Prompt Injection"], providerKey: "Repelloai", }, @@ -458,7 +460,7 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ description: "Defend AI Agentic Guardrails: Indirect/Direct Prompt Injection, Tool Misuse, Malicious MCP and Skills", category: "partner", - logo: `${ASSET_PREFIX}straiker.svg`, + logo: guardrailLogoMap["Straiker"], tags: ["Agentic", "Prompt Injection", "Tool Misuse", "MCP", "Skills"], providerKey: "Straiker", }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_detail.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_detail.test.tsx new file mode 100644 index 00000000000..e17e739267d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_detail.test.tsx @@ -0,0 +1,32 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import GuardrailDetailView from "./guardrail_garden_detail"; +import type { GuardrailCardInfo } from "./guardrail_garden_data"; + +vi.mock("./add_guardrail_form", () => ({ default: () => null })); + +const makeCard = (overrides: Partial = {}): GuardrailCardInfo => ({ + id: "bedrock", + name: "Bedrock Guardrail", + description: "AWS Bedrock Guardrails for content filtering.", + category: "partner", + logo: "/_next/static/media/bedrock.svg", + tags: ["AWS"], + ...overrides, +}); + +const renderDetail = (card: GuardrailCardInfo) => + render(); + +describe("GuardrailDetailView logo", () => { + it("renders the card logo through the shared Logo component with the bundled src", () => { + renderDetail(makeCard()); + expect(screen.getByAltText("Bedrock Guardrail logo")).toHaveAttribute("src", "/_next/static/media/bedrock.svg"); + }); + + it("falls back to a letter avatar when the card has no logo", () => { + renderDetail(makeCard({ logo: "" })); + expect(screen.queryByAltText("Bedrock Guardrail logo")).not.toBeInTheDocument(); + expect(screen.getByText("B")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_detail.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_detail.tsx index c92486bbad9..71c7a527614 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_detail.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_detail.tsx @@ -2,7 +2,7 @@ import React, { useState } from "react"; import { Button } from "antd"; import { ArrowLeftOutlined } from "@ant-design/icons"; import AddGuardrailForm from "./add_guardrail_form"; -import { resolveLogoSrc } from "@/lib/assetPaths"; +import { Logo } from "@/components/molecules/logo/Logo"; import { GUARDRAIL_PRESETS } from "./guardrail_garden_configs"; import { GuardrailCardInfo } from "./guardrail_garden_data"; @@ -60,14 +60,7 @@ const GuardrailDetailView: React.FC = ({ card, onBack, {/* ── Header block (Vertex-style) ── */}
- { - (e.target as HTMLImageElement).style.display = "none"; - }} - /> +

{card.name}

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx index c89fe7277c9..7bb7737e152 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx @@ -81,6 +81,37 @@ describe("Guardrail Info", () => { expect(getByText("Settings")).toBeInTheDocument(); }); + it("should render the provider logo from the bundled guardrail logo map", async () => { + vi.mocked(networking.getGuardrailInfo).mockResolvedValue({ + guardrail_id: "123", + guardrail_name: "Test Guardrail", + litellm_params: { + guardrail: "presidio", + mode: "pre_call", + default_on: true, + }, + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-01-01T00:00:00Z", + guardrail_definition_location: "database", + }); + + vi.mocked(networking.getGuardrailUISettings).mockResolvedValue({ + supported_entities: [], + supported_actions: [], + pii_entity_categories: [], + supported_modes: ["pre_call", "post_call"], + }); + + vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({}); + + const { findByAltText } = render( + {}} accessToken="123" isAdmin={true} />, + ); + + const logo = await findByAltText("Presidio PII logo"); + expect(logo.getAttribute("src")).toContain("microsoft_azure.svg"); + }); + it("should not render the edit button for config guardrails", async () => { // Mock the network responses vi.mocked(networking.getGuardrailInfo).mockResolvedValue({ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx index 1941ec94a60..07df6ff15d9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx @@ -12,6 +12,7 @@ import { Button, Divider, Form, Input, Select, Tooltip } from "antd"; import { CheckIcon, CopyIcon } from "lucide-react"; import React, { useCallback, useEffect, useState } from "react"; import NotificationsManager from "@/components/molecules/notifications_manager"; +import { Logo } from "@/components/molecules/logo/Logo"; import ContentFilterManager, { formatContentFilterDataForAPI } from "./content_filter/ContentFilterManager"; import CustomCodeModal, { EditGuardrailData } from "./custom_code/CustomCodeModal"; import { @@ -524,17 +525,7 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, Provider
- {logo && ( - {`${displayName} { - // Hide broken image - (e.target as HTMLImageElement).style.display = "none"; - }} - /> - )} + {displayName}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx index dd70bb2cf51..12aaba0d696 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx @@ -1,4 +1,30 @@ -import { resolveLogoSrc } from "@/lib/assetPaths"; +import aimSecurityLogo from "../../../../../public/assets/logos/aim_security.jpeg"; +import aktoLogo from "../../../../../public/assets/logos/akto.svg"; +import aporiaLogo from "../../../../../public/assets/logos/aporia.png"; +import bedrockLogo from "../../../../../public/assets/logos/bedrock.svg"; +import catoNetworksLogo from "../../../../../public/assets/logos/cato_networks.svg"; +import ciscoLogo from "../../../../../public/assets/logos/cisco.png"; +import deepkeepLogo from "../../../../../public/assets/logos/deepkeep.svg"; +import enkryptAiLogo from "../../../../../public/assets/logos/enkrypt_ai.avif"; +import googleLogo from "../../../../../public/assets/logos/google.svg"; +import guardrailsAiLogo from "../../../../../public/assets/logos/guardrails_ai.jpeg"; +import javelinLogo from "../../../../../public/assets/logos/javelin.png"; +import lakeraAiLogo from "../../../../../public/assets/logos/lakeraai.jpeg"; +import lassoLogo from "../../../../../public/assets/logos/lasso.png"; +import litellmLogo from "../../../../../public/assets/logos/litellm_logo.jpg"; +import microsoftAzureLogo from "../../../../../public/assets/logos/microsoft_azure.svg"; +import nomaSecurityLogo from "../../../../../public/assets/logos/noma_security.png"; +import openaiSmallLogo from "../../../../../public/assets/logos/openai_small.svg"; +import paloAltoNetworksLogo from "../../../../../public/assets/logos/palo_alto_networks.jpeg"; +import pangeaLogo from "../../../../../public/assets/logos/pangea.png"; +import pillarLogo from "../../../../../public/assets/logos/pillar.jpeg"; +import promptSecurityLogo from "../../../../../public/assets/logos/prompt_security.png"; +import promptguardLogo from "../../../../../public/assets/logos/promptguard.svg"; +import qohashLogo from "../../../../../public/assets/logos/qohash.jpg"; +import repelloAiLogo from "../../../../../public/assets/logos/repelloai.png"; +import straikerLogo from "../../../../../public/assets/logos/straiker.svg"; +import xecguardLogo from "../../../../../public/assets/logos/xecguard.svg"; +import zscalerLogo from "../../../../../public/assets/logos/zscaler.svg"; // Legacy enum - keeping for backward compatibility export enum GuardrailProviders { @@ -136,40 +162,43 @@ export const shouldRenderLLMJudgeFields = (provider: string | null) => { return guardrail_provider_map[provider] === "llm_as_a_judge"; }; -const asset_logos_folder = "/ui/assets/logos/"; +export const guardrailLogoMap = { + "Zscaler AI Guard": zscalerLogo.src, + "Presidio PII": microsoftAzureLogo.src, + "Bedrock Guardrail": bedrockLogo.src, + Lakera: lakeraAiLogo.src, + "Azure Content Safety Prompt Shield": microsoftAzureLogo.src, + "Azure Content Safety Text Moderation": microsoftAzureLogo.src, + "Aporia AI": aporiaLogo.src, + "PANW Prisma AIRS": paloAltoNetworksLogo.src, + "Cisco AI Defense": ciscoLogo.src, + "Noma Security": nomaSecurityLogo.src, + "Javelin Guardrails": javelinLogo.src, + "Pillar Guardrail": pillarLogo.src, + "Google Cloud Model Armor": googleLogo.src, + "Guardrails AI": guardrailsAiLogo.src, + "Lasso Guardrail": lassoLogo.src, + "Pangea Guardrail": pangeaLogo.src, + "AIM Guardrail": aimSecurityLogo.src, + "Cato Networks Guardrail": catoNetworksLogo.src, + "OpenAI Moderation": openaiSmallLogo.src, + EnkryptAI: enkryptAiLogo.src, + "Prompt Security": promptSecurityLogo.src, + PromptGuard: promptguardLogo.src, + XecGuard: xecguardLogo.src, + "LiteLLM Content Filter": litellmLogo.src, + "LiteLLM LLM as a Judge": litellmLogo.src, + Akto: aktoLogo.src, + "DeepKeep AI Firewall": deepkeepLogo.src, + "Qostodian Nexus": qohashLogo.src, + "RepelloAI Argus": repelloAiLogo.src, + Straiker: straikerLogo.src, +} satisfies Record; -export const guardrailLogoMap: Record = { - "Zscaler AI Guard": `${asset_logos_folder}zscaler.svg`, - "Presidio PII": `${asset_logos_folder}microsoft_azure.svg`, - "Bedrock Guardrail": `${asset_logos_folder}bedrock.svg`, - Lakera: `${asset_logos_folder}lakeraai.jpeg`, - "Azure Content Safety Prompt Shield": `${asset_logos_folder}microsoft_azure.svg`, - "Azure Content Safety Text Moderation": `${asset_logos_folder}microsoft_azure.svg`, - "Aporia AI": `${asset_logos_folder}aporia.png`, - "PANW Prisma AIRS": `${asset_logos_folder}palo_alto_networks.jpeg`, - "Cisco AI Defense": `${asset_logos_folder}cisco.png`, - "Noma Security": `${asset_logos_folder}noma_security.png`, - "Javelin Guardrails": `${asset_logos_folder}javelin.png`, - "Pillar Guardrail": `${asset_logos_folder}pillar.jpeg`, - "Google Cloud Model Armor": `${asset_logos_folder}google.svg`, - "Guardrails AI": `${asset_logos_folder}guardrails_ai.jpeg`, - "Lasso Guardrail": `${asset_logos_folder}lasso.png`, - "Pangea Guardrail": `${asset_logos_folder}pangea.png`, - "AIM Guardrail": `${asset_logos_folder}aim_security.jpeg`, - "Cato Networks Guardrail": `${asset_logos_folder}cato_networks.svg`, - "OpenAI Moderation": `${asset_logos_folder}openai_small.svg`, - EnkryptAI: `${asset_logos_folder}enkrypt_ai.avif`, - "Prompt Security": `${asset_logos_folder}prompt_security.png`, - PromptGuard: `${asset_logos_folder}promptguard.svg`, - XecGuard: `${asset_logos_folder}xecguard.svg`, - "LiteLLM Content Filter": `${asset_logos_folder}litellm_logo.jpg`, - "LiteLLM LLM as a Judge": `${asset_logos_folder}litellm_logo.jpg`, - Akto: `${asset_logos_folder}akto.svg`, - "DeepKeep AI Firewall": `${asset_logos_folder}deepkeep.svg`, - "Qostodian Nexus": `${asset_logos_folder}qohash.jpg`, - "RepelloAI Argus": `${asset_logos_folder}repelloai.png`, - Straiker: `${asset_logos_folder}straiker.svg`, -}; +export const getGuardrailLogo = (displayName: string): string | undefined => + Object.prototype.hasOwnProperty.call(guardrailLogoMap, displayName) + ? guardrailLogoMap[displayName as keyof typeof guardrailLogoMap] + : undefined; export const getGuardrailLogoAndName = (guardrailValue: string): { logo: string; displayName: string } => { if (!guardrailValue) { @@ -188,7 +217,7 @@ export const getGuardrailLogoAndName = (guardrailValue: string): { logo: string; // Get the display name from current GuardrailProviders and logo from map const currentProviders = getGuardrailProviders(); const displayName = currentProviders[enumKey as keyof typeof currentProviders]; - const logo = resolveLogoSrc(guardrailLogoMap[displayName as keyof typeof guardrailLogoMap]) ?? ""; + const logo = getGuardrailLogo(displayName ?? "") ?? ""; return { logo, displayName: displayName || guardrailValue }; }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_table.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_table.test.tsx index 4f556e74c16..7612b702391 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_table.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_table.test.tsx @@ -30,6 +30,22 @@ describe("GuardrailTable", () => { } }); + it("renders the provider logo from the bundled guardrail logo map", () => { + render(); + const logo = screen.getByAltText("Presidio PII logo"); + expect(logo.getAttribute("src")).toContain("microsoft_azure.svg"); + }); + + it("falls back to a letter avatar for an unknown provider slug", () => { + const guardrail = makeGuardrail({ + litellm_params: { guardrail: "mystery_guard", mode: "pre_call", default_on: false }, + }); + render(); + expect(screen.getByText("mystery_guard")).toBeInTheDocument(); + expect(screen.queryByAltText("mystery_guard logo")).not.toBeInTheDocument(); + expect(screen.getByText("m")).toBeInTheDocument(); + }); + it("deletes a DB guardrail through the actions menu", async () => { const user = userEvent.setup(); const onDeleteClick = vi.fn(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useSetKeyBlockedState.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useSetKeyBlockedState.test.ts new file mode 100644 index 00000000000..5eb3bdc105d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useSetKeyBlockedState.test.ts @@ -0,0 +1,103 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { renderHook, waitFor } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import React, { ReactNode } from "react"; +import { useSetKeyBlockedState, setKeyBlockedState } from "./useSetKeyBlockedState"; +import { apiClient } from "@/components/networking"; + +vi.mock("@/components/networking", () => ({ + apiClient: { post: vi.fn() }, +})); + +const mockUseAuthorized = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => mockUseAuthorized(), +})); + +const mockPost = vi.mocked(apiClient.post); + +const createWrapper = () => { + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } }); + const wrapper = ({ children }: { children: ReactNode }) => + React.createElement(QueryClientProvider, { client: queryClient }, children); + return { queryClient, wrapper }; +}; + +describe("setKeyBlockedState", () => { + beforeEach(() => { + mockPost.mockReset(); + }); + + it("POSTs the key hash to /key/block when blocking", async () => { + mockPost.mockResolvedValueOnce({ blocked: true }); + + const result = await setKeyBlockedState("sk-access", { keyToken: "hashed-token", blocked: true }); + + expect(mockPost).toHaveBeenCalledWith("/key/block", { + accessToken: "sk-access", + body: { key: "hashed-token" }, + }); + expect(result).toEqual({ blocked: true }); + }); + + it("POSTs the key hash to /key/unblock when unblocking", async () => { + mockPost.mockResolvedValueOnce({ blocked: false }); + + const result = await setKeyBlockedState("sk-access", { keyToken: "hashed-token", blocked: false }); + + expect(mockPost).toHaveBeenCalledWith("/key/unblock", { + accessToken: "sk-access", + body: { key: "hashed-token" }, + }); + expect(result).toEqual({ blocked: false }); + }); + + it("falls back to the requested state when the response has no blocked field", async () => { + mockPost.mockResolvedValueOnce(null); + + const result = await setKeyBlockedState("sk-access", { keyToken: "hashed-token", blocked: true }); + + expect(result).toEqual({ blocked: true }); + }); +}); + +describe("useSetKeyBlockedState", () => { + beforeEach(() => { + mockPost.mockReset(); + mockUseAuthorized.mockReturnValue({ accessToken: "sk-access" }); + }); + + it("invalidates key queries after a successful mutation", async () => { + mockPost.mockResolvedValueOnce({ blocked: true }); + const { queryClient, wrapper } = createWrapper(); + const invalidateSpy = vi.spyOn(queryClient, "invalidateQueries"); + + const { result } = renderHook(() => useSetKeyBlockedState(), { wrapper }); + result.current.mutate({ keyToken: "hashed-token", blocked: true }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: ["keys"] }); + }); + + it("surfaces request failures as mutation errors", async () => { + mockPost.mockRejectedValueOnce(new Error("Key not found.")); + const { wrapper } = createWrapper(); + + const { result } = renderHook(() => useSetKeyBlockedState(), { wrapper }); + result.current.mutate({ keyToken: "missing", blocked: true }); + + await waitFor(() => expect(result.current.isError).toBe(true)); + expect(result.current.error?.message).toBe("Key not found."); + }); + + it("errors without an access token", async () => { + mockUseAuthorized.mockReturnValue({ accessToken: null }); + const { wrapper } = createWrapper(); + + const { result } = renderHook(() => useSetKeyBlockedState(), { wrapper }); + result.current.mutate({ keyToken: "hashed-token", blocked: true }); + + await waitFor(() => expect(result.current.isError).toBe(true)); + expect(mockPost).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useSetKeyBlockedState.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useSetKeyBlockedState.ts new file mode 100644 index 00000000000..792ef567f99 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useSetKeyBlockedState.ts @@ -0,0 +1,45 @@ +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { apiClient } from "@/components/networking"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { keyKeys } from "./useKeys"; + +export interface SetKeyBlockedStateInput { + keyToken: string; + blocked: boolean; +} + +export interface SetKeyBlockedStateResult { + blocked: boolean; +} + +interface BlockKeyResponse { + blocked?: boolean | null; +} + +export const setKeyBlockedState = async ( + accessToken: string, + { keyToken, blocked }: SetKeyBlockedStateInput, +): Promise => { + const response = await apiClient.post(blocked ? "/key/block" : "/key/unblock", { + accessToken, + body: { key: keyToken }, + }); + return { blocked: response?.blocked ?? blocked }; +}; + +export const useSetKeyBlockedState = () => { + const { accessToken } = useAuthorized(); + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: async (input) => { + if (!accessToken) { + throw new Error("Access token is required"); + } + return setKeyBlockedState(accessToken, input); + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: keyKeys.all }); + }, + }); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPLogoSelector.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPLogoSelector.test.tsx index 94b9058b372..67b5d6bfe92 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPLogoSelector.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPLogoSelector.test.tsx @@ -52,4 +52,23 @@ describe("MCPLogoSelector", () => { await user.click(githubButton); expect(onChange).toHaveBeenCalledWith(undefined); }); + + it("should render grid logos from bundled static assets instead of public paths", () => { + render(); + const src = screen.getByAltText("GitHub").getAttribute("src"); + expect(src).toMatch(/^\/_next\//); + expect(src).toContain("github.svg"); + }); + + it("should preview a stored well-known path via its bundled asset", () => { + render(); + const src = screen.getByAltText("Selected logo").getAttribute("src"); + expect(src).toMatch(/^\/_next\//); + expect(src).toContain("github.svg"); + }); + + it("should preview a custom external URL untouched", () => { + render(); + expect(screen.getByAltText("Selected logo").getAttribute("src")).toBe("https://cdn.example.com/logo.png"); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPLogoSelector.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPLogoSelector.tsx index 6f626a1a70b..a67a0dc882d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPLogoSelector.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPLogoSelector.tsx @@ -1,31 +1,51 @@ -import React, { useState } from "react"; +import React from "react"; import { Input, Tooltip } from "antd"; import { InfoCircleOutlined, LinkOutlined } from "@ant-design/icons"; -import { resolveLogoSrc } from "@/lib/assetPaths"; +import { Logo } from "@/components/molecules/logo/Logo"; +import githubLogo from "../../../../../public/assets/logos/github.svg"; +import slackLogo from "../../../../../public/assets/logos/slack.svg"; +import notionLogo from "../../../../../public/assets/logos/notion.svg"; +import linearLogo from "../../../../../public/assets/logos/linear.svg"; +import jiraLogo from "../../../../../public/assets/logos/jira.svg"; +import figmaLogo from "../../../../../public/assets/logos/figma.svg"; +import gmailLogo from "../../../../../public/assets/logos/gmail.svg"; +import googleDriveLogo from "../../../../../public/assets/logos/google_drive.svg"; +import stripeLogo from "../../../../../public/assets/logos/stripe.svg"; +import shopifyLogo from "../../../../../public/assets/logos/shopify.svg"; +import salesforceLogo from "../../../../../public/assets/logos/salesforce.svg"; +import hubspotLogo from "../../../../../public/assets/logos/hubspot.svg"; +import twilioLogo from "../../../../../public/assets/logos/twilio.svg"; +import cloudflareLogo from "../../../../../public/assets/logos/cloudflare.svg"; +import sentryLogo from "../../../../../public/assets/logos/sentry.svg"; +import postgresqlLogo from "../../../../../public/assets/logos/postgresql.svg"; +import snowflakeLogo from "../../../../../public/assets/logos/snowflake.svg"; +import zapierLogo from "../../../../../public/assets/logos/zapier.svg"; +import googleLogo from "../../../../../public/assets/logos/google.svg"; +import gitlabLogo from "../../../../../public/assets/logos/gitlab.svg"; const logos = "/ui/assets/logos/"; -const WELL_KNOWN_LOGOS: { name: string; url: string }[] = [ - { name: "GitHub", url: `${logos}github.svg` }, - { name: "Slack", url: `${logos}slack.svg` }, - { name: "Notion", url: `${logos}notion.svg` }, - { name: "Linear", url: `${logos}linear.svg` }, - { name: "Jira", url: `${logos}jira.svg` }, - { name: "Figma", url: `${logos}figma.svg` }, - { name: "Gmail", url: `${logos}gmail.svg` }, - { name: "Google Drive", url: `${logos}google_drive.svg` }, - { name: "Stripe", url: `${logos}stripe.svg` }, - { name: "Shopify", url: `${logos}shopify.svg` }, - { name: "Salesforce", url: `${logos}salesforce.svg` }, - { name: "HubSpot", url: `${logos}hubspot.svg` }, - { name: "Twilio", url: `${logos}twilio.svg` }, - { name: "Cloudflare", url: `${logos}cloudflare.svg` }, - { name: "Sentry", url: `${logos}sentry.svg` }, - { name: "PostgreSQL", url: `${logos}postgresql.svg` }, - { name: "Snowflake", url: `${logos}snowflake.svg` }, - { name: "Zapier", url: `${logos}zapier.svg` }, - { name: "Google", url: `${logos}google.svg` }, - { name: "GitLab", url: `${logos}gitlab.svg` }, +const WELL_KNOWN_LOGOS: { name: string; url: string; src: string }[] = [ + { name: "GitHub", url: `${logos}github.svg`, src: githubLogo.src }, + { name: "Slack", url: `${logos}slack.svg`, src: slackLogo.src }, + { name: "Notion", url: `${logos}notion.svg`, src: notionLogo.src }, + { name: "Linear", url: `${logos}linear.svg`, src: linearLogo.src }, + { name: "Jira", url: `${logos}jira.svg`, src: jiraLogo.src }, + { name: "Figma", url: `${logos}figma.svg`, src: figmaLogo.src }, + { name: "Gmail", url: `${logos}gmail.svg`, src: gmailLogo.src }, + { name: "Google Drive", url: `${logos}google_drive.svg`, src: googleDriveLogo.src }, + { name: "Stripe", url: `${logos}stripe.svg`, src: stripeLogo.src }, + { name: "Shopify", url: `${logos}shopify.svg`, src: shopifyLogo.src }, + { name: "Salesforce", url: `${logos}salesforce.svg`, src: salesforceLogo.src }, + { name: "HubSpot", url: `${logos}hubspot.svg`, src: hubspotLogo.src }, + { name: "Twilio", url: `${logos}twilio.svg`, src: twilioLogo.src }, + { name: "Cloudflare", url: `${logos}cloudflare.svg`, src: cloudflareLogo.src }, + { name: "Sentry", url: `${logos}sentry.svg`, src: sentryLogo.src }, + { name: "PostgreSQL", url: `${logos}postgresql.svg`, src: postgresqlLogo.src }, + { name: "Snowflake", url: `${logos}snowflake.svg`, src: snowflakeLogo.src }, + { name: "Zapier", url: `${logos}zapier.svg`, src: zapierLogo.src }, + { name: "Google", url: `${logos}google.svg`, src: googleLogo.src }, + { name: "GitLab", url: `${logos}gitlab.svg`, src: gitlabLogo.src }, ]; interface MCPLogoSelectorProps { @@ -34,16 +54,12 @@ interface MCPLogoSelectorProps { } const MCPLogoSelector: React.FC = ({ value, onChange }) => { - const [imgErrors, setImgErrors] = useState>(new Set()); + const selectedWellKnown = WELL_KNOWN_LOGOS.find((l) => l.url === value); const handleSelect = (url: string) => { onChange?.(value === url ? undefined : url); }; - const handleImgError = (url: string) => { - setImgErrors((prev) => new Set(prev).add(url)); - }; - return (
@@ -56,13 +72,10 @@ const MCPLogoSelector: React.FC = ({ value, onChange }) => {/* Preview */} {value && (
- Selected logo { - (e.target as HTMLImageElement).style.display = "none"; - }} />
{value}
@@ -81,8 +94,6 @@ const MCPLogoSelector: React.FC = ({ value, onChange }) =>
{WELL_KNOWN_LOGOS.map((logo) => { const isSelected = value === logo.url; - const hasFailed = imgErrors.has(logo.url); - if (hasFailed) return null; return ( ); @@ -112,7 +118,7 @@ const MCPLogoSelector: React.FC = ({ value, onChange }) => } placeholder="Or paste a custom logo URL..." - value={value && !WELL_KNOWN_LOGOS.some((l) => l.url === value) ? value : ""} + value={value && !selectedWellKnown ? value : ""} onChange={(e) => { const v = e.target.value.trim(); onChange?.(v || undefined); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPServerCard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPServerCard.test.tsx index a0998b587fb..d6343afe219 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPServerCard.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPServerCard.test.tsx @@ -1,8 +1,9 @@ import React from "react"; import { render, screen } from "@testing-library/react"; -import { describe, it, expect, vi } from "vitest"; +import { describe, it, expect, vi, afterEach } from "vitest"; import MCPServerCard from "./MCPServerCard"; import type { MCPServer } from "@/components/mcp_tools/types"; +import { setServerRootPath } from "@/lib/serverRootPath"; const baseServer: MCPServer = { server_id: "srv-1", @@ -43,3 +44,26 @@ describe("MCPServerCard OAuth flow indicator", () => { expect(screen.queryByText("OAuth flow not set")).not.toBeInTheDocument(); }); }); + +describe("MCPServerCard logo", () => { + afterEach(() => { + setServerRootPath("/"); + }); + + it("passes an external logo_url through untouched", () => { + renderCard({ mcp_info: { server_name: "demo_server", logo_url: "https://cdn.example.com/logo.png" } }); + expect(screen.getByAltText("demo_server logo").getAttribute("src")).toBe("https://cdn.example.com/logo.png"); + }); + + it("prefixes a stored asset path with the server root path under a non-root mount", () => { + setServerRootPath("/litellm"); + renderCard({ mcp_info: { server_name: "demo_server", logo_url: "/ui/assets/logos/github.svg" } }); + expect(screen.getByAltText("demo_server logo").getAttribute("src")).toBe("/litellm/ui/assets/logos/github.svg"); + }); + + it("renders a letter avatar when no logo_url is set", () => { + renderCard({ mcp_info: { server_name: "demo_server" } }); + expect(screen.queryByAltText("demo_server logo")).not.toBeInTheDocument(); + expect(screen.getByText("DE")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPServerCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPServerCard.tsx index 4282cdba278..c7dd6e47f76 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPServerCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPServerCard.tsx @@ -1,4 +1,4 @@ -import { useState, type FC, type KeyboardEvent, type MouseEvent } from "react"; +import { type FC, type KeyboardEvent, type MouseEvent } from "react"; import { Dropdown, Tooltip, Typography, Tag } from "antd"; import type { MenuProps } from "antd"; import { @@ -9,6 +9,7 @@ import { ThunderboltOutlined, } from "@ant-design/icons"; import { AUTH_TYPE, type MCPServer } from "@/components/mcp_tools/types"; +import { Logo } from "@/components/molecules/logo/Logo"; import { getMaskedAndFullUrl } from "./utils"; const { Text } = Typography; @@ -52,8 +53,6 @@ const MCPServerCard: FC = ({ const name = server.server_name || alias || server.server_id; // Logo is sourced exclusively from the admin-set `mcp_info.logo_url`. const candidateLogo = server.mcp_info?.logo_url ?? undefined; - const [failedLogoUrl, setFailedLogoUrl] = useState(null); - const logoUrl = candidateLogo && failedLogoUrl !== candidateLogo ? candidateLogo : undefined; const transport = server.transport || "http"; const displayTransport = server.spec_path && transport !== "stdio" ? "openapi" : transport; const authType = server.auth_type || "none"; @@ -148,13 +147,8 @@ const MCPServerCard: FC = ({ className={`group relative flex h-full cursor-pointer flex-col gap-3 rounded-lg p-4 transition-all duration-150 focus:outline-hidden focus-visible:ring-2 focus-visible:ring-blue-400 ${cardClass}`} >
- {logoUrl ? ( - {`${name} setFailedLogoUrl(logoUrl)} - /> + {candidateLogo ? ( + ) : (
{(name || "?").slice(0, 2).toUpperCase()} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/create_mcp_server.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/create_mcp_server.tsx index 9f7639d00c7..b21a5218c20 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/create_mcp_server.tsx @@ -39,10 +39,9 @@ import NotificationsManager from "@/components/molecules/notifications_manager"; import { useMcpOAuthFlow } from "@/hooks/useMcpOAuthFlow"; import { useTestMCPConnection } from "@/hooks/useTestMCPConnection"; import { getSecureItem, setSecureItem } from "@/utils/secureStorage"; -import { resolveLogoSrc } from "@/lib/assetPaths"; +import mcpLogo from "../../../../../public/assets/logos/mcp_logo.png"; -const asset_logos_folder = "/ui/assets/logos/"; -export const mcpLogoImg = `${asset_logos_folder}mcp_logo.png`; +export const mcpLogoImg = mcpLogo.src; interface CreateMCPServerProps { userRole: string; @@ -791,7 +790,7 @@ const CreateMCPServer: React.FC = ({ )} MCP Logo void; +} + +function formatTimestamp(ts?: string): string { + if (!ts) return "—"; + try { + const d = new Date(ts); + return d.toLocaleString(); + } catch { + return ts; + } +} + +export function MemoryDetailDrawer({ row, onClose }: MemoryDetailDrawerProps) { + return ( + + {row.key} + + ) : ( + "Memory" + ) + } + width={720} + destroyOnClose + > + {row && ( + + +
+ + Memory ID + + + {row.memory_id} + +
+
+ + User ID + + {row.user_id ?? "-"} +
+
+ + Team ID + + {row.team_id ?? "-"} +
+
+
+ Value + + {row.value} + +
+ {row.metadata !== undefined && row.metadata !== null && ( +
+ Metadata + + {JSON.stringify(row.metadata, null, 2)} + +
+ )} + ·} wrap size="small" style={{ color: "rgba(0,0,0,0.45)" }}> + + Created {formatTimestamp(row.created_at)} + {row.created_by ? ` by ${row.created_by}` : ""} + + + Updated {formatTimestamp(row.updated_at)} + {row.updated_by ? ` by ${row.updated_by}` : ""} + + +
+ )} +
+ ); +} + +export default MemoryDetailDrawer; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryTable.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryTable.test.tsx new file mode 100644 index 00000000000..f664c650cd4 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryTable.test.tsx @@ -0,0 +1,170 @@ +import { PaginationState } from "@tanstack/react-table"; +import { render, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import React, { useState } from "react"; +import { describe, expect, it, vi } from "vitest"; + +import { MemoryRow } from "@/components/networking"; + +import { MemoryTable } from "./MemoryTable"; + +const makeMemory = (overrides: Partial = {}): MemoryRow => ({ + memory_id: "mem-1", + key: "user:profile", + value: "The user prefers concise answers.", + metadata: null, + user_id: "user-42", + team_id: "team-7", + updated_at: "2024-05-01T12:00:00Z", + ...overrides, +}); + +const baseProps = { + data: [makeMemory()], + isLoading: false, + rowCount: 1, + pagination: { pageIndex: 0, pageSize: 50 } as PaginationState, + onPaginationChange: vi.fn(), + searchValue: "", + onSearchChange: vi.fn(), + isRefreshing: false, + onRefresh: vi.fn(), + hasActiveSearch: false, + onViewClick: vi.fn(), + onEditClick: vi.fn(), + onDeleteClick: vi.fn(), +}; + +describe("MemoryTable", () => { + it("renders every column header", () => { + render(); + for (const header of ["ID", "Name", "Preview", "User ID", "Team ID", "Updated"]) { + expect(screen.getByText(header)).toBeInTheDocument(); + } + }); + + it("opens the detail view when the ID identity cell is clicked", async () => { + const user = userEvent.setup(); + const onViewClick = vi.fn(); + const row = makeMemory({ memory_id: "mem-click" }); + render(); + + await user.click(screen.getByText("mem-click")); + + expect(onViewClick).toHaveBeenCalledTimes(1); + expect(onViewClick).toHaveBeenCalledWith(row); + }); + + it("routes each overflow-menu action to its callback with the row", async () => { + const user = userEvent.setup(); + const onViewClick = vi.fn(); + const onEditClick = vi.fn(); + const onDeleteClick = vi.fn(); + const row = makeMemory({ memory_id: "mem-9" }); + render( + , + ); + + await user.click(screen.getByTestId("memory-actions-mem-9")); + await user.click(await screen.findByTestId("memory-action-edit")); + expect(onEditClick).toHaveBeenCalledWith(row); + expect(onViewClick).not.toHaveBeenCalled(); + expect(onDeleteClick).not.toHaveBeenCalled(); + + await user.click(screen.getByTestId("memory-actions-mem-9")); + await user.click(await screen.findByTestId("memory-action-delete")); + expect(onDeleteClick).toHaveBeenCalledWith(row); + + await user.click(screen.getByTestId("memory-actions-mem-9")); + await user.click(await screen.findByTestId("memory-action-view")); + expect(onViewClick).toHaveBeenCalledWith(row); + }); + + it("shows the empty-only copy when there is no data and no active search", () => { + render(); + expect(screen.getByText("No memories stored yet")).toBeInTheDocument(); + expect(screen.queryByText("No matching memories")).not.toBeInTheDocument(); + }); + + it("shows the filtered-empty copy when a search is active", () => { + render(); + expect(screen.getByText("No matching memories")).toBeInTheDocument(); + expect(screen.queryByText("No memories stored yet")).not.toBeInTheDocument(); + }); + + it("renders loading skeleton rows instead of the empty state while loading", () => { + render(); + expect(screen.getAllByTestId("skeleton-row").length).toBeGreaterThan(0); + expect(screen.queryByText("No memories stored yet")).not.toBeInTheDocument(); + }); + + it("drives the pagination footer from the server rowCount, not the page's row length", () => { + render(); + const range = screen.getByTestId("pagination-range"); + expect(range).toHaveTextContent("Showing 1-50 of 120"); + expect(screen.getByTestId("pagination-page")).toHaveTextContent("Page 1 of 3"); + expect(screen.getByTestId("pagination-next")).toBeEnabled(); + }); + + it("advances the page through the server pagination handler", async () => { + const user = userEvent.setup(); + const onPaginationChange = vi.fn(); + render(); + + await user.click(screen.getByTestId("pagination-next")); + + expect(onPaginationChange).toHaveBeenCalled(); + }); + + it("forwards toolbar search input and refresh to their callbacks", async () => { + const user = userEvent.setup(); + const onSearchChange = vi.fn(); + const onRefresh = vi.fn(); + render(); + + await user.type(screen.getByTestId("datatable-search"), "u"); + expect(onSearchChange).toHaveBeenCalledWith("u"); + + await user.click(screen.getByTestId("datatable-refresh")); + expect(onRefresh).toHaveBeenCalledTimes(1); + }); + + it("keeps the page in range when the rows-per-page selector shrinks the page count", async () => { + const user = userEvent.setup(); + const rowCount = 120; + const seen: PaginationState[] = []; + + function Harness() { + const [pagination, setPagination] = useState({ pageIndex: 4, pageSize: 25 }); + seen.push(pagination); + return ( + + ); + } + + render(); + expect(screen.getByTestId("pagination-page")).toHaveTextContent("Page 5 of 5"); + + await user.click(screen.getByTestId("pagination-page-size")); + await user.click(await screen.findByRole("option", { name: "100" })); + + const final = seen[seen.length - 1]; + expect(final.pageSize).toBe(100); + expect(final.pageIndex).toBeLessThanOrEqual(Math.ceil(rowCount / final.pageSize) - 1); + expect(screen.getByTestId("pagination-page")).toHaveTextContent("Page 2 of 2"); + }); + + it("renders secondary id and date cells for the row", () => { + render(); + const table = screen.getByRole("table"); + expect(within(table).getByText("user-42")).toBeInTheDocument(); + expect(within(table).getByText("team-7")).toBeInTheDocument(); + expect(within(table).getByText("user:profile")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryTable.tsx new file mode 100644 index 00000000000..50dd04ee14c --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryTable.tsx @@ -0,0 +1,94 @@ +"use client"; + +import { OnChangeFn, PaginationState } from "@tanstack/react-table"; +import { Database } from "lucide-react"; +import React, { useMemo } from "react"; + +import { MemoryRow } from "@/components/networking"; +import { DataTable, DataTableToolbar } from "@/components/shared/DataTable"; + +import { getMemoryTableColumns } from "./MemoryTableColumns"; + +interface MemoryTableProps { + data: MemoryRow[]; + isLoading: boolean; + rowCount: number; + pagination: PaginationState; + onPaginationChange: OnChangeFn; + searchValue: string; + onSearchChange: (value: string) => void; + isRefreshing: boolean; + onRefresh: () => void; + hasActiveSearch: boolean; + onViewClick: (row: MemoryRow) => void; + onEditClick: (row: MemoryRow) => void; + onDeleteClick: (row: MemoryRow) => void; +} + +function MemoryEmptyState({ hasActiveSearch }: { hasActiveSearch: boolean }) { + return ( +
+
+ +
+
+ {hasActiveSearch ? "No matching memories" : "No memories stored yet"} +
+
+ {hasActiveSearch + ? "No memories have keys starting with your search." + : "Memories your agents store under /v1/memory will appear here."} +
+
+ ); +} + +export function MemoryTable({ + data, + isLoading, + rowCount, + pagination, + onPaginationChange, + searchValue, + onSearchChange, + isRefreshing, + onRefresh, + hasActiveSearch, + onViewClick, + onEditClick, + onDeleteClick, +}: MemoryTableProps) { + const columns = useMemo(() => { + const columnDeps = { onViewClick, onEditClick, onDeleteClick }; + return getMemoryTableColumns(columnDeps); + }, [onViewClick, onEditClick, onDeleteClick]); + + return ( + row.memory_id} + paginationMode="server" + pagination={pagination} + onPaginationChange={onPaginationChange} + rowCount={rowCount} + isLoading={isLoading} + loadingMessage="Loading memories…" + noDataMessage={} + size="compact" + toolbar={(table) => ( + + )} + /> + ); +} + +export default MemoryTable; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryTableColumns.tsx new file mode 100644 index 00000000000..6b2a6b08704 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryTableColumns.tsx @@ -0,0 +1,150 @@ +"use client"; + +import { ColumnDef } from "@tanstack/react-table"; +import { Eye, MoreHorizontal, Pencil, Trash2 } from "lucide-react"; + +import { MemoryRow } from "@/components/networking"; +import { DateCell, IdCell, IdentityCell } from "@/components/shared/table_cells"; +import { buttonVariants } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { cn } from "@/lib/cva.config"; + +interface MemoryRowActionsProps { + row: MemoryRow; + onViewClick: (row: MemoryRow) => void; + onEditClick: (row: MemoryRow) => void; + onDeleteClick: (row: MemoryRow) => void; +} + +function MemoryRowActions({ row, onViewClick, onEditClick, onDeleteClick }: MemoryRowActionsProps) { + return ( + + + + + + onViewClick(row)}> + + View + + onEditClick(row)}> + + Edit + + + onDeleteClick(row)}> + + Delete + + + + ); +} + +export interface MemoryTableColumnsDeps { + onViewClick: (row: MemoryRow) => void; + onEditClick: (row: MemoryRow) => void; + onDeleteClick: (row: MemoryRow) => void; +} + +export const getMemoryTableColumns = ({ + onViewClick, + onEditClick, + onDeleteClick, +}: MemoryTableColumnsDeps): ColumnDef[] => [ + { + id: "memory_id", + accessorKey: "memory_id", + meta: { title: "ID" }, + header: "ID", + size: 180, + enableSorting: false, + cell: ({ row }) => ( + onViewClick(row.original)} + /> + ), + }, + { + id: "key", + accessorKey: "key", + meta: { title: "Name" }, + header: "Name", + size: 200, + enableSorting: false, + cell: ({ row }) => ( + + {row.original.key} + + ), + }, + { + id: "value", + accessorKey: "value", + meta: { title: "Preview" }, + header: "Preview", + enableSorting: false, + cell: ({ row }) => ( + + {row.original.value || "-"} + + ), + }, + { + id: "user_id", + accessorKey: "user_id", + meta: { title: "User ID" }, + header: "User ID", + size: 160, + enableSorting: false, + cell: ({ row }) => , + }, + { + id: "team_id", + accessorKey: "team_id", + meta: { title: "Team ID" }, + header: "Team ID", + size: 160, + enableSorting: false, + cell: ({ row }) => , + }, + { + id: "updated_at", + accessorKey: "updated_at", + meta: { title: "Updated" }, + header: "Updated", + size: 170, + enableSorting: false, + cell: ({ row }) => , + }, + { + id: "actions", + meta: { className: "text-right", headerClassName: "text-right" }, + header: () => Actions, + size: 64, + enableSorting: false, + enableHiding: false, + cell: ({ row }) => ( +
+ +
+ ), + }, +]; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.test.tsx new file mode 100644 index 00000000000..f415c99225a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.test.tsx @@ -0,0 +1,45 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render } from "@testing-library/react"; +import React from "react"; +import { describe, expect, it, vi } from "vitest"; + +import { MemoryRow } from "@/components/networking"; + +import { MemoryView } from "./MemoryView"; + +interface CapturedTableProps { + isLoading: boolean; + rowCount: number; + data: MemoryRow[]; + hasActiveSearch: boolean; +} + +const captured = vi.hoisted(() => ({ current: null as CapturedTableProps | null })); + +vi.mock("./MemoryTable", () => ({ + MemoryTable: function MemoryTableMock(props: CapturedTableProps) { + captured.current = props; + return
; + }, +})); + +const renderView = (accessToken: string | null) => { + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return render( + + + , + ); +}; + +describe("MemoryView", () => { + it("keeps the table out of the skeleton state when the token is null (disabled query)", () => { + renderView(null); + + expect(captured.current).not.toBeNull(); + expect(captured.current?.isLoading).toBe(false); + expect(captured.current?.data).toEqual([]); + expect(captured.current?.rowCount).toBe(0); + expect(captured.current?.hasActiveSearch).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.tsx index 4ee784f4664..fcb15978f47 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.tsx @@ -1,21 +1,19 @@ "use client"; -import React, { useMemo, useState } from "react"; +import { useDebouncedValue } from "@tanstack/react-pacer/debouncer"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { Button, Card, Drawer, Empty, Input, Space, Table, Typography, message } from "antd"; -import type { ColumnsType } from "antd/es/table"; -import { - DeleteOutlined, - EditOutlined, - EyeOutlined, - PlusOutlined, - ReloadOutlined, - SearchOutlined, -} from "@ant-design/icons"; +import type { PaginationState } from "@tanstack/react-table"; +import { PlusOutlined } from "@ant-design/icons"; +import { Button, Space, Typography, message } from "antd"; +import React, { useCallback, useMemo, useState } from "react"; + import { MemoryRow, createMemory, deleteMemory, fetchMemoryList, updateMemory } from "@/components/networking"; -import { DateCell, IdCell } from "@/components/shared/table_cells"; -import { MemoryEditModal } from "./MemoryEditModal"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; +import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants"; + +import { MemoryDetailDrawer } from "./MemoryDetailDrawer"; +import { MemoryEditModal } from "./MemoryEditModal"; +import { MemoryTable } from "./MemoryTable"; const { Text, Paragraph, Title } = Typography; @@ -25,38 +23,16 @@ interface MemoryViewProps { userRole: string | null; } -function previewValue(value: string, max = 120): string { - if (!value) return ""; - const trimmed = value.trim(); - if (trimmed.length <= max) return trimmed; - return `${trimmed.slice(0, max)}…`; -} - -function formatTimestamp(ts?: string): string { - if (!ts) return "—"; - try { - const d = new Date(ts); - return d.toLocaleString(); - } catch { - return ts; - } -} - -const PAGE_SIZE = 50; +const DEFAULT_PAGE_SIZE = 50; export const MemoryView: React.FC = ({ accessToken }) => { const [searchInput, setSearchInput] = useState(""); - const [appliedSearch, setAppliedSearch] = useState(""); + const [debouncedSearch] = useDebouncedValue(searchInput, { wait: DEBOUNCE_WAIT_MS }); + const [pagination, setPagination] = useState({ pageIndex: 0, pageSize: DEFAULT_PAGE_SIZE }); const [detailRow, setDetailRow] = useState(null); const [editRow, setEditRow] = useState(null); const [deleteRow, setDeleteRow] = useState(null); const [isCreateOpen, setIsCreateOpen] = useState(false); - const [currentPage, setCurrentPage] = useState(1); - - // Reset to page 1 whenever the filter changes. - React.useEffect(() => { - setCurrentPage(1); - }, [appliedSearch]); const queryClient = useQueryClient(); // React Query key prefix for all memory-list variants (paged + filtered). @@ -65,15 +41,15 @@ export const MemoryView: React.FC = ({ accessToken }) => { const MEMORY_LIST_KEY = "memoryList" as const; const { data, isLoading, isFetching } = useQuery({ - queryKey: [MEMORY_LIST_KEY, appliedSearch, currentPage], + queryKey: [MEMORY_LIST_KEY, debouncedSearch, pagination.pageIndex, pagination.pageSize], queryFn: () => { if (!accessToken) throw new Error("Access token required"); // Prefix search matches the Redis-style mental model (namespace scan): // typing "user:" finds "user:profile", "user:prefs", etc. return fetchMemoryList(accessToken, { - keyPrefix: appliedSearch || undefined, - page: currentPage, - pageSize: PAGE_SIZE, + keyPrefix: debouncedSearch || undefined, + page: pagination.pageIndex + 1, + pageSize: pagination.pageSize, }); }, enabled: !!accessToken, @@ -88,7 +64,10 @@ export const MemoryView: React.FC = ({ accessToken }) => { // refetches from scratch (pagination + filter-aware). // - on error: surface the message via antd `message.error`. - const invalidateList = () => queryClient.invalidateQueries({ queryKey: [MEMORY_LIST_KEY] }); + const invalidateList = useCallback( + () => queryClient.invalidateQueries({ queryKey: [MEMORY_LIST_KEY] }), + [queryClient], + ); const createMutation = useMutation({ mutationFn: (args: { key: string; value: string; metadata: unknown }) => { @@ -133,9 +112,14 @@ export const MemoryView: React.FC = ({ accessToken }) => { }, }); - const handleDelete = (row: MemoryRow) => { - setDeleteRow(row); - }; + const handleSearchChange = useCallback((value: string) => { + setSearchInput(value); + setPagination((prev) => ({ ...prev, pageIndex: 0 })); + }, []); + + const handleView = useCallback((row: MemoryRow) => setDetailRow(row), []); + const handleEdit = useCallback((row: MemoryRow) => setEditRow(row), []); + const handleDelete = useCallback((row: MemoryRow) => setDeleteRow(row), []); const confirmDelete = async () => { if (!deleteRow) return; @@ -192,242 +176,43 @@ export const MemoryView: React.FC = ({ accessToken }) => { } }; - const columns: ColumnsType = [ - { - title: "ID", - dataIndex: "memory_id", - key: "memory_id", - width: 140, - render: (_: unknown, r: MemoryRow) => setDetailRow(r)} />, - }, - { - title: "Name", - dataIndex: "key", - key: "key", - width: 200, - render: (k: string) => {k}, - // No client-side sorter: pagination is server-side, so a client sort - // would only reorder the current page and mislead users into thinking - // the whole list is sorted. Backend returns rows ordered by - // `updated_at DESC`; use the prefix filter for discovery by name. - }, - { - title: "Preview", - dataIndex: "value", - key: "value", - render: (v: string) => ( - - {previewValue(v)} - - ), - }, - { - title: "User ID", - dataIndex: "user_id", - key: "user_id", - width: 160, - render: (uid?: string | null) => , - }, - { - title: "Team ID", - dataIndex: "team_id", - key: "team_id", - width: 160, - render: (tid?: string | null) => , - }, - { - title: "Updated", - dataIndex: "updated_at", - key: "updated_at", - width: 180, - render: (ts?: string) => , - // No sorter — backend already returns rows in `updated_at DESC` order, - // and a client-side sorter on a paginated view would only affect the - // current page. - }, - { - title: "", - key: "actions", - width: 140, - render: (_: unknown, r: MemoryRow) => ( - -
- - - - } - value={searchInput} - onChange={(e) => setSearchInput(e.target.value)} - onPressEnter={() => setAppliedSearch(searchInput.trim())} - onClear={() => { - setSearchInput(""); - setAppliedSearch(""); - }} - style={{ width: 280 }} - /> - - - - - - - `${range[0]}–${range[1]} of ${n}`, - onChange: (page) => setCurrentPage(page), - }} - locale={{ - emptyText: ( - - ), - }} - /> - + {/* Detail drawer */} - setDetailRow(null)} - title={ - detailRow ? ( - - {detailRow.key} - - ) : ( - "Memory" - ) - } - width={720} - destroyOnClose - > - {detailRow && ( - - -
- - Memory ID - - - {detailRow.memory_id} - -
-
- - User ID - - {detailRow.user_id ?? "-"} -
-
- - Team ID - - {detailRow.team_id ?? "-"} -
-
-
- Value - - {detailRow.value} - -
- {detailRow.metadata !== undefined && detailRow.metadata !== null && ( -
- Metadata - - {JSON.stringify(detailRow.metadata, null, 2)} - -
- )} - ·} wrap size="small" style={{ color: "rgba(0,0,0,0.45)" }}> - - Created {formatTimestamp(detailRow.created_at)} - {detailRow.created_by ? ` by ${detailRow.created_by}` : ""} - - - Updated {formatTimestamp(detailRow.updated_at)} - {detailRow.updated_by ? ` by ${detailRow.updated_by}` : ""} - - -
- )} -
+ setDetailRow(null)} /> {/* Create / edit modal */} = ({ premiumUser, te const [selectedModelId, setSelectedModelId] = useState(null); const [selectedTeamId, setSelectedTeamId] = useState(null); const [selectedTabIndex, setSelectedTabIndex] = useState(0); - const [healthCurrentPage, setHealthCurrentPage] = useState(1); + const [healthPagination, setHealthPagination] = useState({ + pageIndex: 0, + pageSize: HEALTH_PAGE_SIZE, + }); const queryClient = useQueryClient(); const { data: modelDataResponse, isLoading: isLoadingModels, refetch: refetchModels } = useModelsInfo(); const { data: healthModelDataResponse, isLoading: isLoadingHealthModels } = useModelsInfo( - healthCurrentPage, - HEALTH_PAGE_SIZE, + healthPagination.pageIndex + 1, + healthPagination.pageSize, ); const { data: modelCostMapData, isLoading: isLoadingModelCostMap } = useModelCostMap(); const { data: credentialsResponse, isLoading: isLoadingCredentials } = useCredentials(); @@ -137,14 +141,7 @@ const ModelsAndEndpointsView: React.FC = ({ premiumUser, te return transformModelData(healthModelDataResponse, getProviderFromModel); }, [healthModelDataResponse?.data, getProviderFromModel]); - const healthPaginationMeta = useMemo(() => { - return { - total_count: healthModelDataResponse?.total_count ?? 0, - current_page: healthModelDataResponse?.current_page ?? healthCurrentPage, - total_pages: healthModelDataResponse?.total_pages ?? 1, - size: healthModelDataResponse?.size ?? HEALTH_PAGE_SIZE, - }; - }, [healthModelDataResponse, healthCurrentPage]); + const healthRowCount = healthModelDataResponse?.total_count ?? 0; const isProxyAdmin = userRole && isProxyAdminRole(userRole); const isInternalUser = userRole && internalUserRoles.includes(userRole); @@ -188,7 +185,7 @@ const ModelsAndEndpointsView: React.FC = ({ premiumUser, te const handleRefreshClick = () => { const currentDate = new Date(); setLastRefreshed(currentDate.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" })); - setHealthCurrentPage(1); + setHealthPagination((previous) => ({ ...previous, pageIndex: 0 })); queryClient.invalidateQueries({ queryKey: ["models", "list"] }); refetchModels(); }; @@ -413,10 +410,9 @@ const ModelsAndEndpointsView: React.FC = ({ premiumUser, te setSelectedModelId={setSelectedModelId} teams={teams} isLoading={isLoadingHealthModels} - paginationMeta={healthPaginationMeta} - currentPage={healthCurrentPage} - pageSize={HEALTH_PAGE_SIZE} - onPageChange={setHealthCurrentPage} + pagination={healthPagination} + onPaginationChange={setHealthPagination} + rowCount={healthRowCount} /> ), diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.test.tsx index 814625ff6be..37eeaf4c2af 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.test.tsx @@ -7,8 +7,6 @@ describe("OrganizationFilters", () => { const defaultFilters: FilterState = { org_id: "", org_alias: "", - sort_by: "", - sort_order: "asc", }; it("should render", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.tsx index 5643a4bc51a..6ad2f00fdb0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.tsx @@ -14,8 +14,6 @@ interface OrganizationFiltersProps { type FilterState = { org_id: string; org_alias: string; - sort_by: string; - sort_order: "asc" | "desc"; }; const OrganizationFilters = ({ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.test.tsx new file mode 100644 index 00000000000..d381e5e65ca --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.test.tsx @@ -0,0 +1,57 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen } from "@testing-library/react"; +import React from "react"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/components/vector_store_management/VectorStoreSelector", () => ({ + __esModule: true, + default: () => null, +})); +vi.mock("@/components/mcp_server_management/MCPServerSelector", () => ({ + __esModule: true, + default: () => null, +})); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => ({ + accessToken: null, + userId: null, + userRole: null, + }), +})); +vi.mock("./OrganizationsTable", () => ({ + __esModule: true, + default: (props: { isLoading: boolean }) => ( +
isLoading:{String(props.isLoading)}
+ ), +})); + +import OrganizationsPanel from "./OrganizationsPanel"; + +const renderWithQueryClient = (ui: React.ReactElement) => { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + return render({ui}); +}; + +describe("OrganizationsPanel", () => { + it("gates non-premium users behind the enterprise notice", () => { + renderWithQueryClient(); + + expect(screen.getByText(/LiteLLM Enterprise feature/i)).toBeInTheDocument(); + expect(screen.queryByText("+ Create New Organization")).not.toBeInTheDocument(); + }); + + it("shows the create button for a premium admin", () => { + renderWithQueryClient(); + + expect(screen.getByText("+ Create New Organization")).toBeInTheDocument(); + }); + + it("resolves the loading skeleton to false when the query is disabled (no token)", () => { + renderWithQueryClient(); + + // A disabled React Query keeps isPending true forever; feeding isLoading avoids a stuck skeleton. + expect(screen.getByTestId("organizations-table")).toHaveTextContent("isLoading:false"); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx new file mode 100644 index 00000000000..9f7e029a1d4 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx @@ -0,0 +1,299 @@ +import { organizationKeys, useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrganizations"; +import { useUserModels } from "@/app/(dashboard)/hooks/models/useModels"; +import OrganizationFilters, { FilterState } from "@/app/(dashboard)/organizations/OrganizationFilters"; +import { InfoCircleOutlined } from "@ant-design/icons"; +import { Form, Input, Modal, Select as Select2, Tooltip } from "antd"; +import { useQueryClient } from "@tanstack/react-query"; +import React, { useState } from "react"; +import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; +import MCPServerSelector from "@/components/mcp_server_management/MCPServerSelector"; +import { ModelSelect } from "@/components/ModelSelect/ModelSelect"; +import NotificationsManager from "@/components/molecules/notifications_manager"; +import { organizationCreateCall, organizationDeleteCall } from "@/components/networking"; +import OrganizationInfoView from "@/components/organization/organization_view"; +import NumericalInput from "@/components/shared/numerical_input"; +import { Button } from "@/components/ui/button"; +import VectorStoreSelector from "@/components/vector_store_management/VectorStoreSelector"; + +import OrganizationsTable from "./OrganizationsTable"; + +interface OrganizationsPanelProps { + userRole: string; + accessToken: string | null; + premiumUser: boolean; +} + +const OrganizationsPanel: React.FC = ({ userRole, accessToken, premiumUser }) => { + const [selectedOrgId, setSelectedOrgId] = useState(null); + const [editOrg, setEditOrg] = useState(false); + const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); + const [orgToDelete, setOrgToDelete] = useState(null); + const [isDeleting, setIsDeleting] = useState(false); + const [isOrgModalVisible, setIsOrgModalVisible] = useState(false); + const [form] = Form.useForm(); + const [showFilters, setShowFilters] = useState(false); + const [filters, setFilters] = useState({ org_id: "", org_alias: "" }); + + const queryClient = useQueryClient(); + const { data: organizations = [], isLoading } = useOrganizations({ + org_id: filters.org_id, + org_alias: filters.org_alias, + }); + const { data: userModels = [] } = useUserModels(); + + const searchActive = Boolean(filters.org_id || filters.org_alias); + + const refetchOrganizations = () => queryClient.invalidateQueries({ queryKey: organizationKeys.lists() }); + + const handleFilterChange = (key: keyof FilterState, value: string) => { + setFilters((previousFilters) => ({ ...previousFilters, [key]: value })); + }; + + const handleFilterReset = () => { + setFilters({ org_id: "", org_alias: "" }); + }; + + const handleDelete = (orgId: string | null) => { + if (!orgId) return; + + setOrgToDelete(orgId); + setIsDeleteModalOpen(true); + }; + + const confirmDelete = async () => { + if (!orgToDelete || !accessToken) return; + + try { + setIsDeleting(true); + await organizationDeleteCall(accessToken, orgToDelete); + NotificationsManager.success("Organization deleted successfully"); + + setIsDeleteModalOpen(false); + setOrgToDelete(null); + await refetchOrganizations(); + } catch (error) { + console.error("Error deleting organization:", error); + } finally { + setIsDeleting(false); + } + }; + + const cancelDelete = () => { + setIsDeleteModalOpen(false); + setOrgToDelete(null); + }; + + const handleCreate = async (values: any) => { + try { + if (!accessToken) return; + + // Transform allowed_vector_store_ids and allowed_mcp_servers_and_groups into object_permission + if ( + (values.allowed_vector_store_ids && values.allowed_vector_store_ids.length > 0) || + (values.allowed_mcp_servers_and_groups && + (values.allowed_mcp_servers_and_groups.servers?.length > 0 || + values.allowed_mcp_servers_and_groups.accessGroups?.length > 0)) + ) { + values.object_permission = {}; + if (values.allowed_vector_store_ids && values.allowed_vector_store_ids.length > 0) { + values.object_permission.vector_stores = values.allowed_vector_store_ids; + delete values.allowed_vector_store_ids; + } + if (values.allowed_mcp_servers_and_groups) { + if (values.allowed_mcp_servers_and_groups.servers?.length > 0) { + values.object_permission.mcp_servers = values.allowed_mcp_servers_and_groups.servers; + } + if (values.allowed_mcp_servers_and_groups.accessGroups?.length > 0) { + values.object_permission.mcp_access_groups = values.allowed_mcp_servers_and_groups.accessGroups; + } + delete values.allowed_mcp_servers_and_groups; + } + } + + await organizationCreateCall(accessToken, values); + NotificationsManager.success("Organization created successfully"); + setIsOrgModalVisible(false); + form.resetFields(); + await refetchOrganizations(); + } catch (error) { + console.error("Error creating organization:", error); + } + }; + + const handleCancel = () => { + setIsOrgModalVisible(false); + form.resetFields(); + }; + + if (!premiumUser) { + return ( +
+

+ This is a LiteLLM Enterprise feature, and requires a valid key to use. Get a trial key{" "} + + here + + . +

+
+ ); + } + + return ( +
+ {(userRole === "Admin" || userRole === "Org Admin") && ( + + )} + + {selectedOrgId ? ( + { + setSelectedOrgId(null); + setEditOrg(false); + }} + accessToken={accessToken} + is_org_admin={true} + is_proxy_admin={userRole === "Admin"} + userModels={userModels} + editOrg={editOrg} + /> + ) : ( + <> +

Click on an organization ID to view its details.

+ + { + setSelectedOrgId(organizationId); + setEditOrg(true); + }} + onDeleteClick={handleDelete} + /> + + )} + + +
+ + + + + form.setFieldValue("models", values)} + context="organization" + /> + + + + + + + + daily + weekly + monthly + + + + + + + + + + + Allowed Vector Stores{" "} + + + + + } + name="allowed_vector_store_ids" + className="mt-4" + help="Select vector stores this organization can access. Leave empty for access to all vector stores" + > + form.setFieldValue("allowed_vector_store_ids", values)} + value={form.getFieldValue("allowed_vector_store_ids")} + accessToken={accessToken || ""} + placeholder="Select vector stores (optional)" + /> + + + + Allowed MCP Servers{" "} + + + + + } + name="allowed_mcp_servers_and_groups" + className="mt-4" + help="Select MCP servers and access groups this organization can access." + > + form.setFieldValue("allowed_mcp_servers_and_groups", values)} + value={form.getFieldValue("allowed_mcp_servers_and_groups")} + accessToken={accessToken || ""} + placeholder="Select MCP servers and access groups (optional)" + /> + + + + + + +
+ +
+ +
+ + +
+ ); +}; + +export default OrganizationsPanel; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.test.tsx new file mode 100644 index 00000000000..a06c5c885e3 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.test.tsx @@ -0,0 +1,188 @@ +import { render, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import React from "react"; +import { describe, expect, it, vi } from "vitest"; + +import { Organization } from "@/components/networking"; + +import OrganizationsTable from "./OrganizationsTable"; + +const makeOrganization = (overrides: Partial = {}): Organization => ({ + organization_id: "org-alpha", + organization_alias: "Alpha", + budget_id: "budget-1", + metadata: {}, + models: [], + spend: 0, + model_spend: {}, + created_at: "2023-01-01T00:00:00Z", + created_by: "someone", + updated_at: "2023-01-01T00:00:00Z", + updated_by: "someone", + litellm_budget_table: null, + teams: null, + users: null, + members: null, + ...overrides, +}); + +const baseProps = { + isLoading: false, + userRole: "Admin", + searchActive: false, + onOrganizationClick: vi.fn(), + onEditClick: vi.fn(), + onDeleteClick: vi.fn(), +}; + +describe("OrganizationsTable", () => { + it("renders every column header", () => { + render(); + for (const header of [ + "Organization ID", + "Organization Name", + "Created", + "Spend (USD)", + "Budget (USD)", + "Models", + "TPM / RPM Limits", + "Members", + ]) { + expect(screen.getByText(header)).toBeInTheDocument(); + } + }); + + it("opens the detail view when the organization ID cell is clicked", async () => { + const user = userEvent.setup(); + const onOrganizationClick = vi.fn(); + render( + , + ); + + await user.click(screen.getByText("org-123")); + + expect(onOrganizationClick).toHaveBeenCalledWith("org-123"); + }); + + it("edits and deletes an organization through the ⋯ actions menu (admin)", async () => { + const user = userEvent.setup(); + const onEditClick = vi.fn(); + const onDeleteClick = vi.fn(); + render( + , + ); + + await user.click(screen.getByTestId("organization-actions-org-9")); + await user.click(await screen.findByTestId("organization-action-edit")); + expect(onEditClick).toHaveBeenCalledWith("org-9"); + + await user.click(screen.getByTestId("organization-actions-org-9")); + await user.click(await screen.findByTestId("organization-action-delete")); + expect(onDeleteClick).toHaveBeenCalledWith("org-9"); + }); + + it("hides the row actions menu from non-admins", () => { + render( + , + ); + + expect(screen.queryByTestId("organization-actions-org-9")).not.toBeInTheDocument(); + }); + + it("sorts by created_at descending by default", () => { + render( + , + ); + + const rows = screen.getAllByRole("row"); + // rows[0] is the header row; the newest organization must lead the body. + expect(within(rows[1]).getByText("Newer")).toBeInTheDocument(); + expect(within(rows[2]).getByText("Older")).toBeInTheDocument(); + }); + + it("renders budget, limits, members, and models for a fully-populated organization", () => { + render( + , + ); + + expect(screen.getByText("$100.00")).toBeInTheDocument(); + expect(screen.getByText("TPM: 1000")).toBeInTheDocument(); + expect(screen.getByText("RPM: 60")).toBeInTheDocument(); + expect(screen.getByText("3 Members")).toBeInTheDocument(); + // Five models, three visible -> the shared ModelsCell collapses the rest. + expect(screen.getByText("+2 more")).toBeInTheDocument(); + }); + + it("shows Unlimited budget and All Proxy Models when unset", () => { + render( + , + ); + + expect(screen.getByText("All Proxy Models")).toBeInTheDocument(); + // Budget shows a standalone "Unlimited"; the limits fall back inline. + expect(screen.getByText("Unlimited")).toBeInTheDocument(); + expect(screen.getByText("TPM: Unlimited")).toBeInTheDocument(); + expect(screen.getByText("RPM: Unlimited")).toBeInTheDocument(); + }); + + it("renders loading skeletons instead of rows while loading", () => { + render( + , + ); + + expect(screen.getAllByTestId("skeleton-row").length).toBeGreaterThan(0); + expect(screen.queryByText("ShouldNotShow")).not.toBeInTheDocument(); + }); + + it("uses a search-aware empty state", () => { + const { rerender } = render(); + expect(screen.getByText("No organizations yet")).toBeInTheDocument(); + + rerender(); + expect(screen.getByText("No matching organizations")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.tsx new file mode 100644 index 00000000000..8e68a57d2f7 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.tsx @@ -0,0 +1,75 @@ +"use client"; + +import { SortingState } from "@tanstack/react-table"; +import { Building2, SearchX } from "lucide-react"; +import React, { useMemo, useState } from "react"; + +import { DataTable } from "@/components/shared/DataTable"; +import { Organization } from "@/components/networking"; + +import { getOrganizationsTableColumns } from "./OrganizationsTableColumns"; + +interface OrganizationsTableProps { + organizations: Organization[]; + isLoading: boolean; + userRole: string; + searchActive: boolean; + onOrganizationClick: (organizationId: string) => void; + onEditClick: (organizationId: string) => void; + onDeleteClick: (organizationId: string) => void; +} + +const DEFAULT_SORTING: SortingState = [{ id: "created_at", desc: true }]; + +function EmptyState({ searchActive }: { searchActive: boolean }) { + const Icon = searchActive ? SearchX : Building2; + return ( +
+
+ +
+
+ {searchActive ? "No matching organizations" : "No organizations yet"} +
+
+ {searchActive + ? "No organizations match your search. Try a different name or ID." + : "Create an organization to group teams, models, and budgets."} +
+
+ ); +} + +const OrganizationsTable: React.FC = ({ + organizations, + isLoading, + userRole, + searchActive, + onOrganizationClick, + onEditClick, + onDeleteClick, +}) => { + const [sorting, setSorting] = useState(DEFAULT_SORTING); + + const columns = useMemo(() => { + const deps = { userRole, onOrganizationClick, onEditClick, onDeleteClick }; + return getOrganizationsTableColumns(deps); + }, [userRole, onOrganizationClick, onEditClick, onDeleteClick]); + + return ( + organization.organization_id || String(index)} + sortingMode="client" + sorting={sorting} + onSortingChange={setSorting} + isLoading={isLoading} + loadingMessage="Loading organizations…" + noDataMessage={} + size="compact" + /> + ); +}; + +export default OrganizationsTable; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx new file mode 100644 index 00000000000..31f6a00916c --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx @@ -0,0 +1,186 @@ +"use client"; + +import { ColumnDef } from "@tanstack/react-table"; +import { MoreHorizontal, Pencil, Trash2 } from "lucide-react"; + +import { DataTableSortHeader } from "@/components/shared/DataTable"; +import { DateCell, IdentityCell, ModelsCell, MoneyCell } from "@/components/shared/table_cells"; +import { Organization } from "@/components/networking"; +import { buttonVariants } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { cn } from "@/lib/cva.config"; + +interface OrganizationBudget { + max_budget?: number | null; + tpm_limit?: number | null; + rpm_limit?: number | null; +} + +const getOrganizationBudget = (organization: Organization): OrganizationBudget => + (organization.litellm_budget_table ?? {}) as OrganizationBudget; + +function OrganizationLimitsCell({ organization }: { organization: Organization }) { + const { tpm_limit, rpm_limit } = getOrganizationBudget(organization); + return ( +
+ TPM: {tpm_limit ? tpm_limit : "Unlimited"} + RPM: {rpm_limit ? rpm_limit : "Unlimited"} +
+ ); +} + +interface OrganizationRowActionsProps { + organization: Organization; + onEditClick: (organizationId: string) => void; + onDeleteClick: (organizationId: string) => void; +} + +function OrganizationRowActions({ organization, onEditClick, onDeleteClick }: OrganizationRowActionsProps) { + return ( + + + + + + onEditClick(organization.organization_id)} + > + + Edit + + onDeleteClick(organization.organization_id)} + > + + Delete + + + + ); +} + +export interface OrganizationsTableColumnsDeps { + userRole: string; + onOrganizationClick: (organizationId: string) => void; + onEditClick: (organizationId: string) => void; + onDeleteClick: (organizationId: string) => void; +} + +export const getOrganizationsTableColumns = ({ + userRole, + onOrganizationClick, + onEditClick, + onDeleteClick, +}: OrganizationsTableColumnsDeps): ColumnDef[] => [ + { + id: "organization_id", + accessorKey: "organization_id", + meta: { title: "Organization ID" }, + header: ({ column }) => , + size: 220, + enableSorting: true, + cell: ({ row }) => ( + onOrganizationClick(row.original.organization_id)} + /> + ), + }, + { + id: "organization_alias", + accessorKey: "organization_alias", + meta: { title: "Organization Name" }, + header: ({ column }) => , + size: 200, + enableSorting: true, + cell: ({ row }) => { + const alias = row.original.organization_alias; + return ( + + {alias || "-"} + + ); + }, + }, + { + id: "created_at", + accessorKey: "created_at", + sortingFn: "datetime", + meta: { title: "Created" }, + header: ({ column }) => , + size: 130, + enableSorting: true, + cell: ({ row }) => , + }, + { + id: "spend", + accessorKey: "spend", + meta: { title: "Spend (USD)" }, + header: ({ column }) => , + size: 120, + enableSorting: true, + cell: ({ row }) => , + }, + { + id: "max_budget", + meta: { title: "Budget (USD)" }, + header: "Budget (USD)", + size: 120, + enableSorting: false, + cell: ({ row }) => ( + + ), + }, + { + id: "models", + meta: { title: "Models", skeleton: "chips" }, + header: "Models", + size: 260, + enableSorting: false, + cell: ({ row }) => , + }, + { + id: "limits", + meta: { title: "TPM / RPM Limits" }, + header: "TPM / RPM Limits", + size: 150, + enableSorting: false, + cell: ({ row }) => , + }, + { + id: "members", + meta: { title: "Members" }, + header: "Members", + size: 100, + enableSorting: false, + cell: ({ row }) => {row.original.members?.length ?? 0} Members, + }, + { + id: "actions", + meta: { className: "text-right", headerClassName: "text-right" }, + header: () => Actions, + size: 64, + enableSorting: false, + enableHiding: false, + cell: ({ row }) => + userRole === "Admin" ? ( +
+ +
+ ) : null, + }, +]; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/organizations.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/organizations.test.tsx deleted file mode 100644 index 75a6d30ac2e..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/organizations.test.tsx +++ /dev/null @@ -1,39 +0,0 @@ -import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { render } from "@testing-library/react"; -import React from "react"; -import { describe, expect, it, vi } from "vitest"; - -vi.mock("@/components/vector_store_management/VectorStoreSelector", () => ({ - __esModule: true, - default: () => null, -})); -vi.mock("@/components/mcp_server_management/MCPServerSelector", () => ({ - __esModule: true, - default: () => null, -})); -vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ - default: () => ({ - accessToken: null, - userId: null, - userRole: null, - }), -})); - -import OrganizationsTable from "./organizations"; - -const renderWithQueryClient = (ui: React.ReactElement) => { - const queryClient = new QueryClient({ - defaultOptions: { queries: { retry: false } }, - }); - return render({ui}); -}; - -describe("OrganizationsTable", () => { - it("should render the OrganizationsTable component", () => { - const { getByText } = renderWithQueryClient( - , - ); - - expect(getByText("+ Create New Organization")).toBeInTheDocument(); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/organizations.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/organizations.tsx deleted file mode 100644 index 87d8010759d..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/organizations.tsx +++ /dev/null @@ -1,535 +0,0 @@ -import { organizationKeys, useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrganizations"; -import { useUserModels } from "@/app/(dashboard)/hooks/models/useModels"; -import OrganizationFilters, { FilterState } from "@/app/(dashboard)/organizations/OrganizationFilters"; -import { InfoCircleOutlined } from "@ant-design/icons"; -import { ChevronDownIcon, ChevronRightIcon, RefreshIcon } from "@heroicons/react/outline"; -import { - Badge, - Button, - Card, - Col, - Grid, - Icon, - Tab, - TabGroup, - Table, - TableBody, - TableCell, - TableHead, - TableHeaderCell, - TableRow, - TabList, - TabPanel, - TabPanels, - Text, - TextInput, -} from "@tremor/react"; -import { Form, Input, Modal, Select as Select2, Tooltip } from "antd"; -import { useQueryClient } from "@tanstack/react-query"; -import React, { useState } from "react"; -import { DateCell, IdCell, MoneyCell } from "@/components/shared/table_cells"; -import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; -import TableIconActionButton from "@/components/common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; -import { getModelDisplayName } from "@/components/key_team_helpers/fetch_available_models_team_key"; -import MCPServerSelector from "@/components/mcp_server_management/MCPServerSelector"; -import { ModelSelect } from "@/components/ModelSelect/ModelSelect"; -import NotificationsManager from "@/components/molecules/notifications_manager"; -import { - Organization, - organizationCreateCall, - organizationDeleteCall, - organizationListCall, -} from "@/components/networking"; -import OrganizationInfoView from "@/components/organization/organization_view"; -import NumericalInput from "@/components/shared/numerical_input"; -import VectorStoreSelector from "@/components/vector_store_management/VectorStoreSelector"; - -interface OrganizationsTableProps { - userRole: string; - accessToken: string | null; - lastRefreshed?: string; - handleRefreshClick?: () => void; - premiumUser: boolean; -} - -export const fetchOrganizations = async ( - accessToken: string, - setOrganizations: (organizations: Organization[]) => void, - org_id: string | null = null, - org_alias: string | null = null, -) => { - const organizations = await organizationListCall(accessToken, org_id, org_alias); - setOrganizations(organizations); -}; - -const OrganizationsTable: React.FC = ({ - userRole, - accessToken, - lastRefreshed, - handleRefreshClick, - premiumUser, -}) => { - const [selectedOrgId, setSelectedOrgId] = useState(null); - const [editOrg, setEditOrg] = useState(false); - const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); - const [orgToDelete, setOrgToDelete] = useState(null); - const [isDeleting, setIsDeleting] = useState(false); - const [isOrgModalVisible, setIsOrgModalVisible] = useState(false); - const [form] = Form.useForm(); - const [expandedAccordions, setExpandedAccordions] = useState>({}); - const [showFilters, setShowFilters] = useState(false); - const [filters, setFilters] = useState({ - org_id: "", - org_alias: "", - sort_by: "created_at", - sort_order: "desc", - }); - - const queryClient = useQueryClient(); - const { data: organizations = [] } = useOrganizations({ org_id: filters.org_id, org_alias: filters.org_alias }); - const { data: userModels = [] } = useUserModels(); - - const refetchOrganizations = () => queryClient.invalidateQueries({ queryKey: organizationKeys.lists() }); - - const handleFilterChange = (key: keyof FilterState, value: string) => { - setFilters((previousFilters) => ({ ...previousFilters, [key]: value })); - }; - - const handleFilterReset = () => { - setFilters({ - org_id: "", - org_alias: "", - sort_by: "created_at", - sort_order: "desc", - }); - }; - - const handleDelete = (orgId: string | null) => { - if (!orgId) return; - - setOrgToDelete(orgId); - setIsDeleteModalOpen(true); - }; - - const confirmDelete = async () => { - if (!orgToDelete || !accessToken) return; - - try { - setIsDeleting(true); - await organizationDeleteCall(accessToken, orgToDelete); - NotificationsManager.success("Organization deleted successfully"); - - setIsDeleteModalOpen(false); - setOrgToDelete(null); - await refetchOrganizations(); - } catch (error) { - console.error("Error deleting organization:", error); - } finally { - setIsDeleting(false); - } - }; - - const cancelDelete = () => { - setIsDeleteModalOpen(false); - setOrgToDelete(null); - }; - - const handleCreate = async (values: any) => { - try { - if (!accessToken) return; - - // Transform allowed_vector_store_ids and allowed_mcp_servers_and_groups into object_permission - if ( - (values.allowed_vector_store_ids && values.allowed_vector_store_ids.length > 0) || - (values.allowed_mcp_servers_and_groups && - (values.allowed_mcp_servers_and_groups.servers?.length > 0 || - values.allowed_mcp_servers_and_groups.accessGroups?.length > 0)) - ) { - values.object_permission = {}; - if (values.allowed_vector_store_ids && values.allowed_vector_store_ids.length > 0) { - values.object_permission.vector_stores = values.allowed_vector_store_ids; - delete values.allowed_vector_store_ids; - } - if (values.allowed_mcp_servers_and_groups) { - if (values.allowed_mcp_servers_and_groups.servers?.length > 0) { - values.object_permission.mcp_servers = values.allowed_mcp_servers_and_groups.servers; - } - if (values.allowed_mcp_servers_and_groups.accessGroups?.length > 0) { - values.object_permission.mcp_access_groups = values.allowed_mcp_servers_and_groups.accessGroups; - } - delete values.allowed_mcp_servers_and_groups; - } - } - - await organizationCreateCall(accessToken, values); - NotificationsManager.success("Organization created successfully"); - setIsOrgModalVisible(false); - form.resetFields(); - await refetchOrganizations(); - } catch (error) { - console.error("Error creating organization:", error); - } - }; - - const handleCancel = () => { - setIsOrgModalVisible(false); - form.resetFields(); - }; - - if (!premiumUser) { - return ( -
- - This is a LiteLLM Enterprise feature, and requires a valid key to use. Get a trial key{" "} - - here - - . - -
- ); - } - - return ( -
- -
- {(userRole === "Admin" || userRole === "Org Admin") && ( - - )} - {selectedOrgId ? ( - { - setSelectedOrgId(null); - setEditOrg(false); - }} - accessToken={accessToken} - is_org_admin={true} // You'll need to implement proper org admin check - is_proxy_admin={userRole === "Admin"} - userModels={userModels} - editOrg={editOrg} - /> - ) : ( - - -
- Your Organizations -
-
- {lastRefreshed && Last Refreshed: {lastRefreshed}} - -
-
- - - Click on “Organization ID” to view organization details. - -
- -
-
- -
-
-
- - - Organization ID - Organization Name - Created - Spend (USD) - Budget (USD) - Models - TPM / RPM Limits - Info - Actions - - - - - {organizations && organizations.length > 0 - ? organizations - .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()) - .map((org: Organization) => ( - - - - - {org.organization_alias} - - - - - - - - - - 3 ? "px-0" : ""} - > -
- {Array.isArray(org.models) ? ( -
- {org.models.length === 0 ? ( - - All Proxy Models - - ) : ( - <> -
- {org.models.length > 3 && ( -
- { - setExpandedAccordions((prev) => ({ - ...prev, - [org.organization_id || ""]: - !prev[org.organization_id || ""], - })); - }} - /> -
- )} -
- {org.models.slice(0, 3).map((model, index) => - model === "all-proxy-models" ? ( - - All Proxy Models - - ) : ( - - - {model.length > 30 - ? `${getModelDisplayName(model).slice(0, 30)}...` - : getModelDisplayName(model)} - - - ), - )} - {org.models.length > 3 && - !expandedAccordions[org.organization_id || ""] && ( - - - +{org.models.length - 3}{" "} - {org.models.length - 3 === 1 - ? "more model" - : "more models"} - - - )} - {expandedAccordions[org.organization_id || ""] && ( -
- {org.models.slice(3).map((model, index) => - model === "all-proxy-models" ? ( - - All Proxy Models - - ) : ( - - - {model.length > 30 - ? `${getModelDisplayName(model).slice(0, 30)}...` - : getModelDisplayName(model)} - - - ), - )} -
- )} -
-
- - )} -
- ) : null} -
-
- - - TPM:{" "} - {org.litellm_budget_table?.tpm_limit - ? org.litellm_budget_table?.tpm_limit - : "Unlimited"} -
- RPM:{" "} - {org.litellm_budget_table?.rpm_limit - ? org.litellm_budget_table?.rpm_limit - : "Unlimited"} -
-
- - {org.members?.length || 0} Members - - - {userRole === "Admin" && ( - <> - { - setSelectedOrgId(org.organization_id); - setEditOrg(true); - }} - /> - handleDelete(org.organization_id)} - /> - - )} - -
- )) - : null} -
-
-
- - - - - - )} - - - -
- - - - - form.setFieldValue("models", values)} - context="organization" - /> - - - - - - - - daily - weekly - monthly - - - - - - - - - - - Allowed Vector Stores{" "} - - - - - } - name="allowed_vector_store_ids" - className="mt-4" - help="Select vector stores this organization can access. Leave empty for access to all vector stores" - > - form.setFieldValue("allowed_vector_store_ids", values)} - value={form.getFieldValue("allowed_vector_store_ids")} - accessToken={accessToken || ""} - placeholder="Select vector stores (optional)" - /> - - - - Allowed MCP Servers{" "} - - - - - } - name="allowed_mcp_servers_and_groups" - className="mt-4" - help="Select MCP servers and access groups this organization can access." - > - form.setFieldValue("allowed_mcp_servers_and_groups", values)} - value={form.getFieldValue("allowed_mcp_servers_and_groups")} - accessToken={accessToken || ""} - placeholder="Select MCP servers and access groups (optional)" - /> - - - - - - -
- -
-
-
- - -
- ); -}; - -export default OrganizationsTable; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/page.tsx index 649e54f63eb..a492a572580 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/page.tsx @@ -1,9 +1,9 @@ "use client"; -import OrganizationsTable from "./_components/organizations"; +import OrganizationsPanel from "./_components/OrganizationsPanel"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; export default function OrganizationsPage() { const { accessToken, userRole, premiumUser } = useAuthorized(); - return ; + return ; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.test.tsx new file mode 100644 index 00000000000..7f1d7edc97f --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.test.tsx @@ -0,0 +1,34 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { SearchProviderLabel } from "./CreateSearchTools"; + +describe("SearchProviderLabel", () => { + it("renders the tavily logo from the static bundle, untouched by server-root prefixing", () => { + render(); + const img = screen.getByRole("img", { name: "Tavily logo" }); + expect(img).toHaveAttribute("src", "/_next/static/media/tavily.png"); + }); + + it("renders the exa_ai logo file for the exa_ai slug", () => { + render(); + const img = screen.getByRole("img", { name: "Exa AI logo" }); + expect(img.getAttribute("src")).toContain("exa_ai.png"); + }); + + it("renders the google_pse logo file for the google_pse slug", () => { + render(); + expect(screen.getByRole("img", { name: "Google PSE logo" }).getAttribute("src")).toContain("google_pse.png"); + }); + + it("falls back to a letter avatar for a provider with no bundled logo", () => { + render(); + expect(screen.queryByRole("img")).toBeNull(); + expect(screen.getByText("B")).toBeInTheDocument(); + expect(screen.getByText("Brave Search")).toBeInTheDocument(); + }); + + it("does not guess a legacy /ui/assets/logos/.png url for unknown providers", () => { + const { container } = render(); + expect(container.querySelector("img")).toBeNull(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.tsx b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.tsx index b1cb5eb5581..1eeff00cb1b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.tsx @@ -4,44 +4,37 @@ import { useQuery } from "@tanstack/react-query"; import { Button, TextInput } from "@tremor/react"; import { Form, Input, Modal, Select, Tooltip, Typography } from "antd"; import React, { useState } from "react"; -import { resolveLogoSrc } from "@/lib/assetPaths"; +import { Logo } from "@/components/molecules/logo/Logo"; import NotificationsManager from "@/components/molecules/notifications_manager"; import { createSearchTool, fetchAvailableSearchProviders } from "@/components/networking"; import SearchConnectionTest from "./SearchConnectionTest"; import { AvailableSearchProvider, SearchTool } from "./types"; +import dataforseoLogo from "../../../../../public/assets/logos/dataforseo.png"; +import exaAiLogo from "../../../../../public/assets/logos/exa_ai.png"; +import googlePseLogo from "../../../../../public/assets/logos/google_pse.png"; +import parallelAiLogo from "../../../../../public/assets/logos/parallel_ai.png"; +import perplexityLogo from "../../../../../public/assets/logos/perplexity.png"; +import tavilyLogo from "../../../../../public/assets/logos/tavily.png"; const { TextArea } = Input; -// Search provider logos folder path (matches existing provider logo pattern) -const searchProviderLogosFolder = "/ui/assets/logos/"; - -// Helper function to get logo path for a search provider -const getSearchProviderLogo = (providerName: string): string => { - return `${searchProviderLogosFolder}${providerName}.png`; +const searchProviderLogoMap: Record = { + perplexity: perplexityLogo.src, + tavily: tavilyLogo.src, + parallel_ai: parallelAiLogo.src, + exa_ai: exaAiLogo.src, + google_pse: googlePseLogo.src, + dataforseo: dataforseoLogo.src, }; -// Component to display search provider logo and name interface SearchProviderLabelProps { providerName: string; displayName: string; } -const SearchProviderLabel: React.FC = ({ providerName, displayName }) => ( -
- {/* eslint-disable-next-line @next/next/no-img-element */} - { - e.currentTarget.style.display = "none"; - }} - /> +export const SearchProviderLabel: React.FC = ({ providerName, displayName }) => ( +
+ {displayName}
); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/tool-policies/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/tool-policies/page.tsx index 6aaebaab959..08fded8dca6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/tool-policies/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/tool-policies/page.tsx @@ -4,6 +4,6 @@ import ToolPoliciesView from "@/components/ToolPoliciesView"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; export default function ToolPolicies() { - const { accessToken, userRole } = useAuthorized(); - return ; + const { accessToken } = useAuthorized(); + return ; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx index a5e0488cbb9..cbf3a2cc1f6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx @@ -765,4 +765,37 @@ describe("EntityUsage", () => { }); expect(screen.queryByText(userUuid)).not.toBeInTheDocument(); }); + + it("renders the provider spend table logo from the bundled provider map", async () => { + render(); + + const logo = await screen.findByAltText("openai logo"); + expect(logo.getAttribute("src")).toContain("openai_small"); + }); + + it("renders a letter avatar instead of an img for an unknown provider slug", async () => { + const spendDataUnknownProvider = { + ...mockSpendData, + results: [ + { + ...mockSpendData.results[0], + breakdown: { + ...mockSpendData.results[0].breakdown, + providers: { + "zzz-internal": mockSpendData.results[0].breakdown.providers.openai, + }, + }, + }, + ], + }; + mockTagDailyActivityCall.mockResolvedValue(spendDataUnknownProvider); + + render(); + + await waitFor(() => { + expect(screen.getAllByText("zzz-internal").length).toBeGreaterThan(0); + }); + expect(screen.queryByAltText("zzz-internal logo")).not.toBeInTheDocument(); + expect(screen.getByText("z")).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx index 94a9cedbdf5..534e2be7fe8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx @@ -38,7 +38,7 @@ import { teamDailyActivityCall, userDailyActivityCall, } from "@/components/networking"; -import { getProviderLogoAndName } from "@/components/provider_info_helpers"; +import { Logo } from "@/components/molecules/logo/Logo"; import { usePaginatedDailyActivity } from "../../hooks/usePaginatedDailyActivity"; import { BreakdownMetrics, @@ -774,24 +774,7 @@ const EntityUsage: React.FC = ({ accessToken, entityType, enti
- {provider.provider && ( - {`${provider.provider} { - const target = e.target as HTMLImageElement; - const parent = target.parentElement; - if (parent) { - const fallbackDiv = document.createElement("div"); - fallbackDiv.className = - "w-4 h-4 rounded-full bg-gray-200 flex items-center justify-center text-xs"; - fallbackDiv.textContent = provider.provider?.charAt(0) || "-"; - parent.replaceChild(fallbackDiv, target); - } - }} - /> - )} + {provider.provider && } {provider.provider}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.test.tsx index 996fd58efc1..8dc11babd72 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.test.tsx @@ -1,31 +1,17 @@ -import React from "react"; -import { render, waitFor, screen, fireEvent } from "@testing-library/react"; -import { describe, it, expect, vi, beforeEach } from "vitest"; +/* @vitest-environment jsdom */ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import React from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + import ViewUserDashboard from "./view_users"; +const userListCall = vi.fn(); + // Mock the networking module vi.mock("@/components/networking", () => ({ - userListCall: vi.fn().mockResolvedValue({ - users: [ - { - user_id: "user-1", - user_email: "test@example.com", - user_role: "Admin", - spend: 100.5, - max_budget: null, - key_count: 2, - created_at: "2024-01-01T00:00:00Z", - updated_at: "2024-01-01T00:00:00Z", - sso_user_id: null, - budget_duration: null, - }, - ], - total: 1, - page: 1, - page_size: 25, - total_pages: 1, - }), + userListCall: (...args: unknown[]) => userListCall(...args), userDeleteCall: vi.fn().mockResolvedValue({}), getPossibleUserRoles: vi.fn().mockResolvedValue({ Admin: { ui_label: "Admin" }, @@ -44,6 +30,13 @@ vi.mock("@/components/networking", () => ({ getInternalUserSettings: vi.fn().mockResolvedValue({}), })); +// The detail view has its own test; stub it so this file covers the parent's swap. +vi.mock("./view_users/user_info_view", () => ({ + default: function UserInfoViewMock({ userId, startInEditMode }: { userId: string; startInEditMode?: boolean }) { + return
{`detail:${userId}:${String(Boolean(startInEditMode))}`}
; + }, +})); + // Mock NotificationsManager vi.mock("@/components/molecules/notifications_manager", () => ({ default: { @@ -52,6 +45,21 @@ vi.mock("@/components/molecules/notifications_manager", () => ({ }, })); +const makeUser = (userId: string, email: string) => ({ + user_id: userId, + user_email: email, + user_alias: null, + user_role: "Admin", + spend: 100.5, + max_budget: null, + models: [], + key_count: 2, + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-01-01T00:00:00Z", + sso_user_id: null, + budget_duration: null, +}); + const createQueryClient = () => new QueryClient({ defaultOptions: { @@ -62,105 +70,194 @@ const createQueryClient = () => }, }); -describe("ViewUserDashboard", () => { - const defaultProps = { - accessToken: "test-token", - token: "test-token", - userRole: "Admin", - userID: "admin-user-id", - teams: [], - }; +const defaultProps = { + accessToken: "test-token", + token: "test-token", + userRole: "Admin", + userID: "admin-user-id", + teams: [], +}; +const renderDashboard = () => + render( + + + , + ); + +describe("ViewUserDashboard", () => { beforeEach(() => { vi.clearAllMocks(); + userListCall.mockResolvedValue({ + users: [makeUser("user-1", "test@example.com")], + total: 1, + page: 1, + page_size: 25, + total_pages: 1, + }); }); it("should render the ViewUserDashboard component", async () => { - const queryClient = createQueryClient(); - render( - - - , - ); + renderDashboard(); - // Wait for the component to load (it shows "Loading..." initially) await waitFor(() => { expect(screen.getByText("Users")).toBeInTheDocument(); }); - // Check if main elements are rendered - expect(screen.getByText("Users")).toBeInTheDocument(); - // Use getAllByText since "Default User Settings" appears multiple times - const defaultUserSettingsTabs = screen.getAllByText("Default User Settings"); - expect(defaultUserSettingsTabs.length).toBeGreaterThan(0); + expect(screen.getAllByText("Default User Settings").length).toBeGreaterThan(0); }); - it("should show delete modal after clicking delete user button", async () => { - const queryClient = createQueryClient(); - render( - - - , - ); + it("should show delete modal after choosing delete from the row actions menu", async () => { + const user = userEvent.setup(); + renderDashboard(); - // Wait for the component to load and the table to render await waitFor(() => { - expect(screen.getByText("Users")).toBeInTheDocument(); + expect(screen.getByText("test@example.com")).toBeInTheDocument(); }); - // Wait for the user data to load - await waitFor(() => { - expect(screen.getAllByText("test@example.com").length).toBeGreaterThan(0); - }); - - // Initially, the delete modal should not be visible expect(screen.queryByText("Delete User?")).not.toBeInTheDocument(); - // Find the row containing the user email (use the first one which is in the table) - // The email appears in both the table and potentially in modals, so get the first one from the table - const userEmailCells = screen.getAllByText("test@example.com"); - const userEmailCell = userEmailCells[0]; // First occurrence is in the table - const userRow = userEmailCell.closest("tr"); - expect(userRow).toBeInTheDocument(); - - // Find clickable elements in the actions column (the last column) - const actionCells = userRow?.querySelectorAll("td"); - const actionsCell = actionCells?.[actionCells.length - 1]; - expect(actionsCell).toBeInTheDocument(); - - // Find the action container div with flex gap-2 - const actionContainer = - actionsCell?.querySelector("div.flex.gap-2") || - Array.from(actionsCell?.querySelectorAll("div") || []).find( - (div) => div.className.includes("flex") && div.className.includes("gap"), - ); - - expect(actionContainer).toBeInTheDocument(); - - // Get all direct children of the action container - // These should be Tooltip components wrapping Icon components - const tooltipWrappers = Array.from(actionContainer!.children); - expect(tooltipWrappers.length).toBeGreaterThanOrEqual(2); - - // The delete icon is the second tooltip wrapper (index 1) - // Edit=0, Delete=1, Reset=2 - const deleteTooltipWrapper = tooltipWrappers[1] as HTMLElement; - const clickableElement = deleteTooltipWrapper.querySelector("button, [role='button'], svg") as HTMLElement; - - expect(clickableElement).toBeInTheDocument(); - - fireEvent.click(clickableElement); + await user.click(screen.getByTestId("user-actions-user-1")); + await user.click(await screen.findByTestId("user-action-delete")); await waitFor(() => { expect(screen.getByText("Delete User?")).toBeInTheDocument(); }); - expect( screen.getByText("Are you sure you want to delete this user? This action cannot be undone."), ).toBeInTheDocument(); - const userIdInstances = screen.getAllByText("user-1"); - expect(userIdInstances.length).toBeGreaterThan(0); - const emailInstances = screen.getAllByText("test@example.com"); - expect(emailInstances.length).toBeGreaterThan(0); + expect(screen.getAllByText("user-1").length).toBeGreaterThan(0); + }); + + it("should swap to the detail view when the identity cell is clicked", async () => { + const user = userEvent.setup(); + renderDashboard(); + + await waitFor(() => { + expect(screen.getByText("test@example.com")).toBeInTheDocument(); + }); + + await user.click(screen.getByRole("button", { name: /user-1/ })); + + expect(await screen.findByTestId("user-info-view")).toHaveTextContent("detail:user-1:false"); + expect(screen.queryByText("test@example.com")).not.toBeInTheDocument(); + }); + + it("should open the detail view in edit mode from the row actions menu", async () => { + const user = userEvent.setup(); + renderDashboard(); + + await waitFor(() => { + expect(screen.getByText("test@example.com")).toBeInTheDocument(); + }); + + await user.click(screen.getByTestId("user-actions-user-1")); + await user.click(await screen.findByTestId("user-action-edit")); + + expect(await screen.findByTestId("user-info-view")).toHaveTextContent("detail:user-1:true"); + }); + + describe("bulk edit selection", () => { + beforeEach(() => { + userListCall.mockResolvedValue({ + users: [makeUser("user-1", "ada@example.com"), makeUser("user-2", "grace@example.com")], + total: 2, + page: 1, + page_size: 25, + total_pages: 1, + }); + }); + + it("reveals selection checkboxes only while selection mode is on", async () => { + const user = userEvent.setup(); + renderDashboard(); + + await waitFor(() => { + expect(screen.getByText("ada@example.com")).toBeInTheDocument(); + }); + + expect(screen.queryByTestId("datatable-select-all")).not.toBeInTheDocument(); + + await user.click(screen.getByTestId("toggle-user-selection")); + expect(screen.getByTestId("datatable-select-all")).toBeInTheDocument(); + + await user.click(screen.getByTestId("toggle-user-selection")); + expect(screen.queryByTestId("datatable-select-all")).not.toBeInTheDocument(); + }); + + it("counts the selected rows in the bulk edit button and enables it once a row is picked", async () => { + const user = userEvent.setup(); + renderDashboard(); + + await waitFor(() => { + expect(screen.getByText("ada@example.com")).toBeInTheDocument(); + }); + + await user.click(screen.getByTestId("toggle-user-selection")); + + const bulkEdit = screen.getByTestId("bulk-edit-users"); + expect(bulkEdit).toHaveTextContent("Bulk Edit (0 selected)"); + expect(bulkEdit).toBeDisabled(); + + await user.click(screen.getByTestId("datatable-select-row-user-2")); + expect(screen.getByTestId("bulk-edit-users")).toHaveTextContent("Bulk Edit (1 selected)"); + expect(screen.getByTestId("bulk-edit-users")).not.toBeDisabled(); + + await user.click(screen.getByTestId("datatable-select-all")); + expect(screen.getByTestId("bulk-edit-users")).toHaveTextContent("Bulk Edit (2 selected)"); + }); + + it("clears the selection when selection mode is cancelled", async () => { + const user = userEvent.setup(); + renderDashboard(); + + await waitFor(() => { + expect(screen.getByText("ada@example.com")).toBeInTheDocument(); + }); + + await user.click(screen.getByTestId("toggle-user-selection")); + await user.click(screen.getByTestId("datatable-select-row-user-1")); + expect(screen.getByTestId("bulk-edit-users")).toHaveTextContent("Bulk Edit (1 selected)"); + + await user.click(screen.getByTestId("toggle-user-selection")); + await user.click(screen.getByTestId("toggle-user-selection")); + + expect(screen.getByTestId("bulk-edit-users")).toHaveTextContent("Bulk Edit (0 selected)"); + }); + }); + + describe("server-side query wiring", () => { + it("requests page 1 with the default created_at desc sort", async () => { + renderDashboard(); + + await waitFor(() => { + expect(userListCall).toHaveBeenCalled(); + }); + + const [, userIds, page, pageSize, , , , , sortBy, sortOrder] = userListCall.mock.calls[0]; + expect(userIds).toBeNull(); + expect(page).toBe(1); + expect(pageSize).toBe(25); + expect(sortBy).toBe("created_at"); + expect(sortOrder).toBe("desc"); + }); + + it("sends the clicked column as sort_by and resets to the first page", async () => { + const user = userEvent.setup(); + renderDashboard(); + + await waitFor(() => { + expect(screen.getByText("test@example.com")).toBeInTheDocument(); + }); + + await user.click(screen.getByTestId("sort-header-user_email")); + + await waitFor(() => { + const latest = userListCall.mock.calls[userListCall.mock.calls.length - 1]; + expect(latest[8]).toBe("user_email"); + expect(latest[9]).toBe("asc"); + expect(latest[2]).toBe(1); + }); + }); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.tsx index db3b17d6af3..ce912c09373 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.tsx @@ -1,5 +1,5 @@ import { Tab, TabGroup, TabList, TabPanel, TabPanels } from "@tremor/react"; -import React, { useEffect, useState } from "react"; +import React, { useCallback, useEffect, useMemo, useState } from "react"; import { Button } from "antd"; import BulkEditUserModal from "./BulkEditUsers"; @@ -18,20 +18,24 @@ import OnboardingModal, { InvitationLink } from "@/components/onboarding_link"; import { updateExistingKeys } from "@/utils/dataUtils"; import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants"; import { isAdminRole, isProxyAdminRole } from "@/utils/roles"; -import { useDebouncedState } from "@tanstack/react-pacer/debouncer"; +import { useDebouncedValue } from "@tanstack/react-pacer/debouncer"; import { useQuery, useQueryClient } from "@tanstack/react-query"; -import { Typography } from "antd"; +import { + ColumnFiltersState, + OnChangeFn, + PaginationState, + RowSelectionState, + SortingState, +} from "@tanstack/react-table"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; import NotificationsManager from "@/components/molecules/notifications_manager"; import { modelAvailableCall, userDeleteCall } from "@/components/networking"; import DefaultUserSettings from "./DefaultUserSettings"; -import { columns } from "./view_users/columns"; -import { UserDataTable } from "./view_users/table"; +import { UsersTable } from "./view_users/UsersTable"; +import UserInfoView from "./view_users/user_info_view"; import { UserInfo } from "@/components/networking"; import { Skeleton } from "antd"; -const { Text, Title } = Typography; - interface ViewUserDashboardProps { accessToken: string | null; token: string | null; @@ -41,33 +45,11 @@ interface ViewUserDashboardProps { orgAdminOrgIds?: Array<{ organization_id: string; organization_alias: string }> | null; } -interface FilterState { - email: string; - user_id: string; - user_role: string; - sso_user_id: string; - team: string; - model: string; - min_spend: number | null; - max_spend: number | null; - sort_by: string; - sort_order: "asc" | "desc"; -} - const DEFAULT_PAGE_SIZE = 25; -const initialFilters: FilterState = { - email: "", - user_id: "", - user_role: "", - sso_user_id: "", - team: "", - model: "", - min_spend: null, - max_spend: null, - sort_by: "created_at", - sort_order: "desc", -}; +const DEFAULT_SORT_BY = "created_at"; + +const DEFAULT_SORTING: SortingState = [{ id: DEFAULT_SORT_BY, desc: true }]; const ViewUserDashboard: React.FC = ({ accessToken, @@ -79,34 +61,30 @@ const ViewUserDashboard: React.FC = ({ }) => { const isProxyAdmin = userRole ? isProxyAdminRole(userRole) : false; const queryClient = useQueryClient(); - const [currentPage, setCurrentPage] = useState(1); + + const [pagination, setPagination] = useState({ pageIndex: 0, pageSize: DEFAULT_PAGE_SIZE }); + const [sorting, setSorting] = useState(DEFAULT_SORTING); + const [columnFilters, setColumnFilters] = useState([]); + const [searchInput, setSearchInput] = useState(""); + const [searchEmail] = useDebouncedValue(searchInput, { wait: DEBOUNCE_WAIT_MS }); + + const [rowSelection, setRowSelection] = useState({}); + const [selectionMode, setSelectionMode] = useState(false); + const [isBulkEditModalVisible, setIsBulkEditModalVisible] = useState(false); + + const [selectedUserId, setSelectedUserId] = useState(null); + const [openInEditMode, setOpenInEditMode] = useState(false); + const [editModalVisible, setEditModalVisible] = useState(false); const [selectedUser, setSelectedUser] = useState(null); const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); const [isDeletingUser, setIsDeletingUser] = useState(false); const [userToDelete, setUserToDelete] = useState(null); - const [activeTab, setActiveTab] = useState("users"); - const [filters, setFilters] = useState(initialFilters); - const [debouncedFilters, setDebouncedFilters, debouncer] = useDebouncedState(filters, { wait: DEBOUNCE_WAIT_MS }); const [isInvitationLinkModalVisible, setIsInvitationLinkModalVisible] = useState(false); const [invitationLinkData, setInvitationLinkData] = useState(null); const [baseUrl, setBaseUrl] = useState(null); - const [selectedUsers, setSelectedUsers] = useState([]); - const [isBulkEditModalVisible, setIsBulkEditModalVisible] = useState(false); - const [selectionMode, setSelectionMode] = useState(false); const [userModels, setUserModels] = useState([]); - const handleDelete = (user: UserInfo) => { - setUserToDelete(user); - setIsDeleteModalOpen(true); - }; - - useEffect(() => { - return () => { - debouncer.cancel(); - }; - }, [debouncer]); - useEffect(() => { setBaseUrl(getProxyBaseUrl()); }, []); @@ -130,32 +108,69 @@ const ViewUserDashboard: React.FC = ({ fetchUserModels(); }, [accessToken, userID, userRole]); - const updateFilters = (update: Partial) => { - setFilters((previousFilters) => { - const newFilters = { ...previousFilters, ...update }; - setDebouncedFilters(newFilters); - return newFilters; - }); - }; + const getFilterValue = useCallback( + (columnId: string): string | undefined => { + const entry = columnFilters.find((filter) => filter.id === columnId); + return typeof entry?.value === "string" && entry.value.trim() ? entry.value.trim() : undefined; + }, + [columnFilters], + ); - const handleSortChange = (sortBy: string, sortOrder: "asc" | "desc") => { - updateFilters({ sort_by: sortBy, sort_order: sortOrder }); - }; + const handleSearchChange = useCallback((value: string) => { + setSearchInput(value); + setPagination((previous) => ({ ...previous, pageIndex: 0 })); + setRowSelection({}); + }, []); - const handleResetPassword = async (userId: string) => { - if (!accessToken) { - NotificationsManager.fromBackend("Access token not found"); - return; - } - try { - NotificationsManager.success("Generating password reset link..."); - const data = await invitationCreateCall(accessToken, userId); - setInvitationLinkData(data); - setIsInvitationLinkModalVisible(true); - } catch (error) { - NotificationsManager.fromBackend("Failed to generate password reset link"); - } - }; + const handleSortingChange = useCallback>((updaterOrValue) => { + setSorting(updaterOrValue); + setPagination((previous) => ({ ...previous, pageIndex: 0 })); + setRowSelection({}); + }, []); + + const handleColumnFiltersChange = useCallback>((updaterOrValue) => { + setColumnFilters(updaterOrValue); + setPagination((previous) => ({ ...previous, pageIndex: 0 })); + setRowSelection({}); + }, []); + + const handlePaginationChange = useCallback>((updaterOrValue) => { + setPagination(updaterOrValue); + setRowSelection({}); + }, []); + + const handleUserClick = useCallback((userId: string, openInEdit: boolean = false) => { + setSelectedUserId(userId); + setOpenInEditMode(openInEdit); + }, []); + + const handleCloseUserInfo = useCallback(() => { + setSelectedUserId(null); + setOpenInEditMode(false); + }, []); + + const handleDelete = useCallback((user: UserInfo) => { + setUserToDelete(user); + setIsDeleteModalOpen(true); + }, []); + + const handleResetPassword = useCallback( + async (userId: string) => { + if (!accessToken) { + NotificationsManager.fromBackend("Access token not found"); + return; + } + try { + NotificationsManager.success("Generating password reset link..."); + const data = await invitationCreateCall(accessToken, userId); + setInvitationLinkData(data); + setIsInvitationLinkModalVisible(true); + } catch (error) { + NotificationsManager.fromBackend("Failed to generate password reset link"); + } + }, + [accessToken], + ); const confirmDelete = async () => { if (userToDelete && accessToken) { @@ -220,58 +235,63 @@ const ViewUserDashboard: React.FC = ({ // Close the modal }; - const handlePageChange = async (newPage: number) => { - setCurrentPage(newPage); - }; - const handleToggleSelectionMode = () => { setSelectionMode(!selectionMode); - setSelectedUsers([]); - }; - - const handleSelectionChange = (users: UserInfo[]) => { - setSelectedUsers(users); - }; - - const handleBulkEdit = () => { - if (selectedUsers.length === 0) { - NotificationsManager.fromBackend("Please select users to edit"); - return; - } - - setIsBulkEditModalVisible(true); + setRowSelection({}); }; const handleBulkEditSuccess = () => { // Refresh the user list queryClient.invalidateQueries({ queryKey: ["userList"] }); - setSelectedUsers([]); + setRowSelection({}); setSelectionMode(false); }; + const activeSort = sorting[0]; + const sortBy = activeSort?.id ?? DEFAULT_SORT_BY; + const sortOrder: "asc" | "desc" = activeSort?.desc ?? true ? "desc" : "asc"; + + const userIdFilter = getFilterValue("user_id"); + const ssoUserIdFilter = getFilterValue("sso_user_id"); + const userRoleFilter = getFilterValue("user_role"); + const teamFilter = getFilterValue("team"); + const emailFilter = searchEmail.trim() || null; + + const userListQueryFilters = { + page: pagination.pageIndex + 1, + pageSize: pagination.pageSize, + email: emailFilter, + userId: userIdFilter, + ssoUserId: ssoUserIdFilter, + role: userRoleFilter, + team: teamFilter, + sortBy, + sortOrder, + orgAdminOrgIds, + }; + const userListQuery = useQuery({ - queryKey: ["userList", { debouncedFilter: debouncedFilters, currentPage, orgAdminOrgIds }], + queryKey: ["userList", userListQueryFilters], queryFn: async () => { if (!accessToken) throw new Error("Access token required"); return await userListCall( accessToken, - debouncedFilters.user_id ? [debouncedFilters.user_id] : null, - currentPage, - DEFAULT_PAGE_SIZE, - debouncedFilters.email || null, - debouncedFilters.user_role || null, - debouncedFilters.team || null, - debouncedFilters.sso_user_id || null, - debouncedFilters.sort_by, - debouncedFilters.sort_order, + userIdFilter ? [userIdFilter] : null, + pagination.pageIndex + 1, + pagination.pageSize, + emailFilter, + userRoleFilter ?? null, + teamFilter ?? null, + ssoUserIdFilter ?? null, + sortBy, + sortOrder, orgAdminOrgIds ? orgAdminOrgIds.map((o) => o.organization_id) : null, ); }, enabled: Boolean(accessToken && token && userRole && userID), placeholderData: (previousData) => previousData, }); - const userListResponse = userListQuery.data; const userRolesQuery = useQuery>>({ queryKey: ["userRoles"], @@ -284,28 +304,61 @@ const ViewUserDashboard: React.FC = ({ }); const possibleUIRoles = userRolesQuery.data; - const tableColumns = columns( - possibleUIRoles, - (user) => { - setSelectedUser(user); - setEditModalVisible(true); - }, - handleDelete, - handleResetPassword, - () => {}, // placeholder function, will be overridden in UserDataTable + const users = useMemo(() => userListQuery.data?.users ?? [], [userListQuery.data]); + const totalUserCount = userListQuery.data?.total ?? 0; + + const selectedUsers = useMemo(() => users.filter((user) => rowSelection[user.user_id]), [users, rowSelection]); + + if (selectedUserId) { + return ( + + ); + } + + const usersTable = ( + ); return (
- {userListQuery.isLoading ? ( + {userListQuery.isLoading && ( <> - ) : userID && accessToken ? ( + )} + {!userListQuery.isLoading && userID && accessToken && ( <> {isProxyAdmin && ( = ({ onClick={handleToggleSelectionMode} type={selectionMode ? "primary" : "default"} className="flex items-center" + data-testid="toggle-user-selection" > {selectionMode ? "Cancel Selection" : "Select Users"} @@ -329,57 +383,28 @@ const ViewUserDashboard: React.FC = ({ {isProxyAdmin && selectionMode && ( )} - ) : null} + )}
{isProxyAdmin ? ( - setActiveTab(index === 0 ? "users" : "settings")}> + Users Default User Settings - - { - setSelectedUser(user); - setEditModalVisible(true); - }} - handleDelete={handleDelete} - handleResetPassword={handleResetPassword} - enableSelection={selectionMode} - selectedUsers={selectedUsers} - onSelectionChange={handleSelectionChange} - filters={filters} - updateFilters={updateFilters} - initialFilters={initialFilters} - teams={teams} - userListResponse={userListResponse} - currentPage={currentPage} - handlePageChange={handlePageChange} - /> - + {usersTable} {!userID || !userRole || !accessToken ? ( @@ -398,35 +423,7 @@ const ViewUserDashboard: React.FC = ({ ) : ( - { - setSelectedUser(user); - setEditModalVisible(true); - }} - handleDelete={handleDelete} - handleResetPassword={handleResetPassword} - enableSelection={false} - selectedUsers={[]} - onSelectionChange={handleSelectionChange} - filters={filters} - updateFilters={updateFilters} - initialFilters={initialFilters} - teams={teams} - userListResponse={userListResponse} - currentPage={currentPage} - handlePageChange={handlePageChange} - /> + usersTable )} {/* Existing Modals */} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/UsersTable.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/UsersTable.test.tsx new file mode 100644 index 00000000000..4689ef7cf96 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/UsersTable.test.tsx @@ -0,0 +1,272 @@ +/* @vitest-environment jsdom */ +import type { PaginationState, RowSelectionState, SortingState } from "@tanstack/react-table"; +import { render, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { useState } from "react"; +import { describe, expect, it, vi } from "vitest"; + +import { UserInfo } from "@/components/networking"; + +import { UsersTable } from "./UsersTable"; + +const possibleUIRoles = { + proxy_admin: { ui_label: "Admin" }, + internal_user: { ui_label: "Internal User" }, +}; + +const makeUser = (overrides: Partial = {}): UserInfo => + ({ + user_id: "user-1", + user_email: "ada@example.com", + user_alias: null, + user_role: "proxy_admin", + spend: 12.5, + max_budget: null, + models: [], + key_count: 2, + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-02-01T00:00:00Z", + sso_user_id: null, + budget_duration: null, + ...overrides, + }) as UserInfo; + +interface HarnessOverrides { + data?: UserInfo[]; + rowCount?: number; + isLoading?: boolean; + selectionEnabled?: boolean; + onUserClick?: (userId: string, openInEditMode?: boolean) => void; + onDeleteUser?: (user: UserInfo) => void; + onResetPassword?: (userId: string) => void; + onSortingChange?: ReturnType; +} + +/** + * Renders the table with real selection/sorting state so assertions exercise the + * controlled wiring rather than a stubbed callback. + */ +function Harness({ + data = [makeUser()], + rowCount = 1, + isLoading = false, + selectionEnabled = false, + onUserClick = vi.fn(), + onDeleteUser = vi.fn(), + onResetPassword = vi.fn(), + onSortingChange, +}: HarnessOverrides) { + const [sorting, setSorting] = useState([{ id: "created_at", desc: true }]); + const [pagination, setPagination] = useState({ pageIndex: 0, pageSize: 25 }); + const [rowSelection, setRowSelection] = useState({}); + + return ( + <> + + {Object.keys(rowSelection) + .filter((key) => rowSelection[key]) + .sort() + .join(",")} + + { + setSorting(updater); + onSortingChange?.(updater); + }} + pagination={pagination} + onPaginationChange={setPagination} + columnFilters={[]} + onColumnFiltersChange={vi.fn()} + searchValue="" + onSearchChange={vi.fn()} + selectionEnabled={selectionEnabled} + rowSelection={rowSelection} + onRowSelectionChange={setRowSelection} + onUserClick={onUserClick} + onDeleteUser={onDeleteUser} + onResetPassword={onResetPassword} + /> + + ); +} + +const openRowMenu = async (user: ReturnType, userId: string) => { + await user.click(screen.getByTestId(`user-actions-${userId}`)); +}; + +describe("UsersTable", () => { + it("renders every migrated column header", () => { + render(); + + const headerRow = screen.getAllByRole("row")[0]; + + [ + "User ID", + "Email", + "Status", + "Global Proxy Role", + "User Alias", + "Spend (USD)", + "Budget (USD)", + "SSO ID", + "Virtual Keys", + "Created At", + "Updated At", + ].forEach((header) => { + expect(headerRow.textContent).toContain(header); + }); + }); + + // Sorting is server-side and the backend only accepts these five keys, so a sort + // control on any other column would send an invalid sort_by. Assert the exact set: + // a missing control and an extra one both have to fail. + it("exposes a sort control for exactly the five server-sortable columns", () => { + render(); + + const sortableIds = screen + .getAllByTestId(/^sort-header-/) + .map((node) => (node.getAttribute("data-testid") ?? "").replace("sort-header-", "")) + .sort(); + + expect(sortableIds).toEqual(["created_at", "spend", "user_email", "user_id", "user_role"]); + }); + + it("reports the clicked column to the server sorting handler", async () => { + const user = userEvent.setup(); + const onSortingChange = vi.fn(); + render(); + + await user.click(screen.getByTestId("sort-header-user_email")); + + expect(onSortingChange).toHaveBeenCalledTimes(1); + expect(screen.getByTestId("sort-header-user_email").querySelector("[data-sort-indicator]")).toHaveAttribute( + "data-sort-indicator", + "asc", + ); + }); + + it("opens the detail view from the identity cell without edit mode", async () => { + const user = userEvent.setup(); + const onUserClick = vi.fn(); + render(); + + await user.click(screen.getByRole("button", { name: /user-1/ })); + + expect(onUserClick).toHaveBeenCalledWith("user-1", false); + }); + + it("opens the detail view in edit mode from the row menu", async () => { + const user = userEvent.setup(); + const onUserClick = vi.fn(); + render(); + + await openRowMenu(user, "user-1"); + await user.click(await screen.findByTestId("user-action-edit")); + + expect(onUserClick).toHaveBeenCalledWith("user-1", true); + }); + + it("delegates delete and reset-password from the row menu", async () => { + const user = userEvent.setup(); + const onDeleteUser = vi.fn(); + const onResetPassword = vi.fn(); + render(); + + await openRowMenu(user, "user-1"); + await user.click(await screen.findByTestId("user-action-reset-password")); + expect(onResetPassword).toHaveBeenCalledWith("user-1"); + + await openRowMenu(user, "user-1"); + await user.click(await screen.findByTestId("user-action-delete")); + expect(onDeleteUser).toHaveBeenCalledWith(expect.objectContaining({ user_id: "user-1" })); + }); + + it("renders the SCIM status cell from metadata", () => { + const { rerender } = render(); + expect(screen.getByTestId("user-status-user-1")).toHaveTextContent("Active"); + + rerender()]} />); + expect(screen.getByTestId("user-status-user-1")).toHaveTextContent("Inactive"); + + rerender()]} />); + expect(screen.getByTestId("user-status-user-1")).toHaveTextContent("Active"); + }); + + describe("row selection", () => { + const twoUsers = [ + makeUser({ user_id: "user-1", user_email: "ada@example.com" }), + makeUser({ user_id: "user-2", user_email: "grace@example.com" }), + ]; + + it("hides the selection column until selection mode is on", () => { + const { rerender } = render(); + expect(screen.queryByTestId("datatable-select-all")).not.toBeInTheDocument(); + expect(screen.queryByTestId("datatable-select-row-user-1")).not.toBeInTheDocument(); + + rerender(); + expect(screen.getByTestId("datatable-select-all")).toBeInTheDocument(); + expect(screen.getByTestId("datatable-select-row-user-1")).toBeInTheDocument(); + }); + + it("keys the controlled selection by user id", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByTestId("datatable-select-row-user-2")); + expect(screen.getByTestId("selected-ids")).toHaveTextContent("user-2"); + + await user.click(screen.getByTestId("datatable-select-row-user-1")); + expect(screen.getByTestId("selected-ids")).toHaveTextContent("user-1,user-2"); + + await user.click(screen.getByTestId("datatable-select-row-user-2")); + expect(screen.getByTestId("selected-ids")).toHaveTextContent("user-1"); + }); + + it("selects and clears the whole page from the header checkbox", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByTestId("datatable-select-all")); + expect(screen.getByTestId("selected-ids")).toHaveTextContent("user-1,user-2"); + + await user.click(screen.getByTestId("datatable-select-all")); + expect(screen.getByTestId("selected-ids")).toBeEmptyDOMElement(); + }); + + it("shows an indeterminate header while only part of the page is selected", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByTestId("datatable-select-row-user-1")); + + expect(screen.getByTestId("datatable-select-all")).toHaveAttribute("aria-checked", "mixed"); + }); + }); + + it("renders the empty state when there are no users", () => { + render(); + + expect(screen.getByText("No users found")).toBeInTheDocument(); + }); + + it("shows skeleton rows on the initial load instead of the empty state", () => { + render(); + + expect(screen.getAllByTestId("skeleton-row").length).toBeGreaterThan(0); + expect(screen.queryByText("No users found")).not.toBeInTheDocument(); + }); + + it("keeps the row menu out of the identity cell so only the name and menu act on a row", () => { + render(); + + const rows = screen.getAllByRole("row"); + const dataRow = rows[rows.length - 1]; + expect(within(dataRow).getByTestId("user-actions-user-1")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/UsersTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/UsersTable.tsx new file mode 100644 index 00000000000..26663f5d9e2 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/UsersTable.tsx @@ -0,0 +1,216 @@ +"use client"; + +import { + ColumnFiltersState, + OnChangeFn, + PaginationState, + RowSelectionState, + SortingState, +} from "@tanstack/react-table"; +import { Users } from "lucide-react"; +import { useMemo, useState } from "react"; + +import { UserInfo } from "@/components/networking"; +import { + DataTable, + DataTableFilterDrawer, + DataTableFilterField, + DataTableToolbar, +} from "@/components/shared/DataTable"; +import { SearchSelect } from "@/components/shared/SearchSelect"; +import { Input } from "@/components/ui/input"; + +import { getUsersTableColumns } from "./UsersTableColumns"; + +export interface UsersTableTeamOption { + team_id: string; + team_alias?: string | null; +} + +interface UsersTableProps { + data: UserInfo[]; + rowCount: number; + isLoading: boolean; + possibleUIRoles: Record> | null; + teams: UsersTableTeamOption[] | null; + sorting: SortingState; + onSortingChange: OnChangeFn; + pagination: PaginationState; + onPaginationChange: OnChangeFn; + columnFilters: ColumnFiltersState; + onColumnFiltersChange: OnChangeFn; + searchValue: string; + onSearchChange: (value: string) => void; + selectionEnabled: boolean; + rowSelection: RowSelectionState; + onRowSelectionChange: OnChangeFn; + onUserClick: (userId: string, openInEditMode?: boolean) => void; + onDeleteUser: (user: UserInfo) => void; + onResetPassword: (userId: string) => void; +} + +const FILTER_LABELS: Record = { + user_id: "User ID", + sso_user_id: "SSO ID", + user_role: "Role", + team: "Team", +}; + +function EmptyState() { + return ( +
+
+ +
+
No users found
+
Try adjusting your search or filters.
+
+ ); +} + +export function UsersTable({ + data, + rowCount, + isLoading, + possibleUIRoles, + teams, + sorting, + onSortingChange, + pagination, + onPaginationChange, + columnFilters, + onColumnFiltersChange, + searchValue, + onSearchChange, + selectionEnabled, + rowSelection, + onRowSelectionChange, + onUserClick, + onDeleteUser, + onResetPassword, +}: UsersTableProps) { + const [filtersOpen, setFiltersOpen] = useState(false); + + const columns = useMemo(() => { + const columnDeps = { + possibleUIRoles, + includeSelection: selectionEnabled, + onUserClick, + onDeleteUser, + onResetPassword, + }; + return getUsersTableColumns(columnDeps); + }, [possibleUIRoles, selectionEnabled, onUserClick, onDeleteUser, onResetPassword]); + + const roleOptions = useMemo( + () => + Object.entries(possibleUIRoles ?? {}).map(([role, config]) => ({ + label: config.ui_label || role, + value: role, + })), + [possibleUIRoles], + ); + + const teamOptions = useMemo( + () => + (teams ?? []).map((team) => ({ + label: team.team_alias || team.team_id, + value: team.team_id, + })), + [teams], + ); + + const formatFilterValue = (columnId: string, value: unknown): string => { + const raw = String(value); + if (columnId === "user_role") { + return possibleUIRoles?.[raw]?.ui_label || raw; + } + if (columnId === "team") { + return teams?.find((team) => team.team_id === raw)?.team_alias || raw; + } + return raw; + }; + + return ( + row.user_id} + sortingMode="server" + sorting={sorting} + onSortingChange={onSortingChange} + paginationMode="server" + pagination={pagination} + onPaginationChange={onPaginationChange} + rowCount={rowCount} + filterMode="server" + columnFilters={columnFilters} + onColumnFiltersChange={onColumnFiltersChange} + rowSelection={rowSelection} + onRowSelectionChange={onRowSelectionChange} + isLoading={isLoading} + loadingMessage="Loading users…" + noDataMessage={} + size="compact" + toolbar={(table) => ( + <> + setFiltersOpen(true)} + filterLabels={FILTER_LABELS} + formatFilterValue={formatFilterValue} + /> + + {({ get, set }) => ( + <> + + set("user_id", event.target.value)} + placeholder="Enter user ID…" + data-testid="users-filter-user-id" + /> + + + set("sso_user_id", event.target.value)} + placeholder="Enter SSO ID…" + data-testid="users-filter-sso-id" + /> + + + set("user_role", value)} + placeholder="Select a role…" + emptyText="No roles found" + /> + + + set("team", value)} + placeholder="Select a team…" + emptyText="No teams found" + /> + + + )} + + + )} + /> + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/UsersTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/UsersTableColumns.tsx new file mode 100644 index 00000000000..6c569f205e5 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/UsersTableColumns.tsx @@ -0,0 +1,272 @@ +"use client"; + +import { ColumnDef } from "@tanstack/react-table"; +import { Copy, Info, KeyRound, MoreHorizontal, Pencil, Trash2 } from "lucide-react"; + +import { UserInfo } from "@/components/networking"; +import { createSelectionColumn, DataTableSortHeader } from "@/components/shared/DataTable"; +import { CellTooltip, DateCell, IdentityCell, MoneyCell, StatusBadge } from "@/components/shared/table_cells"; +import { Badge } from "@/components/ui/badge"; +import { buttonVariants } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { cn } from "@/lib/cva.config"; +import { copyToClipboard } from "@/utils/dataUtils"; + +const SSO_ID_HINT = + "SSO ID is the ID of the user in the SSO provider. If the user is not using SSO, this will be null."; + +const SCIM_INACTIVE_HINT = "Deactivated via SCIM (external identity provider). The user's virtual keys are blocked."; + +function isScimInactive(user: UserInfo): boolean { + return (user.metadata as Record | null | undefined)?.scim_active === false; +} + +interface UserRowActionsProps { + user: UserInfo; + onUserClick: (userId: string, openInEditMode?: boolean) => void; + onDeleteUser: (user: UserInfo) => void; + onResetPassword: (userId: string) => void; +} + +function UserRowActions({ user, onUserClick, onDeleteUser, onResetPassword }: UserRowActionsProps) { + return ( + + + + + + onUserClick(user.user_id, true)} data-testid="user-action-edit"> + + Edit user + + onResetPassword(user.user_id)} data-testid="user-action-reset-password"> + + Reset password + + void copyToClipboard(user.user_id, "User ID copied")} + data-testid="user-action-copy" + > + + Copy user ID + + + onDeleteUser(user)} data-testid="user-action-delete"> + + Delete user + + + + ); +} + +export interface UsersTableColumnsDeps { + possibleUIRoles: Record> | null; + includeSelection: boolean; + onUserClick: (userId: string, openInEditMode?: boolean) => void; + onDeleteUser: (user: UserInfo) => void; + onResetPassword: (userId: string) => void; +} + +export const getUsersTableColumns = ({ + possibleUIRoles, + includeSelection, + onUserClick, + onDeleteUser, + onResetPassword, +}: UsersTableColumnsDeps): ColumnDef[] => { + const baseColumns: ColumnDef[] = [ + { + id: "user_id", + accessorKey: "user_id", + meta: { title: "User ID" }, + header: ({ column }) => , + size: 220, + enableSorting: true, + cell: ({ row }) => ( + onUserClick(row.original.user_id, false)} + /> + ), + }, + { + id: "user_email", + accessorKey: "user_email", + meta: { title: "Email" }, + header: ({ column }) => , + size: 220, + enableSorting: true, + cell: ({ row }) => ( + + {row.original.user_email || "-"} + + ), + }, + { + id: "status", + meta: { title: "Status", skeleton: "badge" }, + header: "Status", + size: 110, + enableSorting: false, + cell: ({ row }) => { + if (isScimInactive(row.original)) { + return ( + + ); + } + return ; + }, + }, + { + id: "user_role", + accessorKey: "user_role", + meta: { title: "Global Proxy Role" }, + header: ({ column }) => , + size: 160, + enableSorting: true, + cell: ({ row }) => {possibleUIRoles?.[row.original.user_role]?.ui_label || "-"}, + }, + { + id: "user_alias", + accessorKey: "user_alias", + meta: { title: "User Alias" }, + header: "User Alias", + size: 150, + enableSorting: false, + cell: ({ row }) => ( + + {row.original.user_alias || "-"} + + ), + }, + { + id: "spend", + accessorKey: "spend", + meta: { title: "Spend (USD)", numeric: true }, + header: ({ column }) => , + size: 130, + enableSorting: true, + cell: ({ row }) => , + }, + { + id: "max_budget", + accessorKey: "max_budget", + meta: { title: "Budget (USD)", numeric: true }, + header: "Budget (USD)", + size: 130, + enableSorting: false, + cell: ({ row }) => , + }, + { + id: "sso_user_id", + accessorKey: "sso_user_id", + meta: { title: "SSO ID" }, + header: () => ( + + SSO ID + } + /> + + ), + size: 160, + enableSorting: false, + cell: ({ row }) => ( + + {row.original.sso_user_id ?? "-"} + + ), + }, + { + id: "key_count", + accessorKey: "key_count", + meta: { title: "Virtual Keys", skeleton: "badge" }, + header: "Virtual Keys", + size: 120, + enableSorting: false, + cell: ({ row }) => { + const keyCount = row.original.key_count; + if (keyCount > 0) { + return ( + + {keyCount} {keyCount === 1 ? "Key" : "Keys"} + + ); + } + return ( + + No Keys + + ); + }, + }, + { + id: "created_at", + accessorKey: "created_at", + meta: { title: "Created At" }, + header: ({ column }) => , + size: 130, + enableSorting: true, + cell: ({ row }) => , + }, + { + id: "updated_at", + accessorKey: "updated_at", + meta: { title: "Updated At" }, + header: "Updated At", + size: 130, + enableSorting: false, + cell: ({ row }) => , + }, + { + id: "actions", + meta: { title: "Actions", className: "text-right", headerClassName: "text-right" }, + header: () => Actions, + size: 60, + enableSorting: false, + enableHiding: false, + cell: ({ row }) => ( +
+ +
+ ), + }, + ]; + + if (!includeSelection) { + return baseColumns; + } + + return [ + createSelectionColumn({ + rowAriaLabel: (row) => `Select ${row.original.user_email || row.original.user_id}`, + }), + ...baseColumns, + ]; +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/columns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/columns.tsx deleted file mode 100644 index fc680cb5b1b..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/columns.tsx +++ /dev/null @@ -1,195 +0,0 @@ -import { ColumnDef } from "@tanstack/react-table"; -import { Badge, Grid, Icon } from "@tremor/react"; -import { Tooltip, Checkbox, Tag } from "antd"; -import { UserInfo } from "@/components/networking"; -import { PencilAltIcon, TrashIcon, InformationCircleIcon, RefreshIcon } from "@heroicons/react/outline"; -import { DateCell, IdCell, MoneyCell } from "@/components/shared/table_cells"; - -interface SelectionOptions { - selectedUsers: UserInfo[]; - onSelectUser: (user: UserInfo, isSelected: boolean) => void; - onSelectAll: (isSelected: boolean) => void; - isUserSelected: (user: UserInfo) => boolean; - isAllSelected: boolean; - isIndeterminate: boolean; -} - -export const columns = ( - possibleUIRoles: Record>, - handleEdit: (user: UserInfo) => void, - handleDelete: (user: UserInfo) => void, - handleResetPassword: (userId: string) => void, - handleUserClick: (userId: string, openInEditMode?: boolean) => void, - selectionOptions?: SelectionOptions, -): ColumnDef[] => { - // Backend sortable columns: user_id, user_email, created_at, spend, user_alias, user_role - const baseColumns: ColumnDef[] = [ - { - header: "User ID", - accessorKey: "user_id", - enableSorting: true, - cell: ({ row }) => , - }, - { - header: "Email", - accessorKey: "user_email", - enableSorting: true, - cell: ({ row }) => {row.original.user_email || "-"}, - }, - { - id: "status", - header: "Status", - enableSorting: false, - cell: ({ row }) => { - const isScimInactive = - (row.original.metadata as Record | null | undefined)?.scim_active === false; - if (isScimInactive) { - return ( - - - Inactive - - - ); - } - return ( - - Active - - ); - }, - }, - { - header: "Global Proxy Role", - accessorKey: "user_role", - enableSorting: true, - cell: ({ row }) => {possibleUIRoles?.[row.original.user_role]?.ui_label || "-"}, - }, - { - header: "User Alias", - accessorKey: "user_alias", - enableSorting: false, - cell: ({ row }) => {row.original.user_alias || "-"}, - }, - { - header: "Spend (USD)", - accessorKey: "spend", - enableSorting: true, - cell: ({ row }) => , - }, - { - header: "Budget (USD)", - accessorKey: "max_budget", - enableSorting: false, - cell: ({ row }) => , - }, - { - header: () => ( -
- SSO ID - - - -
- ), - accessorKey: "sso_user_id", - enableSorting: false, - cell: ({ row }) => ( - {row.original.sso_user_id !== null ? row.original.sso_user_id : "-"} - ), - }, - { - header: "Virtual Keys", - accessorKey: "key_count", - enableSorting: false, - cell: ({ row }) => ( - - {row.original.key_count > 0 ? ( - - {row.original.key_count} {row.original.key_count === 1 ? "Key" : "Keys"} - - ) : ( - - No Keys - - )} - - ), - }, - { - header: "Created At", - accessorKey: "created_at", - enableSorting: true, - cell: ({ row }) => , - }, - { - header: "Updated At", - accessorKey: "updated_at", - enableSorting: false, - cell: ({ row }) => , - }, - { - id: "actions", - header: "Actions", - enableSorting: false, - cell: ({ row }) => ( -
- - handleUserClick(row.original.user_id, true)} - className="cursor-pointer hover:text-blue-600" - /> - - - handleDelete(row.original)} - className="cursor-pointer hover:text-red-600" - /> - - - handleResetPassword(row.original.user_id)} - className="cursor-pointer hover:text-green-600" - /> - -
- ), - }, - ]; - - // Add selection column if selection is enabled - if (selectionOptions) { - const { onSelectUser, onSelectAll, isUserSelected, isAllSelected, isIndeterminate } = selectionOptions; - - return [ - { - id: "select", - enableSorting: false, - header: () => ( - onSelectAll(e.target.checked)} - onClick={(e) => e.stopPropagation()} - /> - ), - cell: ({ row }) => ( - onSelectUser(row.original, e.target.checked)} - onClick={(e) => e.stopPropagation()} - /> - ), - }, - ...baseColumns, - ]; - } - - return baseColumns; -}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/table.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/table.test.tsx deleted file mode 100644 index 695aaa30cd7..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/table.test.tsx +++ /dev/null @@ -1,201 +0,0 @@ -import { act, fireEvent, render, screen } from "@testing-library/react"; -import { describe, expect, it, vi } from "vitest"; -import { columns } from "./columns"; -import { UserDataTable } from "./table"; -import { UserInfo } from "@/components/networking"; - -const defaultFilters = { - email: "", - user_id: "", - user_role: "", - sso_user_id: "", - team: "", - model: "", - min_spend: null, - max_spend: null, - sort_by: "", - sort_order: "asc" as const, -}; - -const getDefaultProps = () => ({ - data: [] as any[], - columns: [] as any[], - accessToken: null, - userRole: "Admin", - possibleUIRoles: null as Record> | null, - filters: defaultFilters, - updateFilters: vi.fn(), - initialFilters: defaultFilters, - teams: [] as any[], - handleEdit: vi.fn(), - handleDelete: vi.fn(), - handleResetPassword: vi.fn(), - userListResponse: { users: [], total: 0, page: 1, page_size: 25, total_pages: 1 }, - currentPage: 1, - handlePageChange: vi.fn(), -}); - -describe("UserDataTable", () => { - it("should render the UserDataTable component", () => { - render(); - - expect(screen.getByText("Filters")).toBeInTheDocument(); - }); - - it("should call onSortChange when clicking a sortable header", () => { - const filters = { - ...defaultFilters, - sort_by: "created_at", - sort_order: "desc" as const, - }; - - const onSortChange = vi.fn(); - - const possibleUIRoles = { - admin: { ui_label: "Admin" }, - user: { ui_label: "User" }, - }; - - render( - , - ); - - const emailHeader = screen.getByRole("columnheader", { name: /email/i }); - act(() => { - fireEvent.click(emailHeader); - }); - - expect(onSortChange).toHaveBeenCalledWith("user_email", "desc"); - }); - - it("should show skeleton loaders when isLoading is true", () => { - render(); - - expect(screen.queryByText(/Showing/i)).not.toBeInTheDocument(); - expect(screen.queryByRole("button", { name: /Previous/i })).not.toBeInTheDocument(); - expect(screen.queryByRole("button", { name: /Next/i })).not.toBeInTheDocument(); - }); - - it("should show actual content when isLoading is false", () => { - render(); - - expect(screen.getByText(/Showing/i)).toBeInTheDocument(); - expect(screen.getByRole("button", { name: /Previous/i })).toBeInTheDocument(); - expect(screen.getByRole("button", { name: /Next/i })).toBeInTheDocument(); - }); - - it("should render all column headers", () => { - const possibleUIRoles = { - admin: { ui_label: "Admin" }, - user: { ui_label: "User" }, - }; - - render(); - - [ - "User ID", - "Email", - "Status", - "Global Proxy Role", - "User Alias", - "Spend (USD)", - "Budget (USD)", - "SSO ID", - "Virtual Keys", - "Created At", - "Updated At", - "Actions", - ].forEach((header) => { - expect(screen.getByRole("columnheader", { name: header })).toBeInTheDocument(); - }); - }); - - it("should render the user-row Status cell as Active when scim_active is not set to false", () => { - const possibleUIRoles = { admin: { ui_label: "Admin" } }; - const handlers = { edit: vi.fn(), del: vi.fn(), reset: vi.fn(), click: vi.fn() }; - const cols = columns(possibleUIRoles, handlers.edit, handlers.del, handlers.reset, handlers.click); - const statusCol = cols.find((c) => (c as { id?: string }).id === "status"); - expect(statusCol).toBeDefined(); - - const baseUser: UserInfo = { - user_id: "u-active", - user_email: "active@example.com", - user_alias: null, - user_role: "admin", - spend: 0, - max_budget: null, - models: [], - key_count: 0, - created_at: "", - updated_at: "", - sso_user_id: null, - budget_duration: null, - }; - - const cellNoMetadata = (statusCol as any).cell({ row: { original: baseUser } }); - render(<>{cellNoMetadata}); - expect(screen.getByText("Active")).toBeInTheDocument(); - expect(screen.queryByText("Inactive")).not.toBeInTheDocument(); - }); - - it("should render the user-row Status cell as Inactive when scim_active is false", () => { - const possibleUIRoles = { admin: { ui_label: "Admin" } }; - const cols = columns(possibleUIRoles, vi.fn(), vi.fn(), vi.fn(), vi.fn()); - const statusCol = cols.find((c) => (c as { id?: string }).id === "status")!; - - const inactiveUser: UserInfo = { - user_id: "u-inactive", - user_email: "alex@acme.io", - user_alias: null, - user_role: "internal_user", - spend: 0, - max_budget: null, - models: [], - key_count: 1, - created_at: "", - updated_at: "", - sso_user_id: null, - budget_duration: null, - metadata: { scim_active: false }, - }; - - const cell = (statusCol as any).cell({ row: { original: inactiveUser } }); - render(<>{cell}); - expect(screen.getByText("Inactive")).toBeInTheDocument(); - expect(screen.queryByText("Active")).not.toBeInTheDocument(); - }); - - it("should treat scim_active=true as Active (not Inactive)", () => { - const possibleUIRoles = { admin: { ui_label: "Admin" } }; - const cols = columns(possibleUIRoles, vi.fn(), vi.fn(), vi.fn(), vi.fn()); - const statusCol = cols.find((c) => (c as { id?: string }).id === "status")!; - - const reactivated: UserInfo = { - user_id: "u-rehired", - user_email: "alex@acme.io", - user_alias: null, - user_role: "internal_user", - spend: 0, - max_budget: null, - models: [], - key_count: 1, - created_at: "", - updated_at: "", - sso_user_id: null, - budget_duration: null, - metadata: { scim_active: true }, - }; - - const cell = (statusCol as any).cell({ row: { original: reactivated } }); - render(<>{cell}); - expect(screen.getByText("Active")).toBeInTheDocument(); - expect(screen.queryByText("Inactive")).not.toBeInTheDocument(); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/table.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/table.tsx deleted file mode 100644 index 1ba09243a08..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/table.tsx +++ /dev/null @@ -1,442 +0,0 @@ -import { ColumnDef, flexRender, getCoreRowModel, SortingState, useReactTable } from "@tanstack/react-table"; -import React from "react"; -import { Table, TableHead, TableHeaderCell, TableBody, TableRow, TableCell, Select, SelectItem } from "@tremor/react"; -import { SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon } from "@heroicons/react/outline"; -import { Skeleton } from "antd"; -import { UserInfo } from "@/components/networking"; -import UserInfoView from "./user_info_view"; -import { columns as createColumns } from "./columns"; -import { FilterInput } from "@/components/common_components/Filters/FilterInput"; -import { FiltersButton } from "@/components/common_components/Filters/FiltersButton"; -import { ResetFiltersButton } from "@/components/common_components/Filters/ResetFiltersButton"; -import { Search, User, CircleUserRound } from "lucide-react"; - -interface FilterState { - email: string; - user_id: string; - user_role: string; - sso_user_id: string; - team: string; - model: string; - min_spend: number | null; - max_spend: number | null; - sort_by: string; - sort_order: "asc" | "desc"; -} - -interface UserDataTableProps { - data: UserInfo[]; - columns: ColumnDef[]; - isLoading?: boolean; - onSortChange?: (sortBy: string, sortOrder: "asc" | "desc") => void; - currentSort?: { - sortBy: string; - sortOrder: "asc" | "desc"; - }; - accessToken: string | null; - userRole: string | null; - possibleUIRoles: Record> | null; - handleEdit: (user: UserInfo) => void; - handleDelete: (user: UserInfo) => void; - handleResetPassword: (userId: string) => void; - selectedUsers?: UserInfo[]; - onSelectionChange?: (selectedUsers: UserInfo[]) => void; - enableSelection?: boolean; - // Filter-related props - filters: FilterState; - updateFilters: (update: Partial) => void; - initialFilters: FilterState; - teams: any[] | null; - // Pagination props - userListResponse: any; - currentPage: number; - handlePageChange: (newPage: number) => void; -} - -export function UserDataTable({ - data = [], - columns: originalColumns, - isLoading = false, - onSortChange, - currentSort, - accessToken, - userRole, - possibleUIRoles, - handleEdit, - handleDelete, - handleResetPassword, - selectedUsers = [], - onSelectionChange, - enableSelection = false, - filters, - updateFilters, - initialFilters, - teams, - userListResponse, - currentPage, - handlePageChange, -}: UserDataTableProps) { - const [sorting, setSorting] = React.useState([ - { - id: currentSort?.sortBy || "created_at", - desc: currentSort?.sortOrder === "desc", - }, - ]); - const [selectedUserId, setSelectedUserId] = React.useState(null); - const [openInEditMode, setOpenInEditMode] = React.useState(false); - const [showFilters, setShowFilters] = React.useState(false); - - const handleUserClick = (userId: string, openInEditMode: boolean = false) => { - setSelectedUserId(userId); - setOpenInEditMode(openInEditMode); - }; - - const handleCloseUserInfo = () => { - setSelectedUserId(null); - setOpenInEditMode(false); - }; - - // Selection handlers - const handleSelectUser = (user: UserInfo, isSelected: boolean) => { - if (!onSelectionChange) return; - - if (isSelected) { - onSelectionChange([...selectedUsers, user]); - } else { - onSelectionChange(selectedUsers.filter((u) => u.user_id !== user.user_id)); - } - }; - - const handleSelectAll = (isSelected: boolean) => { - if (!onSelectionChange) return; - - if (isSelected) { - onSelectionChange(data); - } else { - onSelectionChange([]); - } - }; - - const isUserSelected = (user: UserInfo) => { - return selectedUsers.some((u) => u.user_id === user.user_id); - }; - - const isAllSelected = data.length > 0 && selectedUsers.length === data.length; - const isIndeterminate = selectedUsers.length > 0 && selectedUsers.length < data.length; - - // Create columns with the handleUserClick function - const columns = React.useMemo(() => { - if (possibleUIRoles) { - return createColumns( - possibleUIRoles, - handleEdit, - handleDelete, - handleResetPassword, - handleUserClick, - enableSelection - ? { - selectedUsers, - onSelectUser: handleSelectUser, - onSelectAll: handleSelectAll, - isUserSelected, - isAllSelected, - isIndeterminate, - } - : undefined, - ); - } - return originalColumns; - }, [ - possibleUIRoles, - handleEdit, - handleDelete, - handleResetPassword, - handleUserClick, - originalColumns, - enableSelection, - selectedUsers, - isAllSelected, - isIndeterminate, - ]); - - const table = useReactTable({ - data, - columns, - state: { - sorting, - }, - onSortingChange: (updaterOrValue: any) => { - const newSorting = typeof updaterOrValue === "function" ? updaterOrValue(sorting) : updaterOrValue; - setSorting(newSorting); - if (newSorting && Array.isArray(newSorting) && newSorting.length > 0 && newSorting[0]) { - const sortState = newSorting[0]; - if (sortState.id) { - const sortBy = sortState.id; - const sortOrder = sortState.desc ? "desc" : "asc"; - onSortChange?.(sortBy, sortOrder); - } - } else { - // Reset to default sort when no sorting is selected - onSortChange?.("created_at", "desc"); - } - }, - getCoreRowModel: getCoreRowModel(), - manualSorting: true, - enableSorting: true, - }); - - // Update local sorting state when currentSort prop changes - React.useEffect(() => { - if (currentSort) { - setSorting([ - { - id: currentSort.sortBy, - desc: currentSort.sortOrder === "desc", - }, - ]); - } - }, [currentSort]); - - if (selectedUserId) { - return ( - - ); - } - - return ( -
- {/* Filter Section */} -
-
- {/* Search and Filter Controls */} -
- {/* Email Search */} - updateFilters({ email: value })} - icon={Search} - /> - - {/* Filter Button */} - setShowFilters(!showFilters)} - active={showFilters} - hasActiveFilters={!!(filters.user_id || filters.user_role || filters.team)} - /> - - {/* Reset Filters Button */} - { - updateFilters(initialFilters); - }} - /> -
- - {/* Additional Filters */} - {showFilters && ( -
- {/* User ID Search */} - updateFilters({ user_id: value })} - icon={User} - /> - - updateFilters({ sso_user_id: value })} - icon={CircleUserRound} - /> - - {/* Role Dropdown */} -
- -
- - {/* Team Dropdown */} -
- -
-
- )} - - {/* Results Count and Pagination */} -
- {isLoading ? ( - - ) : ( - - Showing{" "} - {userListResponse && userListResponse.users && userListResponse.users.length > 0 - ? (userListResponse.page - 1) * userListResponse.page_size + 1 - : 0}{" "} - -{" "} - {userListResponse && userListResponse.users - ? Math.min(userListResponse.page * userListResponse.page_size, userListResponse.total) - : 0}{" "} - of {userListResponse ? userListResponse.total : 0} results - - )} - - {/* Pagination Buttons */} -
- {isLoading ? ( - <> - - - - ) : ( - <> - - - - )} -
-
-
-
- - {/* Table Section */} -
-
-
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => ( - -
-
- {header.isPlaceholder - ? null - : flexRender(header.column.columnDef.header, header.getContext())} -
- {header.id !== "actions" && header.column.getCanSort() && ( -
- {header.column.getIsSorted() ? ( - { - asc: , - desc: , - }[header.column.getIsSorted() as string] - ) : ( - - )} -
- )} -
-
- ))} -
- ))} -
- - {isLoading ? ( - - -
-

🚅 Loading users...

-
-
-
- ) : data.length > 0 ? ( - table.getRowModel().rows.map((row) => ( - - {row.getVisibleCells().map((cell) => ( - { - if (cell.column.id === "user_id") { - handleUserClick(cell.getValue() as string, false); - } - }} - style={{ - cursor: cell.column.id === "user_id" ? "pointer" : "default", - color: cell.column.id === "user_id" ? "#3b82f6" : "inherit", - }} - > - {flexRender(cell.column.columnDef.cell, cell.getContext())} - - ))} - - )) - ) : ( - - -
-

No users found

-
-
-
- )} -
-
-
-
-
-
- ); -} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx index b5f4b56986d..79ba9b77c14 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx @@ -14,7 +14,7 @@ import { getProviderSpecificFields, VectorStoreFieldConfig, } from "@/components/vector_store_providers"; -import { resolveLogoSrc } from "@/lib/assetPaths"; +import { Logo } from "@/components/molecules/logo/Logo"; import NotificationsManager from "@/components/molecules/notifications_manager"; import S3VectorsConfig from "./S3VectorsConfig"; @@ -294,22 +294,10 @@ const CreateVectorStore: React.FC = ({ accessToken, onSu return (
- {`${providerEnum} { - // Create a div with provider initial as fallback - const target = e.target as HTMLImageElement; - const parent = target.parentElement; - if (parent) { - const fallbackDiv = document.createElement("div"); - fallbackDiv.className = - "w-5 h-5 rounded-full bg-gray-200 flex items-center justify-center text-xs"; - fallbackDiv.textContent = providerDisplayName.charAt(0); - parent.replaceChild(fallbackDiv, target); - } - }} /> {providerDisplayName}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx index e371cbfba42..3eb013d47ad 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx @@ -1,27 +1,34 @@ import { render, screen } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; import { CredentialItem } from "@/components/networking"; +import { Providers, providerLogoMap } from "@/components/provider_info_helpers"; +import { VectorStoreProviders } from "@/components/vector_store_providers"; import VectorStoreForm from "./VectorStoreForm"; vi.mock("@/components/networking"); +const renderForm = () => + render( + , + ); + describe("VectorStoreForm", () => { it("should render the form when visible", () => { - const mockOnCancel = vi.fn(); - const mockOnSuccess = vi.fn(); - const mockAccessToken = "test-token"; - const mockCredentials: CredentialItem[] = []; - - render( - , - ); + renderForm(); expect(screen.getByText("Add New Vector Store")).toBeInTheDocument(); }); + + it("renders the default provider's bundled logo via the shared Logo component", () => { + renderForm(); + + const logo = screen.getByRole("img", { name: `${VectorStoreProviders.Bedrock} logo` }); + expect(logo.getAttribute("src")).toBe(providerLogoMap[Providers.Bedrock]); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx index 82417286738..6cdb895b98d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx @@ -10,7 +10,7 @@ import { getProviderSpecificFields, VectorStoreFieldConfig, } from "@/components/vector_store_providers"; -import { resolveLogoSrc } from "@/lib/assetPaths"; +import { Logo } from "@/components/molecules/logo/Logo"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; import NotificationsManager from "@/components/molecules/notifications_manager"; @@ -130,22 +130,10 @@ const VectorStoreForm: React.FC = ({ return (
- {`${providerEnum} { - // Create a div with provider initial as fallback - const target = e.target as HTMLImageElement; - const parent = target.parentElement; - if (parent) { - const fallbackDiv = document.createElement("div"); - fallbackDiv.className = - "w-5 h-5 rounded-full bg-gray-200 flex items-center justify-center text-xs"; - fallbackDiv.textContent = providerDisplayName.charAt(0); - parent.replaceChild(fallbackDiv, target); - } - }} /> {providerDisplayName}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/vector_store_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/vector_store_info.tsx index 7c27b347eb1..ec20a3fd318 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/vector_store_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/vector_store_info.tsx @@ -10,8 +10,9 @@ import { CredentialItem, } from "@/components/networking"; import { VectorStore } from "@/components/vector_store_management/types"; -import { Providers, providerLogoMap, provider_map } from "@/components/provider_info_helpers"; -import { resolveLogoSrc } from "@/lib/assetPaths"; +import { Providers, provider_map } from "@/components/provider_info_helpers"; +import { getVectorStoreProviderLogoAndName } from "@/components/vector_store_providers"; +import { Logo } from "@/components/molecules/logo/Logo"; import VectorStoreTester from "./VectorStoreTester"; import NotificationsManager from "@/components/molecules/notifications_manager"; @@ -181,23 +182,7 @@ const VectorStoreInfoView: React.FC = ({ return (
- {`${providerEnum} { - // Create a div with provider initial as fallback - const target = e.target as HTMLImageElement; - const parent = target.parentElement; - if (parent) { - const fallbackDiv = document.createElement("div"); - fallbackDiv.className = - "w-5 h-5 rounded-full bg-gray-200 flex items-center justify-center text-xs"; - fallbackDiv.textContent = providerDisplayName.charAt(0); - parent.replaceChild(fallbackDiv, target); - } - }} - /> + {providerDisplayName}
@@ -292,43 +277,11 @@ const VectorStoreInfoView: React.FC = ({
{(() => { const provider = vectorStoreDetails.custom_llm_provider || "bedrock"; - const { displayName, logo } = (() => { - // Find the enum key by matching provider_map values - const enumKey = Object.keys(provider_map).find( - (key) => provider_map[key].toLowerCase() === provider.toLowerCase(), - ); - - if (!enumKey) { - return { displayName: provider, logo: "" }; - } - - // Get the display name from Providers enum and logo from map - const displayName = Providers[enumKey as keyof typeof Providers]; - const logo = resolveLogoSrc(providerLogoMap[displayName]) ?? ""; - - return { displayName, logo }; - })(); + const { displayName, logo } = getVectorStoreProviderLogoAndName(provider); return ( <> - {logo && ( - {`${displayName} { - const target = e.target as HTMLImageElement; - const parent = target.parentElement; - if (parent) { - const fallbackDiv = document.createElement("div"); - fallbackDiv.className = - "w-5 h-5 rounded-full bg-gray-200 flex items-center justify-center text-xs"; - fallbackDiv.textContent = displayName.charAt(0); - parent.replaceChild(fallbackDiv, target); - } - }} - /> - )} + {displayName} ); diff --git a/ui/litellm-dashboard/src/components/SSOModals.test.tsx b/ui/litellm-dashboard/src/components/SSOModals.test.tsx index 365d23f4036..792a964f01c 100644 --- a/ui/litellm-dashboard/src/components/SSOModals.test.tsx +++ b/ui/litellm-dashboard/src/components/SSOModals.test.tsx @@ -472,4 +472,43 @@ describe("SSOModals", () => { expect(NotificationsManager.success).toHaveBeenCalledWith("SSO settings cleared successfully"); expect(mockHandleAddSSOOk).toHaveBeenCalled(); }); + + it("renders provider logos in the SSO provider dropdown", async () => { + const TestWrapper = () => { + const [form] = Form.useForm(); + return ( + {}} + handleAddSSOCancel={() => {}} + handleShowInstructions={() => {}} + handleInstructionsOk={() => {}} + handleInstructionsCancel={() => {}} + form={form} + accessToken={null} + ssoConfigured={false} + /> + ); + }; + + render(); + + fireEvent.mouseDown(screen.getByLabelText("SSO Provider")); + + await waitFor(() => { + expect(screen.getAllByAltText("Google SSO logo").length).toBeGreaterThan(0); + }); + + expect(screen.getAllByAltText("Google SSO logo")[0]).toHaveAttribute("src", expect.stringContaining("google.svg")); + expect(screen.getAllByAltText("Microsoft SSO logo")[0]).toHaveAttribute( + "src", + expect.stringContaining("microsoft_azure.svg"), + ); + expect(screen.getAllByAltText("Okta / Auth0 SSO logo")[0]).toHaveAttribute( + "src", + expect.stringContaining("https://www.okta.com/"), + ); + expect(screen.queryByAltText("Generic SSO logo")).not.toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/SSOModals.tsx b/ui/litellm-dashboard/src/components/SSOModals.tsx index 9b6cd40e0c6..88ce72573d7 100644 --- a/ui/litellm-dashboard/src/components/SSOModals.tsx +++ b/ui/litellm-dashboard/src/components/SSOModals.tsx @@ -4,6 +4,8 @@ import { Text, TextInput } from "@tremor/react"; import { getSSOSettings, updateSSOSettings } from "./networking"; import NotificationsManager from "./molecules/notifications_manager"; import { parseErrorMessage } from "./shared/errorUtils"; +import { Logo } from "@/components/molecules/logo/Logo"; +import { ssoProviderDisplayNames, ssoProviderLogoMap } from "./Settings/AdminSettings/SSOSettings/constants"; interface SSOModalsProps { isAddSSOModalVisible: boolean; @@ -18,13 +20,6 @@ interface SSOModalsProps { ssoConfigured?: boolean; // Add optional prop to indicate if SSO is configured } -const ssoProviderLogoMap: Record = { - google: "https://artificialanalysis.ai/img/logos/google_small.svg", - microsoft: "https://upload.wikimedia.org/wikipedia/commons/a/a8/Microsoft_Azure_Logo.svg", - okta: "https://www.okta.com/sites/default/files/Okta_Logo_BrightBlue_Medium.png", - generic: "", -}; - // Define the SSO provider configuration type interface SSOProviderConfig { envVarMap: Record; @@ -340,17 +335,14 @@ const SSOModals: React.FC = ({
{logo && ( - {value} )} - {value.toLowerCase() === "okta" - ? "Okta / Auth0" - : value.charAt(0).toUpperCase() + value.slice(1)}{" "} - SSO + {ssoProviderDisplayNames[value] || value.charAt(0).toUpperCase() + value.slice(1) + " SSO"}
diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/BaseSSOSettingsForm.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/BaseSSOSettingsForm.test.tsx index c68e2716f5b..21132fff63b 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/BaseSSOSettingsForm.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/BaseSSOSettingsForm.test.tsx @@ -293,4 +293,40 @@ describe("renderProviderFields", () => { expect(result).not.toBeNull(); expect(result?.length).toBe(5); }); + + it("renders provider logos in the dropdown and falls back to a letter avatar on load error", async () => { + const TestWrapper = () => { + const [form] = Form.useForm(); + return ; + }; + + renderWithProviders(); + + await act(async () => { + fireEvent.mouseDown(screen.getByLabelText("SSO Provider")); + }); + + await waitFor(() => { + expect(screen.getAllByAltText("Google SSO logo").length).toBeGreaterThan(0); + }); + + expect(screen.getAllByAltText("Google SSO logo")[0]).toHaveAttribute("src", expect.stringContaining("google.svg")); + expect(screen.getAllByAltText("Microsoft SSO logo")[0]).toHaveAttribute( + "src", + expect.stringContaining("microsoft_azure.svg"), + ); + expect(screen.queryByAltText("Generic SSO logo")).not.toBeInTheDocument(); + + const oktaLogo = screen.getAllByAltText("Okta / Auth0 SSO logo")[0]; + expect(oktaLogo).toHaveAttribute("src", expect.stringContaining("https://www.okta.com/")); + + await act(async () => { + fireEvent.error(oktaLogo); + }); + + await waitFor(() => { + expect(screen.queryByAltText("Okta / Auth0 SSO logo")).not.toBeInTheDocument(); + expect(screen.getByText("O")).toBeInTheDocument(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/BaseSSOSettingsForm.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/BaseSSOSettingsForm.tsx index d16b04466e0..6971c107a73 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/BaseSSOSettingsForm.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/BaseSSOSettingsForm.tsx @@ -4,6 +4,7 @@ import { TextInput } from "@tremor/react"; import { Checkbox, Form, Input, Select } from "antd"; import React from "react"; import { ssoProviderLogoMap, ssoProviderDisplayNames } from "../constants"; +import { Logo } from "@/components/molecules/logo/Logo"; export interface BaseSSOSettingsFormProps { form: any; // Replace with proper Form type if available @@ -117,10 +118,10 @@ const BaseSSOSettingsForm: React.FC = ({ form, onFormS
{logo && ( - {value} )} diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.test.tsx index 5e7908a872b..e585bec4fd5 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.test.tsx @@ -1,14 +1,13 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { render, screen } from "@testing-library/react"; -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import SSOSettings from "./SSOSettings"; +const mockUseSSOSettings = vi.fn(); + // Mock the useSSOSettings hook vi.mock("@/app/(dashboard)/hooks/sso/useSSOSettings", () => ({ - useSSOSettings: () => ({ - data: null, - refetch: vi.fn(), - }), + useSSOSettings: () => mockUseSSOSettings(), })); const createQueryClient = () => @@ -21,17 +20,61 @@ const createQueryClient = () => }, }); -describe("SSOSettings", () => { - it("should render", () => { - const queryClient = createQueryClient(); +const renderSSOSettings = () => { + const queryClient = createQueryClient(); - render( - - - , - ); + return render( + + + , + ); +}; + +const googleConfiguredValues = { + google_client_id: "google-client-id", + google_client_secret: "google-client-secret", + microsoft_client_id: null, + microsoft_client_secret: null, + microsoft_tenant: null, + generic_client_id: null, + generic_client_secret: null, + generic_authorization_endpoint: null, + generic_token_endpoint: null, + generic_userinfo_endpoint: null, + proxy_base_url: null, + user_email: null, + ui_access_mode: null, + role_mappings: null, + team_mappings: null, +}; + +describe("SSOSettings", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockUseSSOSettings.mockReturnValue({ + data: null, + isLoading: false, + refetch: vi.fn(), + }); + }); + + it("should render", () => { + renderSSOSettings(); expect(screen.getByText("SSO Configuration")).toBeInTheDocument(); expect(screen.getByText("Manage Single Sign-On authentication settings")).toBeInTheDocument(); }); + + it("shows the local google logo asset for a google-configured settings payload", () => { + mockUseSSOSettings.mockReturnValue({ + data: { values: googleConfiguredValues }, + isLoading: false, + refetch: vi.fn(), + }); + + renderSSOSettings(); + + const logo = screen.getByAltText("Google SSO logo"); + expect(logo).toHaveAttribute("src", expect.stringContaining("google.svg")); + }); }); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx index 053da380103..e3361050422 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx @@ -4,6 +4,7 @@ import { useSSOSettings, type SSOSettingsValues } from "@/app/(dashboard)/hooks/ import { Button, Card, Descriptions, Space, Tag, Typography } from "antd"; import { Edit, Shield, Trash2 } from "lucide-react"; import { useState } from "react"; +import { Logo } from "@/components/molecules/logo/Logo"; import { ssoProviderDisplayNames, ssoProviderLogoMap } from "./constants"; import AddSSOSettingsModal from "./Modals/AddSSOSettingsModal"; import DeleteSSOSettingsModal from "./Modals/DeleteSSOSettingsModal"; @@ -166,10 +167,10 @@ export default function SSOSettings() {
{ssoProviderLogoMap[selectedProvider] && ( - {selectedProvider} )} {config.providerText} diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/constants.ts b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/constants.ts index e2aa21e4b25..b5f5ccb1b8c 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/constants.ts +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/constants.ts @@ -1,7 +1,10 @@ +import googleLogo from "../../../../../public/assets/logos/google.svg"; +import microsoftAzureLogo from "../../../../../public/assets/logos/microsoft_azure.svg"; + // SSO Provider logos export const ssoProviderLogoMap: Record = { - google: "https://artificialanalysis.ai/img/logos/google_small.svg", - microsoft: "https://upload.wikimedia.org/wikipedia/commons/a/a8/Microsoft_Azure_Logo.svg", + google: googleLogo.src, + microsoft: microsoftAzureLogo.src, okta: "https://www.okta.com/sites/default/files/Okta_Logo_BrightBlue_Medium.png", generic: "", }; diff --git a/ui/litellm-dashboard/src/components/ToolPolicies.tsx b/ui/litellm-dashboard/src/components/ToolPolicies.tsx deleted file mode 100644 index 4468334f813..00000000000 --- a/ui/litellm-dashboard/src/components/ToolPolicies.tsx +++ /dev/null @@ -1,553 +0,0 @@ -"use client"; - -import React, { useCallback, useDeferredValue, useEffect, useMemo, useState } from "react"; -import { Button, Switch, Tooltip } from "antd"; -import { Table, TableHead, TableHeaderCell, TableBody, TableRow, TableCell } from "@tremor/react"; -import { DateCell, IdCell } from "@/components/shared/table_cells"; -import type { SortState } from "./common_components/TableHeaderSortDropdown/TableHeaderSortDropdown"; -import { TableHeaderSortDropdown } from "./common_components/TableHeaderSortDropdown/TableHeaderSortDropdown"; -import FilterComponent, { FilterOption } from "./molecules/filter"; -import { MetricCard } from "./GuardrailsMonitor/MetricCard"; -import { PolicySelect, INPUT_POLICY_OPTIONS, OUTPUT_POLICY_OPTIONS } from "./ToolPolicies/PolicySelect"; -import { fetchToolsList, updateToolPolicy, ToolRow } from "./networking"; - -function getUTCDateKey(date: Date): string { - return `${date.getUTCFullYear()}-${String(date.getUTCMonth() + 1).padStart(2, "0")}-${String(date.getUTCDate()).padStart(2, "0")}`; -} - -function isCreatedInUTCDay(createdAt: string | undefined, utcDateKey: string): boolean { - if (!createdAt) return false; - try { - const d = new Date(createdAt); - return getUTCDateKey(d) === utcDateKey; - } catch { - return false; - } -} - -function countToolsInUTCDay(tools: ToolRow[], utcDateKey: string): number { - return tools.filter((t) => isCreatedInUTCDay(t.created_at, utcDateKey)).length; -} - -function getTrendSubtitle(newToday: number, newYesterday: number): string | undefined { - const diff = newToday - newYesterday; - if (diff === 0) return undefined; - if (diff > 0) return `+${diff} since yesterday`; - return `${diff} since yesterday`; -} - -type SortField = "tool_name" | "input_policy" | "output_policy" | "team_id" | "key_alias" | "created_at" | "call_count"; - -interface FilterValues { - [key: string]: string; -} - -interface ToolPoliciesProps { - accessToken: string | null; - userRole?: string; - onSelectTool?: (toolName: string) => void; -} - -export const ToolPolicies: React.FC = ({ accessToken, onSelectTool }) => { - const [tools, setTools] = useState([]); - const [loading, setLoading] = useState(true); - const [isFetching, setIsFetching] = useState(false); - const [error, setError] = useState(null); - const [savingInput, setSavingInput] = useState(null); - const [savingOutput, setSavingOutput] = useState(null); - - const [searchTerm, setSearchTerm] = useState(""); - const [sortField, setSortField] = useState("created_at"); - const [sortOrder, setSortOrder] = useState<"asc" | "desc">("desc"); - const [currentPage, setCurrentPage] = useState(1); - const [isLiveTail, setIsLiveTail] = useState(true); - const [activeFilters, setActiveFilters] = useState({}); - const pageSize = 50; - - const isFetchingDeferred = useDeferredValue(isFetching); - const isButtonLoading = isFetching || isFetchingDeferred; - - const load = useCallback(async () => { - if (!accessToken) return; - setIsFetching(true); - setError(null); - try { - const rows = await fetchToolsList(accessToken); - setTools(rows); - } catch (e: any) { - setError(e.message ?? "Failed to load tools"); - } finally { - setIsFetching(false); - setLoading(false); - } - }, [accessToken]); - - useEffect(() => { - load(); - }, [load]); - - useEffect(() => { - if (!isLiveTail) return; - const id = setInterval(load, 15000); - return () => clearInterval(id); - }, [isLiveTail, load]); - - const handleInputPolicyChange = async (toolName: string, newPolicy: string) => { - if (!accessToken) return; - setSavingInput(toolName); - try { - await updateToolPolicy(accessToken, toolName, { input_policy: newPolicy }); - setTools((prev) => prev.map((t) => (t.tool_name === toolName ? { ...t, input_policy: newPolicy } : t))); - } catch (e: any) { - alert(`Failed to update input policy: ${e.message}`); - } finally { - setSavingInput(null); - } - }; - - const handleOutputPolicyChange = async (toolName: string, newPolicy: string) => { - if (!accessToken) return; - setSavingOutput(toolName); - try { - await updateToolPolicy(accessToken, toolName, { output_policy: newPolicy }); - setTools((prev) => prev.map((t) => (t.tool_name === toolName ? { ...t, output_policy: newPolicy } : t))); - } catch (e: any) { - alert(`Failed to update output policy: ${e.message}`); - } finally { - setSavingOutput(null); - } - }; - - const handleSortChange = (field: SortField, newState: SortState) => { - if (newState === false) { - setSortField("created_at"); - setSortOrder("desc"); - } else { - setSortField(field); - setSortOrder(newState); - } - setCurrentPage(1); - }; - - const handleApplyFilters = (filters: FilterValues) => { - setActiveFilters(filters); - setCurrentPage(1); - }; - - const handleResetFilters = () => { - setActiveFilters({}); - setCurrentPage(1); - }; - - const teamOptions = Array.from(new Set(tools.map((t) => t.team_id).filter(Boolean))).map((v) => ({ - label: v as string, - value: v as string, - })); - const keyAliasOptions = Array.from(new Set(tools.map((t) => t.key_alias).filter(Boolean))).map((v) => ({ - label: v as string, - value: v as string, - })); - - const filterOptions: FilterOption[] = [ - { - name: "Input Policy", - label: "Input Policy", - options: INPUT_POLICY_OPTIONS.map((o) => ({ label: o.label, value: o.value })), - }, - { - name: "Output Policy", - label: "Output Policy", - options: OUTPUT_POLICY_OPTIONS.map((o) => ({ label: o.label, value: o.value })), - }, - { - name: "Team Name", - label: "Team Name", - options: teamOptions, - }, - { - name: "Key Name", - label: "Key Name", - options: keyAliasOptions, - }, - ]; - - const { newToday, newYesterday, trendSubtitle, totalTools, blockedCount, activeTeamsCount, needsReviewTools } = - useMemo(() => { - const now = new Date(); - const todayKey = getUTCDateKey(now); - const yesterday = new Date(now); - yesterday.setUTCDate(yesterday.getUTCDate() - 1); - const yesterdayKey = getUTCDateKey(yesterday); - - const newToday = countToolsInUTCDay(tools, todayKey); - const newYesterday = countToolsInUTCDay(tools, yesterdayKey); - const trendSubtitle = getTrendSubtitle(newToday, newYesterday); - - const totalTools = tools.length; - const blockedCount = tools.filter((t) => t.input_policy === "blocked").length; - const activeTeamsCount = new Set(tools.map((t) => t.team_id).filter(Boolean)).size; - - const needsReviewTools = tools.filter( - (t) => isCreatedInUTCDay(t.created_at, todayKey) && t.input_policy === "untrusted", - ); - - return { - newToday, - newYesterday, - trendSubtitle, - totalTools, - blockedCount, - activeTeamsCount, - needsReviewTools, - }; - }, [tools]); - - const SortHeader = ({ label, field }: { label: string; field: SortField }) => ( -
- {label} - handleSortChange(field, s)} - /> -
- ); - - const filtered = tools.filter((t) => { - if (searchTerm) { - const q = searchTerm.toLowerCase(); - const matchesSearch = - t.tool_name.toLowerCase().includes(q) || - (t.team_id ?? "").toLowerCase().includes(q) || - (t.key_alias ?? "").toLowerCase().includes(q) || - (t.key_hash ?? "").toLowerCase().includes(q) || - t.input_policy.toLowerCase().includes(q) || - t.output_policy.toLowerCase().includes(q); - if (!matchesSearch) return false; - } - if (activeFilters["Input Policy"] && t.input_policy !== activeFilters["Input Policy"]) return false; - if (activeFilters["Output Policy"] && t.output_policy !== activeFilters["Output Policy"]) return false; - if (activeFilters["Team Name"] && t.team_id !== activeFilters["Team Name"]) return false; - if (activeFilters["Key Name"] && t.key_alias !== activeFilters["Key Name"]) return false; - return true; - }); - - const sorted = [...filtered].sort((a, b) => { - const av = (a as any)[sortField] ?? ""; - const bv = (b as any)[sortField] ?? ""; - if (av < bv) return sortOrder === "desc" ? 1 : -1; - if (av > bv) return sortOrder === "desc" ? -1 : 1; - return 0; - }); - - const totalPages = Math.max(1, Math.ceil(sorted.length / pageSize)); - const paginated = sorted.slice((currentPage - 1) * pageSize, currentPage * pageSize); - - const scrollToToolRow = (toolId: string) => { - const idx = sorted.findIndex((t) => t.tool_id === toolId); - if (idx >= 0) { - const page = Math.floor(idx / pageSize) + 1; - if (page !== currentPage) setCurrentPage(page); - requestAnimationFrame(() => { - setTimeout(() => { - document.getElementById(`tool-row-${toolId}`)?.scrollIntoView({ behavior: "smooth", block: "center" }); - }, 100); - }); - } - }; - - return ( -
-

Tool Policies

- -
- - - - } - /> - - 0 ? "text-red-600" : undefined} - /> - 0 ? activeTeamsCount : "—"} /> -
- - {needsReviewTools.length > 0 && ( -
-

Needs Review

-

- {needsReviewTools.length} new tool{needsReviewTools.length !== 1 ? "s" : ""} discovered that require policy - decisions. -

-
- {needsReviewTools.map((t) => ( - - - {t.tool_name} - - - - ))} -
-
- )} - -
-
-
-
-
- { - setSearchTerm(e.target.value); - setCurrentPage(1); - }} - /> - - - -
- -
- Live Tail - -
- - -
- -
- - Showing {filtered.length === 0 ? 0 : (currentPage - 1) * pageSize + 1} -{" "} - {Math.min(currentPage * pageSize, filtered.length)} of {filtered.length} results - - - Page {currentPage} of {totalPages} - -
- - -
-
-
- -
- -
-
- - {isLiveTail && ( -
- Auto-refreshing every 15 seconds - -
- )} - - {error && ( -
{error}
- )} - - - - - - - - - - - - - - - - - - - - - - - Key Hash - - - - User Agent - - - - {loading ? ( - - - Loading tools… - - - ) : paginated.length === 0 ? ( - - - No tools discovered yet. Make a chat completion that returns tool_calls to start auto-discovery. - - - ) : ( - paginated.map((tool) => ( - - - - - - - - - - - - - - -
- {(tool.call_count ?? 0).toLocaleString()} -
-
- - - - - - - - - {tool.key_alias ?? "-"} - - - - - - {tool.user_agent ?? "-"} - - - -
- )) - )} -
-
- - {totalPages > 1 && ( -
- - Showing {(currentPage - 1) * pageSize + 1} - {Math.min(currentPage * pageSize, sorted.length)} of{" "} - {sorted.length} - -
- - -
-
- )} -
-
- ); -}; - -export default ToolPolicies; diff --git a/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesPanel.test.tsx b/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesPanel.test.tsx new file mode 100644 index 00000000000..721c215cfb1 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesPanel.test.tsx @@ -0,0 +1,321 @@ +import React from "react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { act, render, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { focusManager, QueryClient, QueryClientProvider } from "@tanstack/react-query"; + +import { renderWithProviders, testQueryClient } from "../../../tests/test-utils"; +import type { ToolRow } from "@/components/networking"; +import { ToolPoliciesPanel } from "./ToolPoliciesPanel"; + +const fetchToolsList = vi.fn(); +const updateToolPolicy = vi.fn(); + +vi.mock("@/components/networking", () => ({ + fetchToolsList: (...args: unknown[]) => fetchToolsList(...args), + updateToolPolicy: (...args: unknown[]) => updateToolPolicy(...args), +})); + +const fromBackend = vi.fn(); +vi.mock("@/components/molecules/notifications_manager", () => ({ + default: { fromBackend: (...args: unknown[]) => fromBackend(...args) }, +})); + +const NOW = new Date("2026-07-21T12:00:00Z"); + +const TOOLS: ToolRow[] = [ + { + tool_id: "tool-1", + tool_name: "get_weather", + input_policy: "untrusted", + output_policy: "untrusted", + call_count: 12, + team_id: "team-alpha", + key_alias: "prod-key", + key_hash: "hash-aaa", + user_agent: "curl/8.7.1", + created_at: "2026-07-21T10:00:00Z", + }, + { + tool_id: "tool-2", + tool_name: "search_web", + input_policy: "trusted", + output_policy: "trusted", + call_count: 5, + team_id: "team-beta", + key_alias: "dev-key", + key_hash: "hash-bbb", + created_at: "2026-07-20T10:00:00Z", + }, + { + tool_id: "tool-3", + tool_name: "delete_file", + input_policy: "blocked", + output_policy: "untrusted", + call_count: 100, + key_hash: "hash-ccc", + created_at: "2026-07-19T10:00:00Z", + }, +]; + +const row = (toolId: string): HTMLElement => { + const element = document.querySelector(`[data-row-id="${toolId}"]`); + if (element === null) throw new Error(`row ${toolId} is not rendered`); + return element as HTMLElement; +}; + +const policySelect = (toolId: string, kind: "input" | "output"): HTMLElement => + within(row(toolId)).getAllByRole("combobox")[kind === "input" ? 0 : 1]; + +/** Exact selected-value text. Never assert with toHaveTextContent here: it substring-matches, so "untrusted" satisfies "trusted". */ +const policyValue = (toolId: string, kind: "input" | "output"): string => + policySelect(toolId, kind).closest(".ant-select")?.querySelector(".ant-select-selection-item")?.textContent ?? ""; + +const isSaving = (toolId: string, kind: "input" | "output"): boolean => + policySelect(toolId, kind).closest(".ant-select")?.classList.contains("ant-select-disabled") ?? false; + +const chooseOption = async (user: ReturnType, trigger: HTMLElement, label: string) => { + await user.click(trigger); + const option = await waitFor(() => { + const match = Array.from(document.querySelectorAll(".ant-select-item-option")).find( + (element) => element.textContent === label, + ); + if (match === undefined) throw new Error(`option ${label} not open`); + return match as HTMLElement; + }); + await user.click(option); +}; + +const renderPanel = (onSelectTool = vi.fn()) => + renderWithProviders(); + +const waitForRows = () => waitFor(() => expect(document.querySelector('[data-row-id="tool-1"]')).not.toBeNull()); + +beforeEach(() => { + testQueryClient.clear(); + vi.useFakeTimers({ shouldAdvanceTime: true }); + vi.setSystemTime(NOW); + fetchToolsList.mockReset().mockResolvedValue(TOOLS); + updateToolPolicy.mockReset().mockResolvedValue({}); + fromBackend.mockReset(); + Element.prototype.scrollIntoView = vi.fn(); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("ToolPoliciesPanel data loading", () => { + it("should load tools once and never auto-refresh on a timer", async () => { + renderPanel(); + await waitForRows(); + + await act(async () => { + vi.advanceTimersByTime(60_000); + }); + + expect(fetchToolsList).toHaveBeenCalledTimes(1); + }); + + it("should not refetch when the window regains focus", async () => { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + render( + + + , + ); + await waitForRows(); + + await act(async () => { + focusManager.setFocused(false); + focusManager.setFocused(true); + }); + + expect(fetchToolsList).toHaveBeenCalledTimes(1); + focusManager.setFocused(undefined); + }); + + it("should refetch when the toolbar refresh action is used", async () => { + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + renderPanel(); + await waitForRows(); + + await user.click(screen.getByTestId("datatable-refresh")); + + await waitFor(() => expect(fetchToolsList).toHaveBeenCalledTimes(2)); + }); + + it("should keep rows visible during a refresh instead of falling back to skeletons", async () => { + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + renderPanel(); + await waitForRows(); + + fetchToolsList.mockReturnValue(new Promise(() => {})); + await user.click(screen.getByTestId("datatable-refresh")); + + expect(row("tool-1")).toBeInTheDocument(); + expect(screen.queryAllByTestId("skeleton-row")).toHaveLength(0); + }); + + it("should resolve the loading skeleton when there is no access token", async () => { + renderWithProviders(); + + await waitFor(() => expect(screen.queryAllByTestId("skeleton-row")).toHaveLength(0)); + expect(fetchToolsList).not.toHaveBeenCalled(); + expect(screen.getByText("No tools discovered")).toBeInTheDocument(); + }); + + it("should surface a load failure without wedging the skeleton", async () => { + fetchToolsList.mockRejectedValue(new Error("boom")); + renderPanel(); + + expect(await screen.findByRole("alert")).toHaveTextContent("boom"); + expect(screen.queryAllByTestId("skeleton-row")).toHaveLength(0); + }); +}); + +describe("ToolPoliciesPanel inline policy editing", () => { + it("should patch the input policy and update that row in place", async () => { + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + renderPanel(); + await waitForRows(); + + await chooseOption(user, policySelect("tool-1", "input"), "trusted"); + + expect(updateToolPolicy).toHaveBeenCalledWith("sk-token", "get_weather", { input_policy: "trusted" }); + await waitFor(() => expect(policyValue("tool-1", "input")).toBe("trusted")); + expect(fetchToolsList).toHaveBeenCalledTimes(1); + }); + + it("should patch the output policy from the output column", async () => { + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + renderPanel(); + await waitForRows(); + + await chooseOption(user, policySelect("tool-1", "output"), "trusted"); + + expect(updateToolPolicy).toHaveBeenCalledWith("sk-token", "get_weather", { output_policy: "trusted" }); + }); + + it("should keep every in-flight row disabled when two rows are saved at once", async () => { + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + updateToolPolicy.mockReturnValue(new Promise(() => {})); + renderPanel(); + await waitForRows(); + + await chooseOption(user, policySelect("tool-1", "input"), "trusted"); + await chooseOption(user, policySelect("tool-2", "input"), "blocked"); + + expect(isSaving("tool-2", "input")).toBe(true); + expect(isSaving("tool-1", "input")).toBe(true); + }); + + it("should re-enable only the row whose save finished", async () => { + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + let finishFirst = () => {}; + updateToolPolicy + .mockImplementationOnce(() => new Promise((resolve) => (finishFirst = () => resolve()))) + .mockImplementationOnce(() => new Promise(() => {})); + renderPanel(); + await waitForRows(); + + await chooseOption(user, policySelect("tool-1", "input"), "trusted"); + await chooseOption(user, policySelect("tool-2", "input"), "blocked"); + await act(async () => { + finishFirst(); + }); + + expect(isSaving("tool-1", "input")).toBe(false); + expect(isSaving("tool-2", "input")).toBe(true); + }); + + it("should not let an in-flight refresh clobber a policy that just saved", async () => { + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + let landStaleRefresh = () => {}; + renderPanel(); + await waitForRows(); + + fetchToolsList.mockImplementationOnce( + // resolves with the PRE-save snapshot, i.e. tool-1 still "untrusted" + () => new Promise((resolve) => (landStaleRefresh = () => resolve(TOOLS))), + ); + await user.click(screen.getByTestId("datatable-refresh")); + await chooseOption(user, policySelect("tool-1", "input"), "trusted"); + await waitFor(() => expect(policyValue("tool-1", "input")).toBe("trusted")); + + await act(async () => { + landStaleRefresh(); + }); + await act(async () => { + vi.advanceTimersByTime(100); + }); + + expect(policyValue("tool-1", "input")).toBe("trusted"); + }); + + it("should leave the row untouched and report the failure when the patch is rejected", async () => { + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + updateToolPolicy.mockRejectedValue(new Error("nope")); + renderPanel(); + await waitForRows(); + + await chooseOption(user, policySelect("tool-1", "input"), "trusted"); + + await waitFor(() => expect(fromBackend).toHaveBeenCalledWith("Failed to update input policy: nope")); + expect(policyValue("tool-1", "input")).toBe("untrusted"); + }); + + it("should disable only the one cell that is saving", async () => { + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + updateToolPolicy.mockReturnValue(new Promise(() => {})); + renderPanel(); + await waitForRows(); + + await chooseOption(user, policySelect("tool-1", "input"), "trusted"); + + await waitFor(() => expect(isSaving("tool-1", "input")).toBe(true)); + expect(isSaving("tool-1", "output")).toBe(false); + expect(isSaving("tool-2", "input")).toBe(false); + }); +}); + +describe("ToolPoliciesPanel header chrome", () => { + it("should summarise the loaded tools in the metric cards", async () => { + renderPanel(); + await waitForRows(); + + const metric = (label: string): HTMLElement => { + const card = screen.getByText(label).closest("div.h-full"); + if (card === null) throw new Error(`metric ${label} missing`); + return card as HTMLElement; + }; + + expect(metric("Total Tools Discovered")).toHaveTextContent("3"); + expect(metric("Blocked Tools")).toHaveTextContent("1"); + expect(metric("Active Teams")).toHaveTextContent("2"); + expect(metric("New Today")).toHaveTextContent("1"); + }); + + it("should list only today's untrusted tools for review and scroll to the row", async () => { + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + renderPanel(); + await waitForRows(); + + const banner = screen.getByText("Needs Review").closest("div"); + if (banner === null) throw new Error("needs review banner missing"); + expect(banner).toHaveTextContent("1 new tool discovered"); + expect(within(banner as HTMLElement).queryByText("delete_file")).not.toBeInTheDocument(); + + await user.click(within(banner as HTMLElement).getByRole("button", { name: "Review" })); + + expect(row("tool-1").scrollIntoView).toHaveBeenCalled(); + }); + + it("should hide the review banner when nothing needs a decision", async () => { + fetchToolsList.mockResolvedValue([{ ...TOOLS[0], input_policy: "trusted" }]); + renderPanel(); + await waitForRows(); + + expect(screen.queryByText("Needs Review")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesPanel.tsx b/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesPanel.tsx new file mode 100644 index 00000000000..1b559352469 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesPanel.tsx @@ -0,0 +1,212 @@ +"use client"; + +import { useQuery, useQueryClient, type UseQueryOptions } from "@tanstack/react-query"; +import React, { useCallback, useMemo, useState } from "react"; + +import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard"; +import NotificationsManager from "@/components/molecules/notifications_manager"; +import { fetchToolsList, ToolRow, updateToolPolicy } from "@/components/networking"; + +import { ToolPoliciesTable } from "./ToolPoliciesTable"; + +function getUTCDateKey(date: Date): string { + return `${date.getUTCFullYear()}-${String(date.getUTCMonth() + 1).padStart(2, "0")}-${String(date.getUTCDate()).padStart(2, "0")}`; +} + +function isCreatedInUTCDay(createdAt: string | undefined, utcDateKey: string): boolean { + if (!createdAt) return false; + try { + return getUTCDateKey(new Date(createdAt)) === utcDateKey; + } catch { + return false; + } +} + +function countToolsInUTCDay(tools: ToolRow[], utcDateKey: string): number { + return tools.filter((tool) => isCreatedInUTCDay(tool.created_at, utcDateKey)).length; +} + +function getTrendSubtitle(newToday: number, newYesterday: number): string | undefined { + const diff = newToday - newYesterday; + if (diff === 0) return undefined; + return diff > 0 ? `+${diff} since yesterday` : `${diff} since yesterday`; +} + +function toMessage(error: unknown, fallback: string): string { + return error instanceof Error ? error.message : fallback; +} + +const withTool = (names: ReadonlySet, toolName: string): ReadonlySet => new Set([...names, toolName]); + +const withoutTool = (names: ReadonlySet, toolName: string): ReadonlySet => + new Set([...names].filter((name) => name !== toolName)); + +const TOOLS_QUERY_KEY = "tool-policies"; + +interface ToolPoliciesPanelProps { + accessToken: string | null; + onSelectTool: (toolName: string) => void; +} + +export const ToolPoliciesPanel: React.FC = ({ accessToken, onSelectTool }) => { + const queryClient = useQueryClient(); + const [savingInput, setSavingInput] = useState>(() => new Set()); + const [savingOutput, setSavingOutput] = useState>(() => new Set()); + + const queryKey = useMemo(() => [TOOLS_QUERY_KEY, accessToken], [accessToken]); + + const queryOptions: UseQueryOptions = { + queryKey, + queryFn: async () => (accessToken === null ? [] : fetchToolsList(accessToken)), + enabled: accessToken !== null, + refetchOnWindowFocus: false, + refetchOnReconnect: false, + }; + const query = useQuery(queryOptions); + + const tools = useMemo(() => query.data ?? [], [query.data]); + + // Cancel first: a list fetch that started before this save would otherwise resolve afterwards + // and overwrite the row we just wrote with its pre-save snapshot. + const patchTool = useCallback( + async (toolName: string, patch: Partial) => { + await queryClient.cancelQueries({ queryKey }); + queryClient.setQueryData(queryKey, (previous) => + (previous ?? []).map((tool) => (tool.tool_name === toolName ? { ...tool, ...patch } : tool)), + ); + }, + [queryClient, queryKey], + ); + + const handleInputPolicyChange = useCallback( + async (toolName: string, newPolicy: string) => { + if (accessToken === null) return; + setSavingInput((previous) => withTool(previous, toolName)); + try { + await updateToolPolicy(accessToken, toolName, { input_policy: newPolicy }); + await patchTool(toolName, { input_policy: newPolicy }); + } catch (e) { + NotificationsManager.fromBackend(`Failed to update input policy: ${toMessage(e, "unknown error")}`); + } finally { + setSavingInput((previous) => withoutTool(previous, toolName)); + } + }, + [accessToken, patchTool], + ); + + const handleOutputPolicyChange = useCallback( + async (toolName: string, newPolicy: string) => { + if (accessToken === null) return; + setSavingOutput((previous) => withTool(previous, toolName)); + try { + await updateToolPolicy(accessToken, toolName, { output_policy: newPolicy }); + await patchTool(toolName, { output_policy: newPolicy }); + } catch (e) { + NotificationsManager.fromBackend(`Failed to update output policy: ${toMessage(e, "unknown error")}`); + } finally { + setSavingOutput((previous) => withoutTool(previous, toolName)); + } + }, + [accessToken, patchTool], + ); + + const { newToday, trendSubtitle, totalTools, blockedCount, activeTeamsCount, needsReviewTools } = useMemo(() => { + const now = new Date(); + const todayKey = getUTCDateKey(now); + const yesterday = new Date(now); + yesterday.setUTCDate(yesterday.getUTCDate() - 1); + const today = countToolsInUTCDay(tools, todayKey); + + return { + newToday: today, + trendSubtitle: getTrendSubtitle(today, countToolsInUTCDay(tools, getUTCDateKey(yesterday))), + totalTools: tools.length, + blockedCount: tools.filter((tool) => tool.input_policy === "blocked").length, + activeTeamsCount: new Set(tools.map((tool) => tool.team_id).filter(Boolean)).size, + needsReviewTools: tools.filter( + (tool) => isCreatedInUTCDay(tool.created_at, todayKey) && tool.input_policy === "untrusted", + ), + }; + }, [tools]); + + const scrollToToolRow = (toolId: string) => { + document.querySelector(`[data-row-id="${CSS.escape(toolId)}"]`)?.scrollIntoView({ + behavior: "smooth", + block: "center", + }); + }; + + return ( +
+

Tool Policies

+ +
+ + + + } + /> + + 0 ? "text-red-600" : undefined} + /> + 0 ? activeTeamsCount : "—"} /> +
+ + {needsReviewTools.length > 0 && ( +
+

Needs Review

+

+ {needsReviewTools.length} new tool{needsReviewTools.length !== 1 ? "s" : ""} discovered that require policy + decisions. +

+
+ {needsReviewTools.map((tool) => ( + + + {tool.tool_name} + + + + ))} +
+
+ )} + + {query.isError && ( +
+ {toMessage(query.error, "Failed to load tools")} +
+ )} + + void query.refetch()} + onSelectTool={onSelectTool} + savingInput={savingInput} + savingOutput={savingOutput} + onInputPolicyChange={handleInputPolicyChange} + onOutputPolicyChange={handleOutputPolicyChange} + /> +
+ ); +}; diff --git a/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesTable.test.tsx b/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesTable.test.tsx new file mode 100644 index 00000000000..9d32ad667bd --- /dev/null +++ b/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesTable.test.tsx @@ -0,0 +1,189 @@ +import React from "react"; +import { describe, expect, it, vi } from "vitest"; +import { screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; + +import { renderWithProviders } from "../../../tests/test-utils"; +import type { ToolRow } from "@/components/networking"; +import { ToolPoliciesTable } from "./ToolPoliciesTable"; + +const TOOLS: ToolRow[] = [ + { + tool_id: "tool-1", + tool_name: "get_weather", + input_policy: "untrusted", + output_policy: "untrusted", + call_count: 12, + team_id: "team-alpha", + key_alias: "prod-key", + key_hash: "hash-aaa", + user_agent: "curl/8.7.1", + created_at: "2026-07-21T10:00:00Z", + }, + { + tool_id: "tool-2", + tool_name: "search_web", + input_policy: "trusted", + output_policy: "trusted", + call_count: 5, + team_id: "team-beta", + key_alias: "dev-key", + key_hash: "hash-bbb", + created_at: "2026-07-20T10:00:00Z", + }, + { + tool_id: "tool-3", + tool_name: "delete_file", + input_policy: "blocked", + output_policy: "untrusted", + call_count: 100, + key_hash: "hash-ccc", + created_at: "2026-07-19T10:00:00Z", + }, +]; + +const renderTable = (overrides: Partial> = {}) => { + const props = { + data: TOOLS, + isLoading: false, + isRefreshing: false, + onRefresh: vi.fn(), + onSelectTool: vi.fn(), + savingInput: new Set(), + savingOutput: new Set(), + onInputPolicyChange: vi.fn(), + onOutputPolicyChange: vi.fn(), + ...overrides, + }; + renderWithProviders(); + return props; +}; + +const rowIds = (): (string | null)[] => + Array.from(document.querySelectorAll("tbody tr[data-row-id]")).map((row) => row.getAttribute("data-row-id")); + +const pickFilter = async ( + user: ReturnType, + triggerTestId: string, + optionLabel: string, +): Promise => { + await user.click(screen.getByTestId(triggerTestId)); + await user.click(await screen.findByRole("option", { name: optionLabel })); +}; + +describe("ToolPoliciesTable sorting", () => { + it("should default to newest discovered first", () => { + renderTable(); + + expect(rowIds()).toEqual(["tool-1", "tool-2", "tool-3"]); + }); + + it("should sort by tool name when its header is used", async () => { + const user = userEvent.setup(); + renderTable(); + + await user.click(screen.getByTestId("sort-header-tool_name")); + + expect(rowIds()).toEqual(["tool-3", "tool-1", "tool-2"]); + }); +}); + +describe("ToolPoliciesTable search", () => { + it("should match on tool name", async () => { + const user = userEvent.setup(); + renderTable(); + + await user.type(screen.getByTestId("datatable-search"), "weather"); + + await waitFor(() => expect(rowIds()).toEqual(["tool-1"])); + }); + + it("should match on key hash", async () => { + const user = userEvent.setup(); + renderTable(); + + await user.type(screen.getByTestId("datatable-search"), "hash-bbb"); + + await waitFor(() => expect(rowIds()).toEqual(["tool-2"])); + }); + + it("should not match on user agent", async () => { + const user = userEvent.setup(); + renderTable(); + + await user.type(screen.getByTestId("datatable-search"), "curl"); + + await waitFor(() => expect(rowIds()).toEqual([])); + expect(screen.getByText("No matching tools")).toBeInTheDocument(); + }); +}); + +describe("ToolPoliciesTable filters", () => { + it("should match an input policy exactly rather than as a substring", async () => { + const user = userEvent.setup(); + renderTable(); + + await user.click(screen.getByTestId("datatable-filters-trigger")); + await pickFilter(user, "filter-input-policy", "trusted"); + await user.click(screen.getByTestId("filter-drawer-apply")); + + await waitFor(() => expect(rowIds()).toEqual(["tool-2"])); + }); + + it("should filter by team", async () => { + const user = userEvent.setup(); + renderTable(); + + await user.click(screen.getByTestId("datatable-filters-trigger")); + await pickFilter(user, "filter-team", "team-alpha"); + await user.click(screen.getByTestId("filter-drawer-apply")); + + await waitFor(() => expect(rowIds()).toEqual(["tool-1"])); + expect(screen.getByTestId("filter-chip-team_id")).toHaveTextContent("Team Name:"); + }); + + it("should offer only the teams and keys present in the loaded rows", async () => { + const user = userEvent.setup(); + renderTable(); + + await user.click(screen.getByTestId("datatable-filters-trigger")); + await user.click(screen.getByTestId("filter-team")); + + const teams = (await screen.findAllByRole("option")).map((option) => option.textContent); + expect(teams).toEqual(["All Teams", "team-alpha", "team-beta"]); + }); +}); + +describe("ToolPoliciesTable chrome", () => { + it("should open the detail view from the tool name cell", async () => { + const user = userEvent.setup(); + const { onSelectTool } = renderTable(); + + await user.click(screen.getByRole("button", { name: /get_weather/ })); + + expect(onSelectTool).toHaveBeenCalledWith("get_weather"); + }); + + it("should refresh on demand", async () => { + const user = userEvent.setup(); + const { onRefresh } = renderTable(); + + await user.click(screen.getByTestId("datatable-refresh")); + + expect(onRefresh).toHaveBeenCalledTimes(1); + }); + + it("should explain how discovery works when there are no tools at all", () => { + renderTable({ data: [] }); + + expect(screen.getByText("No tools discovered")).toBeInTheDocument(); + expect(screen.getByText(/tool_calls to start auto-discovery/)).toBeInTheDocument(); + }); + + it("should show skeleton rows while the first load is in flight", () => { + renderTable({ data: [], isLoading: true }); + + expect(screen.getAllByTestId("skeleton-row").length).toBeGreaterThan(0); + expect(screen.queryByText("No tools discovered")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesTable.tsx b/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesTable.tsx new file mode 100644 index 00000000000..bbffb0d5ff5 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesTable.tsx @@ -0,0 +1,199 @@ +"use client"; + +import { ColumnFiltersState } from "@tanstack/react-table"; +import { Wrench } from "lucide-react"; +import { useMemo, useState } from "react"; + +import { ToolRow } from "@/components/networking"; +import { + DataTable, + DataTableFilterDrawer, + DataTableFilterField, + DataTableToolbar, +} from "@/components/shared/DataTable"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; + +import { INPUT_POLICY_OPTIONS, OUTPUT_POLICY_OPTIONS } from "./PolicySelect"; +import { getToolPoliciesTableColumns } from "./ToolPoliciesTableColumns"; + +const ALL_VALUE = "all"; + +const toFilterValue = (value: string | null): string | undefined => + value === null || value === ALL_VALUE ? undefined : value; + +interface ToolPoliciesTableProps { + data: ToolRow[]; + isLoading: boolean; + isRefreshing: boolean; + onRefresh: () => void; + onSelectTool: (toolName: string) => void; + savingInput: ReadonlySet; + savingOutput: ReadonlySet; + onInputPolicyChange: (toolName: string, policy: string) => void; + onOutputPolicyChange: (toolName: string, policy: string) => void; +} + +function ToolPoliciesEmptyState({ filtered }: { filtered: boolean }) { + return ( +
+
+ +
+
+ {filtered ? "No matching tools" : "No tools discovered"} +
+
+ {filtered + ? "No tools match your search or filters." + : "Make a chat completion that returns tool_calls to start auto-discovery."} +
+
+ ); +} + +function uniqueValues(rows: ToolRow[], pick: (row: ToolRow) => string | undefined): string[] { + return Array.from(new Set(rows.map(pick).filter((value): value is string => Boolean(value)))); +} + +export function ToolPoliciesTable({ + data, + isLoading, + isRefreshing, + onRefresh, + onSelectTool, + savingInput, + savingOutput, + onInputPolicyChange, + onOutputPolicyChange, +}: ToolPoliciesTableProps) { + const [globalFilter, setGlobalFilter] = useState(""); + const [columnFilters, setColumnFilters] = useState([]); + const [filtersOpen, setFiltersOpen] = useState(false); + + const columns = useMemo(() => { + const deps = { onSelectTool, savingInput, savingOutput, onInputPolicyChange, onOutputPolicyChange }; + return getToolPoliciesTableColumns(deps); + }, [onSelectTool, savingInput, savingOutput, onInputPolicyChange, onOutputPolicyChange]); + + const teamOptions = useMemo(() => uniqueValues(data, (row) => row.team_id), [data]); + const keyAliasOptions = useMemo(() => uniqueValues(data, (row) => row.key_alias), [data]); + + return ( + row.tool_id} + sortingMode="client" + defaultSorting={[{ id: "created_at", desc: true }]} + paginationMode="client" + pageSizeOptions={[50, 100]} + filterMode="client" + columnFilters={columnFilters} + onColumnFiltersChange={setColumnFilters} + globalFilter={globalFilter} + onGlobalFilterChange={setGlobalFilter} + isLoading={isLoading} + loadingMessage="Loading tools…" + noDataMessage={ 0 || globalFilter !== ""} />} + size="compact" + toolbar={(table) => ( + <> + setFiltersOpen(true)} + showViewOptions={false} + /> + + {({ get, set }) => ( + <> + + + + + + + + + + + + + + )} + + + )} + /> + ); +} diff --git a/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesTableColumns.tsx b/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesTableColumns.tsx new file mode 100644 index 00000000000..29a4708a470 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesTableColumns.tsx @@ -0,0 +1,141 @@ +"use client"; + +import { ColumnDef } from "@tanstack/react-table"; +import { Tooltip } from "antd"; + +import { ToolRow } from "@/components/networking"; +import { DataTableSortHeader } from "@/components/shared/DataTable"; +import { DateCell, IdCell, IdentityCell } from "@/components/shared/table_cells"; + +import { PolicySelect } from "./PolicySelect"; + +interface ToolPoliciesTableColumnsDeps { + onSelectTool: (toolName: string) => void; + savingInput: ReadonlySet; + savingOutput: ReadonlySet; + onInputPolicyChange: (toolName: string, policy: string) => void; + onOutputPolicyChange: (toolName: string, policy: string) => void; +} + +function TruncatedText({ value, className }: { value: string | undefined; className?: string }) { + const text = value ?? "-"; + return ( + + {text} + + ); +} + +export const getToolPoliciesTableColumns = ({ + onSelectTool, + savingInput, + savingOutput, + onInputPolicyChange, + onOutputPolicyChange, +}: ToolPoliciesTableColumnsDeps): ColumnDef[] => [ + { + id: "created_at", + accessorFn: (row) => row.created_at ?? "", + header: ({ column }) => , + size: 170, + enableGlobalFilter: false, + cell: ({ row }) => , + }, + { + id: "tool_name", + accessorFn: (row) => row.tool_name, + header: ({ column }) => , + minSize: 200, + cell: ({ row }) => ( + onSelectTool(row.original.tool_name)} + /> + ), + }, + { + id: "input_policy", + accessorFn: (row) => row.input_policy, + header: ({ column }) => , + size: 140, + filterFn: "equalsString", + meta: { title: "Input Policy", skeleton: "badge" }, + cell: ({ row }) => ( + + ), + }, + { + id: "output_policy", + accessorFn: (row) => row.output_policy, + header: ({ column }) => , + size: 140, + filterFn: "equalsString", + meta: { title: "Output Policy", skeleton: "badge" }, + cell: ({ row }) => ( + + ), + }, + { + id: "call_count", + accessorFn: (row) => row.call_count ?? 0, + header: ({ column }) => , + size: 100, + enableGlobalFilter: false, + meta: { numeric: true }, + cell: ({ row }) => {(row.original.call_count ?? 0).toLocaleString()}, + }, + { + id: "team_id", + accessorFn: (row) => row.team_id ?? "", + header: ({ column }) => , + size: 160, + filterFn: "equalsString", + meta: { title: "Team Name" }, + cell: ({ row }) => , + }, + { + id: "key_hash", + accessorFn: (row) => row.key_hash ?? "", + header: "Key Hash", + size: 150, + enableSorting: false, + cell: ({ row }) => , + }, + { + id: "key_alias", + accessorFn: (row) => row.key_alias ?? "", + header: ({ column }) => , + size: 150, + filterFn: "equalsString", + meta: { title: "Key Name" }, + cell: ({ row }) => , + }, + { + id: "user_agent", + accessorFn: (row) => row.user_agent ?? "", + header: "User Agent", + size: 180, + enableSorting: false, + enableGlobalFilter: false, + cell: ({ row }) => ( + + ), + }, +]; diff --git a/ui/litellm-dashboard/src/components/ToolPoliciesView.test.tsx b/ui/litellm-dashboard/src/components/ToolPoliciesView.test.tsx index 8b2b1d0e4b7..34c697a98d1 100644 --- a/ui/litellm-dashboard/src/components/ToolPoliciesView.test.tsx +++ b/ui/litellm-dashboard/src/components/ToolPoliciesView.test.tsx @@ -14,25 +14,27 @@ vi.mock("@/components/ToolDetail", () => ({ ), })); -vi.mock("@/components/ToolPolicies", () => ({ - ToolPolicies: ({ onSelectTool }: { onSelectTool: (name: string) => void }) => ( -
- Tool Policies Overview - -
- ), +vi.mock("@/components/ToolPolicies/ToolPoliciesPanel", () => ({ + ToolPoliciesPanel: function ToolPoliciesPanelMock({ onSelectTool }: { onSelectTool: (name: string) => void }) { + return ( +
+ Tool Policies Overview + +
+ ); + }, })); describe("ToolPoliciesView", () => { it("should render the overview by default", () => { - renderWithProviders(); + renderWithProviders(); expect(screen.getByText("Tool Policies Overview")).toBeInTheDocument(); }); it("should navigate to tool detail when a tool is selected", async () => { const user = userEvent.setup(); - renderWithProviders(); + renderWithProviders(); await user.click(screen.getByRole("button", { name: /select tool/i })); @@ -42,7 +44,7 @@ describe("ToolPoliciesView", () => { it("should navigate back to overview when back is clicked", async () => { const user = userEvent.setup(); - renderWithProviders(); + renderWithProviders(); await user.click(screen.getByRole("button", { name: /select tool/i })); await user.click(screen.getByRole("button", { name: /back/i })); diff --git a/ui/litellm-dashboard/src/components/ToolPoliciesView.tsx b/ui/litellm-dashboard/src/components/ToolPoliciesView.tsx index 31ea7c7f956..bdff40153b9 100644 --- a/ui/litellm-dashboard/src/components/ToolPoliciesView.tsx +++ b/ui/litellm-dashboard/src/components/ToolPoliciesView.tsx @@ -2,16 +2,15 @@ import React, { useState } from "react"; import { ToolDetail } from "@/components/ToolDetail"; -import { ToolPolicies } from "@/components/ToolPolicies"; +import { ToolPoliciesPanel } from "@/components/ToolPolicies/ToolPoliciesPanel"; type View = { type: "overview" } | { type: "detail"; toolName: string }; interface ToolPoliciesViewProps { accessToken: string | null; - userRole?: string; } -export default function ToolPoliciesView({ accessToken, userRole }: ToolPoliciesViewProps) { +export default function ToolPoliciesView({ accessToken }: ToolPoliciesViewProps) { const [view, setView] = useState({ type: "overview" }); const handleSelectTool = (toolName: string) => { @@ -27,7 +26,7 @@ export default function ToolPoliciesView({ accessToken, userRole }: ToolPolicies {view.type === "detail" ? ( ) : ( - + )}
); diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx index 513054aae7a..95f45ea199e 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx @@ -174,6 +174,13 @@ it("should render VirtualKeysTable component", () => { expect(screen.getByText("Test Key Alias")).toBeInTheDocument(); }); +it("shows the Budget Reset column by default", async () => { + renderWithProviders(); + await waitFor(() => { + expect(screen.getByText("Budget Reset")).toBeInTheDocument(); + }); +}); + it("left-anchors the create-key CTA below the title, between the header and the table toolbar", () => { renderWithProviders(Create New Key} />); @@ -498,8 +505,13 @@ describe("Status column reflects blocked / expiry / scim metadata", () => { renderWithProviders(); + const tag = await screen.findByTestId(`key-status-${mockKey.token_id}`); + expect(tag).toHaveTextContent("Active"); + + const user = userEvent.setup(); + await user.hover(tag); await waitFor(() => { - expect(screen.getByTestId(`key-status-${mockKey.token_id}`)).toHaveTextContent("Active"); + expect(screen.getByText(/not blocked and has not expired/i)).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx index 133ff89a898..fdbc07ee020 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx @@ -46,7 +46,11 @@ const getKeyStatus = (key: KeyResponse): KeyStatus => { if (!Number.isNaN(expiresAt) && expiresAt < Date.now()) { return { tone: "warning", label: "Expired", tooltip: "This key has passed its expiry date." }; } - return { tone: "success", label: "Active" }; + return { + tone: "success", + label: "Active", + tooltip: "This key is not blocked and has not expired.", + }; }; const UserPopoverCell = ({ @@ -359,6 +363,5 @@ export const KEY_TABLE_HIDDEN_COLUMNS: Record = { created_by: false, updated_at: false, expires: false, - budget_reset_at: false, rate_limits: false, }; diff --git a/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx b/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx index 73cfdce5263..a99f8048dca 100644 --- a/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx @@ -10,7 +10,7 @@ import React, { useEffect, useMemo, useState } from "react"; import TeamDropdown from "../common_components/team_dropdown"; import type { Team } from "../key_team_helpers/key_list"; import { type CredentialItem, type ProviderCreateInfo, modelAvailableCall } from "../networking"; -import { Providers, providerLogoMap } from "../provider_info_helpers"; +import { Providers } from "../provider_info_helpers"; import { ProviderLogo } from "../molecules/models/ProviderLogo"; import AdvancedSettings from "./advanced_settings"; import ConditionalPublicModelName from "./conditional_public_model_name"; @@ -181,7 +181,6 @@ const AddModelForm: React.FC = ({ {sortedProviderMetadata.map((providerInfo) => { const displayName = providerInfo.provider_display_name; const providerKey = providerInfo.provider; - const logoSrc = providerLogoMap[displayName] ?? ""; return ( diff --git a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx index 3c6b3829fef..7aa121dcca5 100644 --- a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx @@ -1,19 +1,28 @@ +import arizeLogo from "../../public/assets/logos/arize.png"; +import awsLogo from "../../public/assets/logos/aws.svg"; +import braintrustLogo from "../../public/assets/logos/braintrust.png"; +import datadogLogo from "../../public/assets/logos/datadog.png"; +import galileoLogo from "../../public/assets/logos/galileo.ico"; +import lagoLogo from "../../public/assets/logos/lago.svg"; +import langfuseLogo from "../../public/assets/logos/langfuse.png"; +import langsmithLogo from "../../public/assets/logos/langsmith.png"; +import openmeterLogo from "../../public/assets/logos/openmeter.png"; +import otelLogo from "../../public/assets/logos/otel.png"; + interface CallbackConfig { id: string; displayName: string; - logo: string; + logo?: string; supports_key_team_logging: boolean; dynamic_params: Record; description: string; } -const asset_logos_folder = "/ui/assets/logos/"; - export const CALLBACK_CONFIGS: CallbackConfig[] = [ { id: "arize", displayName: "Arize", - logo: `${asset_logos_folder}arize.png`, + logo: arizeLogo.src, supports_key_team_logging: true, dynamic_params: { arize_api_key: "password", @@ -24,7 +33,7 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ { id: "braintrust", displayName: "Braintrust", - logo: `${asset_logos_folder}braintrust.png`, + logo: braintrustLogo.src, supports_key_team_logging: false, dynamic_params: { braintrust_api_key: "password", @@ -35,7 +44,6 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ { id: "custom_callback_api", displayName: "Custom Callback API", - logo: `${asset_logos_folder}custom.svg`, supports_key_team_logging: true, dynamic_params: { custom_callback_api_url: "text", @@ -46,7 +54,7 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ { id: "galileo", displayName: "Galileo", - logo: `${asset_logos_folder}galileo.ico`, + logo: galileoLogo.src, supports_key_team_logging: false, dynamic_params: { GALILEO_API_KEY: "password", @@ -61,7 +69,7 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ { id: "datadog", displayName: "Datadog", - logo: `${asset_logos_folder}datadog.png`, + logo: datadogLogo.src, supports_key_team_logging: false, dynamic_params: { dd_api_key: "password", @@ -72,7 +80,7 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ { id: "lago", displayName: "Lago", - logo: `${asset_logos_folder}lago.svg`, + logo: lagoLogo.src, supports_key_team_logging: false, dynamic_params: { lago_api_url: "text", @@ -83,7 +91,7 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ { id: "langfuse", displayName: "Langfuse", - logo: `${asset_logos_folder}langfuse.png`, + logo: langfuseLogo.src, supports_key_team_logging: true, dynamic_params: { langfuse_public_key: "text", @@ -95,7 +103,7 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ { id: "langfuse_otel", displayName: "Langfuse OTEL", - logo: `${asset_logos_folder}langfuse.png`, + logo: langfuseLogo.src, supports_key_team_logging: true, dynamic_params: { langfuse_public_key: "text", @@ -107,7 +115,7 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ { id: "langsmith", displayName: "LangSmith", - logo: `${asset_logos_folder}langsmith.png`, + logo: langsmithLogo.src, supports_key_team_logging: true, dynamic_params: { langsmith_api_key: "password", @@ -120,7 +128,7 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ { id: "openmeter", displayName: "OpenMeter", - logo: `${asset_logos_folder}openmeter.png`, + logo: openmeterLogo.src, supports_key_team_logging: false, dynamic_params: { openmeter_api_key: "password", @@ -131,7 +139,7 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ { id: "otel", displayName: "Open Telemetry", - logo: `${asset_logos_folder}otel.png`, + logo: otelLogo.src, supports_key_team_logging: false, dynamic_params: { otel_endpoint: "text", @@ -142,7 +150,7 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ { id: "s3", displayName: "S3", - logo: `${asset_logos_folder}aws.svg`, + logo: awsLogo.src, supports_key_team_logging: false, dynamic_params: { s3_bucket_name: "text", @@ -155,7 +163,7 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ { id: "SQS", displayName: "SQS", - logo: `${asset_logos_folder}aws.svg`, + logo: awsLogo.src, supports_key_team_logging: false, dynamic_params: { sqs_queue_url: "text", diff --git a/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.test.tsx b/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.test.tsx new file mode 100644 index 00000000000..656ef157363 --- /dev/null +++ b/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.test.tsx @@ -0,0 +1,88 @@ +import React from "react"; +import { render, screen, fireEvent } from "@testing-library/react"; +import { describe, it, expect, vi, afterEach } from "vitest"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import MCPAppsPanel from "./MCPAppsPanel"; +import { fetchMCPServers, listMCPTools } from "../networking"; +import type { MCPServer } from "../mcp_tools/types"; +import { setServerRootPath } from "@/lib/serverRootPath"; + +vi.mock("../networking", () => ({ + fetchMCPServers: vi.fn(), + getMCPOAuthUserCredentialStatus: vi.fn(), + listMCPTools: vi.fn(), + deleteMCPOAuthUserCredential: vi.fn(), +})); + +vi.mock("@/hooks/useUserMcpOAuthFlow", () => ({ + useUserMcpOAuthFlow: () => ({ startOAuthFlow: vi.fn(), status: "idle" }), +})); + +const servers = [ + { + server_id: "s-ext", + server_name: "external_logo", + auth_type: "none", + mcp_info: { server_name: "external_logo", logo_url: "https://cdn.example.com/ext.png" }, + }, + { + server_id: "s-local", + server_name: "local_logo", + auth_type: "none", + mcp_info: { server_name: "local_logo", logo_url: "/ui/assets/logos/github.svg" }, + }, + { + server_id: "s-none", + server_name: "no_logo", + auth_type: "none", + }, +] as MCPServer[]; + +const renderPanel = () => + render( + + + , + ); + +describe("MCPAppsPanel logos", () => { + afterEach(() => { + setServerRootPath("/"); + }); + + it("resolves backend logo_url values in the server grid", async () => { + setServerRootPath("/litellm"); + vi.mocked(fetchMCPServers).mockResolvedValue(servers); + vi.mocked(listMCPTools).mockResolvedValue({ tools: [] }); + + renderPanel(); + + expect(await screen.findByText("external_logo")).toBeInTheDocument(); + expect(screen.getByAltText("external_logo logo").getAttribute("src")).toBe("https://cdn.example.com/ext.png"); + expect(screen.getByAltText("local_logo logo").getAttribute("src")).toBe("/litellm/ui/assets/logos/github.svg"); + }); + + it("renders a colored letter avatar for servers without logo_url", async () => { + vi.mocked(fetchMCPServers).mockResolvedValue(servers); + vi.mocked(listMCPTools).mockResolvedValue({ tools: [] }); + + renderPanel(); + + expect(await screen.findByText("no_logo")).toBeInTheDocument(); + expect(screen.queryByAltText("no_logo logo")).not.toBeInTheDocument(); + expect(screen.getByText("N")).toBeInTheDocument(); + }); + + it("resolves the logo_url in the detail header", async () => { + setServerRootPath("/litellm"); + vi.mocked(fetchMCPServers).mockResolvedValue(servers); + vi.mocked(listMCPTools).mockResolvedValue({ tools: [] }); + + renderPanel(); + + fireEvent.click(await screen.findByText("local_logo")); + + expect(await screen.findByRole("heading", { name: "local_logo" })).toBeInTheDocument(); + expect(screen.getByAltText("local_logo logo").getAttribute("src")).toBe("/litellm/ui/assets/logos/github.svg"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.tsx b/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.tsx index 867522090d2..25ced2d62c3 100644 --- a/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.tsx +++ b/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.tsx @@ -14,6 +14,7 @@ import { listMCPTools, } from "../networking"; import { AUTH_TYPE, MCPServer, MCPTool, handleTransport } from "../mcp_tools/types"; +import { Logo } from "@/components/molecules/logo/Logo"; import MessageManager from "@/components/molecules/message_manager"; import { useUserMcpOAuthFlow } from "@/hooks/useUserMcpOAuthFlow"; @@ -270,26 +271,19 @@ const MCPAppsPanel: React.FC = ({ accessToken, selectedServers, onChange
{detailServer.mcp_info?.logo_url ? ( - {`${name} { - const el = e.target as HTMLImageElement; - el.style.display = "none"; - if (el.nextElementSibling) (el.nextElementSibling as HTMLElement).style.display = "flex"; - }} /> - ) : null} -
- {name.charAt(0).toUpperCase()} -
+ ) : ( +
+ {name.charAt(0).toUpperCase()} +
+ )}

{name}

{detailServer.description ?? "MCP server"}

@@ -478,26 +472,19 @@ const MCPAppsPanel: React.FC = ({ accessToken, selectedServers, onChange } ${Math.floor(idx / 2) < Math.floor((filtered.length - 1) / 2) ? "border-b" : ""}`} > {server.mcp_info?.logo_url ? ( - {`${name} { - const el = e.target as HTMLImageElement; - el.style.display = "none"; - if (el.nextElementSibling) (el.nextElementSibling as HTMLElement).style.display = "flex"; - }} /> - ) : null} -
- {name.charAt(0).toUpperCase()} -
+ ) : ( +
+ {name.charAt(0).toUpperCase()} +
+ )}
{name}
diff --git a/ui/litellm-dashboard/src/components/chat/MCPConnectPicker.test.tsx b/ui/litellm-dashboard/src/components/chat/MCPConnectPicker.test.tsx new file mode 100644 index 00000000000..f2912d460f0 --- /dev/null +++ b/ui/litellm-dashboard/src/components/chat/MCPConnectPicker.test.tsx @@ -0,0 +1,55 @@ +import React from "react"; +import { render, screen } from "@testing-library/react"; +import { describe, it, expect, vi, afterEach } from "vitest"; +import MCPConnectPicker from "./MCPConnectPicker"; +import { fetchMCPServers } from "../networking"; +import type { MCPServer } from "../mcp_tools/types"; +import { setServerRootPath } from "@/lib/serverRootPath"; + +vi.mock("../networking", () => ({ + fetchMCPServers: vi.fn(), + listMCPTools: vi.fn(), +})); + +const servers = [ + { + server_id: "s-ext", + server_name: "external_logo", + mcp_info: { server_name: "external_logo", logo_url: "https://cdn.example.com/ext.png" }, + }, + { + server_id: "s-local", + server_name: "local_logo", + mcp_info: { server_name: "local_logo", logo_url: "/ui/assets/logos/github.svg" }, + }, + { + server_id: "s-none", + server_name: "no_logo", + }, +] as MCPServer[]; + +describe("MCPConnectPicker logos", () => { + afterEach(() => { + setServerRootPath("/"); + }); + + it("resolves backend logo_url values through the Logo component", async () => { + setServerRootPath("/litellm"); + vi.mocked(fetchMCPServers).mockResolvedValue(servers); + + render(); + + expect(await screen.findByText("external_logo")).toBeInTheDocument(); + expect(screen.getByAltText("external_logo logo").getAttribute("src")).toBe("https://cdn.example.com/ext.png"); + expect(screen.getByAltText("local_logo logo").getAttribute("src")).toBe("/litellm/ui/assets/logos/github.svg"); + }); + + it("renders no logo at all for servers without logo_url", async () => { + vi.mocked(fetchMCPServers).mockResolvedValue(servers); + + render(); + + expect(await screen.findByText("no_logo")).toBeInTheDocument(); + expect(screen.queryByAltText("no_logo logo")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/chat/MCPConnectPicker.tsx b/ui/litellm-dashboard/src/components/chat/MCPConnectPicker.tsx index abeccb0f041..a353457946c 100644 --- a/ui/litellm-dashboard/src/components/chat/MCPConnectPicker.tsx +++ b/ui/litellm-dashboard/src/components/chat/MCPConnectPicker.tsx @@ -3,6 +3,7 @@ import { Loader2 } from "lucide-react"; import { Switch } from "@/components/ui/switch"; import { Skeleton } from "@/components/ui/skeleton"; import MessageManager from "@/components/molecules/message_manager"; +import { Logo } from "@/components/molecules/logo/Logo"; import { fetchMCPServers, listMCPTools } from "../networking"; import { MCPServer } from "../mcp_tools/types"; @@ -98,13 +99,10 @@ const MCPConnectPicker: React.FC = ({ accessToken, selectedServers, onCha return (
{server.mcp_info?.logo_url && ( - {`${name} { - (e.target as HTMLImageElement).style.display = "none"; - }} /> )}
diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index b0d534e9b7c..355db8bdfc7 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -244,7 +244,13 @@ const menuGroups: MenuGroup[] = [ icon: , external_url: "https://models.litellm.ai/cookbook", }, - { key: "caching", page: "caching", label: "Caching", icon: , roles: all_admin_roles }, + { + key: "caching", + page: "caching", + label: "Response Cache", + icon: , + roles: all_admin_roles, + }, { key: "experimental", page: "experimental", diff --git a/ui/litellm-dashboard/src/components/logging_settings_view.test.tsx b/ui/litellm-dashboard/src/components/logging_settings_view.test.tsx new file mode 100644 index 00000000000..b0a1b54bac7 --- /dev/null +++ b/ui/litellm-dashboard/src/components/logging_settings_view.test.tsx @@ -0,0 +1,40 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { LoggingSettingsView } from "./logging_settings_view"; + +describe("LoggingSettingsView logos", () => { + it("renders the bundled logo for a known logging integration", () => { + render( + , + ); + + expect(screen.getByAltText("Langfuse logo")).toHaveAttribute("src", "/_next/static/media/langfuse.png"); + }); + + it("renders the bundled logo for a disabled callback given by internal slug", () => { + render(); + + expect(screen.getByAltText("Datadog logo")).toHaveAttribute("src", "/_next/static/media/datadog.png"); + }); + + it("renders a letter avatar for an unknown callback name", () => { + render( + , + ); + + expect(document.querySelector("img")).toBeNull(); + expect(screen.getByText("m")).toBeInTheDocument(); + expect(screen.getByText("mystery_callback")).toBeInTheDocument(); + }); + + it("renders a letter avatar for the custom callback API, which has no bundled logo", () => { + render(); + + expect(screen.queryByAltText("Custom Callback API logo")).toBeNull(); + expect(screen.getByText("C")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/logging_settings_view.tsx b/ui/litellm-dashboard/src/components/logging_settings_view.tsx index 5124d98da5d..97eca9d6247 100644 --- a/ui/litellm-dashboard/src/components/logging_settings_view.tsx +++ b/ui/litellm-dashboard/src/components/logging_settings_view.tsx @@ -2,7 +2,7 @@ import React from "react"; import { Tag } from "antd"; import { CogIcon, BanIcon } from "@heroicons/react/outline"; import { callbackInfo, callback_map, reverse_callback_map } from "./callback_info_helpers"; -import { resolveLogoSrc } from "@/lib/assetPaths"; +import { Logo } from "@/components/molecules/logo/Logo"; interface LoggingConfig { callback_name: string; @@ -69,7 +69,6 @@ export function LoggingSettingsView({
{loggingConfigs.map((config, index) => { const displayName = getLoggingDisplayName(config.callback_name); - const logoUrl = resolveLogoSrc(callbackInfo[displayName]?.logo); return (
- {logoUrl ? ( - {displayName} - ) : ( - - )} +
{displayName} @@ -115,7 +114,6 @@ export function LoggingSettingsView({ {disabledCallbacks.map((callbackName, index) => { // Handle both display names and internal values const displayName = reverse_callback_map[callbackName] || callbackName; - const logoUrl = resolveLogoSrc(callbackInfo[displayName]?.logo); return (
- {logoUrl ? ( - {displayName} - ) : ( - - )} +
{displayName} Disabled for this key diff --git a/ui/litellm-dashboard/src/components/model_add/CredentialModal.tsx b/ui/litellm-dashboard/src/components/model_add/CredentialModal.tsx index c92a4a90578..534e06dfe52 100644 --- a/ui/litellm-dashboard/src/components/model_add/CredentialModal.tsx +++ b/ui/litellm-dashboard/src/components/model_add/CredentialModal.tsx @@ -4,8 +4,8 @@ import type { UploadProps } from "antd/es/upload"; import { useState } from "react"; import ProviderSpecificFields from "../add_model/provider_specific_fields"; import { CredentialItem } from "../networking"; -import { Providers, providerLogoMap } from "../provider_info_helpers"; -import { resolveLogoSrc } from "@/lib/assetPaths"; +import { Providers } from "../provider_info_helpers"; +import { Logo } from "@/components/molecules/logo/Logo"; import { resetCredentialFormOnProviderChange } from "./credential_form_helpers"; const { Link } = Typography; @@ -92,22 +92,7 @@ export default function CredentialModal({ {Object.entries(Providers).map(([providerEnum, providerDisplayName]) => (
- {`${providerEnum} { - const target = e.target as HTMLImageElement; - const parent = target.parentElement; - if (parent) { - const fallbackDiv = document.createElement("div"); - fallbackDiv.className = - "w-5 h-5 rounded-full bg-gray-200 flex items-center justify-center text-xs"; - fallbackDiv.textContent = providerDisplayName.charAt(0); - parent.replaceChild(fallbackDiv, target); - } - }} - /> + {providerDisplayName}
diff --git a/ui/litellm-dashboard/src/components/model_add/CredentialsPanel.test.tsx b/ui/litellm-dashboard/src/components/model_add/CredentialsPanel.test.tsx index af38645b00d..93a015c8a36 100644 --- a/ui/litellm-dashboard/src/components/model_add/CredentialsPanel.test.tsx +++ b/ui/litellm-dashboard/src/components/model_add/CredentialsPanel.test.tsx @@ -4,7 +4,7 @@ import userEvent from "@testing-library/user-event"; import { UploadProps } from "antd/es/upload"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { CredentialItem, credentialCreateCall } from "@/components/networking"; +import { CredentialItem, credentialCreateCall, credentialUpdateCall } from "@/components/networking"; import NotificationsManager from "@/components/molecules/notifications_manager"; import CredentialsPanel from "./CredentialsPanel"; @@ -51,11 +51,17 @@ vi.mock("./CredentialModal", () => ({ if (!open) { return null; } + const values = + mode === "edit" + ? { + credential_name: "openai-key", + custom_llm_provider: "openai", + api_key: "sk-1****2345", + api_base: "https://proxy.e2e.example.com/v1", + } + : { credential_name: "new-cred", custom_llm_provider: "openai" }; return ( - ); @@ -179,6 +185,26 @@ describe("CredentialsPanel", () => { expect(NotificationsManager.success).not.toHaveBeenCalled(); }); + it("drops the masked api key from the update payload while keeping the edited api base", async () => { + const user = userEvent.setup(); + mockUseAuthorized.mockReturnValue({ accessToken: "test-token", userRole: "Admin" }); + mockUseCredentials.mockReturnValue({ data: { credentials }, isLoading: false, refetch: vi.fn() }); + vi.mocked(credentialUpdateCall).mockResolvedValueOnce(undefined as never); + + renderPanel(); + + await user.click(screen.getByTestId("credential-actions-openai-key")); + await user.click(await screen.findByTestId("credential-action-edit")); + await user.click(screen.getByTestId("credential-modal-edit-submit")); + + await waitFor(() => { + expect(credentialUpdateCall).toHaveBeenCalled(); + }); + const [, updatedName, payload] = vi.mocked(credentialUpdateCall).mock.calls[0]; + expect(updatedName).toBe("openai-key"); + expect(payload.credential_values).toEqual({ api_base: "https://proxy.e2e.example.com/v1" }); + }); + describe("Admin Viewer write-action gating", () => { // Admin Viewer can VIEW credentials but must not add / edit / delete them. it("hides the Add Credential button but still lists credentials", () => { diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx index 1a4c0ed9ff5..011d0568afd 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.test.tsx @@ -1,6 +1,10 @@ /* @vitest-environment jsdom */ +import type { PaginationState } from "@tanstack/react-table"; import { act, render, screen } from "@testing-library/react"; -import { describe, it, expect, vi, beforeEach } from "vitest"; +import userEvent from "@testing-library/user-event"; +import { useState } from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + import HealthCheckComponent from "./HealthCheckComponent"; const mockIndividualModelHealthCheckCall = vi.fn(); @@ -11,9 +15,57 @@ vi.mock("../networking", () => ({ latestHealthChecksCall: (...args: unknown[]) => mockLatestHealthChecksCall(...args), })); -describe("HealthCheckComponent", () => { - const getDisplayModelName = (model: { model_name?: string }) => model.model_name ?? ""; +const getDisplayModelName = (model: { model_name?: string }) => model.model_name ?? ""; +const makeModel = (id: string, name = "gpt-4") => ({ + model_name: name, + model_info: { id }, + litellm_model_name: name, +}); + +interface HarnessProps { + modelData: { data: ReturnType[] }; + allModelsOnProxy: string[]; + rowCount?: number; + onPageIndexChange?: (pageIndex: number) => void; +} + +/** Holds pagination state so page changes exercise the real controlled wiring. */ +function Harness({ modelData, allModelsOnProxy, rowCount = 1, onPageIndexChange }: HarnessProps) { + const [pagination, setPagination] = useState({ pageIndex: 0, pageSize: 50 }); + + return ( + <> + {pagination.pageIndex} + { + setPagination((previous) => { + const next = typeof updater === "function" ? updater(previous) : updater; + onPageIndexChange?.(next.pageIndex); + return next; + }); + }} + rowCount={rowCount} + /> + + ); +} + +const renderHealthCheck = async (props: HarnessProps) => { + await act(async () => { + render(); + }); + await act(async () => { + await new Promise((r) => setTimeout(r, 0)); + }); +}; + +describe("HealthCheckComponent", () => { beforeEach(() => { vi.clearAllMocks(); mockLatestHealthChecksCall.mockResolvedValue({ latest_health_checks: {} }); @@ -26,29 +78,7 @@ describe("HealthCheckComponent", () => { }); it("should render the health check section", async () => { - const modelData = { - data: [ - { - model_name: "gpt-4", - model_info: { id: "deployment-1" }, - litellm_model_name: "gpt-4", - }, - ], - }; - - await act(async () => { - render( - , - ); - }); - await act(async () => { - await new Promise((r) => setTimeout(r, 0)); - }); + await renderHealthCheck({ modelData: { data: [makeModel("deployment-1")] }, allModelsOnProxy: ["deployment-1"] }); expect(screen.getByText("Model Health Status")).toBeInTheDocument(); expect( @@ -57,176 +87,149 @@ describe("HealthCheckComponent", () => { }); it("should call individualModelHealthCheckCall with model id when run health check is triggered", async () => { - const modelData = { - data: [ - { - model_name: "gpt-4", - model_info: { id: "deployment-abc-123" }, - litellm_model_name: "gpt-4", - }, - ], - }; - render( - , + , ); const runButtons = screen.getAllByTestId("run-health-check-btn"); expect(runButtons.length).toBeGreaterThanOrEqual(1); - const runButton = runButtons[0]; await act(async () => { - runButton.click(); + runButtons[0].click(); }); - await act(async () => { await new Promise((r) => setTimeout(r, 50)); }); - expect(mockIndividualModelHealthCheckCall).toHaveBeenCalledWith("token-123", "deployment-abc-123"); - expect(mockIndividualModelHealthCheckCall).not.toHaveBeenCalledWith("token-123", "gpt-4"); + expect(mockIndividualModelHealthCheckCall).toHaveBeenCalledWith("token", "deployment-abc-123"); + expect(mockIndividualModelHealthCheckCall).not.toHaveBeenCalledWith("token", "gpt-4"); }); - it("should show pagination controls and request the next page", async () => { - const onPageChange = vi.fn(); - const modelData = { - data: [ - { - model_name: "gpt-4", - model_info: { id: "deployment-1" }, - litellm_model_name: "gpt-4", - }, - ], - }; - - await act(async () => { - render( - , - ); - }); - await act(async () => { - await new Promise((r) => setTimeout(r, 0)); + it("should page through results with the shared pagination footer", async () => { + const onPageIndexChange = vi.fn(); + await renderHealthCheck({ + modelData: { data: [makeModel("deployment-1")] }, + allModelsOnProxy: ["deployment-1"], + rowCount: 75, + onPageIndexChange, }); - expect(screen.getByTestId("health-results-count")).toHaveTextContent("Showing 1 - 50 of 75 results"); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 1-50 of 75"); - await act(async () => { - screen.getByRole("button", { name: "Next" }).click(); + const user = userEvent.setup(); + await user.click(screen.getByTestId("pagination-next")); + + expect(onPageIndexChange).toHaveBeenCalledWith(1); + expect(screen.getByTestId("page-index")).toHaveTextContent("1"); + }); + + describe("row selection drives the bulk run", () => { + const twoModels = { data: [makeModel("id-alpha", "alpha"), makeModel("id-beta", "beta")] }; + const bothIds = ["id-alpha", "id-beta"]; + + it("runs only the selected models and labels the button accordingly", async () => { + await renderHealthCheck({ modelData: twoModels, allModelsOnProxy: bothIds, rowCount: 2 }); + const user = userEvent.setup(); + + expect(screen.getByTestId("run-health-checks")).toHaveTextContent("Run All Checks"); + + await user.click(screen.getByTestId("datatable-select-row-id-beta")); + expect(screen.getByTestId("run-health-checks")).toHaveTextContent("Run Selected Checks"); + + await act(async () => { + screen.getByTestId("run-health-checks").click(); + }); + await act(async () => { + await new Promise((r) => setTimeout(r, 50)); + }); + + expect(mockIndividualModelHealthCheckCall).toHaveBeenCalledWith("token", "id-beta"); + expect(mockIndividualModelHealthCheckCall).not.toHaveBeenCalledWith("token", "id-alpha"); }); - expect(onPageChange).toHaveBeenCalledWith(2); + it("falls back to every model on the page when nothing is selected", async () => { + await renderHealthCheck({ modelData: twoModels, allModelsOnProxy: bothIds, rowCount: 2 }); + + await act(async () => { + screen.getByTestId("run-health-checks").click(); + }); + await act(async () => { + await new Promise((r) => setTimeout(r, 50)); + }); + + expect(mockIndividualModelHealthCheckCall).toHaveBeenCalledWith("token", "id-alpha"); + expect(mockIndividualModelHealthCheckCall).toHaveBeenCalledWith("token", "id-beta"); + }); + + it("treats a full page selection as running everything", async () => { + await renderHealthCheck({ modelData: twoModels, allModelsOnProxy: bothIds, rowCount: 2 }); + const user = userEvent.setup(); + + await user.click(screen.getByTestId("datatable-select-all")); + + expect(screen.getByTestId("run-health-checks")).toHaveTextContent("Run All Checks"); + }); + + it("clears the selection from the Clear Selection button", async () => { + await renderHealthCheck({ modelData: twoModels, allModelsOnProxy: bothIds, rowCount: 2 }); + const user = userEvent.setup(); + + expect(screen.queryByTestId("clear-health-selection")).not.toBeInTheDocument(); + + await user.click(screen.getByTestId("datatable-select-row-id-alpha")); + await user.click(screen.getByTestId("clear-health-selection")); + + expect(screen.getByTestId("datatable-select-row-id-alpha")).toHaveAttribute("aria-checked", "false"); + expect(screen.getByTestId("run-health-checks")).toHaveTextContent("Run All Checks"); + }); + + // The pager swaps the underlying rows, so a carried-over selection would target + // models that are no longer on screen. + it("wipes the selection when the page changes", async () => { + await renderHealthCheck({ modelData: twoModels, allModelsOnProxy: bothIds, rowCount: 120 }); + const user = userEvent.setup(); + + await user.click(screen.getByTestId("datatable-select-row-id-alpha")); + expect(screen.getByTestId("clear-health-selection")).toBeInTheDocument(); + + await user.click(screen.getByTestId("pagination-next")); + + expect(screen.queryByTestId("clear-health-selection")).not.toBeInTheDocument(); + expect(screen.getByTestId("datatable-select-row-id-alpha")).toHaveAttribute("aria-checked", "false"); + }); }); describe("latest_health_checks keyed by model id", () => { it("should show status from latest_health_checks when keys match model ids", async () => { - const modelData = { - data: [ - { - model_name: "gpt-4", - model_info: { id: "id-alpha" }, - litellm_model_name: "gpt-4", - }, - { - model_name: "gpt-4", - model_info: { id: "id-beta" }, - litellm_model_name: "gpt-4", - }, - ], - }; - mockLatestHealthChecksCall.mockResolvedValue({ latest_health_checks: { - "id-alpha": { - status: "healthy", - checked_at: "2024-01-15T10:00:00Z", - error_message: null, - }, - "id-beta": { - status: "unhealthy", - checked_at: "2024-01-15T10:05:00Z", - error_message: "Connection failed", - }, + "id-alpha": { status: "healthy", checked_at: "2024-01-15T10:00:00Z", error_message: null }, + "id-beta": { status: "unhealthy", checked_at: "2024-01-15T10:05:00Z", error_message: "Connection failed" }, }, }); - await act(async () => { - render( - , - ); - }); - await act(async () => { - await new Promise((r) => setTimeout(r, 0)); + await renderHealthCheck({ + modelData: { data: [makeModel("id-alpha"), makeModel("id-beta")] }, + allModelsOnProxy: ["id-alpha", "id-beta"], + rowCount: 2, }); expect(mockLatestHealthChecksCall).toHaveBeenCalledWith("token"); - const healthyBadges = screen.getAllByText("healthy"); - const unhealthyBadges = screen.getAllByText("unhealthy"); - expect(healthyBadges.length).toBeGreaterThanOrEqual(1); - expect(unhealthyBadges.length).toBeGreaterThanOrEqual(1); + expect(screen.getAllByText("healthy").length).toBeGreaterThanOrEqual(1); + expect(screen.getAllByText("unhealthy").length).toBeGreaterThanOrEqual(1); }); it("should skip latest_health_checks entries whose key is not a known model id", async () => { - const modelData = { - data: [ - { - model_name: "gpt-4", - model_info: { id: "current-model-id" }, - litellm_model_name: "gpt-4", - }, - ], - }; - mockLatestHealthChecksCall.mockResolvedValue({ latest_health_checks: { - "current-model-id": { - status: "healthy", - checked_at: "2024-01-15T10:00:00Z", - error_message: null, - }, - "deleted-or-unknown-id": { - status: "unhealthy", - checked_at: "2024-01-15T10:05:00Z", - error_message: "Stale entry", - }, + "current-model-id": { status: "healthy", checked_at: "2024-01-15T10:00:00Z", error_message: null }, + "deleted-or-unknown-id": { status: "unhealthy", checked_at: "2024-01-15T10:05:00Z", error_message: "Stale" }, }, }); - await act(async () => { - render( - , - ); - }); - await act(async () => { - await new Promise((r) => setTimeout(r, 0)); + await renderHealthCheck({ + modelData: { data: [makeModel("current-model-id")] }, + allModelsOnProxy: ["current-model-id"], }); expect(screen.getByText("healthy")).toBeInTheDocument(); @@ -234,38 +237,15 @@ describe("HealthCheckComponent", () => { }); it("should not apply status when latest_health_checks key is model name not model id", async () => { - const modelData = { - data: [ - { - model_name: "gpt-4", - model_info: { id: "model-id-123" }, - litellm_model_name: "gpt-4", - }, - ], - }; - mockLatestHealthChecksCall.mockResolvedValue({ latest_health_checks: { - "gpt-4": { - status: "healthy", - checked_at: "2024-01-15T10:00:00Z", - error_message: null, - }, + "gpt-4": { status: "healthy", checked_at: "2024-01-15T10:00:00Z", error_message: null }, }, }); - await act(async () => { - render( - , - ); - }); - await act(async () => { - await new Promise((r) => setTimeout(r, 0)); + await renderHealthCheck({ + modelData: { data: [makeModel("model-id-123")] }, + allModelsOnProxy: ["model-id-123"], }); expect(screen.queryByText("healthy")).not.toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx index 6497f2686cb..46c4a726927 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx @@ -1,24 +1,141 @@ -import React, { useState, useEffect, useRef } from "react"; -import { Title, Text, Button } from "@tremor/react"; +import { OnChangeFn, PaginationState, RowSelectionState } from "@tanstack/react-table"; import { Modal } from "antd"; import { Button as AntdButton } from "antd"; -import { ModelDataTable } from "./table"; -import { healthCheckColumns } from "./health_check_columns"; -import { errorPatterns } from "@/utils/errorPatterns"; -import { individualModelHealthCheckCall, latestHealthChecksCall } from "../networking"; -import { Table as TableInstance } from "@tanstack/react-table"; -import { Team } from "../key_team_helpers/key_list"; +import React, { useCallback, useEffect, useMemo, useState } from "react"; -interface HealthStatus { - status: string; - lastCheck: string; - lastSuccess?: string; - loading: boolean; - error?: string; - fullError?: string; - successResponse?: any; +import { errorPatterns } from "@/utils/errorPatterns"; + +import { Team } from "../key_team_helpers/key_list"; +import { individualModelHealthCheckCall, latestHealthChecksCall } from "../networking"; +import { Button } from "@/components/ui/button"; +import { HealthChecksTable } from "./HealthChecksTable"; +import type { HealthCheckData, HealthStatus } from "./HealthChecksTableColumns"; + +interface LatestHealthCheck { + status?: string; + checked_at?: string | null; + error_message?: string | null; } +const STATUS_TO_ERROR: Record = { + "400": "BadRequestError", + "401": "AuthenticationError", + "403": "ForbiddenError", + "404": "NotFoundError", + "408": "TimeoutError", + "429": "RateLimitError", + "500": "InternalServerError", + "502": "BadGatewayError", + "503": "ServiceUnavailableError", + "504": "GatewayTimeoutError", +}; + +const ERROR_TO_STATUS: Record = { + AuthenticationError: "401", + RateLimitError: "429", + BadRequestError: "400", + InternalServerError: "500", + TimeoutError: "408", + NotFoundError: "404", + ForbiddenError: "403", + ServiceUnavailableError: "503", + BadGatewayError: "502", + GatewayTimeoutError: "504", + ContentPolicyViolationError: "400", +}; + +const KEYWORD_ERRORS: ReadonlyArray<{ pattern: RegExp; label: string }> = [ + { pattern: /missing.*api.*key|invalid.*key|unauthorized/i, label: "AuthenticationError: 401" }, + { pattern: /rate.*limit|too.*many.*requests/i, label: "RateLimitError: 429" }, + { pattern: /timeout|timed.*out/i, label: "TimeoutError: 408" }, + { pattern: /not.*found/i, label: "NotFoundError: 404" }, + { pattern: /forbidden|access.*denied/i, label: "ForbiddenError: 403" }, + { pattern: /internal.*server.*error/i, label: "InternalServerError: 500" }, +]; + +const truncate = (value: string): string => (value.length > 100 ? `${value.substring(0, 97)}...` : value); + +// Helper function to extract meaningful error information +const extractMeaningfulError = (error: unknown): string => { + if (!error) return "Health check failed"; + + const errorStr = typeof error === "string" ? error : JSON.stringify(error); + + // First, look for explicit "ErrorType: StatusCode" patterns + const directPatternMatch = errorStr.match(/(\w+Error):\s*(\d{3})/i); + if (directPatternMatch) { + return `${directPatternMatch[1]}: ${directPatternMatch[2]}`; + } + + // Look for error types and status codes separately, then combine them + const errorTypeMatch = errorStr.match( + /(AuthenticationError|RateLimitError|BadRequestError|InternalServerError|TimeoutError|NotFoundError|ForbiddenError|ServiceUnavailableError|BadGatewayError|ContentPolicyViolationError|\w+Error)/i, + ); + const statusCodeMatch = errorStr.match(/\b(400|401|403|404|408|429|500|502|503|504)\b/); + + if (errorTypeMatch && statusCodeMatch) { + return `${errorTypeMatch[1]}: ${statusCodeMatch[1]}`; + } + + // If we have a status code but no clear error type, map it + if (statusCodeMatch) { + const statusCode = statusCodeMatch[1]; + return `${STATUS_TO_ERROR[statusCode]}: ${statusCode}`; + } + + // If we have an error type but no status code, map error type to expected status code + if (errorTypeMatch) { + const errorType = errorTypeMatch[1]; + const mappedStatus = ERROR_TO_STATUS[errorType]; + if (mappedStatus) { + return `${errorType}: ${mappedStatus}`; + } + return errorType; + } + + // Check for specific error patterns from errorPatterns + for (const { pattern, replacement } of errorPatterns) { + if (pattern.test(errorStr)) { + return replacement; + } + } + + // Look for common error keywords and provide meaningful names with status codes + for (const { pattern, label } of KEYWORD_ERRORS) { + if (pattern.test(errorStr)) { + return label; + } + } + + // Fallback: clean up the error string and return first meaningful part + const cleaned = errorStr + .replace(/[\n\r]+/g, " ") + .replace(/\s+/g, " ") + .trim(); + + // Try to get first meaningful sentence or phrase + const firstSentence = cleaned.split(/[.!?]/)[0]?.trim(); + if (firstSentence && firstSentence.length > 0) { + return truncate(firstSentence); + } + + return truncate(cleaned); +}; + +const toCheckedAtLabel = (checkedAt: string | null | undefined, fallback: string): string => { + if (!checkedAt) { + return fallback; + } + return new Date(checkedAt).toLocaleString(); +}; + +const toLastSuccessLabel = (checkData: LatestHealthCheck, fallback: string): string => { + if (checkData.status !== "healthy") { + return fallback; + } + return toCheckedAtLabel(checkData.checked_at, fallback); +}; + interface HealthCheckComponentProps { accessToken: string | null; modelData: any; @@ -27,15 +144,9 @@ interface HealthCheckComponentProps { setSelectedModelId?: (modelId: string) => void; teams?: Team[] | null; isLoading?: boolean; - paginationMeta?: { - total_count: number; - current_page: number; - total_pages: number; - size: number; - }; - currentPage?: number; - pageSize?: number; - onPageChange?: (page: number) => void; + pagination: PaginationState; + onPaginationChange: OnChangeFn; + rowCount: number; } const HealthCheckComponent: React.FC = ({ @@ -46,14 +157,12 @@ const HealthCheckComponent: React.FC = ({ setSelectedModelId, teams, isLoading = false, - paginationMeta, - currentPage = 1, - pageSize = 50, - onPageChange, + pagination, + onPaginationChange, + rowCount, }) => { const [modelHealthStatuses, setModelHealthStatuses] = useState<{ [key: string]: HealthStatus }>({}); - const [selectedModelsForHealth, setSelectedModelsForHealth] = useState([]); - const [allModelsSelected, setAllModelsSelected] = useState(false); + const [rowSelection, setRowSelection] = useState({}); const [errorModalVisible, setErrorModalVisible] = useState(false); const [selectedErrorDetails, setSelectedErrorDetails] = useState<{ modelName: string; @@ -63,11 +172,9 @@ const HealthCheckComponent: React.FC = ({ const [successModalVisible, setSuccessModalVisible] = useState(false); const [selectedSuccessDetails, setSelectedSuccessDetails] = useState<{ modelName: string; - response: any; + response: unknown; } | null>(null); - const healthTableRef = useRef>(null); - // Initialize health statuses on component mount (keyed by model id) useEffect(() => { if (!accessToken || !modelData?.data) return; @@ -100,8 +207,9 @@ const HealthCheckComponent: React.FC = ({ latestHealthChecks.latest_health_checks && typeof latestHealthChecks.latest_health_checks === "object" ) { - Object.entries(latestHealthChecks.latest_health_checks).forEach(([modelId, checkData]: [string, any]) => { - if (!checkData) return; + Object.entries(latestHealthChecks.latest_health_checks).forEach(([modelId, rawCheck]) => { + if (!rawCheck) return; + const checkData = rawCheck as LatestHealthCheck; // Key is model_id from the backend (guaranteed by DB schema) const modelExists = modelData.data.some((m: any) => m.model_info?.id === modelId); @@ -111,13 +219,8 @@ const HealthCheckComponent: React.FC = ({ healthStatusMap[modelId] = { status: checkData.status || "unknown", - lastCheck: checkData.checked_at ? new Date(checkData.checked_at).toLocaleString() : "None", - lastSuccess: - checkData.status === "healthy" - ? checkData.checked_at - ? new Date(checkData.checked_at).toLocaleString() - : "None" - : "None", + lastCheck: toCheckedAtLabel(checkData.checked_at, "None"), + lastSuccess: toLastSuccessLabel(checkData, "None"), loading: false, error: fullError ? extractMeaningfulError(fullError) : undefined, fullError: fullError, @@ -135,132 +238,73 @@ const HealthCheckComponent: React.FC = ({ initializeHealthStatuses(); }, [accessToken, modelData]); - // Helper function to extract meaningful error information - const extractMeaningfulError = (error: any): string => { - if (!error) return "Health check failed"; + const runIndividualHealthCheck = useCallback( + async (modelId: string) => { + if (!accessToken) return; - let errorStr = typeof error === "string" ? error : JSON.stringify(error); + setModelHealthStatuses((prev) => ({ + ...prev, + [modelId]: { + ...prev[modelId], + loading: true, + status: "checking", + }, + })); - // First, look for explicit "ErrorType: StatusCode" patterns - const directPatternMatch = errorStr.match(/(\w+Error):\s*(\d{3})/i); - if (directPatternMatch) { - return `${directPatternMatch[1]}: ${directPatternMatch[2]}`; - } + try { + const response = await individualModelHealthCheckCall(accessToken, modelId); + const currentTime = new Date().toLocaleString(); - // Look for error types and status codes separately, then combine them - const errorTypeMatch = errorStr.match( - /(AuthenticationError|RateLimitError|BadRequestError|InternalServerError|TimeoutError|NotFoundError|ForbiddenError|ServiceUnavailableError|BadGatewayError|ContentPolicyViolationError|\w+Error)/i, - ); - const statusCodeMatch = errorStr.match(/\b(400|401|403|404|408|429|500|502|503|504)\b/); + if (response.unhealthy_count > 0 && response.unhealthy_endpoints && response.unhealthy_endpoints.length > 0) { + const rawError = response.unhealthy_endpoints[0]?.error || "Health check failed"; + const errorMessage = extractMeaningfulError(rawError); + setModelHealthStatuses((prev) => ({ + ...prev, + [modelId]: { + status: "unhealthy", + lastCheck: currentTime, + lastSuccess: prev[modelId]?.lastSuccess || "None", + loading: false, + error: errorMessage, + fullError: rawError, + }, + })); + } else { + setModelHealthStatuses((prev) => ({ + ...prev, + [modelId]: { + status: "healthy", + lastCheck: currentTime, + lastSuccess: currentTime, + loading: false, + successResponse: response, + }, + })); + } - if (errorTypeMatch && statusCodeMatch) { - return `${errorTypeMatch[1]}: ${statusCodeMatch[1]}`; - } + try { + const latestHealthChecks = await latestHealthChecksCall(accessToken); + const checkData = latestHealthChecks.latest_health_checks?.[modelId] as LatestHealthCheck | undefined; - // If we have a status code but no clear error type, map it - if (statusCodeMatch) { - const statusCode = statusCodeMatch[1]; - const statusToError: { [key: string]: string } = { - "400": "BadRequestError", - "401": "AuthenticationError", - "403": "ForbiddenError", - "404": "NotFoundError", - "408": "TimeoutError", - "429": "RateLimitError", - "500": "InternalServerError", - "502": "BadGatewayError", - "503": "ServiceUnavailableError", - "504": "GatewayTimeoutError", - }; - return `${statusToError[statusCode]}: ${statusCode}`; - } - - // If we have an error type but no status code, map error type to expected status code - if (errorTypeMatch) { - const errorType = errorTypeMatch[1]; - const errorToStatus: { [key: string]: string } = { - AuthenticationError: "401", - RateLimitError: "429", - BadRequestError: "400", - InternalServerError: "500", - TimeoutError: "408", - NotFoundError: "404", - ForbiddenError: "403", - ServiceUnavailableError: "503", - BadGatewayError: "502", - GatewayTimeoutError: "504", - ContentPolicyViolationError: "400", - }; - - const mappedStatus = errorToStatus[errorType]; - if (mappedStatus) { - return `${errorType}: ${mappedStatus}`; - } - return errorType; - } - - // Check for specific error patterns from errorPatterns - for (const { pattern, replacement } of errorPatterns) { - if (pattern.test(errorStr)) { - return replacement; - } - } - - // Look for common error keywords and provide meaningful names with status codes - if (/missing.*api.*key|invalid.*key|unauthorized/i.test(errorStr)) { - return "AuthenticationError: 401"; - } - if (/rate.*limit|too.*many.*requests/i.test(errorStr)) { - return "RateLimitError: 429"; - } - if (/timeout|timed.*out/i.test(errorStr)) { - return "TimeoutError: 408"; - } - if (/not.*found/i.test(errorStr)) { - return "NotFoundError: 404"; - } - if (/forbidden|access.*denied/i.test(errorStr)) { - return "ForbiddenError: 403"; - } - if (/internal.*server.*error/i.test(errorStr)) { - return "InternalServerError: 500"; - } - - // Fallback: clean up the error string and return first meaningful part - const cleaned = errorStr - .replace(/[\n\r]+/g, " ") - .replace(/\s+/g, " ") - .trim(); - - // Try to get first meaningful sentence or phrase - const sentences = cleaned.split(/[.!?]/); - const firstSentence = sentences[0]?.trim(); - - if (firstSentence && firstSentence.length > 0) { - return firstSentence.length > 100 ? firstSentence.substring(0, 97) + "..." : firstSentence; - } - - return cleaned.length > 100 ? cleaned.substring(0, 97) + "..." : cleaned; - }; - - const runIndividualHealthCheck = async (modelId: string) => { - if (!accessToken) return; - - setModelHealthStatuses((prev) => ({ - ...prev, - [modelId]: { - ...prev[modelId], - loading: true, - status: "checking", - }, - })); - - try { - const response = await individualModelHealthCheckCall(accessToken, modelId); - const currentTime = new Date().toLocaleString(); - - if (response.unhealthy_count > 0 && response.unhealthy_endpoints && response.unhealthy_endpoints.length > 0) { - const rawError = response.unhealthy_endpoints[0]?.error || "Health check failed"; + if (checkData) { + const fullError = checkData.error_message || undefined; + setModelHealthStatuses((prev) => ({ + ...prev, + [modelId]: { + status: checkData.status || prev[modelId]?.status || "unknown", + lastCheck: toCheckedAtLabel(checkData.checked_at, prev[modelId]?.lastCheck || "None"), + lastSuccess: toLastSuccessLabel(checkData, prev[modelId]?.lastSuccess || "None"), + loading: false, + error: fullError ? extractMeaningfulError(fullError) : prev[modelId]?.error, + fullError: fullError || prev[modelId]?.fullError, + successResponse: checkData.status === "healthy" ? checkData : prev[modelId]?.successResponse, + }, + })); + } + } catch (dbError) {} + } catch (error) { + const currentTime = new Date().toLocaleString(); + const rawError = error instanceof Error ? error.message : String(error); const errorMessage = extractMeaningfulError(rawError); setModelHealthStatuses((prev) => ({ ...prev, @@ -273,66 +317,18 @@ const HealthCheckComponent: React.FC = ({ fullError: rawError, }, })); - } else { - setModelHealthStatuses((prev) => ({ - ...prev, - [modelId]: { - status: "healthy", - lastCheck: currentTime, - lastSuccess: currentTime, - loading: false, - successResponse: response, - }, - })); } + }, + [accessToken], + ); - try { - const latestHealthChecks = await latestHealthChecksCall(accessToken); - const checkData = latestHealthChecks.latest_health_checks?.[modelId]; - - if (checkData) { - const fullError = checkData.error_message || undefined; - setModelHealthStatuses((prev) => ({ - ...prev, - [modelId]: { - status: checkData.status || prev[modelId]?.status || "unknown", - lastCheck: checkData.checked_at - ? new Date(checkData.checked_at).toLocaleString() - : prev[modelId]?.lastCheck || "None", - lastSuccess: - checkData.status === "healthy" - ? checkData.checked_at - ? new Date(checkData.checked_at).toLocaleString() - : prev[modelId]?.lastSuccess || "None" - : prev[modelId]?.lastSuccess || "None", - loading: false, - error: fullError ? extractMeaningfulError(fullError) : prev[modelId]?.error, - fullError: fullError || prev[modelId]?.fullError, - successResponse: checkData.status === "healthy" ? checkData : prev[modelId]?.successResponse, - }, - })); - } - } catch (dbError) {} - } catch (error) { - const currentTime = new Date().toLocaleString(); - const rawError = error instanceof Error ? error.message : String(error); - const errorMessage = extractMeaningfulError(rawError); - setModelHealthStatuses((prev) => ({ - ...prev, - [modelId]: { - status: "unhealthy", - lastCheck: currentTime, - lastSuccess: prev[modelId]?.lastSuccess || "None", - loading: false, - error: errorMessage, - fullError: rawError, - }, - })); - } - }; + const selectedModelIds = useMemo( + () => Object.keys(rowSelection).filter((modelId) => rowSelection[modelId]), + [rowSelection], + ); const runAllHealthChecks = async () => { - const modelsToCheck = selectedModelsForHealth.length > 0 ? selectedModelsForHealth : all_models_on_proxy; + const modelsToCheck = selectedModelIds.length > 0 ? selectedModelIds : all_models_on_proxy; const loadingStatuses = modelsToCheck.reduce( (acc, modelId) => { @@ -348,14 +344,11 @@ const HealthCheckComponent: React.FC = ({ setModelHealthStatuses((prev) => ({ ...prev, ...loadingStatuses })); - const healthCheckResults: { [key: string]: any } = {}; - const healthCheckPromises = modelsToCheck.map(async (modelId) => { if (!accessToken) return; try { const response = await individualModelHealthCheckCall(accessToken, modelId); - healthCheckResults[modelId] = response; const currentTime = new Date().toLocaleString(); if (response.unhealthy_count > 0 && response.unhealthy_endpoints && response.unhealthy_endpoints.length > 0) { @@ -410,32 +403,26 @@ const HealthCheckComponent: React.FC = ({ const latestHealthChecks = await latestHealthChecksCall(accessToken); if (latestHealthChecks.latest_health_checks) { - Object.entries(latestHealthChecks.latest_health_checks).forEach(([modelId, checkData]: [string, any]) => { - if (modelsToCheck.includes(modelId) && checkData) { - const fullError = checkData.error_message || undefined; - setModelHealthStatuses((prev) => { - const currentStatus = prev[modelId]; - return { - ...prev, - [modelId]: { - status: checkData.status || currentStatus?.status || "unknown", - lastCheck: checkData.checked_at - ? new Date(checkData.checked_at).toLocaleString() - : currentStatus?.lastCheck || "None", - lastSuccess: - checkData.status === "healthy" - ? checkData.checked_at - ? new Date(checkData.checked_at).toLocaleString() - : currentStatus?.lastSuccess || "None" - : currentStatus?.lastSuccess || "None", - loading: false, - error: fullError ? extractMeaningfulError(fullError) : currentStatus?.error, - fullError: fullError || currentStatus?.fullError, - successResponse: checkData.status === "healthy" ? checkData : currentStatus?.successResponse, - }, - }; - }); - } + Object.entries(latestHealthChecks.latest_health_checks).forEach(([modelId, rawCheck]) => { + if (!modelsToCheck.includes(modelId) || !rawCheck) return; + const checkData = rawCheck as LatestHealthCheck; + const fullError = checkData.error_message || undefined; + + setModelHealthStatuses((prev) => { + const currentStatus = prev[modelId]; + return { + ...prev, + [modelId]: { + status: checkData.status || currentStatus?.status || "unknown", + lastCheck: toCheckedAtLabel(checkData.checked_at, currentStatus?.lastCheck || "None"), + lastSuccess: toLastSuccessLabel(checkData, currentStatus?.lastSuccess || "None"), + loading: false, + error: fullError ? extractMeaningfulError(fullError) : currentStatus?.error, + fullError: fullError || currentStatus?.fullError, + successResponse: checkData.status === "healthy" ? checkData : currentStatus?.successResponse, + }, + }; + }); }); } } catch (dbError) { @@ -443,170 +430,116 @@ const HealthCheckComponent: React.FC = ({ } }; - const handleModelSelection = (modelId: string, checked: boolean) => { - if (checked) { - setSelectedModelsForHealth((prev) => [...prev, modelId]); - } else { - setSelectedModelsForHealth((prev) => prev.filter((id) => id !== modelId)); - setAllModelsSelected(false); - } - }; + // Changing the page swaps the underlying rows, so a carried-over selection would + // point at models that are no longer on screen. + const handlePaginationChange = useCallback>( + (updaterOrValue) => { + setRowSelection({}); + setModelHealthStatuses({}); + onPaginationChange(updaterOrValue); + }, + [onPaginationChange], + ); - const handleSelectAll = (checked: boolean) => { - setAllModelsSelected(checked); - if (checked) { - setSelectedModelsForHealth(all_models_on_proxy); - } else { - setSelectedModelsForHealth([]); - } - }; - - const handlePageChange = (page: number) => { - setSelectedModelsForHealth([]); - setAllModelsSelected(false); - setModelHealthStatuses({}); - onPageChange?.(page); - }; - - const showErrorModal = (modelName: string, cleanedError: string, fullError: string) => { - setSelectedErrorDetails({ - modelName, - cleanedError, - fullError, - }); + const showErrorModal = useCallback((modelName: string, cleanedError: string, fullError: string) => { + setSelectedErrorDetails({ modelName, cleanedError, fullError }); setErrorModalVisible(true); - }; + }, []); const closeErrorModal = () => { setErrorModalVisible(false); setSelectedErrorDetails(null); }; - const showSuccessModal = (modelName: string, response: any) => { - setSelectedSuccessDetails({ - modelName, - response, - }); + const showSuccessModal = useCallback((modelName: string, response: unknown) => { + setSelectedSuccessDetails({ modelName, response }); setSuccessModalVisible(true); - }; + }, []); const closeSuccessModal = () => { setSuccessModalVisible(false); setSelectedSuccessDetails(null); }; - const healthTableData = (modelData?.data ?? []).map((model: any) => { - const modelId = model.model_info?.id; - const healthStatus = modelId ? modelHealthStatuses[modelId] : null; - const status = healthStatus || { - status: "none", - lastCheck: "None", - loading: false, - }; - return { - model_name: model.model_name, - model_info: model.model_info, - provider: model.provider, - litellm_model_name: model.litellm_model_name, - health_status: status.status, - last_check: status.lastCheck, - last_success: status.lastSuccess || "None", - health_loading: status.loading, - health_error: status.error, - health_full_error: status.fullError, - }; - }); + const healthTableData = useMemo( + () => + (modelData?.data ?? []).map((model: any) => { + const modelId = model.model_info?.id; + const healthStatus = modelId ? modelHealthStatuses[modelId] : null; + const status = healthStatus || { + status: "none", + lastCheck: "None", + loading: false, + }; + return { + model_name: model.model_name, + model_info: model.model_info, + provider: model.provider, + litellm_model_name: model.litellm_model_name, + health_status: status.status, + last_check: status.lastCheck, + last_success: status.lastSuccess || "None", + health_loading: status.loading, + health_error: status.error, + health_full_error: status.fullError, + }; + }), + [modelData, modelHealthStatuses], + ); - const shouldShowPagination = Boolean(paginationMeta && onPageChange); - const totalCount = paginationMeta?.total_count ?? 0; - const totalPages = paginationMeta?.total_pages ?? 1; - const pageForDisplay = paginationMeta?.current_page ?? currentPage; - const pageSizeForDisplay = paginationMeta?.size ?? pageSize; - const resultsStart = shouldShowPagination && totalCount > 0 ? (pageForDisplay - 1) * pageSizeForDisplay + 1 : 0; - const resultsEnd = shouldShowPagination ? Math.min(pageForDisplay * pageSizeForDisplay, totalCount) : 0; + const isPartialSelection = selectedModelIds.length > 0 && selectedModelIds.length < all_models_on_proxy.length; + const anyCheckRunning = Object.values(modelHealthStatuses).some((status) => status.loading); return (
-
+
- Model Health Status - +

Model Health Status

+

Run health checks on individual models to verify they are working correctly - +

- {selectedModelsForHealth.length > 0 && ( - )}
-
- {shouldShowPagination && ( -
- - {totalCount > 0 - ? `Showing ${resultsStart} - ${resultsEnd} of ${totalCount} results` - : "Showing 0 results"} - - -
- - -
-
- )} - -
+ {/* Error Modal */} = ({ {selectedErrorDetails && (
- Error: -
- {selectedErrorDetails.cleanedError} + Error: +
+ {selectedErrorDetails.cleanedError}
- Full Error Details: -
-
{selectedErrorDetails.fullError}
+ Full Error Details: +
+
{selectedErrorDetails.fullError}
@@ -656,16 +589,16 @@ const HealthCheckComponent: React.FC = ({ {selectedSuccessDetails && (
- Status: -
- Health check passed successfully + Status: +
+ Health check passed successfully
- Response Details: -
-
+              Response Details:
+              
+
                   {JSON.stringify(selectedSuccessDetails.response, null, 2)}
                 
diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTable.test.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTable.test.tsx new file mode 100644 index 00000000000..fc5606d46b4 --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTable.test.tsx @@ -0,0 +1,175 @@ +/* @vitest-environment jsdom */ +import type { PaginationState, RowSelectionState } from "@tanstack/react-table"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { useState } from "react"; +import { describe, expect, it, vi } from "vitest"; + +import { HealthChecksTable } from "./HealthChecksTable"; +import type { HealthCheckData, HealthStatus } from "./HealthChecksTableColumns"; + +const makeRow = (overrides: Partial & { id: string }): HealthCheckData => { + const { id, ...rest } = overrides; + return { + model_name: `model-${id}`, + model_info: { id }, + health_status: "none", + last_check: "None", + last_success: "None", + health_loading: false, + ...rest, + }; +}; + +interface HarnessProps { + data: HealthCheckData[]; + modelHealthStatuses?: Record; + onRunHealthCheck?: (modelId: string) => void; + onSelectModel?: (modelId: string) => void; +} + +function Harness({ data, modelHealthStatuses = {}, onRunHealthCheck = vi.fn(), onSelectModel }: HarnessProps) { + const [pagination, setPagination] = useState({ pageIndex: 0, pageSize: 50 }); + const [rowSelection, setRowSelection] = useState({}); + + return ( + model.model_name} + onRunHealthCheck={onRunHealthCheck} + onShowError={vi.fn()} + onShowSuccess={vi.fn()} + onSelectModel={onSelectModel} + /> + ); +} + +/** Row order by model id, read off the per-row selection checkbox (keyed by getRowId). */ +const rowIds = (): string[] => + screen + .getAllByRole("row") + .slice(1) + .map((row) => row.querySelector('[data-testid^="datatable-select-row-"]')) + .filter((node): node is Element => node !== null) + .map((node) => (node.getAttribute("data-testid") ?? "").replace("datatable-select-row-", "")); + +describe("HealthChecksTable client sorting", () => { + it("orders health status healthy > checking > unknown > unhealthy", async () => { + const user = userEvent.setup(); + render( + , + ); + + await user.click(screen.getByTestId("sort-header-health_status")); + + expect(rowIds()).toEqual(["healthy-row", "checking-row", "unhealthy-row", "weird-row"]); + }); + + it("floats in-progress checks to the top, sinks never-checked, and sorts real checks most-recent-first", async () => { + const user = userEvent.setup(); + render( + , + ); + + await user.click(screen.getByTestId("sort-header-last_check")); + + expect(rowIds()).toEqual(["in-progress", "newer", "older", "never"]); + }); + + // "Never succeeded" is ranked below "None" -- both sink, but not to the same slot. + it("sinks None below real successes and Never succeeded below None", async () => { + const user = userEvent.setup(); + render( + , + ); + + await user.click(screen.getByTestId("sort-header-last_success")); + + expect(rowIds()).toEqual(["newer", "older", "none", "never"]); + }); +}); + +describe("HealthChecksTable rows", () => { + it("renders the live checking cell while a row is loading and disables its run button", () => { + render(); + + expect(screen.getByText("Checking...")).toBeInTheDocument(); + expect(screen.getByTestId("run-health-check-btn")).toBeDisabled(); + }); + + it("runs a health check for the row's model id", async () => { + const user = userEvent.setup(); + const onRunHealthCheck = vi.fn(); + render(); + + await user.click(screen.getByTestId("run-health-check-btn")); + + expect(onRunHealthCheck).toHaveBeenCalledWith("deployment-9"); + }); + + it("opens the model detail from the identity cell", async () => { + const user = userEvent.setup(); + const onSelectModel = vi.fn(); + render(); + + await user.click(screen.getByRole("button", { name: /deployment-9/ })); + + expect(onSelectModel).toHaveBeenCalledWith("deployment-9"); + }); + + it("surfaces the error detail button only when a fuller error exists", () => { + const { rerender } = render( + , + ); + expect(screen.queryByTestId("view-health-error-btn")).not.toBeInTheDocument(); + + rerender( + , + ); + expect(screen.getByTestId("view-health-error-btn")).toBeInTheDocument(); + }); + + it("renders the empty state when the page has no models", () => { + render(); + + expect(screen.getByText("No models found")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTable.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTable.tsx new file mode 100644 index 00000000000..5c0470315bb --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTable.tsx @@ -0,0 +1,92 @@ +"use client"; + +import { OnChangeFn, PaginationState, RowSelectionState, SortingState } from "@tanstack/react-table"; +import { HeartPulse } from "lucide-react"; +import { useMemo, useState } from "react"; + +import { Team } from "@/components/key_team_helpers/key_list"; +import { DataTable } from "@/components/shared/DataTable"; + +import { getHealthChecksTableColumns, type HealthCheckData, type HealthStatus } from "./HealthChecksTableColumns"; + +interface HealthChecksTableProps { + data: HealthCheckData[]; + rowCount: number; + isLoading: boolean; + pagination: PaginationState; + onPaginationChange: OnChangeFn; + rowSelection: RowSelectionState; + onRowSelectionChange: OnChangeFn; + modelHealthStatuses: Record; + getDisplayModelName: (model: HealthCheckData) => string; + onRunHealthCheck: (modelId: string) => void; + onShowError: (modelName: string, cleanedError: string, fullError: string) => void; + onShowSuccess: (modelName: string, response: unknown) => void; + onSelectModel?: (modelId: string) => void; + teams?: Team[] | null; +} + +function EmptyState() { + return ( +
+
+ +
+
No models found
+
Models added to this proxy will show their health here.
+
+ ); +} + +export function HealthChecksTable({ + data, + rowCount, + isLoading, + pagination, + onPaginationChange, + rowSelection, + onRowSelectionChange, + modelHealthStatuses, + getDisplayModelName, + onRunHealthCheck, + onShowError, + onShowSuccess, + onSelectModel, + teams, +}: HealthChecksTableProps) { + const [sorting, setSorting] = useState([]); + + const columns = useMemo(() => { + const columnDeps = { + modelHealthStatuses, + getDisplayModelName, + onRunHealthCheck, + onShowError, + onShowSuccess, + onSelectModel, + teams, + }; + return getHealthChecksTableColumns(columnDeps); + }, [modelHealthStatuses, getDisplayModelName, onRunHealthCheck, onShowError, onShowSuccess, onSelectModel, teams]); + + return ( + row.model_info?.id ?? String(index)} + sortingMode="client" + sorting={sorting} + onSortingChange={setSorting} + paginationMode="server" + pagination={pagination} + onPaginationChange={onPaginationChange} + rowCount={rowCount} + rowSelection={rowSelection} + onRowSelectionChange={onRowSelectionChange} + isLoading={isLoading} + loadingMessage="Loading models…" + noDataMessage={} + size="compact" + /> + ); +} diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx new file mode 100644 index 00000000000..95857488b20 --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthChecksTableColumns.tsx @@ -0,0 +1,413 @@ +"use client"; + +import { ColumnDef } from "@tanstack/react-table"; +import { Info, Play, RefreshCw } from "lucide-react"; + +import { Team } from "@/components/key_team_helpers/key_list"; +import { createSelectionColumn, DataTableSortHeader } from "@/components/shared/DataTable"; +import { IdentityCell, StatusBadge, type StatusTone } from "@/components/shared/table_cells"; +import { cn } from "@/lib/cva.config"; + +export interface HealthStatus { + status: string; + lastCheck: string; + lastSuccess?: string; + loading: boolean; + error?: string; + fullError?: string; + successResponse?: unknown; +} + +export interface HealthCheckData { + model_name: string; + model_info: { + id: string; + created_at?: string; + team_id?: string; + }; + provider?: string; + litellm_model_name?: string; + health_status: string; + last_check: string; + last_success: string; + health_loading: boolean; + health_error?: string; + health_full_error?: string; +} + +const HEALTH_STATUS_TONES: Record = { + healthy: "success", + unhealthy: "error", + checking: "info", + none: "neutral", +}; + +// healthy > checking > unknown > unhealthy, matching the legacy health table ordering. +const HEALTH_STATUS_ORDER: Record = { healthy: 0, checking: 1, unknown: 2, unhealthy: 3 }; + +const NEVER_CHECKED = "Never checked"; +const CHECK_IN_PROGRESS = "Check in progress..."; +const NEVER_SUCCEEDED = "Never succeeded"; +const NONE = "None"; + +function HealthStatusBadge({ status }: { status: string }) { + const tone = HEALTH_STATUS_TONES[status]; + if (!tone) { + return ; + } + return ; +} + +function DotPulse({ className }: { className: string }) { + return ( +
+
+
+
+
+ ); +} + +function DetailButton({ + label, + onClick, + className, + testId, +}: { + label: string; + onClick: () => void; + className: string; + testId: string; +}) { + return ( + + ); +} + +function runButtonLabel(isLoading: boolean, hasExistingStatus: boolean): string { + if (isLoading) { + return "Checking..."; + } + if (hasExistingStatus) { + return "Re-run Health Check"; + } + return "Run Health Check"; +} + +function RunButtonIcon({ isLoading, hasExistingStatus }: { isLoading: boolean; hasExistingStatus: boolean }) { + if (isLoading) { + return ; + } + if (hasExistingStatus) { + return ; + } + return ; +} + +function RunHealthCheckButton({ + model, + onRunHealthCheck, +}: { + model: HealthCheckData; + onRunHealthCheck: (modelId: string) => void; +}) { + const isLoading = model.health_loading; + const hasExistingStatus = Boolean(model.health_status) && model.health_status !== "none"; + const label = runButtonLabel(isLoading, hasExistingStatus); + + return ( + + ); +} + +function compareDatesDesc(rawA: string, rawB: string): number { + const dateA = new Date(rawA).getTime(); + const dateB = new Date(rawB).getTime(); + if (isNaN(dateA) && isNaN(dateB)) { + return 0; + } + if (isNaN(dateA)) { + return 1; + } + if (isNaN(dateB)) { + return -1; + } + return dateB - dateA; +} + +/** + * Ranks the sentinel strings the health table renders in place of a real timestamp. + * `bottom` and `top` are checked in order, so an earlier sentinel outranks a later one + * (e.g. "Never succeeded" sorts below "None"). + */ +function compareSentinels( + rawA: string, + rawB: string, + bottom: readonly string[], + top: readonly string[], +): number | null { + for (const sentinel of bottom) { + if (rawA === sentinel && rawB === sentinel) { + return 0; + } + if (rawA === sentinel) { + return 1; + } + if (rawB === sentinel) { + return -1; + } + } + + for (const sentinel of top) { + if (rawA === sentinel && rawB === sentinel) { + return 0; + } + if (rawA === sentinel) { + return -1; + } + if (rawB === sentinel) { + return 1; + } + } + + return null; +} + +export interface HealthChecksTableColumnsDeps { + modelHealthStatuses: Record; + getDisplayModelName: (model: HealthCheckData) => string; + onRunHealthCheck: (modelId: string) => void; + onShowError: (modelName: string, cleanedError: string, fullError: string) => void; + onShowSuccess: (modelName: string, response: unknown) => void; + onSelectModel?: (modelId: string) => void; + teams?: Team[] | null; +} + +export const getHealthChecksTableColumns = ({ + modelHealthStatuses, + getDisplayModelName, + onRunHealthCheck, + onShowError, + onShowSuccess, + onSelectModel, + teams, +}: HealthChecksTableColumnsDeps): ColumnDef[] => [ + createSelectionColumn({ + rowAriaLabel: (row) => `Select ${row.original.model_info?.id ?? row.original.model_name}`, + }), + { + id: "model_id", + accessorFn: (row) => row.model_info?.id ?? "", + meta: { title: "Model ID" }, + header: ({ column }) => , + size: 220, + enableSorting: true, + sortingFn: "alphanumeric", + cell: ({ row }) => { + const modelId = row.original.model_info?.id ?? ""; + return ( + onSelectModel(modelId) : undefined} + /> + ); + }, + }, + { + id: "model_name", + accessorKey: "model_name", + meta: { title: "Model Name" }, + header: ({ column }) => , + size: 200, + enableSorting: true, + sortingFn: "alphanumeric", + cell: ({ row }) => { + const displayName = getDisplayModelName(row.original) || row.original.model_name; + return ( + + {displayName} + + ); + }, + }, + { + id: "team_id", + accessorFn: (row) => row.model_info?.team_id ?? "", + meta: { title: "Team Alias" }, + header: ({ column }) => , + size: 160, + enableSorting: true, + sortingFn: "alphanumeric", + cell: ({ row }) => { + const teamId = row.original.model_info?.team_id; + if (!teamId) { + return -; + } + const teamAlias = teams?.find((team) => team.team_id === teamId)?.team_alias || teamId; + return ( + + {teamAlias} + + ); + }, + }, + { + id: "health_status", + accessorKey: "health_status", + meta: { title: "Health Status", skeleton: "badge" }, + header: ({ column }) => , + size: 170, + enableSorting: true, + sortingFn: (rowA, rowB) => { + const statusA = (rowA.getValue("health_status") as string) || "unknown"; + const statusB = (rowB.getValue("health_status") as string) || "unknown"; + const orderA = HEALTH_STATUS_ORDER[statusA] ?? 4; + const orderB = HEALTH_STATUS_ORDER[statusB] ?? 4; + return orderA - orderB; + }, + cell: ({ row }) => { + const model = row.original; + + if (model.health_loading) { + return ( +
+ + Checking... +
+ ); + } + + const modelId = model.model_info?.id ?? ""; + const displayName = getDisplayModelName(model) || model.model_name; + const successResponse = modelHealthStatuses[modelId]?.successResponse; + const hasSuccessResponse = model.health_status === "healthy" && successResponse !== undefined; + + return ( +
+ + {hasSuccessResponse && ( + onShowSuccess(displayName, successResponse)} + /> + )} +
+ ); + }, + }, + { + id: "health_error", + accessorKey: "health_error", + meta: { title: "Error Details" }, + header: "Error Details", + size: 240, + enableSorting: false, + cell: ({ row }) => { + const model = row.original; + const modelId = model.model_info?.id ?? ""; + const healthStatus = modelHealthStatuses[modelId]; + + if (!healthStatus?.error) { + return No errors; + } + + const cleanedError = healthStatus.error; + const fullError = healthStatus.fullError || healthStatus.error; + const displayName = getDisplayModelName(model) || model.model_name; + + return ( +
+ + {cleanedError} + + {fullError !== cleanedError && ( + onShowError(displayName, cleanedError, fullError)} + /> + )} +
+ ); + }, + }, + { + id: "last_check", + accessorKey: "last_check", + meta: { title: "Last Check" }, + header: ({ column }) => , + size: 170, + enableSorting: true, + sortingFn: (rowA, rowB) => { + const rawA = (rowA.getValue("last_check") as string) || NEVER_CHECKED; + const rawB = (rowB.getValue("last_check") as string) || NEVER_CHECKED; + const sentinel = compareSentinels(rawA, rawB, [NEVER_CHECKED], [CHECK_IN_PROGRESS]); + return sentinel ?? compareDatesDesc(rawA, rawB); + }, + cell: ({ row }) => ( + + {row.original.health_loading ? CHECK_IN_PROGRESS : row.original.last_check} + + ), + }, + { + id: "last_success", + accessorKey: "last_success", + meta: { title: "Last Success" }, + header: ({ column }) => , + size: 170, + enableSorting: true, + sortingFn: (rowA, rowB) => { + const rawA = (rowA.getValue("last_success") as string) || NEVER_SUCCEEDED; + const rawB = (rowB.getValue("last_success") as string) || NEVER_SUCCEEDED; + const sentinel = compareSentinels(rawA, rawB, [NEVER_SUCCEEDED, NONE], []); + return sentinel ?? compareDatesDesc(rawA, rawB); + }, + cell: ({ row }) => { + const modelId = row.original.model_info?.id ?? ""; + const lastSuccess = modelHealthStatuses[modelId]?.lastSuccess || NONE; + return {lastSuccess}; + }, + }, + { + id: "actions", + meta: { title: "Actions", className: "text-right", headerClassName: "text-right" }, + header: () => Actions, + size: 80, + enableSorting: false, + enableHiding: false, + cell: ({ row }) => ( +
+ +
+ ), + }, +]; diff --git a/ui/litellm-dashboard/src/components/model_dashboard/health_check_columns.tsx b/ui/litellm-dashboard/src/components/model_dashboard/health_check_columns.tsx deleted file mode 100644 index 33c97236f97..00000000000 --- a/ui/litellm-dashboard/src/components/model_dashboard/health_check_columns.tsx +++ /dev/null @@ -1,364 +0,0 @@ -import { ColumnDef } from "@tanstack/react-table"; -import { Tooltip, Checkbox } from "antd"; -import { Text } from "@tremor/react"; -import { InformationCircleIcon, PlayIcon, RefreshIcon } from "@heroicons/react/outline"; -import { Team } from "@/components/key_team_helpers/key_list"; -import { IdCell, StatusBadge, type StatusTone } from "@/components/shared/table_cells"; - -interface HealthCheckData { - model_name: string; - model_info: { - id: string; - created_at?: string; - team_id?: string; - }; - provider?: string; - litellm_model_name?: string; - health_status: string; - last_check: string; - last_success: string; - health_loading: boolean; - health_error?: string; - health_full_error?: string; -} - -const HEALTH_STATUS_TONES: Record = { - healthy: "success", - unhealthy: "error", - checking: "info", - none: "neutral", -}; - -const healthStatusBadge = (status: string): JSX.Element => { - const tone = HEALTH_STATUS_TONES[status]; - return tone ? : ; -}; - -interface HealthStatus { - status: string; - lastCheck: string; - lastSuccess?: string; - loading: boolean; - error?: string; - fullError?: string; - successResponse?: any; -} - -export const healthCheckColumns = ( - modelHealthStatuses: { [key: string]: HealthStatus }, - selectedModelsForHealth: string[], - allModelsSelected: boolean, - handleModelSelection: (modelId: string, checked: boolean) => void, - handleSelectAll: (checked: boolean) => void, - runIndividualHealthCheck: (modelId: string) => void, - getDisplayModelName: (model: any) => string, - showErrorModal?: (modelName: string, cleanedError: string, fullError: string) => void, - showSuccessModal?: (modelName: string, response: any) => void, - setSelectedModelId?: (modelId: string) => void, - teams?: Team[] | null, -): ColumnDef[] => [ - { - header: () => ( -
- 0 && !allModelsSelected} - onChange={(e) => handleSelectAll(e.target.checked)} - onClick={(e) => e.stopPropagation()} - /> - Model ID -
- ), - accessorKey: "model_info.id", - enableSorting: true, - sortingFn: "alphanumeric", - cell: ({ row }) => { - const model = row.original; - const modelId = model.model_info?.id ?? ""; - const isSelected = selectedModelsForHealth.includes(modelId); - - return ( -
- handleModelSelection(modelId, e.target.checked)} - onClick={(e) => e.stopPropagation()} - /> - -
- ); - }, - }, - { - header: "Model Name", - accessorKey: "model_name", - enableSorting: true, - sortingFn: "alphanumeric", - cell: ({ row }) => { - const model = row.original; - const displayName = getDisplayModelName(model) || model.model_name; - - return ( -
- -
{displayName}
-
-
- ); - }, - }, - { - header: "Team Alias", - accessorKey: "model_info.team_id", - enableSorting: true, - sortingFn: "alphanumeric", - cell: ({ row }) => { - const model = row.original; - const teamId = model.model_info?.team_id; - - if (!teamId) { - return -; - } - - const team = teams?.find((t) => t.team_id === teamId); - const teamAlias = team?.team_alias || teamId; - - return ( -
- -
{teamAlias}
-
-
- ); - }, - }, - { - header: "Health Status", - accessorKey: "health_status", - enableSorting: true, - sortingFn: (rowA, rowB, columnId) => { - const statusA = (rowA.getValue("health_status") as string) || "unknown"; - const statusB = (rowB.getValue("health_status") as string) || "unknown"; - - // Define sorting order: healthy > checking > unknown > unhealthy - const statusOrder = { healthy: 0, checking: 1, unknown: 2, unhealthy: 3 }; - const orderA = statusOrder[statusA as keyof typeof statusOrder] ?? 4; - const orderB = statusOrder[statusB as keyof typeof statusOrder] ?? 4; - - return orderA - orderB; - }, - cell: ({ row }) => { - const model = row.original; - const healthStatus = { - status: model.health_status, - loading: model.health_loading, - error: model.health_error, - }; - - if (healthStatus.loading) { - return ( -
-
-
-
-
-
- Checking... -
- ); - } - - const modelId = model.model_info?.id ?? ""; - const displayName = getDisplayModelName(model) || model.model_name; - const hasSuccessResponse = healthStatus.status === "healthy" && modelHealthStatuses[modelId]?.successResponse; - - return ( -
- {healthStatusBadge(healthStatus.status)} - {hasSuccessResponse && showSuccessModal && ( - - - - )} -
- ); - }, - }, - { - header: "Error Details", - accessorKey: "health_error", - enableSorting: false, - cell: ({ row }) => { - const model = row.original; - const modelId = model.model_info?.id ?? ""; - const displayName = getDisplayModelName(model) || model.model_name; - const healthStatus = modelHealthStatuses[modelId]; - - if (!healthStatus?.error) { - return No errors; - } - - const cleanedError = healthStatus.error; - const fullError = healthStatus.fullError || healthStatus.error; - - return ( -
-
- - {cleanedError} - -
- {showErrorModal && fullError !== cleanedError && ( - - - - )} -
- ); - }, - }, - { - header: "Last Check", - accessorKey: "last_check", - enableSorting: true, - sortingFn: (rowA, rowB, columnId) => { - const lastCheckA = (rowA.getValue("last_check") as string) || "Never checked"; - const lastCheckB = (rowB.getValue("last_check") as string) || "Never checked"; - - // Handle special cases - if (lastCheckA === "Never checked" && lastCheckB === "Never checked") return 0; - if (lastCheckA === "Never checked") return 1; // Never checked goes to bottom - if (lastCheckB === "Never checked") return -1; - if (lastCheckA === "Check in progress..." && lastCheckB === "Check in progress...") return 0; - if (lastCheckA === "Check in progress...") return -1; // In progress goes to top - if (lastCheckB === "Check in progress...") return 1; - - // Parse dates for comparison - const dateA = new Date(lastCheckA); - const dateB = new Date(lastCheckB); - - // If dates are invalid, treat as never checked - if (isNaN(dateA.getTime()) && isNaN(dateB.getTime())) return 0; - if (isNaN(dateA.getTime())) return 1; - if (isNaN(dateB.getTime())) return -1; - - // Sort by date (most recent first) - return dateB.getTime() - dateA.getTime(); - }, - cell: ({ row }) => { - const model = row.original; - - return ( - - {model.health_loading ? "Check in progress..." : model.last_check} - - ); - }, - }, - { - header: "Last Success", - accessorKey: "last_success", - enableSorting: true, - sortingFn: (rowA, rowB, columnId) => { - const lastSuccessA = (rowA.getValue("last_success") as string) || "Never succeeded"; - const lastSuccessB = (rowB.getValue("last_success") as string) || "Never succeeded"; - - // Handle special cases - if (lastSuccessA === "Never succeeded" && lastSuccessB === "Never succeeded") return 0; - if (lastSuccessA === "Never succeeded") return 1; // Never succeeded goes to bottom - if (lastSuccessB === "Never succeeded") return -1; - if (lastSuccessA === "None" && lastSuccessB === "None") return 0; - if (lastSuccessA === "None") return 1; // None goes to bottom - if (lastSuccessB === "None") return -1; - - // Parse dates for comparison - const dateA = new Date(lastSuccessA); - const dateB = new Date(lastSuccessB); - - // If dates are invalid, treat as never succeeded - if (isNaN(dateA.getTime()) && isNaN(dateB.getTime())) return 0; - if (isNaN(dateA.getTime())) return 1; - if (isNaN(dateB.getTime())) return -1; - - // Sort by date (most recent first) - return dateB.getTime() - dateA.getTime(); - }, - cell: ({ row }) => { - const model = row.original; - const modelId = model.model_info?.id ?? ""; - const healthStatus = modelHealthStatuses[modelId]; - const lastSuccess = healthStatus?.lastSuccess || "None"; - - return {lastSuccess}; - }, - }, - { - header: "Actions", - id: "actions", - cell: ({ row }) => { - const model = row.original; - const modelId = model.model_info?.id ?? ""; - - const hasExistingStatus = model.health_status && model.health_status !== "none"; - const tooltipText = model.health_loading - ? "Checking..." - : hasExistingStatus - ? "Re-run Health Check" - : "Run Health Check"; - - return ( - - - - ); - }, - enableSorting: false, - }, -]; diff --git a/ui/litellm-dashboard/src/components/model_dashboard/table.tsx b/ui/litellm-dashboard/src/components/model_dashboard/table.tsx deleted file mode 100644 index cd451af31e4..00000000000 --- a/ui/litellm-dashboard/src/components/model_dashboard/table.tsx +++ /dev/null @@ -1,208 +0,0 @@ -import { - ColumnDef, - flexRender, - getCoreRowModel, - getSortedRowModel, - getPaginationRowModel, - SortingState, - useReactTable, - ColumnResizeMode, - VisibilityState, - PaginationState, - OnChangeFn, -} from "@tanstack/react-table"; -import React from "react"; -import { Table, TableHead, TableHeaderCell, TableBody, TableRow, TableCell } from "@tremor/react"; -import { SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon } from "@heroicons/react/outline"; - -// Extend the column meta type to include className -declare module "@tanstack/react-table" { - interface ColumnMeta { - className?: string; - } -} - -interface ModelDataTableProps { - data: TData[]; - columns: ColumnDef[]; - isLoading?: boolean; - defaultSorting?: SortingState; - pagination?: PaginationState; - onPaginationChange?: OnChangeFn; - enablePagination?: boolean; - onRowClick?: (row: TData) => void; -} - -export function ModelDataTable({ - data = [], - columns, - isLoading = false, - defaultSorting = [], - pagination, - onPaginationChange, - enablePagination = false, - onRowClick, -}: ModelDataTableProps) { - const [sorting, setSorting] = React.useState(defaultSorting); - const [columnResizeMode] = React.useState("onChange"); - const [columnSizing, setColumnSizing] = React.useState({}); - const [columnVisibility, setColumnVisibility] = React.useState({}); - - const tableInstance = useReactTable({ - data, - columns, - state: { - sorting, - columnSizing, - columnVisibility, - ...(enablePagination && pagination ? { pagination } : {}), - }, - columnResizeMode, - onSortingChange: setSorting, - onColumnSizingChange: setColumnSizing, - onColumnVisibilityChange: setColumnVisibility, - ...(enablePagination && onPaginationChange ? { onPaginationChange } : {}), - getCoreRowModel: getCoreRowModel(), - getSortedRowModel: getSortedRowModel(), - ...(enablePagination ? { getPaginationRowModel: getPaginationRowModel() } : {}), - enableSorting: true, - enableColumnResizing: true, - defaultColumn: { - minSize: 40, - maxSize: 500, - }, - }); - - const getHeaderText = (header: any): string => { - if (typeof header === "string") { - return header; - } - if (typeof header === "function") { - const headerElement = header(); - if (headerElement && headerElement.props && headerElement.props.children) { - const children = headerElement.props.children; - if (typeof children === "string") { - return children; - } - if (children.props && children.props.children) { - return children.props.children; - } - } - } - return ""; - }; - - return ( -
-
-
- - - {tableInstance.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => ( - -
-
- {header.isPlaceholder - ? null - : flexRender(header.column.columnDef.header, header.getContext())} -
- {header.id !== "actions" && header.column.getCanSort() && ( -
- {header.column.getIsSorted() ? ( - { - asc: , - desc: , - }[header.column.getIsSorted() as string] - ) : ( - - )} -
- )} -
- {header.column.getCanResize() && ( -
- )} - - ))} - - ))} - - - {isLoading ? ( - - -
-

🚅 Loading models...

-
-
-
- ) : tableInstance.getRowModel().rows.length > 0 ? ( - tableInstance.getRowModel().rows.map((row) => ( - onRowClick?.(row.original)} - className={onRowClick ? "cursor-pointer hover:bg-gray-50" : ""} - > - {row.getVisibleCells().map((cell) => ( - - {flexRender(cell.column.columnDef.cell, cell.getContext())} - - ))} - - )) - ) : ( - - -
-

No models found

-
-
-
- )} -
-
-
-
-
- ); -} diff --git a/ui/litellm-dashboard/src/components/model_info_view.test.tsx b/ui/litellm-dashboard/src/components/model_info_view.test.tsx index a496bb05b91..6cf06d759f2 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.test.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.test.tsx @@ -916,4 +916,37 @@ describe("ModelInfoView", () => { expect(screen.getByText(/Created By/)).toBeInTheDocument(); }); }); + + it("renders the provider card logo from the bundled provider map", async () => { + render(, { wrapper }); + + const logo = await screen.findByAltText("openai logo"); + expect(logo.getAttribute("src")).toContain("openai_small"); + }); + + it("renders a letter avatar instead of an img for an unknown provider slug", async () => { + mockUseModelsInfo.mockReturnValue({ + data: { + data: [ + { + ...defaultModelData, + litellm_params: { + ...defaultModelData.litellm_params, + custom_llm_provider: "zzz-internal", + }, + }, + ], + }, + isLoading: false, + error: null, + }); + + render(, { wrapper }); + + await waitFor(() => { + expect(screen.getAllByText("zzz-internal").length).toBeGreaterThan(0); + }); + expect(screen.queryByAltText("zzz-internal logo")).not.toBeInTheDocument(); + expect(screen.getByText("z")).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index 8aaabdc50a2..fe28e0fb40c 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -43,7 +43,7 @@ import { tagListCall, testConnectionRequest, } from "./networking"; -import { getProviderLogoAndName } from "./provider_info_helpers"; +import { Logo } from "@/components/molecules/logo/Logo"; import UpdateModelCredentialsModal from "./update_model_credentials_modal"; import NumericalInput from "./shared/numerical_input"; import { Tag } from "./tag_management/types"; @@ -660,30 +660,7 @@ export default function ModelInfoView({ Provider
- {modelData.provider && ( - {`${modelData.provider} { - const target = e.currentTarget as HTMLImageElement; - const parent = target.parentElement; - if (!parent || !parent.contains(target)) { - return; - } - - try { - const fallbackDiv = document.createElement("div"); - fallbackDiv.className = - "w-4 h-4 rounded-full bg-gray-200 flex items-center justify-center text-xs"; - fallbackDiv.textContent = modelData.provider?.charAt(0) || "-"; - parent.replaceChild(fallbackDiv, target); - } catch (error) { - console.error("Failed to replace provider logo fallback:", error); - } - }} - /> - )} + {modelData.provider && } {modelData.provider || "Not Set"}
diff --git a/ui/litellm-dashboard/src/components/molecules/logo/Logo.test.tsx b/ui/litellm-dashboard/src/components/molecules/logo/Logo.test.tsx new file mode 100644 index 00000000000..c0b52e03a30 --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/logo/Logo.test.tsx @@ -0,0 +1,73 @@ +import React from "react"; +import { describe, expect, it, vi } from "vitest"; +import { act, fireEvent, render, screen } from "@testing-library/react"; +import { Logo } from "./Logo"; +import { Providers, providerLogoMap } from "@/components/provider_info_helpers"; + +vi.mock("@/lib/serverRootPath", () => ({ serverRootPath: "/litellm" })); + +describe("Logo", () => { + it("renders the bundled logo untouched by the server root path for a known provider", () => { + render(); + const img = screen.getByRole("img", { name: "openai logo" }); + expect(img.getAttribute("src")).toBe(providerLogoMap[Providers.OpenAI]); + expect(img.getAttribute("src")).toContain("openai_small"); + }); + + it("renders a letter avatar and no img for an unknown provider", () => { + render(); + expect(screen.getByText("u")).toBeInTheDocument(); + expect(screen.queryByRole("img")).not.toBeInTheDocument(); + }); + + it("renders a dash avatar when src is empty and the label has no characters", () => { + render(); + expect(screen.getByText("-")).toBeInTheDocument(); + expect(screen.queryByRole("img")).not.toBeInTheDocument(); + }); + + it("resolves a backend asset path through the server root path in src mode", () => { + render(); + const img = screen.getByRole("img", { name: "GitHub logo" }); + expect(img.getAttribute("src")).toBe("/litellm/ui/assets/logos/github.svg"); + }); + + it("passes an external https URL through untouched in src mode", () => { + render(); + expect(screen.getByRole("img").getAttribute("src")).toBe("https://cdn.example.com/logo.png"); + }); + + it("swaps to the letter avatar and warns with the failing URL on image error", () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + render(); + const img = screen.getByRole("img", { name: "GitHub logo" }); + + act(() => { + fireEvent.error(img); + }); + + expect(screen.queryByRole("img")).not.toBeInTheDocument(); + expect(screen.getByText("G")).toBeInTheDocument(); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining("/litellm/ui/assets/logos/github.svg")); + warnSpy.mockRestore(); + }); + + it("retries with a new src after a previous src errored", () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { rerender } = render(); + + act(() => { + fireEvent.error(screen.getByRole("img", { name: "Agent logo" })); + }); + expect(screen.queryByRole("img")).not.toBeInTheDocument(); + + rerender(); + const img = screen.getByRole("img", { name: "Agent logo" }); + expect(img.getAttribute("src")).toBe("/litellm/ui/assets/logos/github.svg"); + + rerender(); + expect(screen.queryByRole("img")).not.toBeInTheDocument(); + expect(screen.getByText("A")).toBeInTheDocument(); + warnSpy.mockRestore(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/molecules/logo/Logo.tsx b/ui/litellm-dashboard/src/components/molecules/logo/Logo.tsx new file mode 100644 index 00000000000..f5fb1f0805a --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/logo/Logo.tsx @@ -0,0 +1,34 @@ +import React, { useState } from "react"; +import { getProviderLogoAndName } from "@/components/provider_info_helpers"; +import { resolveLogoSrc } from "@/lib/assetPaths"; + +type LogoProps = { className?: string } & ( + | { provider: string; src?: never; label?: string } + | { provider?: never; src: string | null | undefined; label: string } +); + +export const Logo: React.FC = ({ provider, src, label, className = "w-4 h-4" }) => { + const [erroredSrc, setErroredSrc] = useState(null); + const resolvedSrc = provider !== undefined ? getProviderLogoAndName(provider).logo : resolveLogoSrc(src) ?? ""; + const name = label ?? provider ?? ""; + + if (erroredSrc === resolvedSrc || !resolvedSrc) { + return ( +
+ {name.charAt(0) || "-"} +
+ ); + } + + return ( + {`${name { + console.warn(`Logo failed to load: ${resolvedSrc}`); + setErroredSrc(resolvedSrc); + }} + /> + ); +}; diff --git a/ui/litellm-dashboard/src/components/molecules/models/ProviderLogo.tsx b/ui/litellm-dashboard/src/components/molecules/models/ProviderLogo.tsx index 4a9da15e333..4bc11126ae4 100644 --- a/ui/litellm-dashboard/src/components/molecules/models/ProviderLogo.tsx +++ b/ui/litellm-dashboard/src/components/molecules/models/ProviderLogo.tsx @@ -1,24 +1,11 @@ -import React, { useState } from "react"; -import { getProviderLogoAndName } from "../../provider_info_helpers"; +import React from "react"; +import { Logo } from "@/components/molecules/logo/Logo"; interface ProviderLogoProps { provider: string; className?: string; } -export const ProviderLogo: React.FC = ({ provider, className = "w-4 h-4" }) => { - const [hasError, setHasError] = useState(false); - const { logo } = getProviderLogoAndName(provider); - - const showFallback = hasError || !logo; - - if (showFallback) { - return ( -
- {provider?.charAt(0) || "-"} -
- ); - } - - return {`${provider} setHasError(true)} />; -}; +export const ProviderLogo: React.FC = ({ provider, className = "w-4 h-4" }) => ( + +); diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index d44a491b840..d6e9ba5665c 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -372,7 +372,7 @@ export function getGlobalLitellmHeaderName(): string { return globalLitellmHeaderName; } -const apiClient = createApiClient({ +export const apiClient = createApiClient({ getBaseUrl: getProxyBaseUrl, getAuthHeaderName: getGlobalLitellmHeaderName, onError: handleError, diff --git a/ui/litellm-dashboard/src/components/page_metadata.ts b/ui/litellm-dashboard/src/components/page_metadata.ts index 845e868b917..0f2ff639bb3 100644 --- a/ui/litellm-dashboard/src/components/page_metadata.ts +++ b/ui/litellm-dashboard/src/components/page_metadata.ts @@ -19,6 +19,7 @@ export const pageDescriptions: Record = { "tool-policies": "Configure tool use policies and permissions", "vector-stores": "Manage vector databases for embeddings", new_usage: "View usage analytics and metrics", + "cost-optimization": "Track and configure cost-saving features: prompt compression, caching, and auto routing", logs: "Access request and response logs", "guardrails-monitor": "Monitor guardrail performance and view logs", users: "Manage internal user accounts and permissions", diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 6e19de4a6cc..777cdc62987 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -113,24 +113,42 @@ describe("provider_info_helpers", () => { }); }); - describe("provider logo asset paths", () => { - // Regression: a relative "../ui/assets/logos/" base resolved to - // "/ui/ui/assets/logos/..." (404) on the public model hub at - // /ui/model_hub_table/, which sits a level below the /ui/ SPA. Root-absolute - // paths resolve correctly at any route depth. - it("should expose every provider logo as a root-absolute /ui path", () => { - const logos = Object.values(providerLogoMap); - expect(logos.length).toBeGreaterThan(0); - logos.forEach((logo) => { - expect(logo.startsWith("/ui/assets/logos/")).toBe(true); - expect(logo).not.toContain("../"); + describe("provider logo bundled assets", () => { + it("should map every provider to a bundled logo except the known logoless set, never a raw /ui/assets path", () => { + const knownLogolessProviders = [ + Providers.AUTO_ROUTER, + Providers.BYTEZ, + Providers.CLARIFAI, + Providers.COMPACTIFAI, + Providers.DATAROBOT, + Providers.DOCKER_MODEL_RUNNER, + Providers.DOTPROMPT, + Providers.EMPOWER, + Providers.GALADRIEL, + Providers.GradientAI, + Providers.HEROKU, + Providers.LEMONADE, + Providers.LLAMAFILE, + Providers.MARITALK, + Providers.NLP_CLOUD, + Providers.NSCALE, + Providers.OVHCLOUD, + Providers.PETALS, + Providers.PG_VECTOR, + Providers.PREDIBASE, + Providers.WANDB, + Providers.ZAI, + ]; + const logolessProviders = Object.values(Providers).filter((provider) => !providerLogoMap[provider]); + expect([...logolessProviders].sort()).toEqual([...knownLogolessProviders].sort()); + Object.values(providerLogoMap).forEach((logo) => { + expect(logo?.startsWith("/ui/assets/")).toBe(false); }); }); - it("should resolve a provider logo to a root-absolute path via getProviderLogoAndName", () => { + it("should resolve a provider to its own bundled logo via getProviderLogoAndName", () => { const { logo } = getProviderLogoAndName("openai"); - expect(logo.startsWith("/ui/assets/logos/")).toBe(true); - expect(logo).not.toContain("../"); + expect(logo).toContain("openai_small"); }); }); @@ -430,20 +448,19 @@ describe("getProviderLogoAndName under a custom server_root_path", () => { vi.doUnmock("@/lib/serverRootPath"); }); - // Regression: under SERVER_ROOT_PATH=/litellm the logo must be requested at - // /litellm/ui/assets/logos/... A bare /ui/... path is served off the root and - // 404s behind the reverse proxy. - it("prefixes the server root path onto the resolved logo", async () => { + it("returns the bundled logo URL untouched under a sub-path mount", async () => { vi.resetModules(); vi.doMock("@/lib/serverRootPath", () => ({ serverRootPath: "/litellm" })); - const { getProviderLogoAndName } = await import("./provider_info_helpers"); - expect(getProviderLogoAndName("openai").logo).toBe("/litellm/ui/assets/logos/openai_small.svg"); + const helpers = await import("./provider_info_helpers"); + const { logo } = helpers.getProviderLogoAndName("openai"); + expect(logo).toBe(helpers.providerLogoMap[helpers.Providers.OpenAI]); + expect(logo.startsWith("/litellm")).toBe(false); }); - it("leaves the logo at /ui/... when mounted at the root", async () => { + it("returns the bundled logo URL untouched at the root mount", async () => { vi.resetModules(); vi.doMock("@/lib/serverRootPath", () => ({ serverRootPath: "/" })); - const { getProviderLogoAndName } = await import("./provider_info_helpers"); - expect(getProviderLogoAndName("openai").logo).toBe("/ui/assets/logos/openai_small.svg"); + const helpers = await import("./provider_info_helpers"); + expect(helpers.getProviderLogoAndName("openai").logo).toBe(helpers.providerLogoMap[helpers.Providers.OpenAI]); }); }); diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 3c0286a07f7..7831c3376ac 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -1,4 +1,68 @@ import { resolveLogoSrc } from "@/lib/assetPaths"; +import a2aAgentLogo from "../../public/assets/logos/a2a_agent.png"; +import ai21Logo from "../../public/assets/logos/ai21.svg"; +import aimlApiLogo from "../../public/assets/logos/aiml_api.svg"; +import anthropicLogo from "../../public/assets/logos/anthropic.svg"; +import assemblyaiSmallLogo from "../../public/assets/logos/assemblyai_small.png"; +import basetenLogo from "../../public/assets/logos/baseten.svg"; +import bedrockLogo from "../../public/assets/logos/bedrock.svg"; +import cerebrasLogo from "../../public/assets/logos/cerebras.svg"; +import cloudflareLogo from "../../public/assets/logos/cloudflare.svg"; +import cohereLogo from "../../public/assets/logos/cohere.svg"; +import cometapiLogo from "../../public/assets/logos/cometapi.svg"; +import cursorLogo from "../../public/assets/logos/cursor.svg"; +import databricksLogo from "../../public/assets/logos/databricks.svg"; +import deepgramLogo from "../../public/assets/logos/deepgram.png"; +import deepinfraLogo from "../../public/assets/logos/deepinfra.png"; +import deepseekLogo from "../../public/assets/logos/deepseek.svg"; +import elevenlabsLogo from "../../public/assets/logos/elevenlabs.png"; +import falAiLogo from "../../public/assets/logos/fal_ai.jpg"; +import featherlessLogo from "../../public/assets/logos/featherless.svg"; +import fireworksLogo from "../../public/assets/logos/fireworks.svg"; +import friendliLogo from "../../public/assets/logos/friendli.svg"; +import gigachatLogo from "../../public/assets/logos/gigachat.svg"; +import githubCopilotLogo from "../../public/assets/logos/github_copilot.svg"; +import googleLogo from "../../public/assets/logos/google.svg"; +import groqLogo from "../../public/assets/logos/groq.svg"; +import huggingfaceLogo from "../../public/assets/logos/huggingface.svg"; +import hyperbolicLogo from "../../public/assets/logos/hyperbolic.svg"; +import infinityLogo from "../../public/assets/logos/infinity.png"; +import jinaLogo from "../../public/assets/logos/jina.png"; +import lambdaLogo from "../../public/assets/logos/lambda.svg"; +import lmstudioLogo from "../../public/assets/logos/lmstudio.svg"; +import metaLlamaLogo from "../../public/assets/logos/meta_llama.svg"; +import microsoftAzureLogo from "../../public/assets/logos/microsoft_azure.svg"; +import minimaxLogo from "../../public/assets/logos/minimax.svg"; +import mistralLogo from "../../public/assets/logos/mistral.svg"; +import moonshotLogo from "../../public/assets/logos/moonshot.svg"; +import morphLogo from "../../public/assets/logos/morph.svg"; +import nebiusLogo from "../../public/assets/logos/nebius.svg"; +import novitaLogo from "../../public/assets/logos/novita.svg"; +import nvidiaNimLogo from "../../public/assets/logos/nvidia_nim.svg"; +import nvidiaTritonLogo from "../../public/assets/logos/nvidia_triton.png"; +import ollamaLogo from "../../public/assets/logos/ollama.svg"; +import openaiSmallLogo from "../../public/assets/logos/openai_small.svg"; +import openrouterLogo from "../../public/assets/logos/openrouter.svg"; +import oracleLogo from "../../public/assets/logos/oracle.svg"; +import perplexityAiLogo from "../../public/assets/logos/perplexity-ai.svg"; +import qwenLogo from "../../public/assets/logos/qwen.png"; +import recraftLogo from "../../public/assets/logos/recraft.svg"; +import replicateLogo from "../../public/assets/logos/replicate.svg"; +import runwayLogo from "../../public/assets/logos/runway.png"; +import sambanovaLogo from "../../public/assets/logos/sambanova.svg"; +import sapLogo from "../../public/assets/logos/sap.png"; +import snowflakeLogo from "../../public/assets/logos/snowflake.svg"; +import sonioxLogo from "../../public/assets/logos/soniox.svg"; +import togetheraiLogo from "../../public/assets/logos/togetherai.svg"; +import topazLogo from "../../public/assets/logos/topaz.svg"; +import v0Logo from "../../public/assets/logos/v0.svg"; +import vercelLogo from "../../public/assets/logos/vercel.svg"; +import vllmLogo from "../../public/assets/logos/vllm.png"; +import volcengineLogo from "../../public/assets/logos/volcengine.png"; +import voyageLogo from "../../public/assets/logos/voyage.webp"; +import watsonxLogo from "../../public/assets/logos/watsonx.svg"; +import xaiLogo from "../../public/assets/logos/xai.svg"; +import xinferenceLogo from "../../public/assets/logos/xinference.svg"; export enum Providers { A2A_Agent = "A2A Agent", @@ -222,95 +286,92 @@ export const provider_map: Record = { const standaloneSubproviderSlugs = new Set(["bedrock_mantle"]); -const asset_logos_folder = "/ui/assets/logos/"; - -export const providerLogoMap: Record = { - [Providers.A2A_Agent]: `${asset_logos_folder}a2a_agent.png`, - [Providers.AI21]: `${asset_logos_folder}ai21.svg`, - [Providers.AI21_CHAT]: `${asset_logos_folder}ai21.svg`, - [Providers.AIML]: `${asset_logos_folder}aiml_api.svg`, - [Providers.AIOHTTP_OPENAI]: `${asset_logos_folder}openai_small.svg`, - [Providers.Anthropic]: `${asset_logos_folder}anthropic.svg`, - [Providers.ANTHROPIC_TEXT]: `${asset_logos_folder}anthropic.svg`, - [Providers.AssemblyAI]: `${asset_logos_folder}assemblyai_small.png`, - [Providers.Azure]: `${asset_logos_folder}microsoft_azure.svg`, - [Providers.Azure_AI_Studio]: `${asset_logos_folder}microsoft_azure.svg`, - [Providers.AZURE_TEXT]: `${asset_logos_folder}microsoft_azure.svg`, - [Providers.BASETEN]: `${asset_logos_folder}baseten.svg`, - [Providers.Bedrock]: `${asset_logos_folder}bedrock.svg`, - [Providers.BedrockMantle]: `${asset_logos_folder}bedrock.svg`, - [Providers.SageMaker]: `${asset_logos_folder}bedrock.svg`, - [Providers.Cerebras]: `${asset_logos_folder}cerebras.svg`, - [Providers.CLOUDFLARE]: `${asset_logos_folder}cloudflare.svg`, - [Providers.CODESTRAL]: `${asset_logos_folder}mistral.svg`, - [Providers.Cohere]: `${asset_logos_folder}cohere.svg`, - [Providers.COHERE_CHAT]: `${asset_logos_folder}cohere.svg`, - [Providers.COMETAPI]: `${asset_logos_folder}cometapi.svg`, - [Providers.Cursor]: `${asset_logos_folder}cursor.svg`, - [Providers.Databricks]: `${asset_logos_folder}databricks.svg`, - [Providers.Dashscope]: `${asset_logos_folder}dashscope.svg`, - [Providers.Deepseek]: `${asset_logos_folder}deepseek.svg`, - [Providers.Deepgram]: `${asset_logos_folder}deepgram.png`, - [Providers.DeepInfra]: `${asset_logos_folder}deepinfra.png`, - [Providers.ElevenLabs]: `${asset_logos_folder}elevenlabs.png`, - [Providers.FalAI]: `${asset_logos_folder}fal_ai.jpg`, - [Providers.FEATHERLESS_AI]: `${asset_logos_folder}featherless.svg`, - [Providers.FireworksAI]: `${asset_logos_folder}fireworks.svg`, - [Providers.FRIENDLIAI]: `${asset_logos_folder}friendli.svg`, - [Providers.GIGACHAT]: `${asset_logos_folder}gigachat.svg`, - [Providers.GITHUB_COPILOT]: `${asset_logos_folder}github_copilot.svg`, - [Providers.Google_AI_Studio]: `${asset_logos_folder}google.svg`, - [Providers.GradientAI]: `${asset_logos_folder}gradientai.svg`, - [Providers.Groq]: `${asset_logos_folder}groq.svg`, - [Providers.Hosted_Vllm]: `${asset_logos_folder}vllm.png`, - [Providers.HUGGINGFACE]: `${asset_logos_folder}huggingface.svg`, - [Providers.HYPERBOLIC]: `${asset_logos_folder}hyperbolic.svg`, - [Providers.Infinity]: `${asset_logos_folder}infinity.png`, - [Providers.JinaAI]: `${asset_logos_folder}jina.png`, - [Providers.LAMBDA_AI]: `${asset_logos_folder}lambda.svg`, - [Providers.LM_STUDIO]: `${asset_logos_folder}lmstudio.svg`, - [Providers.LLAMA]: `${asset_logos_folder}meta_llama.svg`, - [Providers.MiniMax]: `${asset_logos_folder}minimax.svg`, - [Providers.MistralAI]: `${asset_logos_folder}mistral.svg`, - [Providers.MOONSHOT]: `${asset_logos_folder}moonshot.svg`, - [Providers.MORPH]: `${asset_logos_folder}morph.svg`, - [Providers.NEBIUS]: `${asset_logos_folder}nebius.svg`, - [Providers.NOVITA]: `${asset_logos_folder}novita.svg`, - [Providers.NVIDIA_NIM]: `${asset_logos_folder}nvidia_nim.svg`, - [Providers.Ollama]: `${asset_logos_folder}ollama.svg`, - [Providers.OLLAMA_CHAT]: `${asset_logos_folder}ollama.svg`, - [Providers.OOBABOOGA]: `${asset_logos_folder}openai_small.svg`, - [Providers.OpenAI]: `${asset_logos_folder}openai_small.svg`, - [Providers.OPENAI_LIKE]: `${asset_logos_folder}openai_small.svg`, - [Providers.OpenAI_Text]: `${asset_logos_folder}openai_small.svg`, - [Providers.OpenAI_Text_Compatible]: `${asset_logos_folder}openai_small.svg`, - [Providers.OpenAI_Compatible]: `${asset_logos_folder}openai_small.svg`, - [Providers.Openrouter]: `${asset_logos_folder}openrouter.svg`, - [Providers.Oracle]: `${asset_logos_folder}oracle.svg`, - [Providers.Perplexity]: `${asset_logos_folder}perplexity-ai.svg`, - [Providers.RECRAFT]: `${asset_logos_folder}recraft.svg`, - [Providers.REPLICATE]: `${asset_logos_folder}replicate.svg`, - [Providers.RunwayML]: `${asset_logos_folder}runwayml.png`, - [Providers.SAGEMAKER_LEGACY]: `${asset_logos_folder}bedrock.svg`, - [Providers.Sambanova]: `${asset_logos_folder}sambanova.svg`, - [Providers.SAP]: `${asset_logos_folder}sap.png`, - [Providers.Snowflake]: `${asset_logos_folder}snowflake.svg`, - [Providers.Soniox]: `${asset_logos_folder}soniox.svg`, - [Providers.TEXT_COMPLETION_CODESTRAL]: `${asset_logos_folder}mistral.svg`, - [Providers.TogetherAI]: `${asset_logos_folder}togetherai.svg`, - [Providers.TOPAZ]: `${asset_logos_folder}topaz.svg`, - [Providers.Triton]: `${asset_logos_folder}nvidia_triton.png`, - [Providers.V0]: `${asset_logos_folder}v0.svg`, - [Providers.VERCEL_AI_GATEWAY]: `${asset_logos_folder}vercel.svg`, - [Providers.Vertex_AI]: `${asset_logos_folder}google.svg`, - [Providers.VERTEX_AI_BETA]: `${asset_logos_folder}google.svg`, - [Providers.VLLM]: `${asset_logos_folder}vllm.png`, - [Providers.VolcEngine]: `${asset_logos_folder}volcengine.png`, - [Providers.Voyage]: `${asset_logos_folder}voyage.webp`, - [Providers.WATSONX]: `${asset_logos_folder}watsonx.svg`, - [Providers.WATSONX_TEXT]: `${asset_logos_folder}watsonx.svg`, - [Providers.xAI]: `${asset_logos_folder}xai.svg`, - [Providers.XINFERENCE]: `${asset_logos_folder}xinference.svg`, +export const providerLogoMap: Partial> = { + [Providers.A2A_Agent]: a2aAgentLogo.src, + [Providers.AI21]: ai21Logo.src, + [Providers.AI21_CHAT]: ai21Logo.src, + [Providers.AIML]: aimlApiLogo.src, + [Providers.AIOHTTP_OPENAI]: openaiSmallLogo.src, + [Providers.Anthropic]: anthropicLogo.src, + [Providers.ANTHROPIC_TEXT]: anthropicLogo.src, + [Providers.AssemblyAI]: assemblyaiSmallLogo.src, + [Providers.Azure]: microsoftAzureLogo.src, + [Providers.Azure_AI_Studio]: microsoftAzureLogo.src, + [Providers.AZURE_TEXT]: microsoftAzureLogo.src, + [Providers.BASETEN]: basetenLogo.src, + [Providers.Bedrock]: bedrockLogo.src, + [Providers.BedrockMantle]: bedrockLogo.src, + [Providers.SageMaker]: bedrockLogo.src, + [Providers.Cerebras]: cerebrasLogo.src, + [Providers.CLOUDFLARE]: cloudflareLogo.src, + [Providers.CODESTRAL]: mistralLogo.src, + [Providers.Cohere]: cohereLogo.src, + [Providers.COHERE_CHAT]: cohereLogo.src, + [Providers.COMETAPI]: cometapiLogo.src, + [Providers.Cursor]: cursorLogo.src, + [Providers.Databricks]: databricksLogo.src, + [Providers.Dashscope]: qwenLogo.src, + [Providers.Deepseek]: deepseekLogo.src, + [Providers.Deepgram]: deepgramLogo.src, + [Providers.DeepInfra]: deepinfraLogo.src, + [Providers.ElevenLabs]: elevenlabsLogo.src, + [Providers.FalAI]: falAiLogo.src, + [Providers.FEATHERLESS_AI]: featherlessLogo.src, + [Providers.FireworksAI]: fireworksLogo.src, + [Providers.FRIENDLIAI]: friendliLogo.src, + [Providers.GIGACHAT]: gigachatLogo.src, + [Providers.GITHUB_COPILOT]: githubCopilotLogo.src, + [Providers.Google_AI_Studio]: googleLogo.src, + [Providers.Groq]: groqLogo.src, + [Providers.Hosted_Vllm]: vllmLogo.src, + [Providers.HUGGINGFACE]: huggingfaceLogo.src, + [Providers.HYPERBOLIC]: hyperbolicLogo.src, + [Providers.Infinity]: infinityLogo.src, + [Providers.JinaAI]: jinaLogo.src, + [Providers.LAMBDA_AI]: lambdaLogo.src, + [Providers.LM_STUDIO]: lmstudioLogo.src, + [Providers.LLAMA]: metaLlamaLogo.src, + [Providers.MiniMax]: minimaxLogo.src, + [Providers.MistralAI]: mistralLogo.src, + [Providers.MOONSHOT]: moonshotLogo.src, + [Providers.MORPH]: morphLogo.src, + [Providers.NEBIUS]: nebiusLogo.src, + [Providers.NOVITA]: novitaLogo.src, + [Providers.NVIDIA_NIM]: nvidiaNimLogo.src, + [Providers.Ollama]: ollamaLogo.src, + [Providers.OLLAMA_CHAT]: ollamaLogo.src, + [Providers.OOBABOOGA]: openaiSmallLogo.src, + [Providers.OpenAI]: openaiSmallLogo.src, + [Providers.OPENAI_LIKE]: openaiSmallLogo.src, + [Providers.OpenAI_Text]: openaiSmallLogo.src, + [Providers.OpenAI_Text_Compatible]: openaiSmallLogo.src, + [Providers.OpenAI_Compatible]: openaiSmallLogo.src, + [Providers.Openrouter]: openrouterLogo.src, + [Providers.Oracle]: oracleLogo.src, + [Providers.Perplexity]: perplexityAiLogo.src, + [Providers.RECRAFT]: recraftLogo.src, + [Providers.REPLICATE]: replicateLogo.src, + [Providers.RunwayML]: runwayLogo.src, + [Providers.SAGEMAKER_LEGACY]: bedrockLogo.src, + [Providers.Sambanova]: sambanovaLogo.src, + [Providers.SAP]: sapLogo.src, + [Providers.Snowflake]: snowflakeLogo.src, + [Providers.Soniox]: sonioxLogo.src, + [Providers.TEXT_COMPLETION_CODESTRAL]: mistralLogo.src, + [Providers.TogetherAI]: togetheraiLogo.src, + [Providers.TOPAZ]: topazLogo.src, + [Providers.Triton]: nvidiaTritonLogo.src, + [Providers.V0]: v0Logo.src, + [Providers.VERCEL_AI_GATEWAY]: vercelLogo.src, + [Providers.Vertex_AI]: googleLogo.src, + [Providers.VERTEX_AI_BETA]: googleLogo.src, + [Providers.VLLM]: vllmLogo.src, + [Providers.VolcEngine]: volcengineLogo.src, + [Providers.Voyage]: voyageLogo.src, + [Providers.WATSONX]: watsonxLogo.src, + [Providers.WATSONX_TEXT]: watsonxLogo.src, + [Providers.xAI]: xaiLogo.src, + [Providers.XINFERENCE]: xinferenceLogo.src, }; export const getProviderLogoAndName = (providerValue: string): { logo: string; displayName: string } => { @@ -338,7 +399,7 @@ export const getProviderLogoAndName = (providerValue: string): { logo: string; d // Get the display name from Providers enum and logo from map const displayName = Providers[enumKey as keyof typeof Providers]; - const logo = resolveLogoSrc(providerLogoMap[displayName as keyof typeof providerLogoMap]) ?? ""; + const logo = resolveLogoSrc(providerLogoMap[displayName]) ?? ""; return { logo, displayName }; }; diff --git a/ui/litellm-dashboard/src/components/settings.test.tsx b/ui/litellm-dashboard/src/components/settings.test.tsx index ebc1a5a7a6c..c9bcc1eb5b9 100644 --- a/ui/litellm-dashboard/src/components/settings.test.tsx +++ b/ui/litellm-dashboard/src/components/settings.test.tsx @@ -1,8 +1,9 @@ -import { act, render, screen, waitFor } from "@testing-library/react"; +import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; +import { Form } from "antd"; import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { alertingSettingsCall, getCallbackConfigsCall, getCallbacksCall } from "./networking"; -import Settings from "./settings"; +import Settings, { backendCallbackLogoSrc, CallbackSelector } from "./settings"; vi.mock("./networking", () => ({ getCallbacksCall: vi.fn(), @@ -232,3 +233,44 @@ describe("Settings", () => { expect(getByText("CloudZero Cost Tracking")).toBeInTheDocument(); }); }); + +describe("backendCallbackLogoSrc", () => { + it("prefixes bare filenames with the assets logo folder", () => { + expect(backendCallbackLogoSrc("datadog.png")).toBe("/ui/assets/logos/datadog.png"); + }); + + it("passes through urls, data uris, and paths untouched", () => { + expect(backendCallbackLogoSrc("https://logos.example.com/x.png")).toBe("https://logos.example.com/x.png"); + expect(backendCallbackLogoSrc("data:image/png;base64,abc")).toBe("data:image/png;base64,abc"); + expect(backendCallbackLogoSrc("/custom/path.png")).toBe("/custom/path.png"); + }); + + it("returns undefined when the backend provides no logo", () => { + expect(backendCallbackLogoSrc(undefined)).toBeUndefined(); + expect(backendCallbackLogoSrc(null)).toBeUndefined(); + expect(backendCallbackLogoSrc("")).toBeUndefined(); + }); +}); + +describe("CallbackSelector logos", () => { + it("resolves backend logos per entry: bare filename, external url, and missing logo", async () => { + const callbackConfigs = [ + { id: "langfuse", displayName: "Langfuse", logo: "langfuse.png" }, + { id: "hosted", displayName: "Hosted", logo: "https://logos.example.com/hosted.png" }, + { id: "nologo", displayName: "NoLogo" }, + ]; + + render( +
+ + , + ); + + fireEvent.mouseDown(screen.getByRole("combobox")); + + expect(await screen.findByAltText("Langfuse logo")).toHaveAttribute("src", "/ui/assets/logos/langfuse.png"); + expect(screen.getByAltText("Hosted logo")).toHaveAttribute("src", "https://logos.example.com/hosted.png"); + expect(screen.queryByAltText("NoLogo logo")).toBeNull(); + expect(screen.getByText("N")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index b3f33133a80..72ddb26f045 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -22,7 +22,7 @@ import React, { useEffect, useState } from "react"; import { Button as Button2, Form, Input, Modal, Select, Typography } from "antd"; import EmailSettings from "./email_settings"; -import { resolveLogoSrc } from "@/lib/assetPaths"; +import { Logo } from "@/components/molecules/logo/Logo"; import NotificationsManager from "./molecules/notifications_manager"; const { Title, Paragraph } = Typography; @@ -56,6 +56,12 @@ interface genericCallbackParams { const assetsLogoFolder = "/ui/assets/logos/"; +export const backendCallbackLogoSrc = (logo: string | null | undefined): string | undefined => { + if (!logo) return undefined; + if (logo.includes("/") || logo.startsWith("data:") || logo.startsWith("http")) return logo; + return `${assetsLogoFolder}${logo}`; +}; + interface DynamicParamsFieldsProps { params: string[]; callbackConfigs: any[]; @@ -131,7 +137,7 @@ interface CallbackSelectorProps { disabled?: boolean; } -const CallbackSelector: React.FC = ({ +export const CallbackSelector: React.FC = ({ callbackConfigs, selectedCallback, onCallbackChange, @@ -156,25 +162,14 @@ const CallbackSelector: React.FC = ({ onChange={onCallbackChange} > {callbackConfigs.map((callbackConfig) => { - const logo = callbackConfig.logo; - const logoSrc = resolveLogoSrc( - logo && (logo.includes("/") || logo.startsWith("data:") || logo.startsWith("http")) - ? logo - : `${assetsLogoFolder}${logo}`, - ); - return (
- {/* eslint-disable-next-line @next/next/no-img-element */} - {`${callbackConfig.displayName} { - e.currentTarget.style.display = "none"; - }} />
{callbackConfig.displayName} diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx index fa0e672026e..c4799594465 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx @@ -18,6 +18,7 @@ import { type OnChangeFn, type Row, type RowData, + type RowSelectionState, type Table, type TableOptions, useReactTable, @@ -70,6 +71,8 @@ export function validateDataTableConfig( const bothSortingSources = props.defaultSorting !== undefined && props.sorting !== undefined; const bothFilterSources = props.defaultColumnFilters !== undefined && props.columnFilters !== undefined; + const controlledSelectionIncomplete = props.rowSelection !== undefined && props.onRowSelectionChange === undefined; + return [ serverSortingIncomplete ? "sortingMode='server' requires both `sorting` and `onSortingChange`." : null, serverPaginationIncomplete @@ -80,6 +83,9 @@ export function validateDataTableConfig( bothFilterSources ? "Provide either `defaultColumnFilters` (uncontrolled) or `columnFilters` (controlled), not both." : null, + controlledSelectionIncomplete + ? "Controlled `rowSelection` requires `onRowSelectionChange`; without it selection changes are dropped." + : null, ].filter((message): message is string => message !== null); } @@ -448,6 +454,9 @@ function useDataTableInstance(props: DataTablePro renderSubComponent, expanded, onExpandedChange, + enableRowSelection, + rowSelection, + onRowSelectionChange, } = props; const sortingState = useControllable(sorting, onSortingChange, defaultSorting ?? []); @@ -462,6 +471,7 @@ function useDataTableInstance(props: DataTablePro ); const globalFilterState = useControllable(globalFilter, onGlobalFilterChange, ""); const expandedState = useControllable(expanded, onExpandedChange, {}); + const rowSelectionState = useControllable(rowSelection, onRowSelectionChange, {}); const [columnVisibility, setColumnVisibility] = useState(defaultColumnVisibility ?? {}); const [columnSizing, setColumnSizing] = useState({}); const columnPinning = React.useMemo(() => derivePinning(columns), [columns]); @@ -476,6 +486,7 @@ function useDataTableInstance(props: DataTablePro columnFilters: filterState.value, globalFilter: globalFilterState.value, expanded: expandedState.value, + rowSelection: rowSelectionState.value, columnVisibility, columnSizing, }, @@ -491,11 +502,13 @@ function useDataTableInstance(props: DataTablePro onColumnFiltersChange: filterState.onChange, onGlobalFilterChange: globalFilterState.onChange, onExpandedChange: expandedState.onChange, + onRowSelectionChange: rowSelectionState.onChange, onColumnVisibilityChange: setColumnVisibility, onColumnSizingChange: setColumnSizing, getCoreRowModel: getCoreRowModel(), ...buildRowModels(sortingMode, paginationMode, filterMode, expansionGuard), ...(getRowId !== undefined ? { getRowId } : {}), + ...(enableRowSelection !== undefined ? { enableRowSelection } : {}), ...(paginationMode === "server" && rowCount !== undefined ? { rowCount } : {}), }; diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTableRowSelection.test.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableRowSelection.test.tsx new file mode 100644 index 00000000000..2464fb93309 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableRowSelection.test.tsx @@ -0,0 +1,137 @@ +import type { ColumnDef, RowSelectionState } from "@tanstack/react-table"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { useState } from "react"; +import { describe, expect, it } from "vitest"; + +import { createSelectionColumn, DataTable, validateDataTableConfig } from "./index"; + +interface Model { + id: string; + name: string; +} + +const data: Model[] = [ + { id: "m1", name: "Alpha" }, + { id: "m2", name: "Beta" }, + { id: "m3", name: "Gamma" }, +]; + +const columns: ColumnDef[] = [ + createSelectionColumn({ rowAriaLabel: (row) => `Select ${row.original.name}` }), + { id: "name", accessorKey: "name", header: "Name", enableSorting: false }, +]; + +const selectAll = () => screen.getByTestId("datatable-select-all"); +const rowBox = (id: string) => screen.getByTestId(`datatable-select-row-${id}`); +const selectedCount = () => screen.getByTestId("count"); + +function ControlledHarness() { + const [rowSelection, setRowSelection] = useState({}); + + return ( + <> + + {Object.keys(rowSelection) + .filter((key) => rowSelection[key]) + .sort() + .join(",")} + + + row.id} + rowSelection={rowSelection} + onRowSelectionChange={setRowSelection} + /> + + ); +} + +describe("DataTable row selection", () => { + it("supports uncontrolled per-row toggle, select-all, and indeterminate", async () => { + const user = userEvent.setup(); + + render( + row.id} + toolbar={(table) => {table.getSelectedRowModel().rows.length}} + />, + ); + + expect(selectedCount()).toHaveTextContent("0"); + + await user.click(rowBox("m1")); + expect(selectedCount()).toHaveTextContent("1"); + expect(selectAll()).toHaveAttribute("aria-checked", "mixed"); + + await user.click(selectAll()); + expect(selectedCount()).toHaveTextContent("3"); + expect(selectAll()).toHaveAttribute("aria-checked", "true"); + + await user.click(selectAll()); + expect(selectedCount()).toHaveTextContent("0"); + }); + + it("keys controlled selection by getRowId so the parent can map back to entities", async () => { + const user = userEvent.setup(); + render(); + + await user.click(rowBox("m2")); + expect(screen.getByTestId("keys")).toHaveTextContent("m2"); + + await user.click(rowBox("m3")); + expect(screen.getByTestId("keys")).toHaveTextContent("m2,m3"); + }); + + it("lets the parent clear the selection, the pattern an external pager needs", async () => { + const user = userEvent.setup(); + render(); + + await user.click(selectAll()); + expect(screen.getByTestId("keys")).toHaveTextContent("m1,m2,m3"); + + await user.click(screen.getByTestId("clear")); + expect(screen.getByTestId("keys")).toBeEmptyDOMElement(); + expect(rowBox("m1")).toHaveAttribute("aria-checked", "false"); + }); + + it("respects an enableRowSelection predicate", async () => { + const user = userEvent.setup(); + + render( + row.id} + enableRowSelection={(row) => row.original.id !== "m2"} + toolbar={(table) => {table.getSelectedRowModel().rows.length}} + />, + ); + + expect(rowBox("m2")).toHaveAttribute("aria-disabled", "true"); + + await user.click(rowBox("m2")); + expect(selectedCount()).toHaveTextContent("0"); + + await user.click(rowBox("m1")); + expect(selectedCount()).toHaveTextContent("1"); + }); + + it("rejects controlled rowSelection without onRowSelectionChange", () => { + const errors = validateDataTableConfig({ data, columns, rowSelection: { m1: true } }); + + expect(errors).toContain( + "Controlled `rowSelection` requires `onRowSelectionChange`; without it selection changes are dropped.", + ); + }); + + it("does not complain when selection is left uncontrolled", () => { + expect(validateDataTableConfig({ data, columns })).toHaveLength(0); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTableSelectionColumn.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableSelectionColumn.tsx new file mode 100644 index 00000000000..da32a01ab0e --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableSelectionColumn.tsx @@ -0,0 +1,53 @@ +"use client"; + +import type { ColumnDef, Row, RowData, Table } from "@tanstack/react-table"; + +import { Checkbox } from "@/components/ui/checkbox"; + +interface SelectionColumnOptions { + rowAriaLabel?: (row: Row) => string; +} + +function SelectAllCheckbox({ table }: { table: Table }) { + const allSelected = table.getIsAllPageRowsSelected(); + const someSelected = table.getIsSomePageRowsSelected(); + + return ( + table.toggleAllPageRowsSelected(Boolean(checked))} + /> + ); +} + +function SelectRowCheckbox({ row, label }: { row: Row; label: string }) { + return ( + row.toggleSelected(Boolean(checked))} + /> + ); +} + +export function createSelectionColumn( + options: SelectionColumnOptions = {}, +): ColumnDef { + const { rowAriaLabel } = options; + + return { + id: "select", + size: 44, + enableSorting: false, + enableHiding: false, + enableResizing: false, + meta: { title: "Select", className: "w-11", headerClassName: "w-11" }, + header: ({ table }) => , + cell: ({ row }) => , + }; +} diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/index.ts b/ui/litellm-dashboard/src/components/shared/DataTable/index.ts index 1ee1eed1258..62ddd1b0742 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/index.ts +++ b/ui/litellm-dashboard/src/components/shared/DataTable/index.ts @@ -3,6 +3,7 @@ import "./columnMeta"; export { DataTable, DataTableConfigError, validateDataTableConfig } from "./DataTable"; export { DataTableFilterDrawer, DataTableFilterField, type FilterDraft } from "./DataTableFilterDrawer"; export { DataTablePagination, DEFAULT_PAGE_SIZE_OPTIONS } from "./DataTablePagination"; +export { createSelectionColumn } from "./DataTableSelectionColumn"; export { DataTableToolbar } from "./DataTableToolbar"; export { DataTableViewOptions } from "./DataTableViewOptions"; export { diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/types.ts b/ui/litellm-dashboard/src/components/shared/DataTable/types.ts index 672ab512ef4..40f3a4df204 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/types.ts +++ b/ui/litellm-dashboard/src/components/shared/DataTable/types.ts @@ -6,6 +6,7 @@ import type { PaginationState, Row, RowData, + RowSelectionState, SortingState, Table, VisibilityState, @@ -59,6 +60,10 @@ export interface DataTableProps { expanded?: ExpandedState; onExpandedChange?: OnChangeFn; + enableRowSelection?: boolean | ((row: Row) => boolean); + rowSelection?: RowSelectionState; + onRowSelectionChange?: OnChangeFn; + onRowClick?: (row: TData) => void; rowClassName?: (row: Row) => string; diff --git a/ui/litellm-dashboard/src/components/shared/form/FormField.test.tsx b/ui/litellm-dashboard/src/components/shared/form/FormField.test.tsx new file mode 100644 index 00000000000..af9122e2bd5 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/form/FormField.test.tsx @@ -0,0 +1,180 @@ +import { zodResolver } from "@hookform/resolvers/zod"; +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import * as React from "react"; +import { useForm } from "react-hook-form"; +import { describe, expect, it, vi } from "vitest"; +import { z } from "zod/v4"; + +import { Input } from "@/components/ui/input"; + +import { FormField } from "./FormField"; + +const schema = z.object({ + team_alias: z.string().min(1, "Please input a team name"), + owner: z.string(), +}); + +type FormInput = z.input; + +const TestForm = ({ + onSubmit, + defaultValues = { team_alias: "team-a", owner: "" }, + description, +}: { + onSubmit: (values: z.output) => void; + defaultValues?: FormInput; + description?: React.ReactNode; +}) => { + const form = useForm>({ + resolver: zodResolver(schema), + defaultValues, + }); + + return ( +
+ + {(field) => } + + +
+ ); +}; + +describe("FormField", () => { + it("associates the label with the control so it is reachable by its accessible name", () => { + render(); + + expect(screen.getByLabelText("Team Name")).toHaveValue("team-a"); + }); + + it("gives each field instance a unique control id", () => { + const Harness = () => { + const form = useForm({ defaultValues: { team_alias: "", owner: "" } }); + return ( + <> + + {(field) => } + + + {(field) => } + + + ); + }; + render(); + + expect(screen.getByLabelText("One").id).not.toBe(screen.getByLabelText("Two").id); + }); + + it("feeds edits back into form state and submits the parsed output", async () => { + const user = userEvent.setup(); + const onSubmit = vi.fn(); + render(); + + await user.clear(screen.getByLabelText("Team Name")); + await user.type(screen.getByLabelText("Team Name"), "team-b"); + await user.click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => expect(onSubmit).toHaveBeenCalledTimes(1)); + expect(onSubmit.mock.calls[0][0]).toEqual({ team_alias: "team-b", owner: "" }); + }); + + it("renders the zod message and blocks submit when validation fails", async () => { + const user = userEvent.setup(); + const onSubmit = vi.fn(); + render(); + + await user.clear(screen.getByLabelText("Team Name")); + await user.click(screen.getByRole("button", { name: "Save" })); + + expect(await screen.findByRole("alert")).toHaveTextContent("Please input a team name"); + expect(onSubmit).not.toHaveBeenCalled(); + }); + + it("marks the control invalid and points aria-describedby at the message", async () => { + const user = userEvent.setup(); + render(); + + await user.clear(screen.getByLabelText("Team Name")); + await user.click(screen.getByRole("button", { name: "Save" })); + + const control = await screen.findByLabelText("Team Name"); + await waitFor(() => expect(control).toHaveAttribute("aria-invalid", "true")); + expect(control.getAttribute("aria-describedby")).toBe(screen.getByRole("alert").id); + }); + + it("leaves a valid control free of aria-invalid", () => { + render(); + + expect(screen.getByLabelText("Team Name")).not.toHaveAttribute("aria-invalid"); + }); + + it("clears the message once the value becomes valid again", async () => { + const user = userEvent.setup(); + render(); + + await user.clear(screen.getByLabelText("Team Name")); + await user.click(screen.getByRole("button", { name: "Save" })); + expect(await screen.findByRole("alert")).toBeInTheDocument(); + + await user.type(screen.getByLabelText("Team Name"), "team-c"); + + await waitFor(() => expect(screen.queryByRole("alert")).not.toBeInTheDocument()); + }); + + it("describes the control by its description when there is no error", () => { + render(); + + const control = screen.getByLabelText("Team Name"); + const describedBy = control.getAttribute("aria-describedby"); + + expect(describedBy).not.toBeNull(); + expect(document.getElementById(describedBy!)).toHaveTextContent("Shown to team members"); + }); + + it("describes the control by both description and error while invalid", async () => { + const user = userEvent.setup(); + render(); + + await user.clear(screen.getByLabelText("Team Name")); + await user.click(screen.getByRole("button", { name: "Save" })); + await screen.findByRole("alert"); + + const ids = screen.getByLabelText("Team Name").getAttribute("aria-describedby")?.split(" ") ?? []; + + expect(ids).toHaveLength(2); + expect(ids).toContain(screen.getByRole("alert").id); + }); + + it("omits aria-describedby entirely when there is no description and no error", () => { + render(); + + expect(screen.getByLabelText("Team Name")).not.toHaveAttribute("aria-describedby"); + }); + + it("hands the control a value and onChange so non-native widgets can be wired", async () => { + const user = userEvent.setup(); + const seen: unknown[] = []; + const Harness = () => { + const form = useForm({ defaultValues: { team_alias: "team-a", owner: "" } }); + return ( + + {(field) => { + seen.push(field.value); + return ( + + ); + }} + + ); + }; + render(); + + await user.click(screen.getByRole("button", { name: "widget" })); + + await waitFor(() => expect(seen.at(-1)).toBe("from-widget")); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/form/FormField.tsx b/ui/litellm-dashboard/src/components/shared/form/FormField.tsx new file mode 100644 index 00000000000..3b9783333cc --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/form/FormField.tsx @@ -0,0 +1,75 @@ +"use client"; + +import * as React from "react"; +import { + Controller, + type Control, + type ControllerRenderProps, + type FieldPath, + type FieldValues, +} from "react-hook-form"; + +import { Field, FieldDescription, FieldError, FieldLabel } from "./field"; + +export type FormFieldControlProps< + TFieldValues extends FieldValues, + TName extends FieldPath, +> = ControllerRenderProps & { + id: string; + "aria-invalid": true | undefined; + "aria-describedby": string | undefined; +}; + +export interface FormFieldProps> { + control: Control; + name: TName; + label?: React.ReactNode; + description?: React.ReactNode; + orientation?: "vertical" | "horizontal" | "responsive"; + className?: string; + children: (control: FormFieldControlProps) => React.ReactNode; +} + +export const FormField = >({ + control, + name, + label, + description, + orientation, + className, + children, +}: FormFieldProps) => { + const reactId = React.useId(); + const controlId = `${reactId}-control`; + const descriptionId = `${reactId}-description`; + const errorId = `${reactId}-error`; + + return ( + { + const invalid = fieldState.error !== undefined; + const describedBy = + [description !== undefined ? descriptionId : undefined, invalid ? errorId : undefined] + .filter((id): id is string => id !== undefined) + .join(" ") || undefined; + const controlProps: FormFieldControlProps = { + ...field, + id: controlId, + "aria-invalid": invalid || undefined, + "aria-describedby": describedBy, + }; + + return ( + + {label !== undefined && {label}} + {children(controlProps)} + {description !== undefined && {description}} + + + ); + }} + /> + ); +}; diff --git a/ui/litellm-dashboard/src/components/shared/form/field.test.tsx b/ui/litellm-dashboard/src/components/shared/form/field.test.tsx new file mode 100644 index 00000000000..54b589ce2f4 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/form/field.test.tsx @@ -0,0 +1,125 @@ +import { render, screen } from "@testing-library/react"; +import * as React from "react"; +import { describe, expect, it } from "vitest"; + +import { + Field, + FieldContent, + FieldDescription, + FieldError, + FieldGroup, + FieldLabel, + FieldLegend, + FieldSeparator, + FieldSet, + FieldTitle, +} from "./field"; + +describe("FieldError", () => { + it("renders nothing when there are no errors and no children", () => { + const { container } = render(); + + expect(container).toBeEmptyDOMElement(); + }); + + it("renders nothing when every error entry is undefined", () => { + const { container } = render(); + + expect(container).toBeEmptyDOMElement(); + }); + + it("renders a single message as plain text, not a list", () => { + render(); + + expect(screen.getByRole("alert")).toHaveTextContent("Required"); + expect(screen.queryByRole("listitem")).not.toBeInTheDocument(); + }); + + it("collapses duplicate messages to a single entry", () => { + render(); + + expect(screen.getByRole("alert")).toHaveTextContent("Required"); + expect(screen.queryByRole("listitem")).not.toBeInTheDocument(); + }); + + it("renders distinct messages as a list", () => { + render(); + + const items = screen.getAllByRole("listitem"); + expect(items.map((item) => item.textContent)).toEqual(["Too short", "Must be lowercase"]); + }); + + it("prefers explicit children over the errors prop", () => { + render(from children); + + expect(screen.getByRole("alert")).toHaveTextContent("from children"); + expect(screen.getByRole("alert")).not.toHaveTextContent("from errors"); + }); + + it("exposes the message to assistive tech via role=alert", () => { + render(); + + expect(screen.getByRole("alert")).toBeInTheDocument(); + }); +}); + +describe("Field", () => { + it("marks itself invalid so descendants can style off it", () => { + render( + + child + , + ); + + expect(screen.getByRole("group")).toHaveAttribute("data-invalid", "true"); + }); + + it("defaults to vertical orientation", () => { + render(); + + expect(screen.getByRole("group")).toHaveAttribute("data-orientation", "vertical"); + }); + + it("honours an explicit orientation", () => { + render(); + + expect(screen.getByRole("group")).toHaveAttribute("data-orientation", "horizontal"); + }); +}); + +describe("field primitives forward refs to their DOM node", () => { + it.each([ + ["Field", Field, HTMLDivElement], + ["FieldContent", FieldContent, HTMLDivElement], + ["FieldDescription", FieldDescription, HTMLParagraphElement], + ["FieldGroup", FieldGroup, HTMLDivElement], + ["FieldLabel", FieldLabel, HTMLLabelElement], + ["FieldSeparator", FieldSeparator, HTMLDivElement], + ["FieldTitle", FieldTitle, HTMLDivElement], + ])("%s", (_name, Component, expected) => { + const ref = React.createRef(); + render(React.createElement(Component as React.ElementType, { ref })); + + expect(ref.current).toBeInstanceOf(expected); + }); + + it("FieldSet and FieldLegend", () => { + const fieldSet = React.createRef(); + const legend = React.createRef(); + render( +
+ Legend +
, + ); + + expect(fieldSet.current).toBeInstanceOf(HTMLFieldSetElement); + expect(legend.current).toBeInstanceOf(HTMLLegendElement); + }); + + it("FieldError", () => { + const ref = React.createRef(); + render(); + + expect(ref.current).toBeInstanceOf(HTMLDivElement); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/form/field.tsx b/ui/litellm-dashboard/src/components/shared/form/field.tsx new file mode 100644 index 00000000000..36ce691827c --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/form/field.tsx @@ -0,0 +1,223 @@ +"use client"; + +import * as React from "react"; +import { type VariantProps } from "cva"; + +import { Label } from "@/components/ui/label"; +import { Separator } from "@/components/ui/separator"; +import { cn, cva } from "@/lib/cva.config"; + +const FieldSet = React.forwardRef>( + ({ className, ...props }, ref) => ( +
[data-slot=checkbox-group]]:gap-3 has-[>[data-slot=radio-group]]:gap-3", + className, + )} + {...props} + /> + ), +); +FieldSet.displayName = "FieldSet"; + +const FieldLegend = React.forwardRef< + HTMLLegendElement, + React.ComponentPropsWithoutRef<"legend"> & { variant?: "legend" | "label" } +>(({ className, variant = "legend", ...props }, ref) => ( + +)); +FieldLegend.displayName = "FieldLegend"; + +const FieldGroup = React.forwardRef>( + ({ className, ...props }, ref) => ( +
+ ), +); +FieldGroup.displayName = "FieldGroup"; + +const fieldVariants = cva({ + base: "group/field flex w-full gap-3 data-[invalid=true]:text-destructive", + variants: { + orientation: { + vertical: "flex-col *:w-full [&>.sr-only]:w-auto", + horizontal: + "flex-row items-center has-[>[data-slot=field-content]]:items-start *:data-[slot=field-label]:flex-auto has-[>[data-slot=field-content]]:[&>[role=checkbox],[role=radio]]:mt-px", + responsive: + "flex-col *:w-full @md/field-group:flex-row @md/field-group:items-center @md/field-group:*:w-auto @md/field-group:has-[>[data-slot=field-content]]:items-start @md/field-group:*:data-[slot=field-label]:flex-auto [&>.sr-only]:w-auto @md/field-group:has-[>[data-slot=field-content]]:[&>[role=checkbox],[role=radio]]:mt-px", + }, + }, + defaultVariants: { + orientation: "vertical", + }, +}); + +const Field = React.forwardRef< + HTMLDivElement, + React.ComponentPropsWithoutRef<"div"> & VariantProps +>(({ className, orientation = "vertical", ...props }, ref) => ( +
+)); +Field.displayName = "Field"; + +const FieldContent = React.forwardRef>( + ({ className, ...props }, ref) => ( +
+ ), +); +FieldContent.displayName = "FieldContent"; + +const FieldLabel = React.forwardRef>( + ({ className, ...props }, ref) => ( +