From caf7f1ccdac8a8d463e71fd112facf6a7fdd2a01 Mon Sep 17 00:00:00 2001
From: Kesku <62210496+kesku@users.noreply.github.com>
Date: Sat, 21 Feb 2026 04:03:52 +0000
Subject: [PATCH] feat(perplexity): update Responses API integration to match
Agent API (#21530)
---
.circleci/config.yml | 625 +-
.circleci/requirements.txt | 3 +-
.dockerignore | 2 +-
.../helm-oci-chart-releaser/action.yml | 19 +-
.github/pull_request_template.md | 1 +
.github/workflows/test-linting.yml | 2 +-
.github/workflows/test-litellm-matrix.yml | 118 +
.github/workflows/test-litellm-ui-build.yml | 32 +
.github/workflows/test-litellm.yml | 14 +-
.github/workflows/test-mcp.yml | 4 +-
.github/workflows/test-model-map.yaml | 15 +
.github/workflows/test_server_root_path.yml | 96 +
.gitignore | 11 +-
.semgrep/rules/README.md | 22 +
.../python/reliability/unbounded-memory.yml | 17 +
.semgrep/rules/python/unbounded-memory.yml | 14 +
.trivyignore | 12 +
AGENTS.md | 8 +-
CLAUDE.md | 1 +
CONTRIBUTING.md | 48 +-
Dockerfile | 43 +-
Makefile | 109 +-
README.md | 15 +-
ci_cd/.grype.yaml | 33 +
ci_cd/security_scans.sh | 24 +-
cookbook/anthropic_agent_sdk/README.md | 144 +
.../anthropic_agent_sdk/agent_with_mcp.py | 140 +
cookbook/anthropic_agent_sdk/common.py | 160 +
.../anthropic_agent_sdk/config.example.yaml | 25 +
cookbook/anthropic_agent_sdk/main.py | 95 +
cookbook/anthropic_agent_sdk/requirements.txt | 2 +
cookbook/livekit_agent_sdk/README.md | 114 +
.../livekit_agent_sdk/config.example.yaml | 21 +
cookbook/livekit_agent_sdk/main.py | 112 +
cookbook/livekit_agent_sdk/requirements.txt | 2 +
.../mock_prompt_management_server/README.md | 293 +
.../mock_prompt_management_server.py | 390 +
cookbook/nova_sonic_realtime.py | 288 +
deploy/charts/litellm-helm/Chart.yaml | 4 +
.../litellm-helm/templates/deployment.yaml | 4 +
.../templates/migrations-job.yaml | 4 +
deploy/charts/litellm-helm/values.yaml | 9 +
docker/Dockerfile.custom_ui | 14 +-
docker/Dockerfile.database | 28 +-
docker/Dockerfile.dev | 28 +-
docker/Dockerfile.non_root | 38 +-
docker/README.md | 5 +-
.../index.md | 1 +
.../blog/claude_code_beta_headers/index.md | 177 +
docs/my-website/blog/claude_opus_4_6/index.md | 730 +
.../blog/claude_sonnet_4_6/index.md | 283 +
.../fastapi_middleware_performance/index.mdx | 220 +
docs/my-website/blog/gemini_3/index.md | 1 +
docs/my-website/blog/gemini_3_flash/index.md | 1 +
.../blog/litellm_observatory/index.md | 136 +
docs/my-website/blog/minimax_m2_5/index.md | 394 +
.../blog/model_cost_map_incident/index.md | 95 +
.../sub_millisecond_proxy_overhead/index.md | 92 +
.../blog/vllm_embeddings_incident/index.md | 117 +
docs/my-website/docs/a2a.md | 227 +-
docs/my-website/docs/a2a_invoking_agents.md | 280 +
.../adding_provider/generic_guardrail_api.md | 7 +
.../generic_prompt_management_api.md | 576 +
.../simple_guardrail_tutorial.md | 7 +-
docs/my-website/docs/benchmarks.md | 31 +-
.../docs/completion/message_sanitization.md | 465 +
docs/my-website/docs/completion/web_search.md | 134 +-
docs/my-website/docs/enterprise.md | 12 +
docs/my-website/docs/evals_api.md | 441 +
.../docs/extras/contributing_code.md | 150 +-
.../integrations/websearch_interception.md | 411 +
docs/my-website/docs/load_test.md | 5 +-
docs/my-website/docs/load_test_advanced.md | 6 +-
docs/my-website/docs/mcp.md | 13 +-
docs/my-website/docs/mcp_oauth.md | 337 +
docs/my-website/docs/mcp_public_internet.md | 251 +
docs/my-website/docs/mcp_semantic_filter.md | 158 +
docs/my-website/docs/mcp_troubleshoot.md | 47 +-
docs/my-website/docs/observability/datadog.md | 9 +
.../observability/langfuse_integration.md | 60 +
docs/my-website/docs/pass_through/bedrock.md | 144 +
.../docs/pass_through/openai_passthrough.md | 25 +-
.../my-website/docs/projects/openai-agents.md | 113 +-
docs/my-website/docs/providers/anthropic.md | 71 +
.../docs/providers/anthropic_tool_search.md | 450 +-
.../providers/azure_ai/azure_model_router.md | 163 +-
docs/my-website/docs/providers/bedrock.md | 2 +-
.../providers/bedrock_realtime_with_audio.md | 362 +
docs/my-website/docs/providers/dashscope.md | 28 +-
docs/my-website/docs/providers/elevenlabs.md | 27 +
docs/my-website/docs/providers/gemini.md | 53 +
.../docs/providers/github_copilot.md | 35 +-
.../docs/providers/litellm_proxy.md | 22 +
docs/my-website/docs/providers/openai.md | 65 +-
.../docs/providers/openai/responses_api.md | 18 +
docs/my-website/docs/providers/perplexity.md | 364 +
docs/my-website/docs/providers/sarvam.md | 92 +
docs/my-website/docs/providers/scaleway.md | 62 +
.../docs/providers/vertex_speech.md | 5 +-
.../docs/providers/watsonx/rerank.md | 52 +
.../my-website/docs/providers/xai_realtime.md | 308 +
docs/my-website/docs/proxy/access_groups.md | 122 +
docs/my-website/docs/proxy/admin_ui_sso.md | 108 +-
docs/my-website/docs/proxy/call_hooks.md | 41 +
docs/my-website/docs/proxy/cli.md | 391 +-
docs/my-website/docs/proxy/cli_sso.md | 31 +
docs/my-website/docs/proxy/config_settings.md | 68 +-
docs/my-website/docs/proxy/configs.md | 1 +
docs/my-website/docs/proxy/custom_pricing.md | 46 +
docs/my-website/docs/proxy/custom_sso.md | 12 +
docs/my-website/docs/proxy/deploy.md | 1 +
docs/my-website/docs/proxy/embedding.md | 10 +
.../docs/proxy/forward_client_headers.md | 46 +
.../proxy/guardrails/custom_code_guardrail.md | 332 +
.../docs/proxy/guardrails/grayswan.md | 197 +-
.../proxy/guardrails/guardrail_policies.md | 396 +
.../docs/proxy/guardrails/onyx_security.md | 3 +
.../docs/proxy/guardrails/policy_tags.md | 139 +
.../docs/proxy/guardrails/policy_templates.md | 296 +
.../docs/proxy/guardrails/quick_start.md | 12 +-
.../docs/proxy/guardrails/zscaler_ai_guard.md | 28 +-
.../docs/proxy/keys_teams_router_settings.md | 150 +
.../docs/proxy/litellm_managed_files.md | 2 +-
docs/my-website/docs/proxy/load_balancing.md | 61 +
docs/my-website/docs/proxy/logging.md | 1 +
docs/my-website/docs/proxy/prod.md | 128 +-
docs/my-website/docs/proxy/prometheus.md | 15 +-
.../docs/proxy/prompt_management.md | 55 +-
.../docs/proxy/pyroscope_profiling.md | 43 +
docs/my-website/docs/proxy/release_cycle.md | 8 +-
docs/my-website/docs/proxy/request_tags.md | 58 +
.../docs/proxy/sync_anthropic_beta_headers.md | 128 +
docs/my-website/docs/proxy/team_budgets.md | 1 -
.../docs/proxy/ui/page_visibility.md | 121 +
docs/my-website/docs/proxy/ui_logs.md | 44 +-
.../docs/proxy/ui_spend_log_settings.md | 92 +
.../docs/proxy/ui_team_soft_budget_alerts.md | 130 +
docs/my-website/docs/proxy/users.md | 11 +-
docs/my-website/docs/proxy/virtual_keys.md | 7 +-
docs/my-website/docs/proxy_auth.md | 333 +
docs/my-website/docs/rag_ingest.md | 78 +-
docs/my-website/docs/realtime.md | 19 +-
docs/my-website/docs/rerank.md | 47 +-
docs/my-website/docs/response_api.md | 157 +-
docs/my-website/docs/routing.md | 41 +-
docs/my-website/docs/search/index.md | 1 +
docs/my-website/docs/traffic_mirroring.md | 83 +
docs/my-website/docs/troubleshoot.md | 27 +-
.../docs/troubleshoot/max_callbacks.md | 68 +
.../docs/troubleshoot/prisma_migrations.md | 113 +
.../docs/troubleshoot/spend_queue_warnings.md | 46 +
.../my-website/docs/troubleshoot/ui_issues.md | 49 +
.../docs/tutorials/claude_agent_sdk.md | 115 +
.../tutorials/claude_code_beta_headers.md | 279 +
.../claude_code_plugin_marketplace.md | 4 +-
.../claude_code_prompt_cache_routing.md | 43 +
docs/my-website/docs/tutorials/claude_mcp.md | 72 +-
.../docs/tutorials/copilotkit_sdk.md | 99 +
.../docs/tutorials/livekit_xai_realtime.md | 190 +
.../docs/tutorials/opencode_integration.md | 3 -
docs/my-website/img/a2a_agent_spend.png | Bin 0 -> 188367 bytes
docs/my-website/img/a2a_trace_grouping.png | Bin 0 -> 396967 bytes
docs/my-website/img/okta_access_policies.png | Bin 0 -> 84047 bytes
.../img/okta_authorization_server.png | Bin 0 -> 53146 bytes
.../img/okta_client_credentials.png | Bin 0 -> 65238 bytes
docs/my-website/img/okta_redirect_uri.png | Bin 0 -> 61377 bytes
docs/my-website/img/okta_security_api.png | Bin 0 -> 38685 bytes
docs/my-website/img/policy_team_attach.png | Bin 0 -> 230432 bytes
docs/my-website/img/policy_test_matching.png | Bin 0 -> 205179 bytes
.../img/release_notes/guard_actions.png | Bin 0 -> 518671 bytes
.../img/release_notes/mcp_internet.png | Bin 0 -> 741206 bytes
docs/my-website/img/ui_access_groups.png | Bin 0 -> 346391 bytes
.../img/ui_granular_router_settings.png | Bin 0 -> 359348 bytes
.../my-website/img/ui_spend_logs_settings.png | Bin 0 -> 351095 bytes
.../img/ui_team_soft_budget_alerts.png | Bin 0 -> 335582 bytes
.../img/ui_team_soft_budget_email_example.png | Bin 0 -> 51534 bytes
docs/my-website/img/ui_tools.png | Bin 0 -> 430362 bytes
docs/my-website/package-lock.json | 7 +
docs/my-website/package.json | 2 +
.../my-website/release_notes/v1.81.0/index.md | 6 +-
docs/my-website/release_notes/v1.81.12.md | 433 +
.../release_notes/v1.81.3-stable/index.md | 423 +
docs/my-website/release_notes/v1.81.6.md | 392 +
docs/my-website/release_notes/v1.81.9.md | 382 +
docs/my-website/sidebars.js | 237 +-
.../BaseHTTPMiddlewareAnimation.tsx | 133 +
.../BenchmarkVisualization.tsx | 337 +
.../MiddlewareDiagrams/PureASGIAnimation.tsx | 67 +
.../components/MiddlewareDiagrams/index.tsx | 3 +
.../MiddlewareDiagrams/styles.module.css | 494 +
docs/my-website/src/pages/troubleshoot.md | 11 -
.../src/theme/BlogListPage/index.js | 123 +
.../src/theme/BlogListPage/styles.module.css | 163 +
...litellm_enterprise-0.1.29-py3-none-any.whl | Bin 0 -> 111358 bytes
.../dist/litellm_enterprise-0.1.29.tar.gz | Bin 0 -> 48839 bytes
...litellm_enterprise-0.1.30-py3-none-any.whl | Bin 0 -> 112487 bytes
.../dist/litellm_enterprise-0.1.30.tar.gz | Bin 0 -> 49964 bytes
...litellm_enterprise-0.1.31-py3-none-any.whl | Bin 0 -> 112741 bytes
.../dist/litellm_enterprise-0.1.31.tar.gz | Bin 0 -> 50205 bytes
...litellm_enterprise-0.1.32-py3-none-any.whl | Bin 0 -> 117096 bytes
.../dist/litellm_enterprise-0.1.32.tar.gz | Bin 0 -> 53648 bytes
enterprise/enterprise_hooks/__init__.py | 5 +
.../send_emails/base_email.py | 119 +-
.../proxy/auth/route_checks.py | 7 +-
.../proxy/common_utils/check_batch_cost.py | 53 +-
.../proxy/hooks/managed_files.py | 469 +-
.../proxy/hooks/managed_vector_stores.py | 464 +
.../proxy/vector_stores/endpoints.py | 2 +
enterprise/pyproject.toml | 4 +-
litellm-js/spend-logs/package.json | 4 +-
litellm-proxy-extras/build_and_publish.md | 127 +
...tellm_proxy_extras-0.4.26-py3-none-any.whl | Bin 0 -> 49777 bytes
.../dist/litellm_proxy_extras-0.4.26.tar.gz | Bin 0 -> 22962 bytes
...tellm_proxy_extras-0.4.27-py3-none-any.whl | Bin 0 -> 50073 bytes
.../dist/litellm_proxy_extras-0.4.27.tar.gz | Bin 0 -> 23269 bytes
...tellm_proxy_extras-0.4.28-py3-none-any.whl | Bin 0 -> 50208 bytes
.../dist/litellm_proxy_extras-0.4.28.tar.gz | Bin 0 -> 23405 bytes
...tellm_proxy_extras-0.4.29-py3-none-any.whl | Bin 0 -> 50734 bytes
.../dist/litellm_proxy_extras-0.4.29.tar.gz | Bin 0 -> 23561 bytes
...tellm_proxy_extras-0.4.30-py3-none-any.whl | Bin 0 -> 51217 bytes
.../dist/litellm_proxy_extras-0.4.30.tar.gz | Bin 0 -> 23752 bytes
...tellm_proxy_extras-0.4.31-py3-none-any.whl | Bin 0 -> 51624 bytes
.../dist/litellm_proxy_extras-0.4.31.tar.gz | Bin 0 -> 23846 bytes
...tellm_proxy_extras-0.4.32-py3-none-any.whl | Bin 0 -> 52112 bytes
.../dist/litellm_proxy_extras-0.4.32.tar.gz | Bin 0 -> 23921 bytes
...tellm_proxy_extras-0.4.33-py3-none-any.whl | Bin 0 -> 52533 bytes
.../dist/litellm_proxy_extras-0.4.33.tar.gz | Bin 0 -> 23973 bytes
...tellm_proxy_extras-0.4.34-py3-none-any.whl | Bin 0 -> 53171 bytes
.../dist/litellm_proxy_extras-0.4.34.tar.gz | Bin 0 -> 24232 bytes
...tellm_proxy_extras-0.4.35-py3-none-any.whl | Bin 0 -> 53616 bytes
.../dist/litellm_proxy_extras-0.4.35.tar.gz | Bin 0 -> 24378 bytes
...tellm_proxy_extras-0.4.36-py3-none-any.whl | Bin 0 -> 54409 bytes
.../dist/litellm_proxy_extras-0.4.36.tar.gz | Bin 0 -> 24664 bytes
...tellm_proxy_extras-0.4.37-py3-none-any.whl | Bin 0 -> 55573 bytes
.../dist/litellm_proxy_extras-0.4.37.tar.gz | Bin 0 -> 25050 bytes
...tellm_proxy_extras-0.4.40-py3-none-any.whl | Bin 0 -> 57362 bytes
.../dist/litellm_proxy_extras-0.4.40.tar.gz | Bin 0 -> 25525 bytes
.../litellm_proxy_extras/_logging.py | 40 +-
.../migration.sql | 10 +-
.../migration.sql | 51 +
.../migration.sql | 13 +
.../migration.sql | 19 +
.../migration.sql | 6 +
.../migration.sql | 3 +
.../migration.sql | 3 +
.../migration.sql | 3 +
.../migration.sql | 8 +
.../migration.sql | 3 +
.../migration.sql | 33 +
.../migration.sql | 22 +
.../migration.sql | 3 +
.../20260214094754_schema_sync/migration.sql | 3 +
.../migration.sql | 2 +
.../migration.sql | 3 +
.../litellm_proxy_extras/schema.prisma | 115 +-
.../litellm_proxy_extras/utils.py | 109 +-
litellm-proxy-extras/migration_runbook.md | 37 +-
litellm-proxy-extras/pyproject.toml | 4 +-
litellm/__init__.py | 75 +-
litellm/_lazy_imports_registry.py | 1058 +-
litellm/_logging.py | 148 +-
litellm/_service_logger.py | 4 +-
litellm/a2a_protocol/__init__.py | 14 +
litellm/a2a_protocol/card_resolver.py | 144 +
.../a2a_protocol/exception_mapping_utils.py | 203 +
litellm/a2a_protocol/exceptions.py | 150 +
litellm/a2a_protocol/main.py | 204 +-
litellm/anthropic_beta_headers_config.json | 182 +
litellm/anthropic_beta_headers_manager.py | 377 +
litellm/batch_completion/main.py | 32 +-
litellm/batches/batch_utils.py | 114 +-
litellm/batches/main.py | 9 +-
litellm/caching/dual_cache.py | 66 +-
litellm/caching/redis_cache.py | 2 +-
litellm/caching/redis_cluster_cache.py | 2 +-
.../transformation.py | 382 +-
litellm/constants.py | 158 +-
litellm/cost_calculator.py | 267 +-
litellm/evals/__init__.py | 33 +
litellm/evals/main.py | 1944 ++
litellm/experimental_mcp_client/client.py | 174 +-
litellm/files/main.py | 19 +-
.../SlackAlerting/slack_alerting.py | 5 +
litellm/integrations/arize/_utils.py | 457 +-
litellm/integrations/arize/arize.py | 35 +
litellm/integrations/arize/arize_phoenix.py | 186 +-
litellm/integrations/braintrust_logging.py | 12 +
.../integrations/braintrust_mock_client.py | 131 +
litellm/integrations/cloudzero/transform.py | 30 +-
litellm/integrations/custom_guardrail.py | 112 +-
litellm/integrations/custom_logger.py | 151 +-
litellm/integrations/datadog/datadog.py | 131 +-
.../datadog/datadog_cost_management.py | 20 +-
.../integrations/datadog/datadog_handler.py | 11 +
.../integrations/datadog/datadog_llm_obs.py | 33 +-
.../datadog/datadog_mock_client.py | 28 +
.../integrations/email_templates/templates.py | 24 +
litellm/integrations/gcs_bucket/gcs_bucket.py | 14 +-
.../gcs_bucket/gcs_bucket_base.py | 13 +
.../gcs_bucket/gcs_bucket_mock_client.py | 192 +
litellm/integrations/helicone.py | 15 +-
litellm/integrations/helicone_mock_client.py | 32 +
litellm/integrations/langfuse/langfuse.py | 28 +-
.../langfuse/langfuse_mock_client.py | 35 +
.../integrations/langfuse/langfuse_otel.py | 96 +-
.../langfuse/langfuse_prompt_management.py | 92 +-
litellm/integrations/langsmith.py | 23 +-
litellm/integrations/langsmith_mock_client.py | 29 +
litellm/integrations/mock_client_factory.py | 216 +
litellm/integrations/opentelemetry.py | 261 +-
litellm/integrations/posthog.py | 45 +-
litellm/integrations/posthog_mock_client.py | 30 +
litellm/integrations/prometheus.py | 556 +-
litellm/integrations/prometheus_services.py | 5 +
litellm/integrations/s3_v2.py | 83 +-
.../websearch_interception/handler.py | 269 +-
.../websearch_interception/tools.py | 56 +-
.../websearch_interception/transformation.py | 190 +-
.../api_route_to_call_types.py | 33 +-
litellm/litellm_core_utils/core_helpers.py | 8 +-
litellm/litellm_core_utils/env_utils.py | 21 +
.../exception_mapping_utils.py | 24 +-
.../litellm_core_utils/get_litellm_params.py | 87 +-
.../get_llm_provider_logic.py | 10 +-
.../litellm_core_utils/get_model_cost_map.py | 201 +-
.../initialize_dynamic_callback_params.py | 54 +-
litellm/litellm_core_utils/litellm_logging.py | 176 +-
.../litellm_core_utils/llm_cost_calc/utils.py | 53 +-
.../convert_dict_to_response.py | 6 +-
.../logging_callback_manager.py | 29 +-
litellm/litellm_core_utils/logging_utils.py | 3 +-
litellm/litellm_core_utils/logging_worker.py | 22 +
.../litellm_core_utils/model_param_helper.py | 12 +-
.../prompt_templates/common_utils.py | 167 +-
.../prompt_templates/factory.py | 462 +-
.../prompt_templates/image_handling.py | 14 +-
litellm/litellm_core_utils/redact_messages.py | 63 +-
litellm/litellm_core_utils/safe_json_dumps.py | 9 +-
.../streaming_chunk_builder_utils.py | 58 +-
.../litellm_core_utils/streaming_handler.py | 130 +-
litellm/litellm_core_utils/token_counter.py | 4 +-
litellm/llms/a2a/__init__.py | 6 +
litellm/llms/a2a/chat/__init__.py | 6 +
.../a2a/chat/guardrail_translation/README.md | 155 +
.../chat/guardrail_translation/__init__.py | 11 +
.../a2a/chat/guardrail_translation/handler.py | 428 +
litellm/llms/a2a/chat/streaming_iterator.py | 103 +
litellm/llms/a2a/chat/transformation.py | 370 +
litellm/llms/a2a/common_utils.py | 152 +
.../chat/guardrail_translation/handler.py | 88 +-
litellm/llms/anthropic/chat/handler.py | 42 +-
litellm/llms/anthropic/chat/transformation.py | 208 +-
litellm/llms/anthropic/common_utils.py | 146 +-
litellm/llms/anthropic/cost_calculation.py | 16 +-
.../adapters/handler.py | 93 +-
.../adapters/streaming_iterator.py | 46 +-
.../adapters/transformation.py | 457 +-
.../messages/transformation.py | 92 +-
litellm/llms/azure/azure.py | 50 +-
litellm/llms/azure/batches/handler.py | 23 +-
.../llms/azure/chat/gpt_5_transformation.py | 8 +-
litellm/llms/azure/chat/gpt_transformation.py | 1 +
litellm/llms/azure/cost_calculation.py | 37 +-
.../llms/azure/responses/transformation.py | 39 +-
.../anthropic/count_tokens/transformation.py | 14 +-
.../anthropic/messages_transformation.py | 1 -
.../llms/azure_ai/anthropic/transformation.py | 2 +-
.../azure_ai/azure_model_router/__init__.py | 4 +
.../azure_model_router/transformation.py | 125 +
litellm/llms/azure_ai/chat/transformation.py | 19 +-
litellm/llms/azure_ai/common_utils.py | 85 +-
litellm/llms/azure_ai/cost_calculator.py | 121 +
.../llms/azure_ai/rerank/transformation.py | 32 +-
litellm/llms/base_llm/chat/transformation.py | 20 +
litellm/llms/base_llm/evals/__init__.py | 7 +
litellm/llms/base_llm/evals/transformation.py | 542 +
.../base_llm/managed_resources/__init__.py | 41 +
.../base_managed_resource.py | 605 +
.../llms/base_llm/managed_resources/utils.py | 364 +
.../base_llm/vector_store/transformation.py | 26 +-
litellm/llms/bedrock/base_aws_llm.py | 163 +-
litellm/llms/bedrock/chat/converse_handler.py | 23 +-
.../bedrock/chat/converse_transformation.py | 428 +-
litellm/llms/bedrock/chat/invoke_handler.py | 8 +-
.../anthropic_claude3_transformation.py | 23 +-
litellm/llms/bedrock/common_utils.py | 76 +-
litellm/llms/bedrock/cost_calculation.py | 13 +-
.../bedrock/embed/cohere_transformation.py | 4 +-
.../anthropic_claude3_transformation.py | 219 +-
litellm/llms/bedrock/realtime/handler.py | 307 +
.../llms/bedrock/realtime/transformation.py | 1156 +
litellm/llms/cerebras/chat.py | 11 +-
.../llms/chatgpt/responses/transformation.py | 21 +-
.../rerank/guardrail_translation/handler.py | 8 +-
.../llms/custom_httpx/aiohttp_transport.py | 58 +-
litellm/llms/custom_httpx/http_handler.py | 68 +-
litellm/llms/custom_httpx/httpx_handler.py | 16 +-
litellm/llms/custom_httpx/llm_http_handler.py | 1387 +-
.../llms/databricks/chat/transformation.py | 36 +-
.../llms/databricks/responses/__init__.py | 0
.../databricks/responses/transformation.py | 100 +
litellm/llms/deprecated_providers/palm.py | 2 +-
litellm/llms/duckduckgo/search/__init__.py | 6 +
.../llms/duckduckgo/search/transformation.py | 252 +
.../llms/fireworks_ai/chat/transformation.py | 4 +
litellm/llms/gemini/chat/transformation.py | 2 +-
litellm/llms/gemini/files/transformation.py | 153 +-
.../llms/gemini/image_edit/transformation.py | 5 +-
.../gemini/image_generation/transformation.py | 2 +
litellm/llms/gigachat/chat/transformation.py | 137 +-
.../github_copilot/chat/transformation.py | 51 +-
litellm/llms/groq/chat/transformation.py | 8 +
.../llms/hosted_vllm/chat/transformation.py | 23 +-
.../hosted_vllm/embedding/transformation.py | 180 +
litellm/llms/minimax/chat/transformation.py | 29 +-
litellm/llms/oci/chat/transformation.py | 50 +-
litellm/llms/ollama/chat/transformation.py | 11 +-
.../llms/openai/chat/gpt_5_transformation.py | 4 +-
.../llms/openai/chat/gpt_transformation.py | 42 +-
.../chat/guardrail_translation/handler.py | 35 +-
litellm/llms/openai/common_utils.py | 75 +-
.../guardrail_translation/handler.py | 21 +-
.../guardrail_translation/__init__.py | 13 +
.../guardrail_translation/handler.py | 179 +
litellm/llms/openai/evals/__init__.py | 7 +
litellm/llms/openai/evals/transformation.py | 426 +
.../guardrail_translation/handler.py | 8 +-
litellm/llms/openai/openai.py | 108 +-
litellm/llms/openai/realtime/handler.py | 80 +-
.../guardrail_translation/handler.py | 84 +-
.../llms/openai/responses/transformation.py | 27 +-
.../speech/guardrail_translation/handler.py | 8 +-
.../guardrail_translation/handler.py | 7 +-
litellm/llms/openai/videos/transformation.py | 45 +-
litellm/llms/openai_like/dynamic_config.py | 24 +-
litellm/llms/openai_like/embedding/handler.py | 3 +-
litellm/llms/openai_like/providers.json | 4 +
.../llms/openrouter/chat/transformation.py | 4 +
.../guardrail_translation/handler.py | 15 +-
litellm/llms/perplexity/responses/__init__.py | 7 +
.../perplexity/responses/transformation.py | 492 +
litellm/llms/s3_vectors/__init__.py | 1 +
.../llms/s3_vectors/vector_stores/__init__.py | 1 +
.../vector_stores/transformation.py | 254 +
.../sagemaker/embedding/transformation.py | 13 +-
litellm/llms/vertex_ai/common_utils.py | 39 +-
.../vertex_ai_context_caching.py | 172 +-
.../llms/vertex_ai/files/transformation.py | 2 +-
.../llms/vertex_ai/gemini/transformation.py | 34 +
.../vertex_and_google_ai_studio_gemini.py | 96 +-
.../vertex_gemini_transformation.py | 2 +
.../vector_stores/rag_api/transformation.py | 9 +-
.../transformation.py | 25 +-
.../anthropic/transformation.py | 69 +-
.../count_tokens/handler.py | 7 +-
.../llama3/transformation.py | 104 +-
litellm/llms/vertex_ai/vertex_llm_base.py | 19 +-
litellm/llms/watsonx/__init__.py | 0
litellm/llms/watsonx/chat/__init__.py | 0
litellm/llms/watsonx/common_utils.py | 2 +
litellm/llms/watsonx/completion/__init__.py | 0
litellm/llms/watsonx/embed/__init__.py | 0
litellm/llms/watsonx/rerank/__init__.py | 0
litellm/llms/watsonx/rerank/transformation.py | 204 +
litellm/llms/xai/chat/transformation.py | 3 +-
litellm/llms/xai/realtime/__init__.py | 5 +
litellm/llms/xai/realtime/handler.py | 38 +
litellm/llms/xai/responses/transformation.py | 132 +-
litellm/llms/zai/chat/transformation.py | 16 +-
litellm/main.py | 267 +-
...odel_prices_and_context_window_backup.json | 4212 +++-
litellm/policy_templates_backup.json | 1883 ++
.../mcp_server/auth/litellm_auth_handler.py | 2 +
.../mcp_server/auth/user_api_key_auth_mcp.py | 148 +-
.../mcp_server/discoverable_endpoints.py | 268 +-
.../guardrail_translation/handler.py | 102 +-
.../_experimental/mcp_server/mcp_debug.py | 329 +
.../mcp_server/mcp_server_manager.py | 319 +-
.../mcp_server/oauth2_token_cache.py | 163 +
.../mcp_server/openapi_to_mcp_generator.py | 34 +-
.../mcp_server/rest_endpoints.py | 297 +-
.../mcp_server/semantic_tool_filter.py | 250 +
.../proxy/_experimental/mcp_server/server.py | 256 +-
.../proxy/_experimental/mcp_server/utils.py | 30 +-
.../proxy/_experimental/out/404/index.html | 1 +
.../_experimental/out/__next.__PAGE__.txt | 31 +
.../proxy/_experimental/out/__next._full.txt | 62 +
.../proxy/_experimental/out/__next._head.txt | 6 +
.../proxy/_experimental/out/__next._index.txt | 7 +
.../proxy/_experimental/out/__next._tree.txt | 5 +
.../C_XKHLw43nx5HaPfGD7XZ/_buildManifest.js | 16 +
.../_clientMiddlewareManifest.json | 1 +
.../_ssgManifest.js | 0
.../_next/static/chunks/00bcc8d30dd19793.js | 9 +
.../_next/static/chunks/00ff280cdb7d7ee5.js | 1 +
.../_next/static/chunks/04b9c7b5c33ea26c.js | 14 +
.../_next/static/chunks/06aaedbe7d27898c.js | 1 +
.../_next/static/chunks/088a4006aa78f150.js | 1 +
.../_next/static/chunks/0a65da2cd24e2ab6.js | 3 +
.../_next/static/chunks/0a671fedee641c02.js | 1 +
.../_next/static/chunks/0a6c418370a8c183.js | 41 +
.../_next/static/chunks/0d1694151d7fdaec.js | 38 +
.../_next/static/chunks/1067d2c077cd73d6.js | 4 +
.../static/chunks/1098-c3e95c9684ff5e95.js | 1 -
.../static/chunks/1108-c2d0c742b6e72436.js | 1 -
.../static/chunks/1128-64fa4a41ccaf67ea.js | 1 -
.../_next/static/chunks/11383a8b78399079.js | 8 +
.../_next/static/chunks/120d96e5e05ab994.js | 2 +
.../_next/static/chunks/121a51d3bbb6f362.js | 7 +
.../_next/static/chunks/1300460219810c10.js | 4 +
.../_next/static/chunks/134f728fa7099e3e.js | 55 +
.../static/chunks/137-c6f74fedf576a11b.js | 1 -
.../static/chunks/1414-2770d1155b664522.js | 1 -
.../static/chunks/1442-024f7e51804e0d7e.js | 1 -
.../static/chunks/1623-54c56cbe1afc3953.js | 1 -
.../static/chunks/1713-b3fdb241d0f3ae7a.js | 1 -
.../_next/static/chunks/179f4b987bc9083f.js | 9 +
.../static/chunks/1901-4d02d1f2a71cdbf7.js | 1 -
.../_next/static/chunks/193ac6435f936582.js | 1 +
.../static/chunks/1994-6637a121c9ee1602.js | 1 -
.../_next/static/chunks/1a01cb4063a7b21e.js | 1 +
.../_next/static/chunks/1ab4ccc7c0ba9eff.js | 14 +
.../_next/static/chunks/1b8186fdb9bf9067.js | 1 +
.../_next/static/chunks/1d3826d625e92c33.js | 2 +
.../_next/static/chunks/1e0e6eb47fe60159.js | 1 +
.../_next/static/chunks/1fe0596a309ad6cf.js | 12 +
.../static/chunks/2068-2c78bfc32dc0de5f.js | 4 -
.../static/chunks/2117-26a589a1115bdd0a.js | 2 -
.../static/chunks/2136-2c0d6e8c18d2c5c4.js | 1 -
.../static/chunks/2202-a83ad035a17401aa.js | 1 -
.../static/chunks/2227-5ae3f36b0a81c5b4.js | 1 -
.../_next/static/chunks/22e715061d511345.js | 8 +
.../static/chunks/2378-252212b7a5e313ce.js | 1 -
.../static/chunks/2409-e94c05c6f11bb939.js | 1 -
.../_next/static/chunks/249ef9d7a08bbfa1.js | 1 +
.../static/chunks/2500-811f2612ec5f6830.js | 1 -
.../_next/static/chunks/25d1ef14bd591cf9.js | 8 +
.../static/chunks/2618-062177b80fc4a38e.js | 1 -
.../static/chunks/2699-38ff37315d78ae04.js | 1 -
.../_next/static/chunks/26adfa4e8ffc85c7.js | 1 +
.../_next/static/chunks/2703702968738794.js | 1 +
.../_next/static/chunks/27c7596aa0326b71.js | 4 +
.../static/chunks/2820-592c2b4ff874a913.js | 1 -
.../static/chunks/2901-0cdd0656eb7463d6.js | 1 -
.../static/chunks/292-aaba6c4e7c8d416d.js | 1 -
.../static/chunks/2926-a9cb83e61fc8ad20.js | 1 -
.../_next/static/chunks/2971c4658f1bcd7d.js | 1 +
.../_next/static/chunks/2f04fe05bcb1c150.js | 1 +
.../_next/static/chunks/2fdd60613421a228.js | 8 +
.../chunks/3014691f-ba91873bc8fe3fad.js | 1 -
.../_next/static/chunks/315cda92f466b9ec.js | 8 +
.../static/chunks/3242-6e6ec7e18f5d698d.js | 1 -
.../static/chunks/3367-58830187e9e5b9fa.js | 1 -
.../static/chunks/337-bb33d149e9f461b3.js | 1 -
.../_next/static/chunks/3454255bdea68dda.js | 1 +
.../static/chunks/3507-14fb4e6cd377d7da.js | 1 -
.../static/chunks/353-e55516ea4730f9d4.js | 1 -
.../static/chunks/3567-9a29feedd7b63950.js | 5 -
.../_next/static/chunks/36ccc2b555a26ad4.js | 4 +
.../static/chunks/3709-34dbb332d3a3ac26.js | 1 -
.../static/chunks/3862-064a3fb795c75b62.js | 1 -
.../_next/static/chunks/3afadb9a550fc886.js | 1 +
.../_next/static/chunks/3b30ab8eaa03bc21.js | 4 +
.../_next/static/chunks/3b4510be1f4cea1f.js | 1 +
.../_next/static/chunks/3d2a01213eb1cc87.js | 1 +
.../_next/static/chunks/3dad14bcec641ba8.js | 1 +
.../_next/static/chunks/3f369c603677cd7a.js | 1 +
.../_next/static/chunks/3f3fa56b5786d58c.css | 1 +
.../_next/static/chunks/3f49d66311c27fe1.js | 105 +
.../_next/static/chunks/403c4d96324c23a6.js | 3 +
.../static/chunks/4077-50cf2a28a79fdcd4.js | 1 -
.../_next/static/chunks/40e89c053e10e01c.js | 1 +
.../static/chunks/4105-9c3c0ee7c494102f.js | 1 -
.../_next/static/chunks/4188d520ca4e5f2b.js | 1 +
.../_next/static/chunks/4262f254ec63c549.js | 1 +
.../static/chunks/4306-f891b96cf0ee333b.js | 1 -
.../static/chunks/4388-2f4ca3419d20af67.js | 1 -
.../static/chunks/4470-3ef8ade20eaf2875.js | 1 -
.../_next/static/chunks/44edba5625a9a9b4.js | 68 +
.../_next/static/chunks/450ebd094f4fa24d.js | 1 +
.../_next/static/chunks/4537761df9dff7f0.js | 1 +
.../static/chunks/4559-52ca85b2d8893149.js | 1 -
.../_next/static/chunks/457923c551f21385.js | 598 +
.../_next/static/chunks/4587f4ad9ebcbb4e.js | 12 +
.../_next/static/chunks/464560f129260d42.js | 420 +
.../_next/static/chunks/46901752d0b0dde9.js | 1 +
.../_next/static/chunks/4758898ae55ecd92.js | 1 +
.../_next/static/chunks/47656bcac78a726c.js | 1 +
.../_next/static/chunks/47ed25bb99ff8a39.js | 1 +
.../static/chunks/4817-59d642defb0e86f2.js | 1 -
.../static/chunks/4865-c1c0885a93c327fa.js | 1 -
.../_next/static/chunks/496b84010c33cf69.js | 1 +
.../_next/static/chunks/4980372eaa37b78b.js | 8 +
.../_next/static/chunks/4995cc30215f504d.js | 1 +
.../_next/static/chunks/4af6a1c366381700.js | 8 +
.../_next/static/chunks/4b385187755a737f.js | 1 +
.../_next/static/chunks/4bacf5b9194c12f5.js | 8 +
.../_next/static/chunks/4e20891f2fd03463.css | 1 +
.../_next/static/chunks/4ed86d695abe3c87.js | 1 +
.../_next/static/chunks/50779d2c65692de7.js | 1 +
.../static/chunks/5105-2998cbe1c9fc8ee4.js | 1 -
.../_next/static/chunks/511809a345b510d8.js | 1 +
.../static/chunks/5144-bbc18c43eade9aef.js | 1 -
.../static/chunks/5202-60292daf4bc5c8fb.js | 1 -
.../static/chunks/5319-5b2d4bf2dc450f99.js | 1 -
.../_next/static/chunks/53218dce8acb3bff.js | 1 +
.../static/chunks/5333-1540faf81c7d7006.js | 1 -
.../_next/static/chunks/5365cf27e8d07577.js | 1 +
.../_next/static/chunks/536cb86ca75d1f30.js | 1 +
.../_next/static/chunks/542a1a209eb732c6.js | 7 +
.../_next/static/chunks/54731bb470e07604.js | 1 +
.../_next/static/chunks/557a369a3f213cfe.js | 1 +
.../_next/static/chunks/5583bc893837fdf8.js | 4 +
.../_next/static/chunks/55c4117d5fcd0aae.js | 8 +
.../_next/static/chunks/565cdfe156dcb380.js | 1 +
.../static/chunks/5695-dbbcbf2da21d2bab.js | 1 -
.../static/chunks/5706-1e314cef9ea5c5d6.js | 1 -
.../_next/static/chunks/570b2e10aa856e54.js | 1 +
.../static/chunks/5733-aa80f52062105ad2.js | 1 -
.../static/chunks/5752-f504fb38ff5e13e8.js | 1 -
.../static/chunks/5869-426268ba6ad0ce0c.js | 1 -
.../_next/static/chunks/58b9eb1766fba8e0.js | 7 +
.../static/chunks/5945-8b3b7713d7f416a2.js | 1 -
.../static/chunks/5975-758334d6641b9c63.js | 1 -
.../static/chunks/5992-287cec06808c74ae.js | 1 -
.../_next/static/chunks/5b2b7fd4dd9a44f3.js | 4 +
.../_next/static/chunks/5b9c0b6d6c814e58.js | 1 +
.../_next/static/chunks/5d1f33f9fa668633.js | 179 +
.../_next/static/chunks/5d3e07ae5afa6fa6.js | 105 +
.../_next/static/chunks/5d547ead001142ce.js | 7 +
.../_next/static/chunks/5db1c5d0d0e548b4.js | 1 +
.../_next/static/chunks/5eb6648cefff2d8a.js | 1 +
.../_next/static/chunks/5f9c3b92a016f382.js | 14 +
.../_next/static/chunks/6008d176e68995d6.js | 1 +
.../static/chunks/605-102c0e6d8bb7517c.js | 1 -
.../_next/static/chunks/617bc18095fe8025.js | 7 +
.../static/chunks/6285-68f82c874b184eba.js | 1 -
.../_next/static/chunks/6367dd1d1cf7eeef.js | 1 +
.../static/chunks/6399-ccf9cdbdcd5f7abb.js | 1 -
.../_next/static/chunks/63f40e445646cfa6.js | 1 +
.../_next/static/chunks/64f1a2ef9113d86f.js | 13 +
.../static/chunks/6537-f70f2c4278e93458.js | 1 -
.../static/chunks/6600-0ec5e2dc66d8b41a.js | 1 -
.../static/chunks/6609-3e081758ffbe3786.js | 1 -
.../static/chunks/665-d94073042ee5b874.js | 1 -
.../static/chunks/6653-e61fdc06093fc0a8.js | 1 -
.../_next/static/chunks/66a190706fc6c35a.js | 167 +
.../_next/static/chunks/67570d9401e62846.js | 3 +
.../_next/static/chunks/6774f9c1f201e744.js | 1 +
.../_next/static/chunks/69aeba649b0dc90f.js | 139 +
.../_next/static/chunks/6c4c97f1ea6e7d77.js | 4 +
.../_next/static/chunks/6d587e6e43260fc9.js | 19 +
.../_next/static/chunks/6e033c78c15ab9a6.js | 4 +
.../static/chunks/7138-5b134dc8ad670770.js | 1 -
.../_next/static/chunks/72250192fd3153b7.js | 1 +
.../static/chunks/7271-46e4c11ee6b0a4d6.js | 8 -
.../_next/static/chunks/730305e005d7bd1d.js | 105 +
.../_next/static/chunks/738c339383c3b4b6.js | 1 +
.../_next/static/chunks/74982774ef38dcdb.js | 2 +
.../static/chunks/7526-da6b2857a3ca248d.js | 1 -
.../static/chunks/766-baf0336e8ba5c686.js | 1 -
.../static/chunks/7688-ca173ea41812cf94.js | 1 -
.../_next/static/chunks/76a83e13dfaf23db.js | 2 +
.../_next/static/chunks/77d897b03fb96fa0.js | 8 +
.../static/chunks/7851-c10cbe6fcac2f9d6.js | 1 -
.../static/chunks/7906-59ba450db59c8efa.js | 1 -
.../static/chunks/7926-108623e14caeb770.js | 1 -
.../_next/static/chunks/7ad0165018dc89ce.js | 1 +
.../_next/static/chunks/7b788dd93ad868b3.js | 1 +
.../_next/static/chunks/7e2badb3d178f837.js | 13 +
.../_next/static/chunks/7e3f5ce4b2a613d4.js | 1 +
.../_next/static/chunks/7e417dd24c8becd0.js | 1 +
.../_next/static/chunks/7f9e9c54ac262de2.js | 1 +
.../static/chunks/8014-d6138fce46bba1e2.js | 1 -
.../_next/static/chunks/8015668aa5f04beb.js | 1 +
.../static/chunks/8049-cb52b16664f13e28.js | 1 -
.../static/chunks/8135-881fe2cea0032570.js | 1 -
.../_next/static/chunks/814136f5b55e06b6.js | 7 +
.../static/chunks/8143-9e4312f059e9ed27.js | 1 -
.../static/chunks/816-37c57b39f4e7ece1.js | 1 -
.../_next/static/chunks/81bf20526995284e.js | 1 +
.../_next/static/chunks/81e224efc874dea6.js | 8 +
.../static/chunks/8211-8dd5691abf54d0ca.js | 1 -
.../_next/static/chunks/82a6c2af12705c46.js | 1 +
.../_next/static/chunks/82ef36abe5e2e833.js | 1 +
.../_next/static/chunks/831fda51c425b4a8.js | 1 +
.../_next/static/chunks/8354d717e34ebd6f.js | 86 +
.../_next/static/chunks/841e807b7dbb7e4f.js | 8 +
.../static/chunks/8473-7749355a9a4b1818.js | 1 -
.../_next/static/chunks/84884fbf517f5d74.js | 1 +
.../static/chunks/8529-6b66d5dba2148164.js | 1 -
.../static/chunks/8565-5c05f6bbb9d0662f.js | 1 -
.../static/chunks/8582-3a775364dbf07fa8.js | 1 -
.../_next/static/chunks/88c74f8b4b20d25a.js | 1 +
.../_next/static/chunks/8992001a9a91bc67.js | 1 +
.../_next/static/chunks/89b9f8dbb6f0d490.js | 1 +
.../_next/static/chunks/8a607e531e36f204.js | 8 +
.../_next/static/chunks/8e12212d7a0aeaee.js | 4 +
.../_next/static/chunks/9022b46fabff1181.js | 1 +
.../static/chunks/9028-2bfc9f09930a0d61.js | 1 -
.../static/chunks/9039-e44ff08ca4f37a12.js | 1 -
.../static/chunks/9140-09af618948244b82.js | 1 -
.../static/chunks/9145-9507437d5b599cea.js | 1 -
.../static/chunks/9264-fd8ab51d702e9535.js | 1 -
.../_next/static/chunks/92cf5d832080641f.js | 13 +
.../_next/static/chunks/93032856602932c1.js | 420 +
.../_next/static/chunks/936738f40fc24cc1.js | 8 +
.../_next/static/chunks/937c3b6cb00f6b79.js | 13 +
.../static/chunks/9409-6eefc92a7f8433ff.js | 1 -
.../static/chunks/9584-9d4fd7b3d6a7c9e7.js | 1 -
.../_next/static/chunks/97efd6e1c67bedcb.js | 421 +
.../static/chunks/9818-6f03d7efd4fb8533.js | 1 -
.../_next/static/chunks/983036f73d37142a.js | 7 +
.../static/chunks/9841-721a173be76941d1.js | 1 -
.../_next/static/chunks/99be180c22b927f8.js | 167 +
.../_next/static/chunks/99cf9cf99df5ccfc.js | 1 +
.../_next/static/chunks/9cf03e6d4b5b806e.js | 420 +
.../_next/static/chunks/9d9fbd3add7d0f88.js | 1 +
.../_next/static/chunks/9f5ccd929375c1d6.js | 1 +
.../_next/static/chunks/a0f302271a793712.js | 4 +
.../_next/static/chunks/a1c3d7b907b7b731.js | 8 +
.../_next/static/chunks/a1ef280b7ad5ae6a.js | 3 +
.../_next/static/chunks/a21582fe1f52b973.js | 2 +
.../_next/static/chunks/a382857dbbcea5d1.js | 50 +
.../_next/static/chunks/a44b0c08814c45ae.js | 1 +
.../_next/static/chunks/a6bf78649679c265.js | 1 +
.../_next/static/chunks/a6dad97d9634a72d.js | 1 +
.../static/chunks/a6dad97d9634a72d.js.map | 1 +
.../_next/static/chunks/a7aecb91c09b0e9a.js | 216 +
.../_next/static/chunks/a7b79d0fe43dcbd0.js | 1 +
.../_next/static/chunks/a7c0a41b6156d9b2.js | 1 +
.../_next/static/chunks/a966296c3a6b28f6.js | 2 +
.../_next/static/chunks/a9ebedc318fa36dc.js | 1 +
.../_next/static/chunks/ab7a826839e7e423.js | 1 +
.../api-reference/page-a6a3e9e67b671303.js | 1 -
.../api-playground/page-67c4f150eba92e64.js | 1 -
.../budgets/page-e471b9b73cfc894b.js | 1 -
.../caching/page-d31cc105402ab7e0.js | 1 -
.../old-usage/page-5e097dbb8ce40bb4.js | 1 -
.../prompts/page-8236f1efda3366f0.js | 1 -
.../tag-management/page-5627fd94813402eb.js | 1 -
.../guardrails/page-060e61cb783d32ef.js | 1 -
.../(dashboard)/layout-f7f722423efd1c5b.js | 1 -
.../(dashboard)/logs/page-5a10d46ca991b83e.js | 1 -
.../model-hub/page-1479dcb217587498.js | 1 -
.../page-b69988590beaa5c8.js | 1 -
.../organizations/page-56a03e123f452d60.js | 1 -
.../playground/page-fc3dff494dc4db08.js | 1 -
.../admin-settings/page-746658933a633902.js | 1 -
.../page-e3df74ef5ac0dcd5.js | 1 -
.../router-settings/page-ffa3245ebcbbc02a.js | 1 -
.../ui-theme/page-d833946961b065a5.js | 1 -
.../teams/page-8d1a71afa9e9ff16.js | 1 -
.../test-key/page-63cd64b722408984.js | 1 -
.../mcp-servers/page-7dd2ea6f1433d41f.js | 1 -
.../vector-stores/page-0f618cc2d6cae794.js | 1 -
.../usage/page-d7532f354d44803b.js | 1 -
.../users/page-6d3643cef6c068ee.js | 1 -
.../virtual-keys/page-73444bcb0cfe86b7.js | 1 -
.../app/_not-found/page-e8d298ce4e35f231.js | 1 -
.../chunks/app/layout-bed96765a7fb7bdd.js | 1 -
.../chunks/app/login/page-61bfa80619b62f6b.js | 1 -
.../oauth/callback/page-01be1cae3559363d.js | 1 -
.../app/model_hub/page-92347d2021ca8580.js | 1 -
.../model_hub_table/page-2ad344049541235f.js | 1 -
.../app/onboarding/page-17ecf5bf068f8157.js | 1 -
.../chunks/app/page-587b0acf34f8c747.js | 1 -
.../_next/static/chunks/b5bcd87b218a6bcd.js | 14 +
.../_next/static/chunks/b64beb414bc36659.js | 1 +
.../_next/static/chunks/b85f190e8626c49c.js | 1 +
.../_next/static/chunks/ba5a05afc286361c.js | 8 +
.../_next/static/chunks/bdf355b41816a002.js | 82 +
.../_next/static/chunks/be340f56c7da1645.js | 1 +
.../_next/static/chunks/bf880fd979d4a2e6.js | 84 +
.../_next/static/chunks/c058ac3e89dc33df.js | 1 +
.../_next/static/chunks/c19d75622900fb62.js | 1 +
.../_next/static/chunks/c3d0c3b532b01699.js | 8 +
.../_next/static/chunks/c4111e97b0095227.js | 1 +
.../_next/static/chunks/c43ea300e1f2db88.js | 1 +
.../_next/static/chunks/c4452a79c69324a6.js | 17 +
.../_next/static/chunks/c6a3593fb6892e17.js | 1 +
.../_next/static/chunks/c7b74067c01ee971.js | 2 +
.../_next/static/chunks/c91982ee39ef0f77.js | 1 +
.../_next/static/chunks/c93c5c533dba84d1.js | 1 +
.../_next/static/chunks/c9af2deb434988d6.js | 8 +
.../_next/static/chunks/ca22b37c24b4d34a.js | 35 +
.../_next/static/chunks/cb8e6ba28461af15.js | 4 +
.../_next/static/chunks/ce8464047a8ce464.js | 1 +
.../_next/static/chunks/cf68fd1f1761ba48.js | 10 +
.../_next/static/chunks/cfc22f1e9e2830a5.js | 1 +
.../_next/static/chunks/d24f23929997cfa1.js | 1 +
.../_next/static/chunks/d2aa91699d95f4b3.js | 1 +
.../_next/static/chunks/d4240d7bae1e2b30.js | 1 +
.../_next/static/chunks/d4710ffa8fe96c6a.js | 2 +
.../_next/static/chunks/d64d74932cb225a3.js | 1 +
.../_next/static/chunks/d682c064a60ae3d6.css | 1 +
.../_next/static/chunks/d96012bcfc98706a.js | 1 +
.../_next/static/chunks/d991de8f2cd90aca.js | 1 +
.../_next/static/chunks/db89710f0ce96e05.js | 1 +
.../_next/static/chunks/dbca964212122d58.js | 1 +
.../_next/static/chunks/de9cdee2e8c8fa36.js | 1 +
.../_next/static/chunks/e007904603a33bc5.js | 10 +
.../_next/static/chunks/e0d42088ec18edc9.js | 420 +
.../_next/static/chunks/e1c5d2e47c042b8a.js | 50 +
.../_next/static/chunks/e1f23fd814ac3500.js | 4 +
.../_next/static/chunks/e1fe71b9ff3d3857.js | 8 +
.../chunks/e228588e-635e9029d9d88215.js | 1 -
.../_next/static/chunks/e3bc795c751bb99a.js | 41 +
.../_next/static/chunks/e71fe358fd0c350f.js | 7 +
.../_next/static/chunks/e8ed72789c2b42ff.js | 39 +
.../_next/static/chunks/e96398764f77c728.js | 2 +
.../_next/static/chunks/eaf91f44e099fe65.js | 1 +
.../_next/static/chunks/eea976cf4a05fc92.js | 55 +
.../_next/static/chunks/ef41b5b82a37e553.js | 8 +
.../_next/static/chunks/f628c4bfd7854ec0.js | 21 +
.../_next/static/chunks/f98b25d79cd05714.js | 7 +
.../_next/static/chunks/fa8a1b9b6454c116.js | 1 +
.../_next/static/chunks/fb981bf7548d9de3.js | 2 +
.../chunks/fd9d1056-a07bacd8fcc2728b.js | 1 -
.../_next/static/chunks/fe750aa0bf04912c.js | 21 +
.../_next/static/chunks/fea300adfdeaf3b9.js | 1 +
.../_next/static/chunks/ffd416b6dab7092c.js | 12 +
.../_next/static/chunks/ffe482191cf04a55.js | 1 +
.../chunks/main-app-c6945ec5b2d5e671.js | 1 -
.../static/chunks/main-e4e168e4dfadea03.js | 1 -
.../chunks/turbopack-901b35f89c1f6751.js | 4 +
.../out/_next/static/css/83c095d0528a2e35.css | 3 -
.../media/1bffadaabf893a1e-s.7cd81963.woff2 | Bin 0 -> 85272 bytes
.../media/2bbe8d2671613f1f-s.76dcb0b2.woff2 | Bin 0 -> 10280 bytes
.../media/2c55a0e60120577a-s.2a48534a.woff2 | Bin 0 -> 25844 bytes
.../media/5476f68d60460930-s.c995e352.woff2 | Bin 0 -> 19044 bytes
.../media/83afe278b6a6bb3c-s.p.3a6ba036.woff2 | Bin 0 -> 48432 bytes
.../media/9c72aa0f40e4eef8-s.18a48cbc.woff2 | Bin 0 -> 18744 bytes
.../media/ad66f9afd8947f86-s.7a40eb73.woff2 | Bin 0 -> 11272 bytes
.../_next/static/media/favicon.1d32c690.ico | Bin 0 -> 6387 bytes
.../zHD7JXLXiWgn1NPp82VmF/_buildManifest.js | 1 -
.../proxy/_experimental/out/_not-found.txt | 16 +
.../out/_not-found/__next._full.txt | 16 +
.../out/_not-found/__next._head.txt | 6 +
.../out/_not-found/__next._index.txt | 7 +
.../_not-found/__next._not-found.__PAGE__.txt | 5 +
.../out/_not-found/__next._not-found.txt | 4 +
.../out/_not-found/__next._tree.txt | 3 +
.../_experimental/out/_not-found/index.html | 1 +
.../proxy/_experimental/out/api-reference.txt | 40 +-
...KGRhc2hib2FyZCk.api-reference.__PAGE__.txt | 9 +
.../__next.!KGRhc2hib2FyZCk.api-reference.txt | 4 +
.../api-reference/__next.!KGRhc2hib2FyZCk.txt | 7 +
.../out/api-reference/__next._full.txt | 27 +
.../out/api-reference/__next._head.txt | 6 +
.../out/api-reference/__next._index.txt | 7 +
.../out/api-reference/__next._tree.txt | 4 +
.../out/api-reference/index.html | 2 +-
.../out/assets/logos/s3_vector.png | Bin 0 -> 191076 bytes
.../out/assets/logos/zscaler.svg | 5 +
.../out/experimental/api-playground.txt | 42 +-
...k.experimental.api-playground.__PAGE__.txt | 9 +
...2hib2FyZCk.experimental.api-playground.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.experimental.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.txt | 7 +
.../api-playground/__next._full.txt | 29 +
.../api-playground/__next._head.txt | 6 +
.../api-playground/__next._index.txt | 7 +
.../api-playground/__next._tree.txt | 4 +
.../experimental/api-playground/index.html | 2 +-
.../out/experimental/budgets.txt | 42 +-
...ib2FyZCk.experimental.budgets.__PAGE__.txt | 9 +
....!KGRhc2hib2FyZCk.experimental.budgets.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.experimental.txt | 4 +
.../budgets/__next.!KGRhc2hib2FyZCk.txt | 7 +
.../out/experimental/budgets/__next._full.txt | 29 +
.../out/experimental/budgets/__next._head.txt | 6 +
.../experimental/budgets/__next._index.txt | 7 +
.../out/experimental/budgets/__next._tree.txt | 4 +
.../out/experimental/budgets/index.html | 2 +-
.../out/experimental/caching.txt | 42 +-
...ib2FyZCk.experimental.caching.__PAGE__.txt | 9 +
....!KGRhc2hib2FyZCk.experimental.caching.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.experimental.txt | 4 +
.../caching/__next.!KGRhc2hib2FyZCk.txt | 7 +
.../out/experimental/caching/__next._full.txt | 29 +
.../out/experimental/caching/__next._head.txt | 6 +
.../experimental/caching/__next._index.txt | 7 +
.../out/experimental/caching/__next._tree.txt | 4 +
.../out/experimental/caching/index.html | 2 +-
.../out/experimental/claude-code-plugins.txt | 29 +
...erimental.claude-code-plugins.__PAGE__.txt | 9 +
...FyZCk.experimental.claude-code-plugins.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.experimental.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.txt | 7 +
.../claude-code-plugins/__next._full.txt | 29 +
.../claude-code-plugins/__next._head.txt | 6 +
.../claude-code-plugins/__next._index.txt | 7 +
.../claude-code-plugins/__next._tree.txt | 4 +
.../claude-code-plugins/index.html | 1 +
.../out/experimental/old-usage.txt | 42 +-
...2FyZCk.experimental.old-usage.__PAGE__.txt | 9 +
...KGRhc2hib2FyZCk.experimental.old-usage.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.experimental.txt | 4 +
.../old-usage/__next.!KGRhc2hib2FyZCk.txt | 7 +
.../experimental/old-usage/__next._full.txt | 29 +
.../experimental/old-usage/__next._head.txt | 6 +
.../experimental/old-usage/__next._index.txt | 7 +
.../experimental/old-usage/__next._tree.txt | 4 +
.../out/experimental/old-usage/index.html | 2 +-
.../out/experimental/prompts.txt | 42 +-
...ib2FyZCk.experimental.prompts.__PAGE__.txt | 9 +
....!KGRhc2hib2FyZCk.experimental.prompts.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.experimental.txt | 4 +
.../prompts/__next.!KGRhc2hib2FyZCk.txt | 7 +
.../out/experimental/prompts/__next._full.txt | 29 +
.../out/experimental/prompts/__next._head.txt | 6 +
.../experimental/prompts/__next._index.txt | 7 +
.../out/experimental/prompts/__next._tree.txt | 4 +
.../out/experimental/prompts/index.html | 2 +-
.../out/experimental/tag-management.txt | 42 +-
...k.experimental.tag-management.__PAGE__.txt | 9 +
...2hib2FyZCk.experimental.tag-management.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.experimental.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.txt | 7 +
.../tag-management/__next._full.txt | 29 +
.../tag-management/__next._head.txt | 6 +
.../tag-management/__next._index.txt | 7 +
.../tag-management/__next._tree.txt | 4 +
.../experimental/tag-management/index.html | 2 +-
.../proxy/_experimental/out/guardrails.txt | 40 +-
...t.!KGRhc2hib2FyZCk.guardrails.__PAGE__.txt | 9 +
.../__next.!KGRhc2hib2FyZCk.guardrails.txt | 4 +
.../guardrails/__next.!KGRhc2hib2FyZCk.txt | 7 +
.../out/guardrails/__next._full.txt | 27 +
.../out/guardrails/__next._head.txt | 6 +
.../out/guardrails/__next._index.txt | 7 +
.../out/guardrails/__next._tree.txt | 4 +
.../_experimental/out/guardrails/index.html | 1 +
litellm/proxy/_experimental/out/index.html | 2 +-
litellm/proxy/_experimental/out/index.txt | 69 +-
litellm/proxy/_experimental/out/login.txt | 28 +-
.../_experimental/out/login/__next._full.txt | 21 +
.../_experimental/out/login/__next._head.txt | 6 +
.../_experimental/out/login/__next._index.txt | 7 +
.../_experimental/out/login/__next._tree.txt | 4 +
.../out/login/__next.login.__PAGE__.txt | 9 +
.../_experimental/out/login/__next.login.txt | 4 +
.../proxy/_experimental/out/login/index.html | 2 +-
litellm/proxy/_experimental/out/logs.txt | 41 +-
.../__next.!KGRhc2hib2FyZCk.logs.__PAGE__.txt | 10 +
.../out/logs/__next.!KGRhc2hib2FyZCk.logs.txt | 4 +
.../out/logs/__next.!KGRhc2hib2FyZCk.txt | 7 +
.../_experimental/out/logs/__next._full.txt | 28 +
.../_experimental/out/logs/__next._head.txt | 6 +
.../_experimental/out/logs/__next._index.txt | 7 +
.../_experimental/out/logs/__next._tree.txt | 5 +
.../proxy/_experimental/out/logs/index.html | 2 +-
.../_experimental/out/mcp/oauth/callback.txt | 28 +-
.../out/mcp/oauth/callback/__next._full.txt | 21 +
.../out/mcp/oauth/callback/__next._head.txt | 6 +
.../out/mcp/oauth/callback/__next._index.txt | 7 +
.../out/mcp/oauth/callback/__next._tree.txt | 4 +
.../__next.mcp.oauth.callback.__PAGE__.txt | 9 +
.../callback/__next.mcp.oauth.callback.txt | 4 +
.../mcp/oauth/callback/__next.mcp.oauth.txt | 4 +
.../out/mcp/oauth/callback/__next.mcp.txt | 4 +
.../out/mcp/oauth/callback/index.html | 2 +-
litellm/proxy/_experimental/out/model-hub.txt | 40 +-
...xt.!KGRhc2hib2FyZCk.model-hub.__PAGE__.txt | 9 +
.../__next.!KGRhc2hib2FyZCk.model-hub.txt | 4 +
.../out/model-hub/__next.!KGRhc2hib2FyZCk.txt | 7 +
.../out/model-hub/__next._full.txt | 27 +
.../out/model-hub/__next._head.txt | 6 +
.../out/model-hub/__next._index.txt | 7 +
.../out/model-hub/__next._tree.txt | 4 +
.../_experimental/out/model-hub/index.html | 2 +-
litellm/proxy/_experimental/out/model_hub.txt | 30 +-
.../out/model_hub/__next._full.txt | 23 +
.../out/model_hub/__next._head.txt | 6 +
.../out/model_hub/__next._index.txt | 7 +
.../out/model_hub/__next._tree.txt | 4 +
.../model_hub/__next.model_hub.__PAGE__.txt | 9 +
.../out/model_hub/__next.model_hub.txt | 4 +
.../_experimental/out/model_hub/index.html | 1 +
.../_experimental/out/model_hub_table.txt | 35 +-
.../out/model_hub_table/__next._full.txt | 28 +
.../out/model_hub_table/__next._head.txt | 6 +
.../out/model_hub_table/__next._index.txt | 7 +
.../out/model_hub_table/__next._tree.txt | 4 +
.../__next.model_hub_table.__PAGE__.txt | 9 +
.../__next.model_hub_table.txt | 4 +
.../out/model_hub_table/index.html | 2 +-
.../out/models-and-endpoints.txt | 40 +-
...ib2FyZCk.models-and-endpoints.__PAGE__.txt | 9 +
....!KGRhc2hib2FyZCk.models-and-endpoints.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.txt | 7 +
.../out/models-and-endpoints/__next._full.txt | 27 +
.../out/models-and-endpoints/__next._head.txt | 6 +
.../models-and-endpoints/__next._index.txt | 7 +
.../out/models-and-endpoints/__next._tree.txt | 4 +
.../out/models-and-endpoints/index.html | 2 +-
.../proxy/_experimental/out/onboarding.txt | 28 +-
.../out/onboarding/__next._full.txt | 21 +
.../out/onboarding/__next._head.txt | 6 +
.../out/onboarding/__next._index.txt | 7 +
.../out/onboarding/__next._tree.txt | 4 +
.../onboarding/__next.onboarding.__PAGE__.txt | 9 +
.../out/onboarding/__next.onboarding.txt | 4 +
.../_experimental/out/onboarding/index.html | 1 +
.../proxy/_experimental/out/organizations.txt | 40 +-
...KGRhc2hib2FyZCk.organizations.__PAGE__.txt | 9 +
.../__next.!KGRhc2hib2FyZCk.organizations.txt | 4 +
.../organizations/__next.!KGRhc2hib2FyZCk.txt | 7 +
.../out/organizations/__next._full.txt | 27 +
.../out/organizations/__next._head.txt | 6 +
.../out/organizations/__next._index.txt | 7 +
.../out/organizations/__next._tree.txt | 4 +
.../out/organizations/index.html | 2 +-
.../proxy/_experimental/out/playground.txt | 40 +-
...t.!KGRhc2hib2FyZCk.playground.__PAGE__.txt | 9 +
.../__next.!KGRhc2hib2FyZCk.playground.txt | 4 +
.../playground/__next.!KGRhc2hib2FyZCk.txt | 7 +
.../out/playground/__next._full.txt | 27 +
.../out/playground/__next._head.txt | 6 +
.../out/playground/__next._index.txt | 7 +
.../out/playground/__next._tree.txt | 4 +
.../_experimental/out/playground/index.html | 2 +-
litellm/proxy/_experimental/out/policies.txt | 27 +
...ext.!KGRhc2hib2FyZCk.policies.__PAGE__.txt | 9 +
.../__next.!KGRhc2hib2FyZCk.policies.txt | 4 +
.../out/policies/__next.!KGRhc2hib2FyZCk.txt | 7 +
.../out/policies/__next._full.txt | 27 +
.../out/policies/__next._head.txt | 6 +
.../out/policies/__next._index.txt | 7 +
.../out/policies/__next._tree.txt | 4 +
.../_experimental/out/policies/index.html | 1 +
.../out/settings/admin-settings.txt | 42 +-
...FyZCk.settings.admin-settings.__PAGE__.txt | 9 +
...GRhc2hib2FyZCk.settings.admin-settings.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.settings.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.txt | 7 +
.../settings/admin-settings/__next._full.txt | 29 +
.../settings/admin-settings/__next._head.txt | 6 +
.../settings/admin-settings/__next._index.txt | 7 +
.../settings/admin-settings/__next._tree.txt | 4 +
.../out/settings/admin-settings/index.html | 2 +-
.../out/settings/logging-and-alerts.txt | 42 +-
...k.settings.logging-and-alerts.__PAGE__.txt | 9 +
...2hib2FyZCk.settings.logging-and-alerts.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.settings.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.txt | 7 +
.../logging-and-alerts/__next._full.txt | 29 +
.../logging-and-alerts/__next._head.txt | 6 +
.../logging-and-alerts/__next._index.txt | 7 +
.../logging-and-alerts/__next._tree.txt | 4 +
.../settings/logging-and-alerts/index.html | 2 +-
.../out/settings/router-settings.txt | 42 +-
...yZCk.settings.router-settings.__PAGE__.txt | 9 +
...Rhc2hib2FyZCk.settings.router-settings.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.settings.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.txt | 7 +
.../settings/router-settings/__next._full.txt | 29 +
.../settings/router-settings/__next._head.txt | 6 +
.../router-settings/__next._index.txt | 7 +
.../settings/router-settings/__next._tree.txt | 4 +
.../out/settings/router-settings/index.html | 2 +-
.../_experimental/out/settings/ui-theme.txt | 41 +-
.../__next.!KGRhc2hib2FyZCk.settings.txt | 4 +
...c2hib2FyZCk.settings.ui-theme.__PAGE__.txt | 9 +
...ext.!KGRhc2hib2FyZCk.settings.ui-theme.txt | 4 +
.../ui-theme/__next.!KGRhc2hib2FyZCk.txt | 7 +
.../out/settings/ui-theme/__next._full.txt | 28 +
.../out/settings/ui-theme/__next._head.txt | 6 +
.../out/settings/ui-theme/__next._index.txt | 7 +
.../out/settings/ui-theme/__next._tree.txt | 4 +
.../out/settings/ui-theme/index.html | 2 +-
litellm/proxy/_experimental/out/teams.txt | 40 +-
...__next.!KGRhc2hib2FyZCk.teams.__PAGE__.txt | 9 +
.../teams/__next.!KGRhc2hib2FyZCk.teams.txt | 4 +
.../out/teams/__next.!KGRhc2hib2FyZCk.txt | 7 +
.../_experimental/out/teams/__next._full.txt | 27 +
.../_experimental/out/teams/__next._head.txt | 6 +
.../_experimental/out/teams/__next._index.txt | 7 +
.../_experimental/out/teams/__next._tree.txt | 4 +
.../proxy/_experimental/out/teams/index.html | 2 +-
litellm/proxy/_experimental/out/test-key.txt | 40 +-
...ext.!KGRhc2hib2FyZCk.test-key.__PAGE__.txt | 9 +
.../__next.!KGRhc2hib2FyZCk.test-key.txt | 4 +
.../out/test-key/__next.!KGRhc2hib2FyZCk.txt | 7 +
.../out/test-key/__next._full.txt | 27 +
.../out/test-key/__next._head.txt | 6 +
.../out/test-key/__next._index.txt | 7 +
.../out/test-key/__next._tree.txt | 4 +
.../_experimental/out/test-key/index.html | 2 +-
.../_experimental/out/tools/mcp-servers.txt | 41 +-
...c2hib2FyZCk.tools.mcp-servers.__PAGE__.txt | 9 +
...ext.!KGRhc2hib2FyZCk.tools.mcp-servers.txt | 4 +
.../__next.!KGRhc2hib2FyZCk.tools.txt | 4 +
.../mcp-servers/__next.!KGRhc2hib2FyZCk.txt | 7 +
.../out/tools/mcp-servers/__next._full.txt | 28 +
.../out/tools/mcp-servers/__next._head.txt | 6 +
.../out/tools/mcp-servers/__next._index.txt | 7 +
.../out/tools/mcp-servers/__next._tree.txt | 4 +
.../out/tools/mcp-servers/index.html | 2 +-
.../_experimental/out/tools/vector-stores.txt | 42 +-
.../__next.!KGRhc2hib2FyZCk.tools.txt | 4 +
...hib2FyZCk.tools.vector-stores.__PAGE__.txt | 9 +
...t.!KGRhc2hib2FyZCk.tools.vector-stores.txt | 4 +
.../vector-stores/__next.!KGRhc2hib2FyZCk.txt | 7 +
.../out/tools/vector-stores/__next._full.txt | 29 +
.../out/tools/vector-stores/__next._head.txt | 6 +
.../out/tools/vector-stores/__next._index.txt | 7 +
.../out/tools/vector-stores/__next._tree.txt | 4 +
.../out/tools/vector-stores/index.html | 2 +-
litellm/proxy/_experimental/out/usage.txt | 40 +-
.../out/usage/__next.!KGRhc2hib2FyZCk.txt | 7 +
...__next.!KGRhc2hib2FyZCk.usage.__PAGE__.txt | 9 +
.../usage/__next.!KGRhc2hib2FyZCk.usage.txt | 4 +
.../_experimental/out/usage/__next._full.txt | 27 +
.../_experimental/out/usage/__next._head.txt | 6 +
.../_experimental/out/usage/__next._index.txt | 7 +
.../_experimental/out/usage/__next._tree.txt | 4 +
.../proxy/_experimental/out/usage/index.html | 2 +-
litellm/proxy/_experimental/out/users.txt | 40 +-
.../out/users/__next.!KGRhc2hib2FyZCk.txt | 7 +
...__next.!KGRhc2hib2FyZCk.users.__PAGE__.txt | 9 +
.../users/__next.!KGRhc2hib2FyZCk.users.txt | 4 +
.../_experimental/out/users/__next._full.txt | 27 +
.../_experimental/out/users/__next._head.txt | 6 +
.../_experimental/out/users/__next._index.txt | 7 +
.../_experimental/out/users/__next._tree.txt | 4 +
.../proxy/_experimental/out/users/index.html | 2 +-
.../proxy/_experimental/out/virtual-keys.txt | 40 +-
.../virtual-keys/__next.!KGRhc2hib2FyZCk.txt | 7 +
...!KGRhc2hib2FyZCk.virtual-keys.__PAGE__.txt | 9 +
.../__next.!KGRhc2hib2FyZCk.virtual-keys.txt | 4 +
.../out/virtual-keys/__next._full.txt | 27 +
.../out/virtual-keys/__next._head.txt | 6 +
.../out/virtual-keys/__next._index.txt | 7 +
.../out/virtual-keys/__next._tree.txt | 4 +
.../_experimental/out/virtual-keys/index.html | 2 +-
litellm/proxy/_new_secret_config.yaml | 22 +
litellm/proxy/_types.py | 142 +-
.../proxy/agent_endpoints/a2a_endpoints.py | 139 +-
litellm/proxy/agent_endpoints/a2a_routing.py | 53 +
.../auth/agent_permission_handler.py | 235 +-
.../agent_endpoints/model_list_helpers.py | 96 +
.../proxy/anthropic_endpoints/endpoints.py | 17 +
litellm/proxy/auth/auth_checks.py | 645 +-
litellm/proxy/auth/auth_utils.py | 24 +-
litellm/proxy/auth/handle_jwt.py | 16 +-
litellm/proxy/auth/ip_address_utils.py | 155 +
litellm/proxy/auth/login_utils.py | 74 +-
litellm/proxy/auth/model_checks.py | 31 +-
litellm/proxy/auth/route_checks.py | 13 +-
litellm/proxy/auth/user_api_key_auth.py | 249 +-
litellm/proxy/batches_endpoints/endpoints.py | 187 +-
litellm/proxy/cached_logo.jpg | Bin 50535 -> 24694 bytes
litellm/proxy/client/cli/commands/auth.py | 239 +-
litellm/proxy/common_request_processing.py | 303 +-
.../proxy/common_utils/cache_coordinator.py | 192 +
litellm/proxy/common_utils/callback_utils.py | 55 +
.../proxy/common_utils/http_parsing_utils.py | 97 +
.../common_utils/key_rotation_manager.py | 29 +-
.../proxy/common_utils/performance_utils.py | 4 +-
litellm/proxy/compliance_checks.py | 221 +
litellm/proxy/custom_sso.py | 9 +-
litellm/proxy/db/db_spend_update_writer.py | 210 +-
.../db_transaction_queue/base_update_queue.py | 8 +-
.../daily_spend_update_queue.py | 3 +-
.../spend_update_queue.py | 8 +-
.../ui_discovery_endpoints.py | 6 +-
.../example_config_yaml/otel_test_config.yaml | 4 +-
.../pipeline_test_guardrails.py | 69 +
.../spend_tracking_config.yaml | 2 +-
.../test_pipeline_config.yaml | 64 +
litellm/proxy/google_endpoints/endpoints.py | 2 +-
.../proxy/guardrails/guardrail_endpoints.py | 408 +-
.../guardrail_hooks/bedrock_guardrails.py | 81 +-
.../guardrail_hooks/custom_code/__init__.py | 65 +
.../custom_code/custom_code_guardrail.py | 404 +
.../guardrail_hooks/custom_code/primitives.py | 832 +
.../guardrail_hooks/enkryptai/enkryptai.py | 6 +-
.../generic_guardrail_api/__init__.py | 3 +
.../generic_guardrail_api/example_config.yaml | 1 +
.../generic_guardrail_api.py | 245 +-
.../guardrail_hooks/grayswan/grayswan.py | 81 +-
.../hiddenlayer/hiddenlayer.py | 6 +-
.../guardrails/guardrail_hooks/lakera_ai.py | 2 +-
.../guardrail_hooks/lakera_ai_v2.py | 2 +-
.../litellm_content_filter/__init__.py | 2 +
.../categories/bias_racial.yaml | 507 +-
.../categories/harm_toxic_abuse.json | 4724 ++++
.../categories/harm_toxic_abuse_au.json | 345 +
.../categories/harm_toxic_abuse_de.json | 587 +
.../categories/harm_toxic_abuse_es.json | 614 +
.../categories/harm_toxic_abuse_fr.json | 821 +
.../categories/harmful_child_safety.yaml | 156 +
.../prompt_injection_data_exfiltration.yaml | 123 +
.../prompt_injection_jailbreak.yaml | 140 +
.../prompt_injection_malicious_code.yaml | 147 +
.../categories/prompt_injection_sql.yaml | 120 +
.../prompt_injection_system_prompt.yaml | 145 +
.../litellm_content_filter/content_filter.py | 945 +-
.../litellm_content_filter/patterns.json | 138 +-
.../litellm_content_filter/patterns.py | 32 +-
.../airline_brand_protection.yaml | 89 +
.../aviation_safety_topics.yaml | 120 +
.../eu_ai_act_art5_biometric_profiling.yaml | 148 +
...eu_ai_act_art5_biometric_profiling_fr.yaml | 157 +
.../eu_ai_act_art5_emotion_recognition.yaml | 143 +
...eu_ai_act_art5_emotion_recognition_fr.yaml | 132 +
.../eu_ai_act_art5_manipulation.yaml | 95 +
.../eu_ai_act_art5_manipulation_fr.yaml | 99 +
.../eu_ai_act_art5_social_scoring.yaml | 118 +
.../eu_ai_act_art5_social_scoring_fr.yaml | 126 +
.../eu_ai_act_art5_vulnerability.yaml | 112 +
.../eu_ai_act_art5_vulnerability_fr.yaml | 109 +
.../policy_templates/eu_ai_act_article5.yaml | 157 +
.../eu_ai_act_article5_fr.yaml | 242 +
.../uae_anti_discrimination.yaml | 71 +
.../uae_cultural_sensitivity.yaml | 72 +
.../guardrail_hooks/mcp_security/__init__.py | 45 +
.../mcp_security/mcp_security_guardrail.py | 114 +
.../model_armor/model_armor.py | 45 +-
.../guardrails/guardrail_hooks/onyx/onyx.py | 17 +-
.../guardrail_hooks/openai/moderations.py | 360 +-
.../guardrail_hooks/pangea/pangea.py | 11 +-
.../guardrails/guardrail_hooks/presidio.py | 241 +-
.../prompt_security/prompt_security.py | 6 +-
.../guardrail_hooks/qualifire/qualifire.py | 12 +-
.../unified_guardrail/unified_guardrail.py | 128 +-
.../zscaler_ai_guard/__init__.py | 4 +
.../zscaler_ai_guard/zscaler_ai_guard.py | 163 +-
.../proxy/guardrails/guardrail_registry.py | 21 +-
.../shared_health_check_manager.py | 3 +-
.../health_endpoints/_health_endpoints.py | 47 +-
litellm/proxy/hooks/batch_rate_limiter.py | 88 +-
.../proxy/hooks/dynamic_rate_limiter_v3.py | 35 +-
.../proxy/hooks/key_management_event_hooks.py | 34 +-
.../hooks/mcp_semantic_filter/ARCHITECTURE.md | 96 +
.../hooks/mcp_semantic_filter/__init__.py | 9 +
.../proxy/hooks/mcp_semantic_filter/hook.py | 353 +
.../proxy/hooks/model_max_budget_limiter.py | 43 +-
.../proxy/hooks/proxy_track_cost_callback.py | 12 +-
litellm/proxy/image_endpoints/endpoints.py | 6 +
litellm/proxy/litellm_pre_call_utils.py | 391 +-
.../access_group_endpoints.py | 380 +
.../budget_management_endpoints.py | 39 +-
.../common_daily_activity.py | 95 +-
.../management_endpoints/common_utils.py | 171 +-
.../compliance_endpoints.py | 79 +
.../internal_user_endpoints.py | 71 +-
.../key_management_endpoints.py | 983 +-
.../mcp_management_endpoints.py | 259 +-
.../management_endpoints/policy_endpoints.py | 642 +
.../management_endpoints/scim/scim_v2.py | 302 +
.../management_endpoints/team_endpoints.py | 202 +-
litellm/proxy/management_endpoints/types.py | 16 +-
litellm/proxy/management_endpoints/ui_sso.py | 414 +-
litellm/proxy/mcp_registry.json | 426 +
.../middleware/prometheus_auth_middleware.py | 33 +-
.../proxy/openai_evals_endpoints/__init__.py | 7 +
.../proxy/openai_evals_endpoints/endpoints.py | 1067 +
.../openai_files_endpoints/common_utils.py | 149 +
.../openai_files_endpoints/files_endpoints.py | 69 +-
.../pass_through_endpoints/architecture.md | 71 +
.../llm_passthrough_endpoints.py | 203 +-
.../pass_through_endpoints.py | 281 +-
litellm/proxy/policy_engine/__init__.py | 60 +
litellm/proxy/policy_engine/architecture.md | 54 +
.../policy_engine/attachment_registry.py | 498 +
.../policy_engine/condition_evaluator.py | 111 +
litellm/proxy/policy_engine/init_policies.py | 276 +
.../proxy/policy_engine/pipeline_executor.py | 216 +
.../proxy/policy_engine/policy_endpoints.py | 640 +
litellm/proxy/policy_engine/policy_matcher.py | 181 +
.../proxy/policy_engine/policy_registry.py | 600 +
.../policy_engine/policy_resolve_endpoints.py | 405 +
.../proxy/policy_engine/policy_resolver.py | 289 +
.../proxy/policy_engine/policy_validator.py | 401 +
litellm/proxy/proxy_cli.py | 14 +-
litellm/proxy/proxy_config.yaml | 85 +-
litellm/proxy/proxy_server.py | 2362 +-
litellm/proxy/rag_endpoints/endpoints.py | 225 +-
.../proxy/response_api_endpoints/endpoints.py | 2 +
.../proxy/response_polling/polling_handler.py | 12 +-
litellm/proxy/route_llm_request.py | 137 +-
litellm/proxy/schema.prisma | 128 +-
litellm/proxy/search_endpoints/endpoints.py | 82 +-
.../search_tool_management.py | 77 +-
.../spend_management_endpoints.py | 291 +-
.../spend_tracking/spend_tracking_utils.py | 94 +-
.../proxy_setting_endpoints.py | 124 +-
litellm/proxy/utils.py | 399 +-
.../proxy/vector_store_endpoints/endpoints.py | 111 +-
.../management_endpoints.py | 418 +-
.../vector_store_files_endpoints/endpoints.py | 323 +-
litellm/proxy_auth/__init__.py | 30 +
litellm/proxy_auth/credentials.py | 240 +
litellm/rag/ingestion/__init__.py | 6 +
litellm/rag/ingestion/base_ingestion.py | 17 +-
.../rag/ingestion/file_parsers/__init__.py | 9 +
.../rag/ingestion/file_parsers/pdf_parser.py | 70 +
litellm/rag/ingestion/s3_vectors_ingestion.py | 573 +
litellm/rag/ingestion/vertex_ai_ingestion.py | 478 +
litellm/rag/main.py | 29 +-
litellm/realtime_api/main.py | 71 +-
litellm/rerank_api/main.py | 29 +-
.../handler.py | 2 +
.../streaming_iterator.py | 37 +-
.../transformation.py | 140 +-
litellm/responses/main.py | 47 +-
.../responses/mcp/chat_completions_handler.py | 380 +-
.../mcp/litellm_proxy_mcp_handler.py | 46 +-
litellm/responses/streaming_iterator.py | 6 +
litellm/responses/utils.py | 17 +-
litellm/router.py | 671 +-
litellm/router_strategy/budget_limiter.py | 97 +-
litellm/router_strategy/lowest_tpm_rpm_v2.py | 9 +-
litellm/router_utils/common_utils.py | 5 +-
litellm/router_utils/get_retry_from_policy.py | 10 +-
.../deployment_affinity_check.py | 396 +
.../pre_call_checks/model_rate_limit_check.py | 373 +
.../prompt_caching_deployment_check.py | 3 +-
.../responses_api_deployment_check.py | 12 +
litellm/router_utils/search_api_router.py | 3 +-
litellm/scheduler.py | 11 +
.../secret_managers/aws_secret_manager_v2.py | 161 +-
.../hashicorp_secret_manager.py | 126 +-
litellm/secret_managers/main.py | 20 +-
litellm/types/access_group.py | 39 +
litellm/types/guardrails.py | 85 +-
litellm/types/integrations/cloudzero.py | 16 +-
litellm/types/integrations/prometheus.py | 134 +-
litellm/types/llms/anthropic.py | 15 +-
litellm/types/llms/bedrock.py | 31 +
litellm/types/llms/custom_http.py | 1 +
litellm/types/llms/oci.py | 67 +-
litellm/types/llms/openai.py | 59 +-
litellm/types/llms/openai_evals.py | 417 +
litellm/types/llms/watsonx.py | 1 +
litellm/types/llms/xai.py | 23 +
.../types/mcp_server/mcp_server_manager.py | 12 +
litellm/types/policy_engine.py | 36 +
litellm/types/proxy/compliance_endpoints.py | 33 +
.../ui_discovery_endpoints.py | 1 +
.../guardrail_hooks/generic_guardrail_api.py | 37 +-
.../proxy/guardrails/guardrail_hooks/onyx.py | 5 +
.../guardrail_hooks/zscaler_ai_guard.py | 133 +
.../key_management_endpoints.py | 42 +
.../proxy/management_endpoints/scim_v2.py | 66 +-
.../proxy/management_endpoints/ui_sso.py | 20 +
litellm/types/proxy/policy_engine/__init__.py | 101 +
.../proxy/policy_engine/pipeline_types.py | 98 +
.../types/proxy/policy_engine/policy_types.py | 324 +
.../proxy/policy_engine/resolver_types.py | 386 +
.../proxy/policy_engine/validation_types.py | 80 +
litellm/types/rag.py | 46 +-
litellm/types/router.py | 43 +-
litellm/types/search.py | 1 +
litellm/types/utils.py | 101 +-
litellm/types/vector_stores.py | 4 +
litellm/utils.py | 268 +-
model_prices_and_context_window.json | 4054 +++-
package.json | 4 +-
poetry.lock | 1450 +-
policy_templates.json | 1883 ++
provider_endpoints_support.json | 105 +-
proxy_config.yaml | 7 -
proxy_server_config.yaml | 2 +-
pyproject.toml | 23 +-
requirements.txt | 18 +-
schema.prisma | 113 +-
.../local_vertex_agent.py | 0
.../{ => local_only_agent_tests}/test_a2a.py | 0
.../test_a2a_completion_bridge.py | 0
tests/agent_tests/test_a2a_agent.py | 83 +
.../test_batch_custom_pricing.py | 131 +
tests/batches_tests/test_batch_rate_limits.py | 669 +
.../test_batches_logging_unit_tests.py | 339 +
.../check_guardrail_apply_decorator.py | 132 +
tests/code_coverage_tests/check_licenses.py | 11 +-
.../enforce_llms_folder_style.py | 5 +-
tests/code_coverage_tests/liccheck.ini | 3 +
tests/code_coverage_tests/license_cache.json | 2 +-
.../code_coverage_tests/recursive_detector.py | 5 +
.../router_code_coverage.py | 7 +-
.../test_prometheus_logging_callbacks.py | 198 +-
.../integrations/test_prometheus.py | 139 +-
.../proxy/auth/test_route_checks.py | 135 +
.../test_bedrock_apply_guardrail.py | 77 +-
.../proxy/hooks/test_managed_files.py | 1230 +-
.../test_eu_ai_act_article5.py | 257 +
.../test_eu_ai_act_french_3_scenarios.py | 292 +
tests/guardrails_tests/test_lakera_v2.py | 74 +-
.../guardrails_tests/test_zscaler_ai_guard.py | 226 +-
.../image_gen_tests/test_image_generation.py | 84 +-
.../test_litellm_proxy_extras_utils.py | 17 +-
.../test_anthropic_reasoning_effort.py | 64 +
.../anthropic/test_anthropic_schema_filter.py | 155 +
.../llms/bedrock/test_nova_imported_models.py | 92 +
.../oci/chat/test_oci_chat_transformation.py | 37 +-
.../mcp_server/test_discoverable_endpoints.py | 1108 +
.../management_endpoints/test_common_utils.py | 159 +
.../proxy/test_init_litellm_callbacks.py | 175 +
...t_batch_completion_models_all_responses.py | 118 +
.../test_bedrock_extended_beta_models.py | 170 +
tests/litellm/test_proxy_auth.py | 204 +
.../test_router_retry_backoff_headers.py | 88 +
.../test_stream_chunk_builder_images.py | 242 +
.../test_bedrock_converse_dedup_factory.py | 447 +
tests/litellm_utils_tests/test_hashicorp.py | 380 +
tests/litellm_utils_tests/test_utils.py | 1 +
.../base_responses_api.py | 144 +-
.../test_azure_responses_api.py | 4 +
...t_base_responses_api_streaming_iterator.py | 120 +
.../test_openai_responses_api.py | 3 +
tests/llm_translation/realtime/__init__.py | 0
.../realtime/base_realtime_tests.py | 426 +
.../{ => realtime}/test_openai_realtime.py | 0
.../realtime/test_openai_realtime_simple.py | 29 +
.../realtime/test_xai_realtime.py | 34 +
tests/llm_translation/test_a2a.py | 132 +
.../test_anthropic_completion.py | 4 +-
tests/llm_translation/test_azure_ai.py | 28 +-
tests/llm_translation/test_azure_openai.py | 15 +
.../test_bedrock_anthropic_regression.py | 526 +
.../test_bedrock_completion.py | 14 +-
.../test_bedrock_invoke_tests.py | 4 +-
tests/llm_translation/test_evals_api.py | 237 +
tests/llm_translation/test_gemini.py | 17 +
tests/llm_translation/test_gigachat.py | 288 +-
tests/llm_translation/test_groq.py | 126 +-
.../test_hosted_vllm_embedding_e2e.py | 152 +
.../test_convert_dict_to_chat_completion.py | 187 +
.../test_model_cost_map_resilience.py | 291 +
tests/llm_translation/test_openai.py | 4 +-
tests/llm_translation/test_optional_params.py | 90 +-
tests/llm_translation/test_xai.py | 26 +
tests/local_testing/test_add_update_models.py | 2 +
.../test_amazing_vertex_completion.py | 184 +-
tests/local_testing/test_auth_utils.py | 19 +
...test_batch_completion_return_exceptions.py | 12 +-
tests/local_testing/test_completion.py | 21 +-
.../test_completion_with_retries.py | 1 +
tests/local_testing/test_embedding.py | 4 +-
tests/local_testing/test_get_model_info.py | 10 +-
tests/local_testing/test_literalai.py | 72 -
.../test_openai_moderations_hook.py | 70 +-
.../test_router_max_parallel_requests.py | 4 +-
tests/local_testing/test_router_utils.py | 4 +-
tests/local_testing/test_scheduler.py | 38 +
tests/local_testing/test_streaming.py | 6 +-
tests/local_testing/test_timeout.py | 91 -
.../test_dynamic_otel_keys.py | 52 +
.../test_logging_redaction_e2e_test.py | 9 +-
.../test_otel_logging.py | 72 +-
tests/logging_callback_tests/test_posthog.py | 143 +
.../test_standard_logging_payload.py | 16 +
...tandard_logging_payload_excluded_fields.py | 415 +
.../test_unit_tests_init_callbacks.py | 26 +
.../mcp_tests/test_aresponses_api_with_mcp.py | 6 +
tests/mcp_tests/test_mcp_auth_priority.py | 7 +-
tests/mcp_tests/test_mcp_chat_completions.py | 725 +-
tests/mcp_tests/test_mcp_client_unit.py | 25 +-
tests/mcp_tests/test_mcp_logging.py | 3 +-
tests/mcp_tests/test_mcp_server.py | 219 +-
tests/mcp_tests/test_oauth2_mcp_config.yaml | 14 +
tests/mcp_tests/test_openapi_spec_path_url.py | 92 +
.../test_semantic_tool_filter_e2e.py | 93 +
.../test_openai_batches_endpoint.py | 316 +-
tests/otel_tests/test_prometheus.py | 134 +-
.../base_anthropic_unified_messages_test.py | 61 +
.../test_anthropic_messages_passthrough.py | 141 +-
.../test_anthropic_messages_tool_search.py | 20 +-
.../test_bedrock_tool_use_beta_header.py | 44 +-
.../test_passthrough_registry_updates.py | 145 +
tests/proxy_admin_ui_tests/package.json | 4 +-
.../test_key_management.py | 56 +-
.../ui_unit_tests/log_filter_logic.test.tsx | 87 -
.../ui_unit_tests/package.json | 4 +-
.../test_all_beta_headers.py | 169 +
.../test_claude_agent_sdk.py | 124 +
.../test_config.yaml | 53 +
tests/proxy_unit_tests/test_auth_checks.py | 308 +-
tests/proxy_unit_tests/test_get_image.py | 89 +
tests/proxy_unit_tests/test_jwt.py | 11 +-
.../test_prisma_client_backoff_retry.py | 14 +
.../test_proxy_custom_logger.py | 119 +-
.../test_proxy_pass_user_config.py | 4 +
tests/proxy_unit_tests/test_proxy_routes.py | 5 +
tests/proxy_unit_tests/test_proxy_server.py | 18 +-
.../test_proxy_setting_guardrails.py | 3 +-
tests/proxy_unit_tests/test_proxy_utils.py | 231 +-
.../test_response_polling_handler.py | 142 +-
.../proxy_unit_tests/test_server_root_path.py | 64 +
.../test_ui_path_detection.py | 157 +
...test_unit_test_max_model_budget_limiter.py | 63 +-
.../test_user_api_key_auth.py | 7 +-
.../test_zero_cost_model_budget_bypass.py | 590 +
.../test_get_model_list_alias_optimization.py | 50 +
.../test_router_acancel_batch.py | 53 +
.../test_router_helper_utils.py | 29 +
tests/search_tests/test_duckduckgo_search.py | 356 +
.../a2a_protocol/test_card_resolver.py | 90 +
tests/test_litellm/caching/test_dual_cache.py | 58 +
...responses_transformation_transformation.py | 256 +-
...responses_transformation_transformation.py | 102 +
tests/test_litellm/conftest.py | 210 +-
.../containers/test_container_api.py | 131 +-
.../containers/test_container_integration.py | 27 +-
.../send_emails/test_resend_email.py | 128 +-
.../send_emails/test_sendgrid_email.py | 88 +-
.../test_afile_retrieve_returns_unified_id.py | 67 +
.../test_batch_retrieve_input_file_id.py | 75 +
..._retrieve_returns_unified_input_file_id.py | 124 +
.../test_deleted_file_returns_403_not_404.py | 119 +
.../proxy/test_file_deletion_blocking.py | 525 +
.../proxy/test_managed_files_access_check.py | 200 +
.../proxy/test_managed_files_hook.py | 167 +
.../context_management_and_shell.json | 20 +
.../test_mcp_client.py | 72 +-
.../google_genai/test_google_genai_adapter.py | 4 +-
.../google_genai/test_google_genai_handler.py | 19 +-
.../arize/test_arize_otel_coexistence.py | 169 +
.../integrations/arize/test_arize_phoenix.py | 61 +-
.../integrations/arize/test_arize_utils.py | 21 +-
.../datadog/test_datadog_tags_regression.py | 91 +
.../test_langfuse_prompt_management.py | 16 +-
.../integrations/test_custom_guardrail.py | 97 +
.../integrations/test_langfuse.py | 66 +-
.../integrations/test_langfuse_otel.py | 461 +-
.../integrations/test_opentelemetry.py | 20 +
.../test_prometheus_client_ip_user_agent.py | 235 +
.../integrations/test_prometheus_labels.py | 249 +-
.../test_prometheus_missing_metrics.py | 77 +
.../test_prometheus_none_metadata.py | 176 +
.../integrations/test_prometheus_services.py | 58 +
.../test_responses_background_cost.py | 15 +
tests/test_litellm/integrations/test_s3_v2.py | 180 +
.../test_websearch_chat_completion.py | 398 +
.../llm_cost_calc/test_llm_cost_calc_utils.py | 6 +-
.../test_tool_call_cost_tracking.py | 53 +
...ore_utils_prompt_templates_common_utils.py | 56 +
...llm_core_utils_prompt_templates_factory.py | 151 +
.../test_exception_mapping_utils.py | 9 +
.../test_get_litellm_params.py | 127 +
.../litellm_core_utils/test_image_handling.py | 79 +
.../test_litellm_logging.py | 295 +
.../test_redact_messages.py | 145 +
.../test_safe_json_dumps.py | 32 +
.../test_streaming_chunk_builder_utils.py | 72 +-
.../litellm_core_utils/test_token_counter.py | 35 +-
.../test_anthropic_guardrail_handler.py | 236 +
.../test_anthropic_chat_transformation.py | 788 +-
...al_pass_through_adapters_transformation.py | 871 +-
...erimental_pass_through_messages_handler.py | 83 +-
...t_anthropic_messages_structured_outputs.py | 2 +-
.../anthropic/test_anthropic_common_utils.py | 452 +-
.../test_anthropic_structured_output.py | 43 +-
.../anthropic/test_azure_ai_cache_pricing.py | 42 +
.../anthropic/test_message_sanitization.py | 380 +
.../test_azure_chat_gpt_transformation.py | 13 +
.../chat/test_azure_gpt5_transformation.py | 11 +
.../test_azure_image_generation_init.py | 197 +
.../response/test_azure_transformation.py | 183 +-
.../llms/azure/test_azure_common_utils.py | 3 +-
.../azure/test_azure_exception_mapping.py | 147 +-
.../test_azure_tts_transformation.py | 12 +-
.../chat/test_azure_ai_transformation.py | 52 +
...e_anthropic_count_tokens_transformation.py | 111 +
.../test_azure_anthropic_transformation.py | 94 +
.../test_azure_ai_rerank_transformation.py | 100 +
.../azure_ai/test_azure_ai_cost_calculator.py | 346 +
...ations_anthropic_claude3_transformation.py | 143 +
.../chat/test_converse_transformation.py | 536 +-
.../test_converse_transformation_nova_2.py | 613 +-
.../chat/test_streaming_choice_index.py | 114 +
.../expected_bedrock_batch_completions.jsonl | 4 +-
.../test_bedrock_realtime_transformation.py | 646 +
.../bedrock/test_anthropic_beta_support.py | 2 +-
.../llms/bedrock/test_base_aws_llm.py | 342 +-
.../test_chatgpt_responses_transformation.py | 39 +
.../custom_httpx/test_aiohttp_transport.py | 32 +
.../llms/custom_httpx/test_http_handler.py | 277 +-
.../test_databricks_chat_transformation.py | 43 +
.../llms/databricks/responses/__init__.py | 0
...est_databricks_responses_transformation.py | 149 +
.../test_fireworks_ai_chat_transformation.py | 29 +
.../llms/gemini/files/__init__.py | 1 +
.../files/test_gemini_files_transformation.py | 298 +
.../test_gemini_image_edit_transformation.py | 2 +-
.../test_hosted_vllm_chat_transformation.py | 96 +-
.../chat/test_hosted_vllm_ssl_verify.py | 152 +
.../test_hosted_vllm_embedding_ssl_verify.py | 140 +
...st_hosted_vllm_embedding_transformation.py | 293 +
.../test_huggingface_embedding_handler.py | 20 +-
.../test_meta_llama_chat_transformation.py | 82 +-
.../oci/chat/test_oci_chat_transformation.py | 108 +
.../oci/chat/test_oci_cohere_tool_calls.py | 211 +
.../ollama/test_ollama_chat_transformation.py | 130 +-
.../test_openai_guardrail_handler.py | 148 +
.../chat/test_openai_gpt_transformation.py | 176 +-
.../test_embeddings_guardrail_handler.py | 83 +
.../llms/openai/evals/__init__.py | 1 +
.../evals/test_openai_evals_transformation.py | 257 +
.../llms/openai/realtime/README.md | 82 +
.../realtime/test_openai_realtime_handler.py | 8 +-
...test_openai_responses_guardrail_handler.py | 178 +
.../test_openai_responses_transformation.py | 123 +
.../llms/openai/test_gpt5_transformation.py | 17 +
.../llms/openai_like/embedding/__init__.py | 1 +
.../embedding/test_openai_like_embedding.py | 378 +
.../llms/openai_like/test_json_providers.py | 41 +
...est_perplexity_responses_transformation.py | 381 +
.../test_publicai_chat_transformation.py | 16 +-
.../test_litellm/llms/s3_vectors/__init__.py | 1 +
.../llms/s3_vectors/vector_stores/__init__.py | 1 +
.../test_s3_vectors_transformation.py | 115 +
.../test_sagemaker_embedding_voyage.py | 102 +-
.../llms/test_cache_control_and_reasoning.py | 281 +
tests/test_litellm/llms/test_lifecycle_fix.py | 46 +
.../test_vertex_ai_context_caching.py | 353 +
.../test_vertex_ai_gemini_transformation.py | 266 +-
...test_vertex_and_google_ai_studio_gemini.py | 396 +
...rtex_ai_image_generation_transformation.py | 41 +
.../test_vertex_ai_rerank_integration.py | 106 +-
.../test_vertex_ai_rerank_transformation.py | 128 +-
.../vertex_ai/test_vertex_ai_common_utils.py | 30 +
..._vertex_ai_anthropic_image_url_handling.py | 192 +
...artner_models_anthropic_messages_config.py | 117 +
...partner_models_anthropic_transformation.py | 378 +-
.../test_vertex_ai_gpt_oss_transformation.py | 128 +-
...ai_partner_models_llama3_transformation.py | 92 +-
.../vertex_ai_partner_models/qwen/__init__.py | 0
.../test_vertex_ai_qwen_global_endpoint.py | 287 +
...est_volcengine_responses_transformation.py | 5 +-
.../llms/watsonx/rerank/__init__.py | 0
.../watsonx/rerank/test_watsonx_rerank.py | 224 +
.../test_litellm/llms/watsonx/test_watsonx.py | 42 +-
.../llms/watsonx/test_watsonx_common_utils.py | 242 +
.../test_xai_responses_transformation.py | 213 +-
.../auth/test_user_api_key_auth_mcp.py | 357 +-
.../test_mcp_guardrail_handler.py | 54 +-
.../mcp_server/test_discoverable_endpoints.py | 543 +-
.../mcp_server/test_jwt_mcp_enforcement.py | 480 +
.../mcp_server/test_jwt_mcp_simple.py | 277 +
.../mcp_server/test_mcp_debug.py | 252 +
.../mcp_server/test_mcp_discovery.py | 182 +
.../mcp_server/test_mcp_server.py | 85 +-
.../mcp_server/test_mcp_server_manager.py | 151 +-
.../mcp_server/test_mcp_stale_session.py | 440 +
.../mcp_server/test_oauth2_token_cache.py | 157 +
.../mcp_server/test_rest_endpoints.py | 511 +-
.../mcp_server/test_semantic_tool_filter.py | 394 +
.../auth/test_agent_permission_handler.py | 56 +-
.../agent_endpoints/test_a2a_endpoints.py | 4 +-
.../test_model_list_helpers.py | 110 +
.../anthropic_endpoints/test_endpoints.py | 20 +
.../proxy/auth/test_auth_checks.py | 139 +-
.../proxy/auth/test_auth_utils.py | 23 +
.../test_litellm/proxy/auth/test_cli_auth.py | 203 +
.../proxy/auth/test_login_utils.py | 477 +-
.../proxy/auth/test_mcp_ip_filtering.py | 93 +
.../proxy/auth/test_model_checks.py | 24 +
.../auth/test_object_permission_loading.py | 175 +
.../proxy/auth/test_route_checks.py | 255 +-
.../proxy/auth/test_user_api_key_auth.py | 151 +-
.../common_utils/test_http_parsing_utils.py | 64 +
.../common_utils/test_key_rotation_manager.py | 193 +-
.../test_base_update_queue.py | 5 +-
.../test_spend_update_queue.py | 33 +
.../proxy/db/test_db_spend_update_writer.py | 167 +-
.../test_ui_discovery_endpoints.py | 31 +
.../test_interactions_agent_param.py | 75 +
.../content_filter/test_content_filter.py | 1218 +-
.../content_filter/test_eu_patterns.py | 90 +
.../content_filter/test_gdpr_policy_e2e.py | 293 +
.../content_filter/test_patterns.py | 25 +-
.../openai/test_moderations.py | 351 +-
.../test_openai_moderation_streaming.py | 172 +
.../test_generic_guardrail_api.py | 214 +-
.../guardrail_hooks/test_grayswan.py | 131 +-
.../guardrail_hooks/test_mcp_security.py | 184 +
.../guardrail_hooks/test_model_armor.py | 385 +-
.../guardrails/guardrail_hooks/test_onyx.py | 302 +-
.../guardrail_hooks/test_presidio.py | 312 +-
.../test_unified_guardrail.py | 100 +-
.../guardrails/test_guardrail_endpoints.py | 105 +
.../guardrails/test_pillar_guardrails.py | 90 +-
.../health_endpoints/test_health_endpoints.py | 170 +
.../hooks/test_image_generation_guardrails.py | 293 +
.../hooks/test_key_management_event_hooks.py | 249 +
.../test_post_call_response_headers_hook.py | 197 +
...st_post_call_streaming_hook_integration.py | 273 +
...test_post_call_success_hook_integration.py | 260 +
.../hooks/test_proxy_track_cost_callback.py | 74 +
.../proxy/image_endpoints/test_endpoints.py | 4 +
.../scim/test_scim_v2_discovery.py | 300 +
.../test_search_tool_management.py | 546 +
.../test_access_group_endpoints.py | 884 +
.../test_budget_endpoints.py | 55 +-
.../test_common_daily_activity.py | 254 +
.../management_endpoints/test_common_utils.py | 488 +
.../test_compliance_endpoints.py | 387 +
.../test_internal_user_endpoints.py | 234 +-
.../test_key_management_endpoints.py | 1571 +-
.../test_mcp_management_endpoints.py | 513 +-
.../test_policy_endpoints.py | 589 +
.../test_team_endpoints.py | 534 +
.../proxy/management_endpoints/test_ui_sso.py | 1414 +-
.../test_prometheus_auth_middleware_asgi.py | 24 +
.../test_files_endpoint.py | 178 +
.../test_llm_pass_through_endpoints.py | 176 +
.../test_pass_through_endpoints.py | 316 +-
.../test_vertex_passthrough_load_balancing.py | 72 +
.../proxy/policy_engine/__init__.py | 0
.../policy_engine/test_attachment_registry.py | 335 +
.../policy_engine/test_condition_evaluator.py | 113 +
.../policy_engine/test_pipeline_executor.py | 484 +
.../policy_engine/test_policy_matcher.py | 160 +
.../policy_engine/test_policy_resolver.py | 193 +
.../policy_engine/test_policy_validator.py | 85 +
.../proxy/rag_endpoints/__init__.py | 0
.../proxy/rag_endpoints/test_rag_endpoints.py | 130 +
.../test_spend_management_endpoints.py | 1004 +-
.../test_spend_tracking_utils.py | 120 +
.../proxy/test_api_key_masking_in_errors.py | 136 +
.../proxy/test_chat_completion_metadata.py | 154 +
.../proxy/test_common_request_processing.py | 262 +-
.../proxy/test_litellm_pre_call_utils.py | 311 +-
tests/test_litellm/proxy/test_proxy_cli.py | 82 +-
tests/test_litellm/proxy/test_proxy_server.py | 1511 +-
tests/test_litellm/proxy/test_proxy_utils.py | 46 +
tests/test_litellm/proxy/test_pyroscope.py | 147 +
.../proxy/test_response_model_sanitization.py | 217 +
.../proxy/test_route_a2a_models.py | 105 +
.../proxy/test_route_llm_request.py | 123 +-
.../test_proxy_setting_endpoints.py | 110 +
.../test_vector_store_access_control.py | 87 +
.../test_vector_store_endpoints.py | 469 +
.../test_litellm_completion_responses.py | 209 +-
...test_tool_call_streaming_transformation.py | 161 +
.../mcp/test_chat_completions_handler.py | 661 +-
.../responses/test_metadata_codex_callback.py | 176 +
.../test_responses_api_request_body.py | 103 +
.../responses/test_responses_utils.py | 75 +
.../test_budget_limiter_hotpath.py | 232 +
.../test_deployment_affinity_check.py | 659 +
.../test_aws_secret_manager_rotation.py | 109 +
.../test_secret_managers_main.py | 107 +-
.../test_litellm/test_a2a_registry_lookup.py | 73 +
.../test_anthropic_beta_headers_filtering.py | 430 +
.../test_claude_opus_4_6_config.py | 210 +
tests/test_litellm/test_constants.py | 10 +-
.../test_cost_calculation_log_level.py | 109 +-
tests/test_litellm/test_cost_calculator.py | 468 +-
.../test_deepseek_model_metadata.py | 180 +
.../test_litellm/test_eager_tiktoken_load.py | 29 +
tests/test_litellm/test_exception_exports.py | 31 +
tests/test_litellm/test_logging.py | 162 +-
tests/test_litellm/test_main.py | 172 +-
tests/test_litellm/test_model_param_helper.py | 33 +
.../test_responses_api_bridge_non_stream.py | 380 +-
tests/test_litellm/test_router.py | 212 +
.../test_enforce_model_rate_limits.py | 315 +
.../test_router_model_cost_isolation.py | 264 +
tests/test_litellm/test_router_redis_init.py | 56 +
.../test_router_silent_experiment.py | 214 +
tests/test_litellm/test_service_logger.py | 97 +
tests/test_litellm/test_ssl_verify_unit.py | 51 +-
tests/test_litellm/test_utils.py | 276 +-
tests/test_litellm/test_video_generation.py | 315 +-
.../test_xai_responses_auto_routing.py | 264 +
tests/test_litellm/types/__init__.py | 0
tests/test_litellm/types/proxy/__init__.py | 0
.../types/proxy/policy_engine/__init__.py | 0
.../policy_engine/test_pipeline_types.py | 152 +
.../policy_engine/test_resolver_types.py | 102 +
...test_vector_store_create_provider_logic.py | 19 +-
.../test_vector_store_registry.py | 34 +-
tests/test_otel_thread_leak.py | 90 +
tests/test_presidio_latency.py | 73 +
tests/test_proxy_server_non_root.py | 7 +-
tests/test_service_logger_otel.py | 28 +
tests/test_team.py | 2 +
.../rag/test_rag_s3_vectors.py | 107 +
.../rag/test_rag_vertex_ai.py | 229 +-
.../test_s3_vectors_vector_store.py | 42 +
ui/litellm-dashboard/e2e_tests/constants.ts | 5 +
ui/litellm-dashboard/e2e_tests/globalSetup.ts | 4 +-
.../e2e_tests/tests/keys/deleteKey.spec.ts | 25 +
.../tests/keys/regenerateKey.spec.ts | 21 +
.../tests/keys/updateKeyLimits.spec.ts | 27 +
.../e2e_tests/tests/login/login.spec.ts | 4 +-
.../tests/navigation/sidebar.spec.ts | 12 +
ui/litellm-dashboard/knip.json | 18 +
ui/litellm-dashboard/next.config.mjs | 16 +-
ui/litellm-dashboard/package-lock.json | 17931 ++--------------
ui/litellm-dashboard/package.json | 28 +-
.../public/assets/logos/s3_vector.png | Bin 0 -> 191076 bytes
.../public/assets/logos/zscaler.svg | 5 +
.../scripts/e2e_tests/neonHelperScripts.ts | 17 +-
.../app/(dashboard)/components/Sidebar2.tsx | 330 +-
.../components/SidebarProvider.tsx | 44 +-
.../accessGroups/useAccessGroupDetails.ts | 63 +
.../accessGroups/useAccessGroups.test.ts | 242 +
.../hooks/accessGroups/useAccessGroups.ts | 70 +
.../accessGroups/useCreateAccessGroup.ts | 68 +
.../accessGroups/useDeleteAccessGroup.ts | 55 +
.../hooks/accessGroups/useEditAccessGroup.ts | 77 +
.../cloudzero/useCloudZeroCreate.test.ts | 325 +
.../cloudzero/useCloudZeroDryRun.test.ts | 239 +
.../cloudzero/useCloudZeroExport.test.ts | 239 +
.../cloudzero/useCloudZeroSettings.test.ts | 675 +
.../hooks/cloudzero/useCloudZeroSettings.ts | 2 +-
.../hooks/logDetails/useLogDetails.ts | 30 +
.../useMCPSemanticFilterSettings.ts | 19 +
.../useUpdateMCPSemanticFilterSettings.ts | 25 +
.../mcpServers/useMCPAccessGroups.test.ts | 124 +
.../hooks/mcpServers/useMCPServers.test.ts | 134 +
.../hooks/models/useModels.test.ts | 252 +-
.../app/(dashboard)/hooks/models/useModels.ts | 47 +-
.../hooks/proxyConfig/useProxyConfig.test.ts | 554 +
.../hooks/proxyConfig/useProxyConfig.ts | 180 +
.../hooks/sso/useEditSSOSettings.test.ts | 312 +
.../hooks/sso/useSSOSettings.test.ts | 310 +
.../(dashboard)/hooks/sso/useSSOSettings.ts | 5 +
.../useStoreRequestInSpendLogs.ts | 63 +
.../(dashboard)/hooks/teams/useTeams.test.ts | 530 +-
.../app/(dashboard)/hooks/teams/useTeams.ts | 2 +-
.../hooks/uiConfig/useUIConfig.test.ts | 2 +
.../hooks/uiSettings/useUISettings.test.ts | 72 +-
.../hooks/uiSettings/useUISettings.ts | 5 +-
.../uiSettings/useUpdateUISettings.test.ts | 240 +
.../(dashboard)/hooks/useAuthorized.test.ts | 97 +-
.../app/(dashboard)/hooks/useAuthorized.ts | 40 +-
.../hooks/useDisableShowNewBadge.test.ts | 180 +
.../hooks/useDisableShowPrompts.test.ts | 180 +
.../hooks/useDisableShowPrompts.ts | 35 +
.../hooks/useDisableUsageIndicator.test.ts | 190 +
.../hooks/useDisableUsageIndicator.ts | 33 +
.../(dashboard)/hooks/users/useUsers.test.ts | 339 +
.../app/(dashboard)/hooks/users/useUsers.ts | 41 +
.../src/app/(dashboard)/layout.tsx | 16 +-
.../ModelsAndEndpointsView.tsx | 4 +-
.../components/AllModelsTab.test.tsx | 22 +-
.../components/AllModelsTab.tsx | 308 +-
.../ModelAnalyticsTab/FilterByContent.tsx | 134 -
.../ModelAnalyticsTab/ModelAnalyticsTab.tsx | 474 -
.../utils/modelDataTransformer.test.ts | 69 +
.../utils/modelDataTransformer.ts | 8 +-
.../src/app/(dashboard)/playground/page.tsx | 5 +
.../src/app/(dashboard)/policies/page.tsx | 17 +
.../settings/admin-settings/page.tsx | 17 +-
.../src/app/(dashboard)/teams/TeamsView.tsx | 2 +-
.../components/modals/CreateTeamModal.tsx | 48 +-
ui/litellm-dashboard/src/app/globals.css | 28 -
ui/litellm-dashboard/src/app/layout.tsx | 6 +-
.../src/app/login/LoginPage.test.tsx | 98 +-
.../src/app/login/LoginPage.tsx | 33 +-
.../src/app/mcp/oauth/callback/page.tsx | 12 +-
.../src/app/model_hub/page.tsx | 17 +-
.../src/app/model_hub_table/page.tsx | 17 +-
.../src/app/onboarding/page.tsx | 12 +-
ui/litellm-dashboard/src/app/page.tsx | 443 +-
.../components/AIHub/ModelHubTable.test.tsx | 153 +-
.../src/components/AIHub/ModelHubTable.tsx | 35 +-
.../AccessGroupsDetailsPage.test.tsx | 384 +
.../AccessGroups/AccessGroupsDetailsPage.tsx | 345 +
.../AccessGroupsModal/AccessGroupBaseForm.tsx | 159 +
.../AccessGroupCreateModal.tsx | 67 +
.../AccessGroupEditModal.tsx | 85 +
.../AccessGroups/AccessGroupsPage.test.tsx | 321 +
.../AccessGroups/AccessGroupsPage.tsx | 404 +
.../src/components/AccessGroups/types.ts | 46 +
.../src/components/AdminPanel.test.tsx | 325 +
.../src/components/AdminPanel.tsx | 373 +
.../src/components/BulkEditUsers.test.tsx | 343 +
.../{bulk_edit_user.tsx => BulkEditUsers.tsx} | 10 +-
.../src/components/CreateUserButton.test.tsx | 264 +
...e_user_button.tsx => CreateUserButton.tsx} | 151 +-
.../EntityUsageExport/ExportTypeSelector.tsx | 10 +-
.../src/components/EntityUsageExport/types.ts | 2 +-
.../EntityUsageExport/utils.test.ts | 560 +
.../src/components/EntityUsageExport/utils.ts | 91 +
.../ModelSelect/ModelSelect.test.tsx | 682 +-
.../components/ModelSelect/ModelSelect.tsx | 23 +-
.../PaginatedModelSelect.test.tsx | 301 +
.../PaginatedModelSelect.tsx | 143 +
.../CommunityEngagementButtons.test.tsx | 50 +
.../CommunityEngagementButtons.tsx | 36 +
.../Navbar/UserDropdown/UserDropdown.test.tsx | 289 +
.../Navbar/UserDropdown/UserDropdown.tsx | 180 +
.../src/components/OldTeams.test.tsx | 109 +-
.../src/components/OldTeams.tsx | 105 +-
.../src/components/SSOModals.tsx | 6 +-
.../CreateSearchTools.tsx} | 26 +-
.../SearchConnectionTest.tsx} | 6 +-
.../SearchTools/SearchToolColumn.tsx | 114 +
.../SearchTools/SearchToolTester.test.tsx | 425 +
.../SearchToolTester.tsx} | 0
.../SearchTools/SearchToolView.test.tsx | 278 +
.../SearchToolView.tsx} | 30 +-
.../SearchTools/SearchTools.test.tsx | 242 +
.../SearchTools.tsx} | 100 +-
.../src/components/SearchTools/index.tsx | 6 +
.../{search_tools => SearchTools}/types.tsx | 1 +
.../MCPSemanticFilterSettings.tsx | 309 +
.../MCPSemanticFilterTestPanel.tsx | 164 +
.../semanticFilterTestUtils.ts | 95 +
.../Modals/BaseSSOSettingsForm.test.tsx | 111 +
.../Modals/BaseSSOSettingsForm.tsx | 37 +
.../Modals/DeleteSSOSettingsModal.tsx | 1 +
.../Modals/EditSSOSettingsModal.test.tsx | 106 +
.../Modals/EditSSOSettingsModal.tsx | 11 +
.../AdminSettings/SSOSettings/SSOSettings.tsx | 22 +-
.../AdminSettings/SSOSettings/utils.test.ts | 144 +
.../AdminSettings/SSOSettings/utils.ts | 14 +-
.../UISettings/PageVisibilitySettings.tsx | 146 +
.../UISettings/UISettings.test.tsx | 75 +
.../AdminSettings/UISettings/UISettings.tsx | 56 +-
.../Fallbacks/AddFallbacks.test.tsx | 309 +
.../RouterSettings/Fallbacks/AddFallbacks.tsx | 169 +
.../Fallbacks/AddFallbacksModal.test.tsx | 56 +
.../Fallbacks/AddFallbacksModal.tsx | 52 +
.../Fallbacks/FallbackGroupConfig.tsx | 208 +
.../Fallbacks/FallbackSelectionForm.test.tsx | 233 +
.../Fallbacks/FallbackSelectionForm.tsx | 132 +
.../Fallbacks/Fallbacks.test.tsx | 382 +
.../RouterSettings/Fallbacks/Fallbacks.tsx | 334 +
.../src/components/TeamSSOSettings.test.tsx | 669 +-
.../src/components/TeamSSOSettings.tsx | 22 +-
.../src/components/UsageIndicator.test.tsx | 187 +
...usage_indicator.tsx => UsageIndicator.tsx} | 171 +-
.../EntityUsage/SpendByProvider.test.tsx | 238 +
.../EntityUsage/SpendByProvider.tsx | 131 +
.../components/KeyModelUsageView.test.tsx | 298 +
.../components/KeyModelUsageView.tsx | 108 +
.../components/UsagePageView.test.tsx | 675 +-
.../UsagePage/components/UsagePageView.tsx | 252 +-
.../src/components/UsagePage/types.ts | 10 +
.../VirtualKeysPage/VirtualKeysTable.test.tsx | 172 +-
.../VirtualKeysPage/VirtualKeysTable.tsx | 77 +-
.../src/components/activity_metrics.test.tsx | 155 +
.../src/components/activity_metrics.tsx | 83 +-
.../src/components/add_fallbacks.test.tsx | 47 -
.../src/components/add_fallbacks.tsx | 250 -
.../add_model/RouterConfigBuilder.test.tsx | 296 +
.../add_model/RouterConfigBuilder.tsx | 277 +
.../add_model/add_auto_router_tab.tsx | 2 +-
.../add_model/router_config_builder.tsx | 281 -
.../src/components/admins.tsx | 673 -
.../src/components/budgets/budget_modal.tsx | 2 +-
.../components/budgets/edit_budget_modal.tsx | 2 +-
.../components/bulk_create_users_button.tsx | 32 +-
.../src/components/cloudzero_export_modal.tsx | 2 +-
.../common_components/AccessGroupSelector.tsx | 112 +
.../DefaultProxyAdminTag.tsx | 24 +
.../DeleteResourceModal.test.tsx | 187 +-
.../common_components/DeleteResourceModal.tsx | 34 +-
.../KeyLifecycleSettings.test.tsx | 383 +
.../KeyLifecycleSettings.tsx | 12 +-
.../common_components/NewBadge.test.tsx | 35 +
.../components/common_components/NewBadge.tsx | 12 +-
.../RouterSettingsAccordion.tsx | 372 +
.../TableHeaderSortDropdown.test.tsx | 148 +
.../TableHeaderSortDropdown.tsx | 85 +
.../components/common_components/all_view.tsx | 153 -
.../common_components/chartUtils.test.tsx | 382 +
.../common_components/default_org.tsx | 6 -
.../common_components/user_form.tsx | 59 -
.../components/create_user_button.test.tsx | 35 -
.../src/components/dashboard_default_team.tsx | 103 -
.../src/components/delete_model_button.tsx | 55 -
.../edit_auto_router_modal.tsx | 4 +-
.../edit_model/edit_model_modal.tsx | 214 -
.../src/components/edit_user.tsx | 2 +-
.../src/components/enter_proxy_url.tsx | 69 -
.../src/components/fallbacks.test.tsx | 95 -
.../src/components/fallbacks.tsx | 212 -
.../src/components/general_settings.tsx | 2 +-
.../src/components/guardrails.tsx | 49 +-
.../guardrails/add_guardrail_form.tsx | 176 +-
.../content_filter/CategoryTable.tsx | 147 +
.../ContentCategoryConfiguration.tsx | 121 +-
.../ContentFilterConfiguration.tsx | 17 +-
.../content_filter/ContentFilterDisplay.tsx | 35 +-
.../ContentFilterManager.test.tsx | 505 +-
.../content_filter/ContentFilterManager.tsx | 120 +-
.../content_filter/CustomPatternModal.tsx | 15 +-
.../content_filter/KeywordModal.tsx | 15 +-
.../content_filter/KeywordTable.tsx | 14 +-
.../content_filter/PatternModal.tsx | 7 +-
.../content_filter/PatternTable.tsx | 12 +-
.../custom_code/CustomCodeEditor.tsx | 188 +
.../custom_code/CustomCodeModal.tsx | 832 +
.../custom_code/CustomCodePlayground.tsx | 588 +
.../custom_code/custom_code_constants.ts | 194 +
.../guardrails/custom_code/index.ts | 1 +
.../guardrails/guardrail_info.test.tsx | 126 +-
.../components/guardrails/guardrail_info.tsx | 163 +-
.../guardrails/guardrail_info_helpers.tsx | 1 +
.../guardrails/guardrail_optional_params.tsx | 18 +-
.../guardrails/guardrail_provider_fields.tsx | 7 +-
.../components/key_team_helpers/key_list.tsx | 1 +
.../organization_search_fn.tsx | 23 -
.../key_team_helpers/team_search_fn.tsx | 19 -
.../src/components/leftnav.tsx | 592 +-
.../src/components/mcp_connection_test.tsx | 279 -
.../mcp_tools/MCPNetworkSettings.tsx | 152 +
.../MCPPermissionManagement.test.tsx | 10 +-
.../mcp_tools/MCPPermissionManagement.tsx | 24 +
.../mcp_tools/MCPToolArgumentsForm.tsx | 345 +
.../components/mcp_tools/OAuthFormFields.tsx | 166 +
.../mcp_tools/StdioConfiguration.tsx | 9 +-
.../mcp_tools/create_mcp_server.tsx | 216 +-
.../src/components/mcp_tools/mcp_connect.tsx | 28 +-
.../components/mcp_tools/mcp_discovery.tsx | 320 +
.../mcp_tools/mcp_server_columns.tsx | 18 +-
.../mcp_tools/mcp_server_edit.test.tsx | 154 +
.../components/mcp_tools/mcp_server_edit.tsx | 308 +-
.../components/mcp_tools/mcp_server_view.tsx | 25 +-
.../components/mcp_tools/mcp_servers.test.tsx | 4 +
.../src/components/mcp_tools/mcp_servers.tsx | 38 +-
.../src/components/mcp_tools/types.tsx | 39 +-
.../src/components/mcp_tools/utils.tsx | 4 +-
.../model_add/CredentialDeleteModal.tsx | 90 -
.../model_add/reuse_credentials.tsx | 2 +-
.../model_dashboard/HealthCheckComponent.tsx | 4 +
.../model_dashboard/all_models_table.tsx | 211 +
.../model_dashboard/health_check_columns.tsx | 27 +
.../src/components/model_dashboard/table.tsx | 11 +-
.../src/components/model_info_view.test.tsx | 741 +-
.../src/components/model_info_view.tsx | 129 +-
.../src/components/molecules/filter.test.tsx | 527 +
.../src/components/molecules/filter.tsx | 25 +-
.../molecules/models/columns.test.tsx | 935 +
.../components/molecules/models/columns.tsx | 654 +-
.../molecules/notifications_manager.tsx | 38 +-
.../src/components/navbar.test.tsx | 83 +-
.../src/components/navbar.tsx | 204 +-
.../src/components/networking.tsx | 1392 +-
.../src/components/onboarding_link.tsx | 2 +-
.../organisms/create_key_button.test.tsx | 111 +-
.../organisms/create_key_button.tsx | 135 +-
.../organisms/regenerate_key_modal.tsx | 18 +
.../components/organization/add_org_admin.tsx | 113 -
.../organization/organization_view.tsx | 36 +-
.../organization/view_members_of_org.tsx | 74 -
.../src/components/page_metadata.ts | 45 +
.../src/components/page_utils.test.ts | 241 +
.../src/components/page_utils.ts | 75 +
.../chat_ui/AdditionalModelSettings.test.tsx | 53 +-
.../chat_ui/AdditionalModelSettings.tsx | 45 +-
.../playground/chat_ui/ChatUI.test.tsx | 64 +
.../components/playground/chat_ui/ChatUI.tsx | 391 +-
.../playground/chat_ui/CodeSnippets.test.tsx | 3 +-
.../playground/chat_ui/CodeSnippets.tsx | 4 +
.../playground/chat_ui/chatConstants.ts | 1 +
.../chat_ui/mode_endpoint_mapping.tsx | 1 +
.../components/playground/chat_ui/types.ts | 58 -
.../playground/compareUI/CompareUI.tsx | 1 +
.../playground/complianceUI/ComplianceUI.tsx | 1376 ++
.../playground/llm_calls/a2a_send_message.tsx | 6 +
.../llm_calls/anthropic_messages.tsx | 2 +
.../llm_calls/chat_completion.test.tsx | 70 +-
.../playground/llm_calls/chat_completion.tsx | 86 +-
.../llm_calls/responses_api.test.tsx | 29 +-
.../playground/llm_calls/responses_api.tsx | 2 +
.../components/policies/PolicySelector.tsx | 77 +
.../policies/add_attachment_form.tsx | 318 +
.../components/policies/add_policy_form.tsx | 569 +
.../components/policies/attachment_table.tsx | 310 +
.../policies/build_attachment_data.ts | 25 +
.../policies/guardrail_selection_modal.tsx | 249 +
.../components/policies/impact_popover.tsx | 84 +
.../policies/impact_preview_alert.tsx | 61 +
.../src/components/policies/index.tsx | 593 +
.../policies/pipeline_flow_builder.tsx | 999 +
.../src/components/policies/policy_info.tsx | 224 +
.../src/components/policies/policy_table.tsx | 309 +
.../components/policies/policy_templates.tsx | 302 +
.../components/policies/policy_test_panel.tsx | 254 +
.../policies/template_parameter_modal.tsx | 127 +
.../src/components/policies/types.ts | 101 +
.../prompts/prompt_editor_view/utils.ts | 211 +-
.../src/components/public_model_hub.test.tsx | 22 +-
.../src/components/public_model_hub.tsx | 52 +-
.../src/components/request_model_access.tsx | 2 +-
.../ReliabilityRetriesSection.tsx | 8 +-
.../router_settings/RouterSettingsForm.tsx | 84 +
.../src/components/router_settings/index.tsx | 88 +-
.../src/components/search_tools/index.tsx | 6 -
.../search_tools/search_tool_columns.tsx | 78 -
.../components/survey/NudgePrompt.test.tsx | 101 +
.../src/components/survey/NudgePrompt.tsx | 73 +-
.../components/CreateTagModal.tsx | 2 +-
.../src/components/team/TeamInfo.test.tsx | 729 +
.../src/components/team/TeamInfo.tsx | 1316 ++
.../components/team/TeamMemberTab.test.tsx | 386 +
.../src/components/team/TeamMemberTab.tsx | 233 +
.../components/team/available_teams.test.tsx | 3 +-
.../src/components/team/available_teams.tsx | 11 +-
.../team/tabVisibilityUtils.test.ts | 74 +
.../src/components/team/tabVisibilityUtils.ts | 59 +
.../src/components/team/team_info.test.tsx | 572 -
.../src/components/team/team_info.tsx | 1124 -
.../components/team/team_member_view.test.tsx | 193 -
.../src/components/team/team_member_view.tsx | 193 -
.../templates/key_edit_view.test.tsx | 464 +-
.../components/templates/key_edit_view.tsx | 115 +-
.../templates/key_info_view.test.tsx | 174 +-
.../components/templates/key_info_view.tsx | 152 +-
.../components/templates/model_dashboard.tsx | 1755 --
.../src/components/usage_indicator.test.tsx | 86 -
.../src/components/user_edit_view.test.tsx | 513 +
.../src/components/user_edit_view.tsx | 64 +-
.../CreateVectorStore.test.tsx | 276 +
.../CreateVectorStore.tsx | 434 +
.../DocumentsTable.test.tsx | 102 +
.../DocumentsTable.tsx | 98 +
.../S3VectorsConfig.test.tsx | 203 +
.../S3VectorsConfig.tsx | 192 +
.../TestVectorStoreTab.test.tsx | 90 +
.../TestVectorStoreTab.tsx | 75 +
.../VectorStoreForm.tsx | 11 +-
.../VectorStoreTable.test.tsx | 12 +-
.../VectorStoreTable.tsx | 26 +
.../vector_store_management/index.tsx | 56 +-
.../vector_store_management/types.tsx | 45 +-
.../src/components/vector_store_providers.tsx | 37 +
.../view_logs/ConfigInfoMessage.tsx | 16 +-
.../view_logs/CostBreakdownViewer.tsx | 50 +-
.../GuardrailViewer/CompliancePanel.tsx | 195 +
.../GuardrailViewer/GuardrailViewer.test.tsx | 24 +-
.../GuardrailViewer/GuardrailViewer.tsx | 779 +-
.../LogDetailsDrawer/CollapsibleMessage.tsx | 88 +
.../LogDetailsDrawer/DrawerHeader.tsx | 205 +
.../LogDetailsDrawer/HistorySection.tsx | 62 +
.../LogDetailsDrawer/HistoryTree.tsx | 83 +
.../LogDetailsDrawer/InputCard.test.tsx | 219 +
.../view_logs/LogDetailsDrawer/InputCard.tsx | 92 +
.../view_logs/LogDetailsDrawer/JsonViewer.tsx | 35 +
.../LogDetailsDrawer/LogDetailContent.tsx | 509 +
.../LogDetailsDrawer/LogDetailsDrawer.tsx | 394 +
.../LogDetailsDrawer/MessageBlock.tsx | 104 +
.../LogDetailsDrawer/MessageCard.tsx | 199 +
.../LogDetailsDrawer/OutputCard.test.tsx | 173 +
.../view_logs/LogDetailsDrawer/OutputCard.tsx | 104 +
.../LogDetailsDrawer/PrettyMessagesView.tsx | 41 +
.../LogDetailsDrawer/SectionHeader.tsx | 100 +
.../LogDetailsDrawer/SimpleMessageBlock.tsx | 74 +
.../LogDetailsDrawer/SimpleToolCallBlock.tsx | 65 +
.../view_logs/LogDetailsDrawer/TokenFlow.tsx | 22 +
.../LogDetailsDrawer/ToolCallBlock.tsx | 78 +
.../LogDetailsDrawer/ToolCallCard.tsx | 79 +
.../LogDetailsDrawer/TruncatedValue.tsx | 35 +
.../view_logs/LogDetailsDrawer/constants.ts | 39 +
.../view_logs/LogDetailsDrawer/index.ts | 4 +
.../LogDetailsDrawer/prettyMessagesTypes.ts | 28 +
.../LogDetailsDrawer/prettyMessagesUtils.ts | 126 +
.../LogDetailsDrawer/useKeyboardNavigation.ts | 87 +
.../view_logs/LogDetailsDrawer/utils.ts | 93 +
.../view_logs/RequestResponsePanel.test.tsx | 74 +
.../view_logs/RequestResponsePanel.tsx | 4 +-
.../src/components/view_logs/SessionView.tsx | 193 -
.../SpendLogsSettingsModal.test.tsx | 484 +
.../SpendLogsSettingsModal.tsx | 156 +
.../ToolsSection/FormattedToolView.tsx | 124 +
.../view_logs/ToolsSection/JsonToolView.tsx | 39 +
.../ToolsSection/ToolExpandedContent.tsx | 52 +
.../view_logs/ToolsSection/ToolItem.tsx | 74 +
.../ToolsSection/ToolsSection.test.tsx | 117 +
.../view_logs/ToolsSection/ToolsSection.tsx | 65 +
.../view_logs/ToolsSection/index.ts | 7 +
.../view_logs/ToolsSection/types.ts | 42 +
.../view_logs/ToolsSection/utils.test.ts | 293 +
.../view_logs/ToolsSection/utils.ts | 130 +
.../src/components/view_logs/TypeBadges.tsx | 30 +
.../view_logs/VectorStoreViewer.tsx | 38 +-
.../src/components/view_logs/columns.tsx | 190 +-
.../src/components/view_logs/constants.ts | 24 +
.../src/components/view_logs/country_cell.tsx | 29 -
.../src/components/view_logs/index.test.tsx | 103 +-
.../src/components/view_logs/index.tsx | 433 +-
.../view_logs/log_filter_logic.test.tsx | 682 +
.../components/view_logs/log_filter_logic.tsx | 59 +-
.../src/components/view_logs/table.tsx | 31 +-
.../src/components/view_logs/utils.ts | 20 +
.../src/components/view_users.tsx | 10 +-
.../src/contexts/AntdGlobalProvider.tsx | 24 +
.../src/data/compliancePrompts.ts | 310 +
.../src/hooks/useTestMCPConnection.tsx | 6 +-
.../src/utils/jwtUtils.test.ts | 79 +-
ui/litellm-dashboard/src/utils/jwtUtils.ts | 15 +
ui/litellm-dashboard/tsconfig.json | 27 +-
ui/litellm-dashboard/tsconfig.tsbuildinfo | 2 +-
ui/litellm-dashboard/vitest.config.ts | 1 +
2196 files changed, 221591 insertions(+), 43440 deletions(-)
create mode 100644 .github/workflows/test-litellm-matrix.yml
create mode 100644 .github/workflows/test-litellm-ui-build.yml
create mode 100644 .github/workflows/test-model-map.yaml
create mode 100644 .github/workflows/test_server_root_path.yml
create mode 100644 .semgrep/rules/README.md
create mode 100644 .semgrep/rules/python/reliability/unbounded-memory.yml
create mode 100644 .semgrep/rules/python/unbounded-memory.yml
create mode 100644 .trivyignore
create mode 100644 cookbook/anthropic_agent_sdk/README.md
create mode 100644 cookbook/anthropic_agent_sdk/agent_with_mcp.py
create mode 100644 cookbook/anthropic_agent_sdk/common.py
create mode 100644 cookbook/anthropic_agent_sdk/config.example.yaml
create mode 100644 cookbook/anthropic_agent_sdk/main.py
create mode 100644 cookbook/anthropic_agent_sdk/requirements.txt
create mode 100644 cookbook/livekit_agent_sdk/README.md
create mode 100644 cookbook/livekit_agent_sdk/config.example.yaml
create mode 100644 cookbook/livekit_agent_sdk/main.py
create mode 100644 cookbook/livekit_agent_sdk/requirements.txt
create mode 100644 cookbook/mock_prompt_management_server/README.md
create mode 100644 cookbook/mock_prompt_management_server/mock_prompt_management_server.py
create mode 100644 cookbook/nova_sonic_realtime.py
create mode 100644 docs/my-website/blog/claude_code_beta_headers/index.md
create mode 100644 docs/my-website/blog/claude_opus_4_6/index.md
create mode 100644 docs/my-website/blog/claude_sonnet_4_6/index.md
create mode 100644 docs/my-website/blog/fastapi_middleware_performance/index.mdx
create mode 100644 docs/my-website/blog/litellm_observatory/index.md
create mode 100644 docs/my-website/blog/minimax_m2_5/index.md
create mode 100644 docs/my-website/blog/model_cost_map_incident/index.md
create mode 100644 docs/my-website/blog/sub_millisecond_proxy_overhead/index.md
create mode 100644 docs/my-website/blog/vllm_embeddings_incident/index.md
create mode 100644 docs/my-website/docs/a2a_invoking_agents.md
create mode 100644 docs/my-website/docs/adding_provider/generic_prompt_management_api.md
create mode 100644 docs/my-website/docs/completion/message_sanitization.md
create mode 100644 docs/my-website/docs/evals_api.md
create mode 100644 docs/my-website/docs/integrations/websearch_interception.md
create mode 100644 docs/my-website/docs/mcp_oauth.md
create mode 100644 docs/my-website/docs/mcp_public_internet.md
create mode 100644 docs/my-website/docs/mcp_semantic_filter.md
create mode 100644 docs/my-website/docs/providers/bedrock_realtime_with_audio.md
create mode 100644 docs/my-website/docs/providers/sarvam.md
create mode 100644 docs/my-website/docs/providers/scaleway.md
create mode 100644 docs/my-website/docs/providers/watsonx/rerank.md
create mode 100644 docs/my-website/docs/providers/xai_realtime.md
create mode 100644 docs/my-website/docs/proxy/access_groups.md
create mode 100644 docs/my-website/docs/proxy/guardrails/custom_code_guardrail.md
create mode 100644 docs/my-website/docs/proxy/guardrails/guardrail_policies.md
create mode 100644 docs/my-website/docs/proxy/guardrails/policy_tags.md
create mode 100644 docs/my-website/docs/proxy/guardrails/policy_templates.md
create mode 100644 docs/my-website/docs/proxy/keys_teams_router_settings.md
create mode 100644 docs/my-website/docs/proxy/pyroscope_profiling.md
create mode 100644 docs/my-website/docs/proxy/request_tags.md
create mode 100644 docs/my-website/docs/proxy/sync_anthropic_beta_headers.md
create mode 100644 docs/my-website/docs/proxy/ui/page_visibility.md
create mode 100644 docs/my-website/docs/proxy/ui_spend_log_settings.md
create mode 100644 docs/my-website/docs/proxy/ui_team_soft_budget_alerts.md
create mode 100644 docs/my-website/docs/proxy_auth.md
create mode 100644 docs/my-website/docs/traffic_mirroring.md
create mode 100644 docs/my-website/docs/troubleshoot/max_callbacks.md
create mode 100644 docs/my-website/docs/troubleshoot/prisma_migrations.md
create mode 100644 docs/my-website/docs/troubleshoot/spend_queue_warnings.md
create mode 100644 docs/my-website/docs/troubleshoot/ui_issues.md
create mode 100644 docs/my-website/docs/tutorials/claude_agent_sdk.md
create mode 100644 docs/my-website/docs/tutorials/claude_code_beta_headers.md
create mode 100644 docs/my-website/docs/tutorials/claude_code_prompt_cache_routing.md
create mode 100644 docs/my-website/docs/tutorials/copilotkit_sdk.md
create mode 100644 docs/my-website/docs/tutorials/livekit_xai_realtime.md
create mode 100644 docs/my-website/img/a2a_agent_spend.png
create mode 100644 docs/my-website/img/a2a_trace_grouping.png
create mode 100644 docs/my-website/img/okta_access_policies.png
create mode 100644 docs/my-website/img/okta_authorization_server.png
create mode 100644 docs/my-website/img/okta_client_credentials.png
create mode 100644 docs/my-website/img/okta_redirect_uri.png
create mode 100644 docs/my-website/img/okta_security_api.png
create mode 100644 docs/my-website/img/policy_team_attach.png
create mode 100644 docs/my-website/img/policy_test_matching.png
create mode 100644 docs/my-website/img/release_notes/guard_actions.png
create mode 100644 docs/my-website/img/release_notes/mcp_internet.png
create mode 100644 docs/my-website/img/ui_access_groups.png
create mode 100644 docs/my-website/img/ui_granular_router_settings.png
create mode 100644 docs/my-website/img/ui_spend_logs_settings.png
create mode 100644 docs/my-website/img/ui_team_soft_budget_alerts.png
create mode 100644 docs/my-website/img/ui_team_soft_budget_email_example.png
create mode 100644 docs/my-website/img/ui_tools.png
create mode 100644 docs/my-website/release_notes/v1.81.12.md
create mode 100644 docs/my-website/release_notes/v1.81.3-stable/index.md
create mode 100644 docs/my-website/release_notes/v1.81.6.md
create mode 100644 docs/my-website/release_notes/v1.81.9.md
create mode 100644 docs/my-website/src/components/MiddlewareDiagrams/BaseHTTPMiddlewareAnimation.tsx
create mode 100644 docs/my-website/src/components/MiddlewareDiagrams/BenchmarkVisualization.tsx
create mode 100644 docs/my-website/src/components/MiddlewareDiagrams/PureASGIAnimation.tsx
create mode 100644 docs/my-website/src/components/MiddlewareDiagrams/index.tsx
create mode 100644 docs/my-website/src/components/MiddlewareDiagrams/styles.module.css
delete mode 100644 docs/my-website/src/pages/troubleshoot.md
create mode 100644 docs/my-website/src/theme/BlogListPage/index.js
create mode 100644 docs/my-website/src/theme/BlogListPage/styles.module.css
create mode 100644 enterprise/dist/litellm_enterprise-0.1.29-py3-none-any.whl
create mode 100644 enterprise/dist/litellm_enterprise-0.1.29.tar.gz
create mode 100644 enterprise/dist/litellm_enterprise-0.1.30-py3-none-any.whl
create mode 100644 enterprise/dist/litellm_enterprise-0.1.30.tar.gz
create mode 100644 enterprise/dist/litellm_enterprise-0.1.31-py3-none-any.whl
create mode 100644 enterprise/dist/litellm_enterprise-0.1.31.tar.gz
create mode 100644 enterprise/dist/litellm_enterprise-0.1.32-py3-none-any.whl
create mode 100644 enterprise/dist/litellm_enterprise-0.1.32.tar.gz
create mode 100644 enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py
create mode 100644 litellm-proxy-extras/build_and_publish.md
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.26-py3-none-any.whl
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.26.tar.gz
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.27-py3-none-any.whl
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.27.tar.gz
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.28-py3-none-any.whl
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.28.tar.gz
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.29-py3-none-any.whl
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.29.tar.gz
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.30-py3-none-any.whl
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.30.tar.gz
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.31-py3-none-any.whl
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.31.tar.gz
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.32-py3-none-any.whl
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.32.tar.gz
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.33-py3-none-any.whl
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.33.tar.gz
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.34-py3-none-any.whl
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.34.tar.gz
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35-py3-none-any.whl
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35.tar.gz
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36-py3-none-any.whl
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36.tar.gz
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.37-py3-none-any.whl
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.37.tar.gz
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.40-py3-none-any.whl
create mode 100644 litellm-proxy-extras/dist/litellm_proxy_extras-0.4.40.tar.gz
create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260123131407_add_policy_tables_and_policies_field/migration.sql
create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260131150814_add_team_user_to_vector_stores/migration.sql
create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260203120000_add_deprecated_verification_token_table/migration.sql
create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260205091235_allow_team_guardrail_config/migration.sql
create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260205144610_add_soft_budget_to_team_table/migration.sql
create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260207093506_add_available_on_public_internet_to_mcp_servers/migration.sql
create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260207110613_add_soft_budget_to_deleted_teams_table/migration.sql
create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260209085821_add_verificationtoken_indexes/migration.sql
create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260212103349_adjust_tags_policy_table/migration.sql
create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql
create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260213105436_add_managed_vector_store_table/migration.sql
create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260213170952_access_group_change_to_model_name/migration.sql
create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260214094754_schema_sync/migration.sql
create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260214124140_baseline_diff/migration.sql
create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260214163027_add_pipeline_to_policy_table/migration.sql
create mode 100644 litellm/a2a_protocol/card_resolver.py
create mode 100644 litellm/a2a_protocol/exception_mapping_utils.py
create mode 100644 litellm/a2a_protocol/exceptions.py
create mode 100644 litellm/anthropic_beta_headers_config.json
create mode 100644 litellm/anthropic_beta_headers_manager.py
create mode 100644 litellm/evals/__init__.py
create mode 100644 litellm/evals/main.py
create mode 100644 litellm/integrations/braintrust_mock_client.py
create mode 100644 litellm/integrations/datadog/datadog_mock_client.py
create mode 100644 litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py
create mode 100644 litellm/integrations/helicone_mock_client.py
create mode 100644 litellm/integrations/langfuse/langfuse_mock_client.py
create mode 100644 litellm/integrations/langsmith_mock_client.py
create mode 100644 litellm/integrations/mock_client_factory.py
create mode 100644 litellm/integrations/posthog_mock_client.py
create mode 100644 litellm/litellm_core_utils/env_utils.py
create mode 100644 litellm/llms/a2a/__init__.py
create mode 100644 litellm/llms/a2a/chat/__init__.py
create mode 100644 litellm/llms/a2a/chat/guardrail_translation/README.md
create mode 100644 litellm/llms/a2a/chat/guardrail_translation/__init__.py
create mode 100644 litellm/llms/a2a/chat/guardrail_translation/handler.py
create mode 100644 litellm/llms/a2a/chat/streaming_iterator.py
create mode 100644 litellm/llms/a2a/chat/transformation.py
create mode 100644 litellm/llms/a2a/common_utils.py
create mode 100644 litellm/llms/azure_ai/azure_model_router/__init__.py
create mode 100644 litellm/llms/azure_ai/azure_model_router/transformation.py
create mode 100644 litellm/llms/azure_ai/cost_calculator.py
create mode 100644 litellm/llms/base_llm/evals/__init__.py
create mode 100644 litellm/llms/base_llm/evals/transformation.py
create mode 100644 litellm/llms/base_llm/managed_resources/__init__.py
create mode 100644 litellm/llms/base_llm/managed_resources/base_managed_resource.py
create mode 100644 litellm/llms/base_llm/managed_resources/utils.py
create mode 100644 litellm/llms/bedrock/realtime/handler.py
create mode 100644 litellm/llms/bedrock/realtime/transformation.py
rename ui/litellm-dashboard/src/components/teams.tsx => litellm/llms/databricks/responses/__init__.py (100%)
create mode 100644 litellm/llms/databricks/responses/transformation.py
create mode 100644 litellm/llms/duckduckgo/search/__init__.py
create mode 100644 litellm/llms/duckduckgo/search/transformation.py
create mode 100644 litellm/llms/hosted_vllm/embedding/transformation.py
create mode 100644 litellm/llms/openai/embeddings/guardrail_translation/__init__.py
create mode 100644 litellm/llms/openai/embeddings/guardrail_translation/handler.py
create mode 100644 litellm/llms/openai/evals/__init__.py
create mode 100644 litellm/llms/openai/evals/transformation.py
create mode 100644 litellm/llms/perplexity/responses/__init__.py
create mode 100644 litellm/llms/perplexity/responses/transformation.py
create mode 100644 litellm/llms/s3_vectors/__init__.py
create mode 100644 litellm/llms/s3_vectors/vector_stores/__init__.py
create mode 100644 litellm/llms/s3_vectors/vector_stores/transformation.py
create mode 100644 litellm/llms/watsonx/__init__.py
create mode 100644 litellm/llms/watsonx/chat/__init__.py
create mode 100644 litellm/llms/watsonx/completion/__init__.py
create mode 100644 litellm/llms/watsonx/embed/__init__.py
create mode 100644 litellm/llms/watsonx/rerank/__init__.py
create mode 100644 litellm/llms/watsonx/rerank/transformation.py
create mode 100644 litellm/llms/xai/realtime/__init__.py
create mode 100644 litellm/llms/xai/realtime/handler.py
create mode 100644 litellm/policy_templates_backup.json
create mode 100644 litellm/proxy/_experimental/mcp_server/mcp_debug.py
create mode 100644 litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py
create mode 100644 litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py
create mode 100644 litellm/proxy/_experimental/out/404/index.html
create mode 100644 litellm/proxy/_experimental/out/__next.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/_next/static/C_XKHLw43nx5HaPfGD7XZ/_buildManifest.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/C_XKHLw43nx5HaPfGD7XZ/_clientMiddlewareManifest.json
rename litellm/proxy/_experimental/out/_next/static/{zHD7JXLXiWgn1NPp82VmF => C_XKHLw43nx5HaPfGD7XZ}/_ssgManifest.js (100%)
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/00bcc8d30dd19793.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/00ff280cdb7d7ee5.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/04b9c7b5c33ea26c.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/06aaedbe7d27898c.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/088a4006aa78f150.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0a65da2cd24e2ab6.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0a671fedee641c02.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0a6c418370a8c183.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0d1694151d7fdaec.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1067d2c077cd73d6.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1098-c3e95c9684ff5e95.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1108-c2d0c742b6e72436.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1128-64fa4a41ccaf67ea.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/11383a8b78399079.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/120d96e5e05ab994.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/121a51d3bbb6f362.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1300460219810c10.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/134f728fa7099e3e.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/137-c6f74fedf576a11b.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1414-2770d1155b664522.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1442-024f7e51804e0d7e.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1623-54c56cbe1afc3953.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1713-b3fdb241d0f3ae7a.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/179f4b987bc9083f.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1901-4d02d1f2a71cdbf7.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/193ac6435f936582.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1994-6637a121c9ee1602.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1a01cb4063a7b21e.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1ab4ccc7c0ba9eff.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1b8186fdb9bf9067.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1d3826d625e92c33.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1e0e6eb47fe60159.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1fe0596a309ad6cf.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2068-2c78bfc32dc0de5f.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2117-26a589a1115bdd0a.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2136-2c0d6e8c18d2c5c4.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2202-a83ad035a17401aa.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2227-5ae3f36b0a81c5b4.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/22e715061d511345.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2378-252212b7a5e313ce.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2409-e94c05c6f11bb939.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/249ef9d7a08bbfa1.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2500-811f2612ec5f6830.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/25d1ef14bd591cf9.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2618-062177b80fc4a38e.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2699-38ff37315d78ae04.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/26adfa4e8ffc85c7.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2703702968738794.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/27c7596aa0326b71.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2820-592c2b4ff874a913.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2901-0cdd0656eb7463d6.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/292-aaba6c4e7c8d416d.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2926-a9cb83e61fc8ad20.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2971c4658f1bcd7d.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2f04fe05bcb1c150.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2fdd60613421a228.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3014691f-ba91873bc8fe3fad.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/315cda92f466b9ec.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3242-6e6ec7e18f5d698d.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3367-58830187e9e5b9fa.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/337-bb33d149e9f461b3.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3454255bdea68dda.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3507-14fb4e6cd377d7da.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/353-e55516ea4730f9d4.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3567-9a29feedd7b63950.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/36ccc2b555a26ad4.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3709-34dbb332d3a3ac26.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3862-064a3fb795c75b62.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3afadb9a550fc886.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3b30ab8eaa03bc21.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3b4510be1f4cea1f.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3d2a01213eb1cc87.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3dad14bcec641ba8.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3f369c603677cd7a.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3f3fa56b5786d58c.css
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3f49d66311c27fe1.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/403c4d96324c23a6.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4077-50cf2a28a79fdcd4.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/40e89c053e10e01c.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4105-9c3c0ee7c494102f.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4188d520ca4e5f2b.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4262f254ec63c549.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4306-f891b96cf0ee333b.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4388-2f4ca3419d20af67.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4470-3ef8ade20eaf2875.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/44edba5625a9a9b4.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/450ebd094f4fa24d.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4537761df9dff7f0.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4559-52ca85b2d8893149.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/457923c551f21385.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4587f4ad9ebcbb4e.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/464560f129260d42.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/46901752d0b0dde9.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4758898ae55ecd92.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/47656bcac78a726c.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/47ed25bb99ff8a39.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4817-59d642defb0e86f2.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4865-c1c0885a93c327fa.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/496b84010c33cf69.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4980372eaa37b78b.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4995cc30215f504d.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4af6a1c366381700.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4b385187755a737f.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4bacf5b9194c12f5.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4e20891f2fd03463.css
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4ed86d695abe3c87.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/50779d2c65692de7.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5105-2998cbe1c9fc8ee4.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/511809a345b510d8.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5144-bbc18c43eade9aef.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5202-60292daf4bc5c8fb.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5319-5b2d4bf2dc450f99.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/53218dce8acb3bff.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5333-1540faf81c7d7006.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5365cf27e8d07577.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/536cb86ca75d1f30.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/542a1a209eb732c6.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/54731bb470e07604.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/557a369a3f213cfe.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5583bc893837fdf8.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/55c4117d5fcd0aae.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/565cdfe156dcb380.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5695-dbbcbf2da21d2bab.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5706-1e314cef9ea5c5d6.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/570b2e10aa856e54.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5733-aa80f52062105ad2.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5752-f504fb38ff5e13e8.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5869-426268ba6ad0ce0c.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/58b9eb1766fba8e0.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5945-8b3b7713d7f416a2.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5975-758334d6641b9c63.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5992-287cec06808c74ae.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5b2b7fd4dd9a44f3.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5b9c0b6d6c814e58.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5d1f33f9fa668633.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5d3e07ae5afa6fa6.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5d547ead001142ce.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5db1c5d0d0e548b4.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5eb6648cefff2d8a.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5f9c3b92a016f382.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6008d176e68995d6.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/605-102c0e6d8bb7517c.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/617bc18095fe8025.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6285-68f82c874b184eba.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6367dd1d1cf7eeef.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6399-ccf9cdbdcd5f7abb.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/63f40e445646cfa6.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/64f1a2ef9113d86f.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6537-f70f2c4278e93458.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6600-0ec5e2dc66d8b41a.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6609-3e081758ffbe3786.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/665-d94073042ee5b874.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6653-e61fdc06093fc0a8.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/66a190706fc6c35a.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/67570d9401e62846.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6774f9c1f201e744.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/69aeba649b0dc90f.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6c4c97f1ea6e7d77.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6d587e6e43260fc9.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6e033c78c15ab9a6.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7138-5b134dc8ad670770.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/72250192fd3153b7.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7271-46e4c11ee6b0a4d6.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/730305e005d7bd1d.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/738c339383c3b4b6.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/74982774ef38dcdb.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7526-da6b2857a3ca248d.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/766-baf0336e8ba5c686.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7688-ca173ea41812cf94.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/76a83e13dfaf23db.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/77d897b03fb96fa0.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7851-c10cbe6fcac2f9d6.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7906-59ba450db59c8efa.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7926-108623e14caeb770.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7ad0165018dc89ce.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7b788dd93ad868b3.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7e2badb3d178f837.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7e3f5ce4b2a613d4.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7e417dd24c8becd0.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7f9e9c54ac262de2.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8014-d6138fce46bba1e2.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8015668aa5f04beb.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8049-cb52b16664f13e28.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8135-881fe2cea0032570.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/814136f5b55e06b6.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8143-9e4312f059e9ed27.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/816-37c57b39f4e7ece1.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/81bf20526995284e.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/81e224efc874dea6.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8211-8dd5691abf54d0ca.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/82a6c2af12705c46.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/82ef36abe5e2e833.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/831fda51c425b4a8.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8354d717e34ebd6f.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/841e807b7dbb7e4f.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8473-7749355a9a4b1818.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/84884fbf517f5d74.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8529-6b66d5dba2148164.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8565-5c05f6bbb9d0662f.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8582-3a775364dbf07fa8.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/88c74f8b4b20d25a.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8992001a9a91bc67.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/89b9f8dbb6f0d490.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8a607e531e36f204.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8e12212d7a0aeaee.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9022b46fabff1181.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9028-2bfc9f09930a0d61.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9039-e44ff08ca4f37a12.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9140-09af618948244b82.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9145-9507437d5b599cea.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9264-fd8ab51d702e9535.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/92cf5d832080641f.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/93032856602932c1.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/936738f40fc24cc1.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/937c3b6cb00f6b79.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9409-6eefc92a7f8433ff.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9584-9d4fd7b3d6a7c9e7.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/97efd6e1c67bedcb.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9818-6f03d7efd4fb8533.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/983036f73d37142a.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9841-721a173be76941d1.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/99be180c22b927f8.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/99cf9cf99df5ccfc.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9cf03e6d4b5b806e.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9d9fbd3add7d0f88.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9f5ccd929375c1d6.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/a0f302271a793712.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/a1c3d7b907b7b731.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/a1ef280b7ad5ae6a.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/a21582fe1f52b973.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/a382857dbbcea5d1.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/a44b0c08814c45ae.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/a6bf78649679c265.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/a6dad97d9634a72d.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/a6dad97d9634a72d.js.map
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/a7aecb91c09b0e9a.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/a7b79d0fe43dcbd0.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/a7c0a41b6156d9b2.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/a966296c3a6b28f6.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/a9ebedc318fa36dc.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/ab7a826839e7e423.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/api-reference/page-a6a3e9e67b671303.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/experimental/api-playground/page-67c4f150eba92e64.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/experimental/budgets/page-e471b9b73cfc894b.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/experimental/caching/page-d31cc105402ab7e0.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/experimental/old-usage/page-5e097dbb8ce40bb4.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/experimental/prompts/page-8236f1efda3366f0.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/experimental/tag-management/page-5627fd94813402eb.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/guardrails/page-060e61cb783d32ef.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/layout-f7f722423efd1c5b.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/logs/page-5a10d46ca991b83e.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/model-hub/page-1479dcb217587498.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/models-and-endpoints/page-b69988590beaa5c8.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/organizations/page-56a03e123f452d60.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/playground/page-fc3dff494dc4db08.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/settings/admin-settings/page-746658933a633902.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/settings/logging-and-alerts/page-e3df74ef5ac0dcd5.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/settings/router-settings/page-ffa3245ebcbbc02a.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/settings/ui-theme/page-d833946961b065a5.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/teams/page-8d1a71afa9e9ff16.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/test-key/page-63cd64b722408984.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/tools/mcp-servers/page-7dd2ea6f1433d41f.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/tools/vector-stores/page-0f618cc2d6cae794.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/usage/page-d7532f354d44803b.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/users/page-6d3643cef6c068ee.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/virtual-keys/page-73444bcb0cfe86b7.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/_not-found/page-e8d298ce4e35f231.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/layout-bed96765a7fb7bdd.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/login/page-61bfa80619b62f6b.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/mcp/oauth/callback/page-01be1cae3559363d.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub/page-92347d2021ca8580.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub_table/page-2ad344049541235f.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/onboarding/page-17ecf5bf068f8157.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/page-587b0acf34f8c747.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/b5bcd87b218a6bcd.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/b64beb414bc36659.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/b85f190e8626c49c.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/ba5a05afc286361c.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/bdf355b41816a002.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/be340f56c7da1645.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/bf880fd979d4a2e6.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/c058ac3e89dc33df.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/c19d75622900fb62.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/c3d0c3b532b01699.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/c4111e97b0095227.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/c43ea300e1f2db88.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/c4452a79c69324a6.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/c6a3593fb6892e17.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/c7b74067c01ee971.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/c91982ee39ef0f77.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/c93c5c533dba84d1.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/c9af2deb434988d6.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/ca22b37c24b4d34a.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/cb8e6ba28461af15.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/ce8464047a8ce464.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/cf68fd1f1761ba48.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/cfc22f1e9e2830a5.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/d24f23929997cfa1.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/d2aa91699d95f4b3.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/d4240d7bae1e2b30.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/d4710ffa8fe96c6a.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/d64d74932cb225a3.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/d682c064a60ae3d6.css
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/d96012bcfc98706a.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/d991de8f2cd90aca.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/db89710f0ce96e05.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/dbca964212122d58.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/de9cdee2e8c8fa36.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/e007904603a33bc5.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/e0d42088ec18edc9.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/e1c5d2e47c042b8a.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/e1f23fd814ac3500.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/e1fe71b9ff3d3857.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/e228588e-635e9029d9d88215.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/e3bc795c751bb99a.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/e71fe358fd0c350f.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/e8ed72789c2b42ff.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/e96398764f77c728.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/eaf91f44e099fe65.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/eea976cf4a05fc92.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/ef41b5b82a37e553.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/f628c4bfd7854ec0.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/f98b25d79cd05714.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/fa8a1b9b6454c116.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/fb981bf7548d9de3.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/fd9d1056-a07bacd8fcc2728b.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/fe750aa0bf04912c.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/fea300adfdeaf3b9.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/ffd416b6dab7092c.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/ffe482191cf04a55.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/main-app-c6945ec5b2d5e671.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/main-e4e168e4dfadea03.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/turbopack-901b35f89c1f6751.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/css/83c095d0528a2e35.css
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/1bffadaabf893a1e-s.7cd81963.woff2
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/2bbe8d2671613f1f-s.76dcb0b2.woff2
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/2c55a0e60120577a-s.2a48534a.woff2
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/5476f68d60460930-s.c995e352.woff2
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/9c72aa0f40e4eef8-s.18a48cbc.woff2
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/ad66f9afd8947f86-s.7a40eb73.woff2
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/favicon.1d32c690.ico
delete mode 100644 litellm/proxy/_experimental/out/_next/static/zHD7JXLXiWgn1NPp82VmF/_buildManifest.js
create mode 100644 litellm/proxy/_experimental/out/_not-found.txt
create mode 100644 litellm/proxy/_experimental/out/_not-found/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/_not-found/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/_not-found/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/_not-found/__next._not-found.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/_not-found/__next._not-found.txt
create mode 100644 litellm/proxy/_experimental/out/_not-found/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/_not-found/index.html
create mode 100644 litellm/proxy/_experimental/out/api-reference/__next.!KGRhc2hib2FyZCk.api-reference.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/api-reference/__next.!KGRhc2hib2FyZCk.api-reference.txt
create mode 100644 litellm/proxy/_experimental/out/api-reference/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/api-reference/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/api-reference/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/api-reference/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/api-reference/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/assets/logos/s3_vector.png
create mode 100644 litellm/proxy/_experimental/out/assets/logos/zscaler.svg
create mode 100644 litellm/proxy/_experimental/out/experimental/api-playground/__next.!KGRhc2hib2FyZCk.experimental.api-playground.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/api-playground/__next.!KGRhc2hib2FyZCk.experimental.api-playground.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/api-playground/__next.!KGRhc2hib2FyZCk.experimental.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/api-playground/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/api-playground/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/api-playground/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/api-playground/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/api-playground/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/budgets/__next.!KGRhc2hib2FyZCk.experimental.budgets.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/budgets/__next.!KGRhc2hib2FyZCk.experimental.budgets.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/budgets/__next.!KGRhc2hib2FyZCk.experimental.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/budgets/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/budgets/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/budgets/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/budgets/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/budgets/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/caching/__next.!KGRhc2hib2FyZCk.experimental.caching.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/caching/__next.!KGRhc2hib2FyZCk.experimental.caching.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/caching/__next.!KGRhc2hib2FyZCk.experimental.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/caching/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/caching/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/caching/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/caching/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/caching/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/claude-code-plugins.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/claude-code-plugins/__next.!KGRhc2hib2FyZCk.experimental.claude-code-plugins.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/claude-code-plugins/__next.!KGRhc2hib2FyZCk.experimental.claude-code-plugins.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/claude-code-plugins/__next.!KGRhc2hib2FyZCk.experimental.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/claude-code-plugins/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/claude-code-plugins/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/claude-code-plugins/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/claude-code-plugins/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/claude-code-plugins/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/claude-code-plugins/index.html
create mode 100644 litellm/proxy/_experimental/out/experimental/old-usage/__next.!KGRhc2hib2FyZCk.experimental.old-usage.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/old-usage/__next.!KGRhc2hib2FyZCk.experimental.old-usage.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/old-usage/__next.!KGRhc2hib2FyZCk.experimental.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/old-usage/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/old-usage/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/old-usage/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/old-usage/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/old-usage/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/prompts/__next.!KGRhc2hib2FyZCk.experimental.prompts.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/prompts/__next.!KGRhc2hib2FyZCk.experimental.prompts.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/prompts/__next.!KGRhc2hib2FyZCk.experimental.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/prompts/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/prompts/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/prompts/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/prompts/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/prompts/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/tag-management/__next.!KGRhc2hib2FyZCk.experimental.tag-management.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/tag-management/__next.!KGRhc2hib2FyZCk.experimental.tag-management.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/tag-management/__next.!KGRhc2hib2FyZCk.experimental.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/tag-management/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/tag-management/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/tag-management/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/tag-management/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/experimental/tag-management/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/guardrails/__next.!KGRhc2hib2FyZCk.guardrails.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/guardrails/__next.!KGRhc2hib2FyZCk.guardrails.txt
create mode 100644 litellm/proxy/_experimental/out/guardrails/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/guardrails/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/guardrails/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/guardrails/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/guardrails/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/guardrails/index.html
create mode 100644 litellm/proxy/_experimental/out/login/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/login/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/login/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/login/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/login/__next.login.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/login/__next.login.txt
create mode 100644 litellm/proxy/_experimental/out/logs/__next.!KGRhc2hib2FyZCk.logs.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/logs/__next.!KGRhc2hib2FyZCk.logs.txt
create mode 100644 litellm/proxy/_experimental/out/logs/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/logs/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/logs/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/logs/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/logs/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/__next.mcp.oauth.callback.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/__next.mcp.oauth.callback.txt
create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/__next.mcp.oauth.txt
create mode 100644 litellm/proxy/_experimental/out/mcp/oauth/callback/__next.mcp.txt
create mode 100644 litellm/proxy/_experimental/out/model-hub/__next.!KGRhc2hib2FyZCk.model-hub.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/model-hub/__next.!KGRhc2hib2FyZCk.model-hub.txt
create mode 100644 litellm/proxy/_experimental/out/model-hub/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/model-hub/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/model-hub/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/model-hub/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/model-hub/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/model_hub/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/model_hub/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/model_hub/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/model_hub/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/model_hub/__next.model_hub.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/model_hub/__next.model_hub.txt
create mode 100644 litellm/proxy/_experimental/out/model_hub/index.html
create mode 100644 litellm/proxy/_experimental/out/model_hub_table/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/model_hub_table/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/model_hub_table/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/model_hub_table/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/model_hub_table/__next.model_hub_table.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/model_hub_table/__next.model_hub_table.txt
create mode 100644 litellm/proxy/_experimental/out/models-and-endpoints/__next.!KGRhc2hib2FyZCk.models-and-endpoints.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/models-and-endpoints/__next.!KGRhc2hib2FyZCk.models-and-endpoints.txt
create mode 100644 litellm/proxy/_experimental/out/models-and-endpoints/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/models-and-endpoints/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/models-and-endpoints/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/models-and-endpoints/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/models-and-endpoints/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/onboarding/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/onboarding/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/onboarding/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/onboarding/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/onboarding/__next.onboarding.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/onboarding/__next.onboarding.txt
create mode 100644 litellm/proxy/_experimental/out/onboarding/index.html
create mode 100644 litellm/proxy/_experimental/out/organizations/__next.!KGRhc2hib2FyZCk.organizations.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/organizations/__next.!KGRhc2hib2FyZCk.organizations.txt
create mode 100644 litellm/proxy/_experimental/out/organizations/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/organizations/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/organizations/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/organizations/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/organizations/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/playground/__next.!KGRhc2hib2FyZCk.playground.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/playground/__next.!KGRhc2hib2FyZCk.playground.txt
create mode 100644 litellm/proxy/_experimental/out/playground/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/playground/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/playground/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/playground/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/playground/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/policies.txt
create mode 100644 litellm/proxy/_experimental/out/policies/__next.!KGRhc2hib2FyZCk.policies.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/policies/__next.!KGRhc2hib2FyZCk.policies.txt
create mode 100644 litellm/proxy/_experimental/out/policies/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/policies/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/policies/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/policies/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/policies/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/policies/index.html
create mode 100644 litellm/proxy/_experimental/out/settings/admin-settings/__next.!KGRhc2hib2FyZCk.settings.admin-settings.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/settings/admin-settings/__next.!KGRhc2hib2FyZCk.settings.admin-settings.txt
create mode 100644 litellm/proxy/_experimental/out/settings/admin-settings/__next.!KGRhc2hib2FyZCk.settings.txt
create mode 100644 litellm/proxy/_experimental/out/settings/admin-settings/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/settings/admin-settings/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/settings/admin-settings/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/settings/admin-settings/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/settings/admin-settings/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/settings/logging-and-alerts/__next.!KGRhc2hib2FyZCk.settings.logging-and-alerts.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/settings/logging-and-alerts/__next.!KGRhc2hib2FyZCk.settings.logging-and-alerts.txt
create mode 100644 litellm/proxy/_experimental/out/settings/logging-and-alerts/__next.!KGRhc2hib2FyZCk.settings.txt
create mode 100644 litellm/proxy/_experimental/out/settings/logging-and-alerts/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/settings/logging-and-alerts/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/settings/logging-and-alerts/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/settings/logging-and-alerts/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/settings/logging-and-alerts/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/settings/router-settings/__next.!KGRhc2hib2FyZCk.settings.router-settings.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/settings/router-settings/__next.!KGRhc2hib2FyZCk.settings.router-settings.txt
create mode 100644 litellm/proxy/_experimental/out/settings/router-settings/__next.!KGRhc2hib2FyZCk.settings.txt
create mode 100644 litellm/proxy/_experimental/out/settings/router-settings/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/settings/router-settings/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/settings/router-settings/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/settings/router-settings/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/settings/router-settings/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/settings/ui-theme/__next.!KGRhc2hib2FyZCk.settings.txt
create mode 100644 litellm/proxy/_experimental/out/settings/ui-theme/__next.!KGRhc2hib2FyZCk.settings.ui-theme.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/settings/ui-theme/__next.!KGRhc2hib2FyZCk.settings.ui-theme.txt
create mode 100644 litellm/proxy/_experimental/out/settings/ui-theme/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/settings/ui-theme/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/settings/ui-theme/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/settings/ui-theme/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/settings/ui-theme/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/teams/__next.!KGRhc2hib2FyZCk.teams.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/teams/__next.!KGRhc2hib2FyZCk.teams.txt
create mode 100644 litellm/proxy/_experimental/out/teams/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/teams/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/teams/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/teams/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/teams/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/test-key/__next.!KGRhc2hib2FyZCk.test-key.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/test-key/__next.!KGRhc2hib2FyZCk.test-key.txt
create mode 100644 litellm/proxy/_experimental/out/test-key/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/test-key/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/test-key/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/test-key/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/test-key/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/tools/mcp-servers/__next.!KGRhc2hib2FyZCk.tools.mcp-servers.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/tools/mcp-servers/__next.!KGRhc2hib2FyZCk.tools.mcp-servers.txt
create mode 100644 litellm/proxy/_experimental/out/tools/mcp-servers/__next.!KGRhc2hib2FyZCk.tools.txt
create mode 100644 litellm/proxy/_experimental/out/tools/mcp-servers/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/tools/mcp-servers/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/tools/mcp-servers/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/tools/mcp-servers/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/tools/mcp-servers/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/tools/vector-stores/__next.!KGRhc2hib2FyZCk.tools.txt
create mode 100644 litellm/proxy/_experimental/out/tools/vector-stores/__next.!KGRhc2hib2FyZCk.tools.vector-stores.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/tools/vector-stores/__next.!KGRhc2hib2FyZCk.tools.vector-stores.txt
create mode 100644 litellm/proxy/_experimental/out/tools/vector-stores/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/tools/vector-stores/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/tools/vector-stores/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/tools/vector-stores/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/tools/vector-stores/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/usage/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/usage/__next.!KGRhc2hib2FyZCk.usage.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/usage/__next.!KGRhc2hib2FyZCk.usage.txt
create mode 100644 litellm/proxy/_experimental/out/usage/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/usage/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/usage/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/usage/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/users/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/users/__next.!KGRhc2hib2FyZCk.users.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/users/__next.!KGRhc2hib2FyZCk.users.txt
create mode 100644 litellm/proxy/_experimental/out/users/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/users/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/users/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/users/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/virtual-keys/__next.!KGRhc2hib2FyZCk.txt
create mode 100644 litellm/proxy/_experimental/out/virtual-keys/__next.!KGRhc2hib2FyZCk.virtual-keys.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/virtual-keys/__next.!KGRhc2hib2FyZCk.virtual-keys.txt
create mode 100644 litellm/proxy/_experimental/out/virtual-keys/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/virtual-keys/__next._head.txt
create mode 100644 litellm/proxy/_experimental/out/virtual-keys/__next._index.txt
create mode 100644 litellm/proxy/_experimental/out/virtual-keys/__next._tree.txt
create mode 100644 litellm/proxy/agent_endpoints/a2a_routing.py
create mode 100644 litellm/proxy/agent_endpoints/model_list_helpers.py
create mode 100644 litellm/proxy/auth/ip_address_utils.py
create mode 100644 litellm/proxy/common_utils/cache_coordinator.py
create mode 100644 litellm/proxy/compliance_checks.py
create mode 100644 litellm/proxy/example_config_yaml/pipeline_test_guardrails.py
create mode 100644 litellm/proxy/example_config_yaml/test_pipeline_config.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/custom_code/__init__.py
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/custom_code/primitives.py
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/harm_toxic_abuse.json
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/harm_toxic_abuse_au.json
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/harm_toxic_abuse_de.json
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/harm_toxic_abuse_es.json
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/harm_toxic_abuse_fr.json
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/harmful_child_safety.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_data_exfiltration.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_jailbreak.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_malicious_code.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_sql.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/categories/prompt_injection_system_prompt.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/airline_brand_protection.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/aviation_safety_topics.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/eu_ai_act_art5_biometric_profiling.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/eu_ai_act_art5_biometric_profiling_fr.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/eu_ai_act_art5_emotion_recognition.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/eu_ai_act_art5_emotion_recognition_fr.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/eu_ai_act_art5_manipulation.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/eu_ai_act_art5_manipulation_fr.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/eu_ai_act_art5_social_scoring.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/eu_ai_act_art5_social_scoring_fr.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/eu_ai_act_art5_vulnerability.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/eu_ai_act_art5_vulnerability_fr.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/eu_ai_act_article5.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/eu_ai_act_article5_fr.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/uae_anti_discrimination.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/policy_templates/uae_cultural_sensitivity.yaml
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/mcp_security/__init__.py
create mode 100644 litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py
create mode 100644 litellm/proxy/hooks/mcp_semantic_filter/ARCHITECTURE.md
create mode 100644 litellm/proxy/hooks/mcp_semantic_filter/__init__.py
create mode 100644 litellm/proxy/hooks/mcp_semantic_filter/hook.py
create mode 100644 litellm/proxy/management_endpoints/access_group_endpoints.py
create mode 100644 litellm/proxy/management_endpoints/compliance_endpoints.py
create mode 100644 litellm/proxy/management_endpoints/policy_endpoints.py
create mode 100644 litellm/proxy/mcp_registry.json
create mode 100644 litellm/proxy/openai_evals_endpoints/__init__.py
create mode 100644 litellm/proxy/openai_evals_endpoints/endpoints.py
create mode 100644 litellm/proxy/pass_through_endpoints/architecture.md
create mode 100644 litellm/proxy/policy_engine/__init__.py
create mode 100644 litellm/proxy/policy_engine/architecture.md
create mode 100644 litellm/proxy/policy_engine/attachment_registry.py
create mode 100644 litellm/proxy/policy_engine/condition_evaluator.py
create mode 100644 litellm/proxy/policy_engine/init_policies.py
create mode 100644 litellm/proxy/policy_engine/pipeline_executor.py
create mode 100644 litellm/proxy/policy_engine/policy_endpoints.py
create mode 100644 litellm/proxy/policy_engine/policy_matcher.py
create mode 100644 litellm/proxy/policy_engine/policy_registry.py
create mode 100644 litellm/proxy/policy_engine/policy_resolve_endpoints.py
create mode 100644 litellm/proxy/policy_engine/policy_resolver.py
create mode 100644 litellm/proxy/policy_engine/policy_validator.py
create mode 100644 litellm/proxy_auth/__init__.py
create mode 100644 litellm/proxy_auth/credentials.py
create mode 100644 litellm/rag/ingestion/file_parsers/__init__.py
create mode 100644 litellm/rag/ingestion/file_parsers/pdf_parser.py
create mode 100644 litellm/rag/ingestion/s3_vectors_ingestion.py
create mode 100644 litellm/rag/ingestion/vertex_ai_ingestion.py
create mode 100644 litellm/router_utils/pre_call_checks/deployment_affinity_check.py
create mode 100644 litellm/router_utils/pre_call_checks/model_rate_limit_check.py
create mode 100644 litellm/types/access_group.py
create mode 100644 litellm/types/llms/openai_evals.py
create mode 100644 litellm/types/llms/xai.py
create mode 100644 litellm/types/policy_engine.py
create mode 100644 litellm/types/proxy/compliance_endpoints.py
create mode 100644 litellm/types/proxy/guardrails/guardrail_hooks/zscaler_ai_guard.py
create mode 100644 litellm/types/proxy/management_endpoints/key_management_endpoints.py
create mode 100644 litellm/types/proxy/policy_engine/__init__.py
create mode 100644 litellm/types/proxy/policy_engine/pipeline_types.py
create mode 100644 litellm/types/proxy/policy_engine/policy_types.py
create mode 100644 litellm/types/proxy/policy_engine/resolver_types.py
create mode 100644 litellm/types/proxy/policy_engine/validation_types.py
create mode 100644 policy_templates.json
delete mode 100644 proxy_config.yaml
rename tests/agent_tests/{ => local_only_agent_tests}/local_vertex_agent.py (100%)
rename tests/agent_tests/{ => local_only_agent_tests}/test_a2a.py (100%)
rename tests/agent_tests/{ => local_only_agent_tests}/test_a2a_completion_bridge.py (100%)
create mode 100644 tests/agent_tests/test_a2a_agent.py
create mode 100644 tests/batches_tests/test_batch_custom_pricing.py
create mode 100644 tests/code_coverage_tests/check_guardrail_apply_decorator.py
create mode 100644 tests/guardrails_tests/test_eu_ai_act_article5.py
create mode 100644 tests/guardrails_tests/test_eu_ai_act_french_3_scenarios.py
create mode 100644 tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py
create mode 100644 tests/litellm/llms/anthropic/test_anthropic_schema_filter.py
create mode 100644 tests/litellm/llms/bedrock/test_nova_imported_models.py
create mode 100644 tests/litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
create mode 100644 tests/litellm/proxy/management_endpoints/test_common_utils.py
create mode 100644 tests/litellm/proxy/test_init_litellm_callbacks.py
create mode 100644 tests/litellm/test_batch_completion_models_all_responses.py
create mode 100644 tests/litellm/test_bedrock_extended_beta_models.py
create mode 100644 tests/litellm/test_proxy_auth.py
create mode 100644 tests/litellm/test_router_retry_backoff_headers.py
create mode 100644 tests/litellm/test_stream_chunk_builder_images.py
create mode 100644 tests/litellm_core_utils/test_bedrock_converse_dedup_factory.py
create mode 100644 tests/llm_translation/realtime/__init__.py
create mode 100644 tests/llm_translation/realtime/base_realtime_tests.py
rename tests/llm_translation/{ => realtime}/test_openai_realtime.py (100%)
create mode 100644 tests/llm_translation/realtime/test_openai_realtime_simple.py
create mode 100644 tests/llm_translation/realtime/test_xai_realtime.py
create mode 100644 tests/llm_translation/test_a2a.py
create mode 100644 tests/llm_translation/test_bedrock_anthropic_regression.py
create mode 100644 tests/llm_translation/test_evals_api.py
create mode 100644 tests/llm_translation/test_hosted_vllm_embedding_e2e.py
create mode 100644 tests/llm_translation/test_model_cost_map_resilience.py
delete mode 100644 tests/local_testing/test_literalai.py
create mode 100644 tests/logging_callback_tests/test_dynamic_otel_keys.py
create mode 100644 tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py
create mode 100644 tests/mcp_tests/test_oauth2_mcp_config.yaml
create mode 100644 tests/mcp_tests/test_openapi_spec_path_url.py
create mode 100644 tests/mcp_tests/test_semantic_tool_filter_e2e.py
create mode 100644 tests/pass_through_unit_tests/test_passthrough_registry_updates.py
delete mode 100644 tests/proxy_admin_ui_tests/ui_unit_tests/log_filter_logic.test.tsx
create mode 100644 tests/proxy_e2e_anthropic_messages_tests/test_all_beta_headers.py
create mode 100644 tests/proxy_e2e_anthropic_messages_tests/test_claude_agent_sdk.py
create mode 100644 tests/proxy_e2e_anthropic_messages_tests/test_config.yaml
create mode 100644 tests/proxy_unit_tests/test_get_image.py
create mode 100644 tests/proxy_unit_tests/test_server_root_path.py
create mode 100644 tests/proxy_unit_tests/test_ui_path_detection.py
create mode 100644 tests/proxy_unit_tests/test_zero_cost_model_budget_bypass.py
create mode 100644 tests/router_unit_tests/test_get_model_list_alias_optimization.py
create mode 100644 tests/router_unit_tests/test_router_acancel_batch.py
create mode 100644 tests/search_tests/test_duckduckgo_search.py
create mode 100644 tests/test_litellm/a2a_protocol/test_card_resolver.py
create mode 100644 tests/test_litellm/caching/test_dual_cache.py
create mode 100644 tests/test_litellm/enterprise/proxy/test_afile_retrieve_returns_unified_id.py
create mode 100644 tests/test_litellm/enterprise/proxy/test_batch_retrieve_input_file_id.py
create mode 100644 tests/test_litellm/enterprise/proxy/test_batch_retrieve_returns_unified_input_file_id.py
create mode 100644 tests/test_litellm/enterprise/proxy/test_deleted_file_returns_403_not_404.py
create mode 100644 tests/test_litellm/enterprise/proxy/test_file_deletion_blocking.py
create mode 100644 tests/test_litellm/enterprise/proxy/test_managed_files_access_check.py
create mode 100644 tests/test_litellm/enterprise/proxy/test_managed_files_hook.py
create mode 100644 tests/test_litellm/expected_responses_api_request/context_management_and_shell.json
create mode 100644 tests/test_litellm/integrations/arize/test_arize_otel_coexistence.py
create mode 100644 tests/test_litellm/integrations/datadog/test_datadog_tags_regression.py
create mode 100644 tests/test_litellm/integrations/test_prometheus_client_ip_user_agent.py
create mode 100644 tests/test_litellm/integrations/test_prometheus_missing_metrics.py
create mode 100644 tests/test_litellm/integrations/test_prometheus_none_metadata.py
create mode 100644 tests/test_litellm/integrations/websearch_interception/test_websearch_chat_completion.py
create mode 100644 tests/test_litellm/litellm_core_utils/test_get_litellm_params.py
create mode 100644 tests/test_litellm/litellm_core_utils/test_redact_messages.py
create mode 100644 tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py
rename test_anthropic_messages_structured_outputs_minimal.py => tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_structured_outputs.py (98%)
create mode 100644 tests/test_litellm/llms/anthropic/test_azure_ai_cache_pricing.py
create mode 100644 tests/test_litellm/llms/anthropic/test_message_sanitization.py
create mode 100644 tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_count_tokens_transformation.py
create mode 100644 tests/test_litellm/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py
create mode 100644 tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py
create mode 100644 tests/test_litellm/llms/bedrock/chat/test_streaming_choice_index.py
create mode 100644 tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py
create mode 100644 tests/test_litellm/llms/databricks/responses/__init__.py
create mode 100644 tests/test_litellm/llms/databricks/responses/test_databricks_responses_transformation.py
create mode 100644 tests/test_litellm/llms/gemini/files/__init__.py
create mode 100644 tests/test_litellm/llms/gemini/files/test_gemini_files_transformation.py
create mode 100644 tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_ssl_verify.py
create mode 100644 tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_ssl_verify.py
create mode 100644 tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_transformation.py
create mode 100644 tests/test_litellm/llms/openai/embeddings/guardrail_translation/test_embeddings_guardrail_handler.py
create mode 100644 tests/test_litellm/llms/openai/evals/__init__.py
create mode 100644 tests/test_litellm/llms/openai/evals/test_openai_evals_transformation.py
create mode 100644 tests/test_litellm/llms/openai/realtime/README.md
create mode 100644 tests/test_litellm/llms/openai_like/embedding/__init__.py
create mode 100644 tests/test_litellm/llms/openai_like/embedding/test_openai_like_embedding.py
create mode 100644 tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py
create mode 100644 tests/test_litellm/llms/s3_vectors/__init__.py
create mode 100644 tests/test_litellm/llms/s3_vectors/vector_stores/__init__.py
create mode 100644 tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py
create mode 100644 tests/test_litellm/llms/test_cache_control_and_reasoning.py
create mode 100644 tests/test_litellm/llms/test_lifecycle_fix.py
create mode 100644 tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/qwen/__init__.py
create mode 100644 tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/qwen/test_vertex_ai_qwen_global_endpoint.py
create mode 100644 tests/test_litellm/llms/watsonx/rerank/__init__.py
create mode 100644 tests/test_litellm/llms/watsonx/rerank/test_watsonx_rerank.py
create mode 100644 tests/test_litellm/llms/watsonx/test_watsonx_common_utils.py
create mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/test_jwt_mcp_enforcement.py
create mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/test_jwt_mcp_simple.py
create mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_debug.py
create mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_discovery.py
create mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py
create mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/test_oauth2_token_cache.py
create mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py
create mode 100644 tests/test_litellm/proxy/agent_endpoints/test_model_list_helpers.py
create mode 100644 tests/test_litellm/proxy/auth/test_cli_auth.py
create mode 100644 tests/test_litellm/proxy/auth/test_mcp_ip_filtering.py
create mode 100644 tests/test_litellm/proxy/auth/test_object_permission_loading.py
create mode 100644 tests/test_litellm/proxy/google_endpoints/test_interactions_agent_param.py
create mode 100644 tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_eu_patterns.py
create mode 100644 tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_gdpr_policy_e2e.py
create mode 100644 tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_openai_moderation_streaming.py
create mode 100644 tests/test_litellm/proxy/guardrails/guardrail_hooks/test_mcp_security.py
create mode 100644 tests/test_litellm/proxy/hooks/test_image_generation_guardrails.py
create mode 100644 tests/test_litellm/proxy/hooks/test_post_call_response_headers_hook.py
create mode 100644 tests/test_litellm/proxy/hooks/test_post_call_streaming_hook_integration.py
create mode 100644 tests/test_litellm/proxy/hooks/test_post_call_success_hook_integration.py
create mode 100644 tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_discovery.py
create mode 100644 tests/test_litellm/proxy/management_endpoints/search_endpoints/test_search_tool_management.py
create mode 100644 tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py
create mode 100644 tests/test_litellm/proxy/management_endpoints/test_common_utils.py
create mode 100644 tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py
create mode 100644 tests/test_litellm/proxy/management_endpoints/test_policy_endpoints.py
create mode 100644 tests/test_litellm/proxy/middleware/test_prometheus_auth_middleware_asgi.py
create mode 100644 tests/test_litellm/proxy/policy_engine/__init__.py
create mode 100644 tests/test_litellm/proxy/policy_engine/test_attachment_registry.py
create mode 100644 tests/test_litellm/proxy/policy_engine/test_condition_evaluator.py
create mode 100644 tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
create mode 100644 tests/test_litellm/proxy/policy_engine/test_policy_matcher.py
create mode 100644 tests/test_litellm/proxy/policy_engine/test_policy_resolver.py
create mode 100644 tests/test_litellm/proxy/policy_engine/test_policy_validator.py
create mode 100644 tests/test_litellm/proxy/rag_endpoints/__init__.py
create mode 100644 tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py
create mode 100644 tests/test_litellm/proxy/test_api_key_masking_in_errors.py
create mode 100644 tests/test_litellm/proxy/test_chat_completion_metadata.py
create mode 100644 tests/test_litellm/proxy/test_pyroscope.py
create mode 100644 tests/test_litellm/proxy/test_response_model_sanitization.py
create mode 100644 tests/test_litellm/proxy/test_route_a2a_models.py
create mode 100644 tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_access_control.py
create mode 100644 tests/test_litellm/responses/test_metadata_codex_callback.py
create mode 100644 tests/test_litellm/responses/test_responses_api_request_body.py
create mode 100644 tests/test_litellm/router_strategy/test_budget_limiter_hotpath.py
create mode 100644 tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py
create mode 100644 tests/test_litellm/secret_managers/test_aws_secret_manager_rotation.py
create mode 100644 tests/test_litellm/test_a2a_registry_lookup.py
create mode 100644 tests/test_litellm/test_anthropic_beta_headers_filtering.py
create mode 100644 tests/test_litellm/test_claude_opus_4_6_config.py
create mode 100644 tests/test_litellm/test_deepseek_model_metadata.py
create mode 100644 tests/test_litellm/test_exception_exports.py
create mode 100644 tests/test_litellm/test_model_param_helper.py
create mode 100644 tests/test_litellm/test_router/test_enforce_model_rate_limits.py
create mode 100644 tests/test_litellm/test_router_model_cost_isolation.py
create mode 100644 tests/test_litellm/test_router_redis_init.py
create mode 100644 tests/test_litellm/test_router_silent_experiment.py
create mode 100644 tests/test_litellm/test_service_logger.py
create mode 100644 tests/test_litellm/test_xai_responses_auto_routing.py
create mode 100644 tests/test_litellm/types/__init__.py
create mode 100644 tests/test_litellm/types/proxy/__init__.py
create mode 100644 tests/test_litellm/types/proxy/policy_engine/__init__.py
create mode 100644 tests/test_litellm/types/proxy/policy_engine/test_pipeline_types.py
create mode 100644 tests/test_litellm/types/proxy/policy_engine/test_resolver_types.py
create mode 100644 tests/test_otel_thread_leak.py
create mode 100644 tests/test_presidio_latency.py
create mode 100644 tests/vector_store_tests/rag/test_rag_s3_vectors.py
create mode 100644 tests/vector_store_tests/test_s3_vectors_vector_store.py
create mode 100644 ui/litellm-dashboard/e2e_tests/tests/keys/deleteKey.spec.ts
create mode 100644 ui/litellm-dashboard/e2e_tests/tests/keys/regenerateKey.spec.ts
create mode 100644 ui/litellm-dashboard/e2e_tests/tests/keys/updateKeyLimits.spec.ts
create mode 100644 ui/litellm-dashboard/knip.json
create mode 100644 ui/litellm-dashboard/public/assets/logos/s3_vector.png
create mode 100644 ui/litellm-dashboard/public/assets/logos/zscaler.svg
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/accessGroups/useAccessGroupDetails.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/accessGroups/useAccessGroups.test.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/accessGroups/useAccessGroups.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/accessGroups/useCreateAccessGroup.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/accessGroups/useDeleteAccessGroup.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/accessGroups/useEditAccessGroup.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroCreate.test.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroDryRun.test.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroExport.test.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings.test.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/logDetails/useLogDetails.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpSemanticFilterSettings/useMCPSemanticFilterSettings.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpSemanticFilterSettings/useUpdateMCPSemanticFilterSettings.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPAccessGroups.test.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServers.test.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/proxyConfig/useProxyConfig.test.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/proxyConfig/useProxyConfig.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useEditSSOSettings.test.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useSSOSettings.test.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/storeRequestInSpendLogs/useStoreRequestInSpendLogs.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/uiSettings/useUpdateUISettings.test.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/useDisableShowNewBadge.test.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/useDisableShowPrompts.test.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/useDisableShowPrompts.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/useDisableUsageIndicator.test.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/useDisableUsageIndicator.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/users/useUsers.test.ts
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/users/useUsers.ts
delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelAnalyticsTab/FilterByContent.tsx
delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelAnalyticsTab/ModelAnalyticsTab.tsx
create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/policies/page.tsx
create mode 100644 ui/litellm-dashboard/src/components/AccessGroups/AccessGroupsDetailsPage.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/AccessGroups/AccessGroupsDetailsPage.tsx
create mode 100644 ui/litellm-dashboard/src/components/AccessGroups/AccessGroupsModal/AccessGroupBaseForm.tsx
create mode 100644 ui/litellm-dashboard/src/components/AccessGroups/AccessGroupsModal/AccessGroupCreateModal.tsx
create mode 100644 ui/litellm-dashboard/src/components/AccessGroups/AccessGroupsModal/AccessGroupEditModal.tsx
create mode 100644 ui/litellm-dashboard/src/components/AccessGroups/AccessGroupsPage.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/AccessGroups/AccessGroupsPage.tsx
create mode 100644 ui/litellm-dashboard/src/components/AccessGroups/types.ts
create mode 100644 ui/litellm-dashboard/src/components/AdminPanel.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/AdminPanel.tsx
create mode 100644 ui/litellm-dashboard/src/components/BulkEditUsers.test.tsx
rename ui/litellm-dashboard/src/components/{bulk_edit_user.tsx => BulkEditUsers.tsx} (98%)
create mode 100644 ui/litellm-dashboard/src/components/CreateUserButton.test.tsx
rename ui/litellm-dashboard/src/components/{create_user_button.tsx => CreateUserButton.tsx} (80%)
create mode 100644 ui/litellm-dashboard/src/components/ModelSelect/PaginatedModelSelect/PaginatedModelSelect.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/ModelSelect/PaginatedModelSelect/PaginatedModelSelect.tsx
create mode 100644 ui/litellm-dashboard/src/components/Navbar/CommunityEngagementButtons/CommunityEngagementButtons.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/Navbar/CommunityEngagementButtons/CommunityEngagementButtons.tsx
create mode 100644 ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx
rename ui/litellm-dashboard/src/components/{search_tools/create_search_tool.tsx => SearchTools/CreateSearchTools.tsx} (97%)
rename ui/litellm-dashboard/src/components/{search_tools/search_connection_test.tsx => SearchTools/SearchConnectionTest.tsx} (98%)
create mode 100644 ui/litellm-dashboard/src/components/SearchTools/SearchToolColumn.tsx
create mode 100644 ui/litellm-dashboard/src/components/SearchTools/SearchToolTester.test.tsx
rename ui/litellm-dashboard/src/components/{search_tools/search_tool_tester.tsx => SearchTools/SearchToolTester.tsx} (100%)
create mode 100644 ui/litellm-dashboard/src/components/SearchTools/SearchToolView.test.tsx
rename ui/litellm-dashboard/src/components/{search_tools/search_tool_view.tsx => SearchTools/SearchToolView.tsx} (86%)
create mode 100644 ui/litellm-dashboard/src/components/SearchTools/SearchTools.test.tsx
rename ui/litellm-dashboard/src/components/{search_tools/search_tools.tsx => SearchTools/SearchTools.tsx} (78%)
create mode 100644 ui/litellm-dashboard/src/components/SearchTools/index.tsx
rename ui/litellm-dashboard/src/components/{search_tools => SearchTools}/types.tsx (96%)
create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings.tsx
create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterTestPanel.tsx
create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/semanticFilterTestUtils.ts
create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/PageVisibilitySettings.tsx
create mode 100644 ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.tsx
create mode 100644 ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacksModal.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacksModal.tsx
create mode 100644 ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx
create mode 100644 ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx
create mode 100644 ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx
create mode 100644 ui/litellm-dashboard/src/components/UsageIndicator.test.tsx
rename ui/litellm-dashboard/src/components/{usage_indicator.tsx => UsageIndicator.tsx} (73%)
create mode 100644 ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/SpendByProvider.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/SpendByProvider.tsx
create mode 100644 ui/litellm-dashboard/src/components/UsagePage/components/KeyModelUsageView.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/UsagePage/components/KeyModelUsageView.tsx
delete mode 100644 ui/litellm-dashboard/src/components/add_fallbacks.test.tsx
delete mode 100644 ui/litellm-dashboard/src/components/add_fallbacks.tsx
create mode 100644 ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.tsx
delete mode 100644 ui/litellm-dashboard/src/components/add_model/router_config_builder.tsx
delete mode 100644 ui/litellm-dashboard/src/components/admins.tsx
create mode 100644 ui/litellm-dashboard/src/components/common_components/AccessGroupSelector.tsx
create mode 100644 ui/litellm-dashboard/src/components/common_components/DefaultProxyAdminTag.tsx
create mode 100644 ui/litellm-dashboard/src/components/common_components/KeyLifecycleSettings.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx
create mode 100644 ui/litellm-dashboard/src/components/common_components/TableHeaderSortDropdown/TableHeaderSortDropdown.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/common_components/TableHeaderSortDropdown/TableHeaderSortDropdown.tsx
delete mode 100644 ui/litellm-dashboard/src/components/common_components/all_view.tsx
create mode 100644 ui/litellm-dashboard/src/components/common_components/chartUtils.test.tsx
delete mode 100644 ui/litellm-dashboard/src/components/common_components/default_org.tsx
delete mode 100644 ui/litellm-dashboard/src/components/common_components/user_form.tsx
delete mode 100644 ui/litellm-dashboard/src/components/create_user_button.test.tsx
delete mode 100644 ui/litellm-dashboard/src/components/dashboard_default_team.tsx
delete mode 100644 ui/litellm-dashboard/src/components/delete_model_button.tsx
delete mode 100644 ui/litellm-dashboard/src/components/edit_model/edit_model_modal.tsx
delete mode 100644 ui/litellm-dashboard/src/components/enter_proxy_url.tsx
delete mode 100644 ui/litellm-dashboard/src/components/fallbacks.test.tsx
delete mode 100644 ui/litellm-dashboard/src/components/fallbacks.tsx
create mode 100644 ui/litellm-dashboard/src/components/guardrails/content_filter/CategoryTable.tsx
create mode 100644 ui/litellm-dashboard/src/components/guardrails/custom_code/CustomCodeEditor.tsx
create mode 100644 ui/litellm-dashboard/src/components/guardrails/custom_code/CustomCodeModal.tsx
create mode 100644 ui/litellm-dashboard/src/components/guardrails/custom_code/CustomCodePlayground.tsx
create mode 100644 ui/litellm-dashboard/src/components/guardrails/custom_code/custom_code_constants.ts
create mode 100644 ui/litellm-dashboard/src/components/guardrails/custom_code/index.ts
delete mode 100644 ui/litellm-dashboard/src/components/key_team_helpers/organization_search_fn.tsx
delete mode 100644 ui/litellm-dashboard/src/components/key_team_helpers/team_search_fn.tsx
delete mode 100644 ui/litellm-dashboard/src/components/mcp_connection_test.tsx
create mode 100644 ui/litellm-dashboard/src/components/mcp_tools/MCPNetworkSettings.tsx
create mode 100644 ui/litellm-dashboard/src/components/mcp_tools/MCPToolArgumentsForm.tsx
create mode 100644 ui/litellm-dashboard/src/components/mcp_tools/OAuthFormFields.tsx
create mode 100644 ui/litellm-dashboard/src/components/mcp_tools/mcp_discovery.tsx
create mode 100644 ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx
delete mode 100644 ui/litellm-dashboard/src/components/model_add/CredentialDeleteModal.tsx
create mode 100644 ui/litellm-dashboard/src/components/model_dashboard/all_models_table.tsx
create mode 100644 ui/litellm-dashboard/src/components/molecules/filter.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/molecules/models/columns.test.tsx
delete mode 100644 ui/litellm-dashboard/src/components/organization/add_org_admin.tsx
delete mode 100644 ui/litellm-dashboard/src/components/organization/view_members_of_org.tsx
create mode 100644 ui/litellm-dashboard/src/components/page_metadata.ts
create mode 100644 ui/litellm-dashboard/src/components/page_utils.test.ts
create mode 100644 ui/litellm-dashboard/src/components/page_utils.ts
create mode 100644 ui/litellm-dashboard/src/components/playground/complianceUI/ComplianceUI.tsx
create mode 100644 ui/litellm-dashboard/src/components/policies/PolicySelector.tsx
create mode 100644 ui/litellm-dashboard/src/components/policies/add_attachment_form.tsx
create mode 100644 ui/litellm-dashboard/src/components/policies/add_policy_form.tsx
create mode 100644 ui/litellm-dashboard/src/components/policies/attachment_table.tsx
create mode 100644 ui/litellm-dashboard/src/components/policies/build_attachment_data.ts
create mode 100644 ui/litellm-dashboard/src/components/policies/guardrail_selection_modal.tsx
create mode 100644 ui/litellm-dashboard/src/components/policies/impact_popover.tsx
create mode 100644 ui/litellm-dashboard/src/components/policies/impact_preview_alert.tsx
create mode 100644 ui/litellm-dashboard/src/components/policies/index.tsx
create mode 100644 ui/litellm-dashboard/src/components/policies/pipeline_flow_builder.tsx
create mode 100644 ui/litellm-dashboard/src/components/policies/policy_info.tsx
create mode 100644 ui/litellm-dashboard/src/components/policies/policy_table.tsx
create mode 100644 ui/litellm-dashboard/src/components/policies/policy_templates.tsx
create mode 100644 ui/litellm-dashboard/src/components/policies/policy_test_panel.tsx
create mode 100644 ui/litellm-dashboard/src/components/policies/template_parameter_modal.tsx
create mode 100644 ui/litellm-dashboard/src/components/policies/types.ts
create mode 100644 ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.tsx
delete mode 100644 ui/litellm-dashboard/src/components/search_tools/index.tsx
delete mode 100644 ui/litellm-dashboard/src/components/search_tools/search_tool_columns.tsx
create mode 100644 ui/litellm-dashboard/src/components/survey/NudgePrompt.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/team/TeamInfo.tsx
create mode 100644 ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx
create mode 100644 ui/litellm-dashboard/src/components/team/tabVisibilityUtils.test.ts
create mode 100644 ui/litellm-dashboard/src/components/team/tabVisibilityUtils.ts
delete mode 100644 ui/litellm-dashboard/src/components/team/team_info.test.tsx
delete mode 100644 ui/litellm-dashboard/src/components/team/team_info.tsx
delete mode 100644 ui/litellm-dashboard/src/components/team/team_member_view.test.tsx
delete mode 100644 ui/litellm-dashboard/src/components/team/team_member_view.tsx
delete mode 100644 ui/litellm-dashboard/src/components/templates/model_dashboard.tsx
delete mode 100644 ui/litellm-dashboard/src/components/usage_indicator.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/user_edit_view.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/vector_store_management/CreateVectorStore.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/vector_store_management/CreateVectorStore.tsx
create mode 100644 ui/litellm-dashboard/src/components/vector_store_management/DocumentsTable.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/vector_store_management/DocumentsTable.tsx
create mode 100644 ui/litellm-dashboard/src/components/vector_store_management/S3VectorsConfig.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/vector_store_management/S3VectorsConfig.tsx
create mode 100644 ui/litellm-dashboard/src/components/vector_store_management/TestVectorStoreTab.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/vector_store_management/TestVectorStoreTab.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/CompliancePanel.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/CollapsibleMessage.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/DrawerHeader.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/HistorySection.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/HistoryTree.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/InputCard.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/InputCard.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/JsonViewer.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/MessageBlock.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/MessageCard.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/OutputCard.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/OutputCard.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SectionHeader.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SimpleMessageBlock.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SimpleToolCallBlock.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TokenFlow.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ToolCallBlock.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/ToolCallCard.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TruncatedValue.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/constants.ts
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/index.ts
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesTypes.ts
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesUtils.ts
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/useKeyboardNavigation.ts
create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/utils.ts
delete mode 100644 ui/litellm-dashboard/src/components/view_logs/SessionView.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/SpendLogsSettingsModal/SpendLogsSettingsModal.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/SpendLogsSettingsModal/SpendLogsSettingsModal.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/ToolsSection/FormattedToolView.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/ToolsSection/JsonToolView.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/ToolsSection/ToolExpandedContent.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/ToolsSection/ToolItem.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/ToolsSection/ToolsSection.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/ToolsSection/ToolsSection.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/ToolsSection/index.ts
create mode 100644 ui/litellm-dashboard/src/components/view_logs/ToolsSection/types.ts
create mode 100644 ui/litellm-dashboard/src/components/view_logs/ToolsSection/utils.test.ts
create mode 100644 ui/litellm-dashboard/src/components/view_logs/ToolsSection/utils.ts
create mode 100644 ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/constants.ts
delete mode 100644 ui/litellm-dashboard/src/components/view_logs/country_cell.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/log_filter_logic.test.tsx
create mode 100644 ui/litellm-dashboard/src/components/view_logs/utils.ts
create mode 100644 ui/litellm-dashboard/src/contexts/AntdGlobalProvider.tsx
create mode 100644 ui/litellm-dashboard/src/data/compliancePrompts.ts
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 02a9e3b0714..188b02c9f1c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -21,9 +21,7 @@ commands:
- run:
name: "Install local version of litellm-enterprise"
command: |
- cd enterprise
- python -m pip install -e .
- cd ..
+ pip install --force-reinstall --no-deps -e enterprise/
setup_litellm_test_deps:
steps:
- checkout
@@ -112,14 +110,32 @@ jobs:
python -m mypy .
cd ..
no_output_timeout: 10m
- local_testing:
+
+ semgrep:
docker:
- image: cimg/python:3.12
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
+ steps:
+ - checkout
+ - setup_google_dns
+ - run:
+ name: Install Semgrep
+ command: pip install semgrep
+ - run:
+ name: Run Semgrep (custom rules only)
+ command: semgrep scan --config .semgrep/rules . --error
+ local_testing_part1:
+ docker:
+ - image: cimg/python:3.12
+ auth:
+ username: ${DOCKERHUB_USERNAME}
+ password: ${DOCKERHUB_PASSWORD}
+ working_directory: ~/project
+ parallelism: 4
steps:
- checkout
- setup_google_dns
@@ -205,20 +221,32 @@ jobs:
# Run pytest and generate JUnit XML report
- run:
- name: Run tests
+ name: Run tests (Part 1 - A-M)
command: |
- pwd
- ls
- # Add --timeout to kill hanging tests after 300s (5 min)
- # Add -v to show test names as they run for debugging
- # Add --tb=short for shorter tracebacks
- python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=20 -k "not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache" -n 4 --timeout=300 --timeout_method=thread
+ mkdir test-results
+
+ # Discover test files (A-M)
+ TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py")
+
+ echo "$TEST_FILES" | circleci tests run \
+ --split-by=timings \
+ --verbose \
+ --command="xargs python -m pytest \
+ -vv \
+ --cov=litellm \
+ --cov-report=xml \
+ --junitxml=test-results/junit.xml \
+ --durations=20 \
+ -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \
+ -n 4 \
+ --timeout=300 \
+ --timeout_method=thread"
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
- mv coverage.xml local_testing_coverage.xml
- mv .coverage local_testing_coverage
+ mv coverage.xml local_testing_part1_coverage.xml
+ mv .coverage local_testing_part1_coverage
# Store test results
- store_test_results:
@@ -226,8 +254,136 @@ jobs:
- persist_to_workspace:
root: .
paths:
- - local_testing_coverage.xml
- - local_testing_coverage
+ - local_testing_part1_coverage.xml
+ - local_testing_part1_coverage
+ local_testing_part2:
+ docker:
+ - image: cimg/python:3.12
+ auth:
+ username: ${DOCKERHUB_USERNAME}
+ password: ${DOCKERHUB_PASSWORD}
+ working_directory: ~/project
+ parallelism: 4
+ steps:
+ - checkout
+ - setup_google_dns
+ - run:
+ name: Show git commit hash
+ command: |
+ echo "Git commit hash: $CIRCLE_SHA1"
+
+ - restore_cache:
+ keys:
+ - v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
+ - run:
+ name: Install Dependencies
+ command: |
+ python -m pip install --upgrade pip
+ python -m pip install -r .circleci/requirements.txt
+ pip install "pytest==7.3.1"
+ pip install "pytest-retry==1.6.3"
+ pip install "pytest-asyncio==0.21.1"
+ pip install "pytest-cov==5.0.0"
+ pip install "mypy==1.18.2"
+ pip install "google-generativeai==0.3.2"
+ pip install "google-cloud-aiplatform==1.43.0"
+ pip install pyarrow
+ pip install "boto3==1.36.0"
+ pip install "aioboto3==13.4.0"
+ pip install langchain
+ pip install lunary==0.2.5
+ pip install "azure-identity==1.16.1"
+ pip install "langfuse==2.59.7"
+ pip install "logfire==0.29.0"
+ pip install numpydoc
+ pip install traceloop-sdk==0.21.1
+ pip install opentelemetry-api==1.25.0
+ pip install opentelemetry-sdk==1.25.0
+ pip install opentelemetry-exporter-otlp==1.25.0
+ pip install openai==1.100.1
+ pip install prisma==0.11.0
+ pip install "detect_secrets==1.5.0"
+ pip install "httpx==0.24.1"
+ pip install "respx==0.22.0"
+ pip install fastapi
+ pip install "gunicorn==21.2.0"
+ pip install "anyio==4.2.0"
+ pip install "aiodynamo==23.10.1"
+ pip install "asyncio==3.4.3"
+ pip install "apscheduler==3.10.4"
+ pip install "PyGithub==1.59.1"
+ pip install argon2-cffi
+ pip install "pytest-mock==3.12.0"
+ pip install python-multipart
+ pip install google-cloud-aiplatform
+ pip install prometheus-client==0.20.0
+ pip install "pydantic==2.10.2"
+ pip install "diskcache==5.6.1"
+ pip install "Pillow==10.3.0"
+ pip install "jsonschema==4.22.0"
+ pip install "pytest-xdist==3.6.1"
+ pip install "pytest-timeout==2.2.0"
+ pip install "websockets==13.1.0"
+ pip install semantic_router --no-deps
+ pip install aurelio_sdk --no-deps
+ pip uninstall posthog -y
+ - setup_litellm_enterprise_pip
+ - save_cache:
+ paths:
+ - ./venv
+ key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
+ - run:
+ name: Run prisma ./docker/entrypoint.sh
+ command: |
+ set +e
+ chmod +x docker/entrypoint.sh
+ ./docker/entrypoint.sh
+ set -e
+ - run:
+ name: Black Formatting
+ command: |
+ cd litellm
+ python -m pip install black
+ python -m black .
+ cd ..
+
+ # Run pytest and generate JUnit XML report
+ - run:
+ name: Run tests (Part 2 - N-Z)
+ command: |
+ mkdir test-results
+
+ # Discover test files (N-Z)
+ TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py")
+
+ echo "$TEST_FILES" | circleci tests run \
+ --split-by=timings \
+ --verbose \
+ --command="xargs python -m pytest \
+ -vv \
+ --cov=litellm \
+ --cov-report=xml \
+ --junitxml=test-results/junit.xml \
+ --durations=20 \
+ -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \
+ -n 4 \
+ --timeout=300 \
+ --timeout_method=thread"
+ no_output_timeout: 120m
+ - run:
+ name: Rename the coverage files
+ command: |
+ mv coverage.xml local_testing_part2_coverage.xml
+ mv .coverage local_testing_part2_coverage
+
+ # Store test results
+ - store_test_results:
+ path: test-results
+ - persist_to_workspace:
+ root: .
+ paths:
+ - local_testing_part2_coverage.xml
+ - local_testing_part2_coverage
langfuse_logging_unit_tests:
docker:
- image: cimg/python:3.11
@@ -499,7 +655,6 @@ jobs:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
-
steps:
- checkout
- setup_google_dns
@@ -513,6 +668,7 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
+ pip install "pytest-xdist==3.6.1"
pip install semantic_router --no-deps
pip install aurelio_sdk --no-deps
# Run pytest and generate JUnit XML report
@@ -575,8 +731,8 @@ jobs:
- run:
name: Rename the coverage files
command: |
- mv coverage.xml litellm_router_coverage.xml
- mv .coverage litellm_router_coverage
+ mv coverage.xml litellm_router_unit_coverage.xml
+ mv .coverage litellm_router_unit_coverage
# Store test results
- store_test_results:
path: test-results
@@ -584,8 +740,8 @@ jobs:
- persist_to_workspace:
root: .
paths:
- - litellm_router_coverage.xml
- - litellm_router_coverage
+ - litellm_router_unit_coverage.xml
+ - litellm_router_unit_coverage
litellm_security_tests:
machine:
image: ubuntu-2204:2023.10.1
@@ -1115,7 +1271,15 @@ jobs:
ls
# Add --timeout to kill hanging tests after 120s (2 min)
# Add --durations=20 to show 20 slowest tests for debugging
- python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
+ # Subdirectories with dedicated jobs (maintain this list as new jobs are added)
+ IGNORE_DIRS=(
+ "tests/llm_translation/realtime"
+ )
+ IGNORE_ARGS=""
+ for dir in "${IGNORE_DIRS[@]}"; do
+ IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir"
+ done
+ python -m pytest -vv tests/llm_translation $IGNORE_ARGS --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
no_output_timeout: 120m
- run:
name: Rename the coverage files
@@ -1131,6 +1295,54 @@ jobs:
paths:
- llm_translation_coverage.xml
- llm_translation_coverage
+ realtime_translation_testing:
+ docker:
+ - image: cimg/python:3.11
+ auth:
+ username: ${DOCKERHUB_USERNAME}
+ password: ${DOCKERHUB_PASSWORD}
+ working_directory: ~/project
+
+ steps:
+ - checkout
+ - setup_google_dns
+ - run:
+ name: Install Dependencies
+ command: |
+ python -m pip install --upgrade pip
+ python -m pip install -r requirements.txt
+ pip install "pytest==7.3.1"
+ pip install "pytest-retry==1.6.3"
+ pip install "pytest-cov==5.0.0"
+ pip install "pytest-asyncio==0.21.1"
+ pip install "respx==0.22.0"
+ pip install "pytest-xdist==3.6.1"
+ pip install "pytest-timeout==2.2.0"
+ pip install "websockets"
+ # Run pytest and generate JUnit XML report
+ - run:
+ name: Run realtime tests
+ command: |
+ pwd
+ ls
+ # Add --timeout to kill hanging tests after 120s (2 min)
+ # Add --durations=20 to show 20 slowest tests for debugging
+ python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
+ no_output_timeout: 120m
+ - run:
+ name: Rename the coverage files
+ command: |
+ mv coverage.xml realtime_translation_coverage.xml
+ mv .coverage realtime_translation_coverage
+
+ # Store test results
+ - store_test_results:
+ path: test-results
+ - persist_to_workspace:
+ root: .
+ paths:
+ - realtime_translation_coverage.xml
+ - realtime_translation_coverage
mcp_testing:
docker:
- image: cimg/python:3.11
@@ -1176,6 +1388,51 @@ jobs:
paths:
- mcp_coverage.xml
- mcp_coverage
+ agent_testing:
+ docker:
+ - image: cimg/python:3.11
+ auth:
+ username: ${DOCKERHUB_USERNAME}
+ password: ${DOCKERHUB_PASSWORD}
+ working_directory: ~/project
+
+ steps:
+ - checkout
+ - setup_google_dns
+ - run:
+ name: Install Dependencies
+ command: |
+ python -m pip install --upgrade pip
+ python -m pip install -r requirements.txt
+ pip install "pytest==7.3.1"
+ pip install "pytest-retry==1.6.3"
+ pip install "pytest-cov==5.0.0"
+ pip install "pytest-asyncio==0.21.1"
+ pip install "respx==0.22.0"
+ pip install "pydantic==2.11.0"
+ pip install "a2a-sdk"
+ # Run pytest and generate JUnit XML report
+ - run:
+ name: Run tests
+ command: |
+ pwd
+ ls
+ python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+ no_output_timeout: 120m
+ - run:
+ name: Rename the coverage files
+ command: |
+ mv coverage.xml agent_coverage.xml
+ mv .coverage agent_coverage
+
+ # Store test results
+ - store_test_results:
+ path: test-results
+ - persist_to_workspace:
+ root: .
+ paths:
+ - agent_coverage.xml
+ - agent_coverage
guardrails_testing:
docker:
- image: cimg/python:3.11
@@ -1397,7 +1654,7 @@ jobs:
- search_coverage.xml
- search_coverage
# Split litellm_mapped_tests into 3 parallel jobs for 3x faster execution
- litellm_mapped_tests_proxy:
+ litellm_mapped_tests_proxy_part1:
docker:
- image: cimg/python:3.11
auth:
@@ -1408,23 +1665,53 @@ jobs:
steps:
- setup_litellm_test_deps
- run:
- name: Run proxy tests
+ name: Run proxy tests part 1 (high-volume directories)
command: |
prisma generate
- python -m pytest tests/test_litellm/proxy --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
- no_output_timeout: 120m
+ export PYTHONUNBUFFERED=1
+ python -m pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/client tests/test_litellm/proxy/auth --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy-part1.xml --durations=10 -n 8 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A
+ no_output_timeout: 60m
- run:
name: Rename the coverage files
command: |
- mv coverage.xml litellm_proxy_tests_coverage.xml
- mv .coverage litellm_proxy_tests_coverage
+ mv coverage.xml litellm_proxy_tests_part1_coverage.xml
+ mv .coverage litellm_proxy_tests_part1_coverage
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- - litellm_proxy_tests_coverage.xml
- - litellm_proxy_tests_coverage
+ - litellm_proxy_tests_part1_coverage.xml
+ - litellm_proxy_tests_part1_coverage
+ litellm_mapped_tests_proxy_part2:
+ docker:
+ - image: cimg/python:3.11
+ auth:
+ username: ${DOCKERHUB_USERNAME}
+ password: ${DOCKERHUB_PASSWORD}
+ working_directory: ~/project
+ resource_class: xlarge
+ steps:
+ - setup_litellm_test_deps
+ - run:
+ name: Run proxy tests part 2 (all other tests)
+ command: |
+ prisma generate
+ export PYTHONUNBUFFERED=1
+ python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 8 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A
+ no_output_timeout: 60m
+ - run:
+ name: Rename the coverage files
+ command: |
+ mv coverage.xml litellm_proxy_tests_part2_coverage.xml
+ mv .coverage litellm_proxy_tests_part2_coverage
+ - store_test_results:
+ path: test-results
+ - persist_to_workspace:
+ root: .
+ paths:
+ - litellm_proxy_tests_part2_coverage.xml
+ - litellm_proxy_tests_part2_coverage
litellm_mapped_tests_llms:
docker:
- image: cimg/python:3.11
@@ -1465,7 +1752,7 @@ jobs:
- run:
name: Run core tests
command: |
- python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
+ python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --ignore=tests/test_litellm/experimental_mcp_client --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
no_output_timeout: 120m
- run:
name: Rename the coverage files
@@ -1506,6 +1793,33 @@ jobs:
paths:
- litellm_core_utils_tests_coverage.xml
- litellm_core_utils_tests_coverage
+ litellm_mapped_tests_mcps:
+ docker:
+ - image: cimg/python:3.11
+ auth:
+ username: ${DOCKERHUB_USERNAME}
+ password: ${DOCKERHUB_PASSWORD}
+ working_directory: ~/project
+ resource_class: xlarge
+ steps:
+ - setup_litellm_test_deps
+ - run:
+ name: Run MCP client tests
+ command: |
+ python -m pytest tests/test_litellm/experimental_mcp_client --cov=litellm --cov-report=xml --junitxml=test-results/junit-mcps.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
+ no_output_timeout: 120m
+ - run:
+ name: Rename the coverage files
+ command: |
+ mv coverage.xml litellm_mcps_tests_coverage.xml
+ mv .coverage litellm_mcps_tests_coverage
+ - store_test_results:
+ path: test-results
+ - persist_to_workspace:
+ root: .
+ paths:
+ - litellm_mcps_tests_coverage.xml
+ - litellm_mcps_tests_coverage
litellm_mapped_tests_integrations:
docker:
- image: cimg/python:3.11
@@ -1743,13 +2057,14 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
+ pip install "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
- python -m pytest -vv tests/image_gen_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5
+ python -m pytest -vv tests/image_gen_tests -n 4 --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
@@ -1792,6 +2107,7 @@ jobs:
pip install "mlflow==2.17.2"
pip install "anthropic==0.52.0"
pip install "blockbuster==1.5.24"
+ pip install "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- setup_litellm_enterprise_pip
- run:
@@ -1799,7 +2115,7 @@ jobs:
command: |
pwd
ls
- python -m pytest -vv tests/logging_callback_tests --cov=litellm --cov-report=xml -s -v --junitxml=test-results/junit.xml --durations=5
+ python -m pytest -vv tests/logging_callback_tests --cov=litellm -n 4 --cov-report=xml -s -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
@@ -2034,6 +2350,7 @@ jobs:
- run: python ./tests/code_coverage_tests/router_code_coverage.py
- run: python ./tests/code_coverage_tests/test_chat_completion_imports.py
- run: python ./tests/code_coverage_tests/info_log_check.py
+ - run: python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py
- run: python ./tests/code_coverage_tests/test_ban_set_verbose.py
- run: python ./tests/code_coverage_tests/code_qa_check_tests.py
- run: python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
@@ -2192,6 +2509,8 @@ jobs:
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.100.1"
+ pip install "litellm[proxy]"
+ pip install "pytest-xdist==3.6.1"
- run:
name: Install dockerize
command: |
@@ -2268,7 +2587,7 @@ jobs:
command: |
pwd
ls
- python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
+ python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
no_output_timeout: 120m
# Store test results
@@ -3263,6 +3582,112 @@ jobs:
- store_test_results:
path: test-results
+ proxy_e2e_anthropic_messages_tests:
+ machine:
+ image: ubuntu-2204:2023.10.1
+ resource_class: xlarge
+ working_directory: ~/project
+ steps:
+ - checkout
+ - setup_google_dns
+ - run:
+ name: Install Docker CLI (In case it's not already installed)
+ command: |
+ curl -fsSL https://get.docker.com | sh
+ sudo usermod -aG docker $USER
+ docker version
+ - run:
+ name: Install Python 3.10
+ command: |
+ curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
+ bash miniconda.sh -b -p $HOME/miniconda
+ export PATH="$HOME/miniconda/bin:$PATH"
+ conda init bash
+ source ~/.bashrc
+ conda create -n myenv python=3.10 -y
+ conda activate myenv
+ python --version
+ - run:
+ name: Install Dependencies
+ command: |
+ export PATH="$HOME/miniconda/bin:$PATH"
+ source $HOME/miniconda/etc/profile.d/conda.sh
+ conda activate myenv
+ pip install "pytest==7.3.1"
+ pip install "pytest-asyncio==0.21.1"
+ pip install "boto3==1.36.0"
+ pip install "httpx==0.27.0"
+ pip install "claude-agent-sdk"
+ pip install -r requirements.txt
+ - run:
+ name: Install dockerize
+ command: |
+ wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+ sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+ rm dockerize-linux-amd64-v0.6.1.tar.gz
+ - run:
+ name: Start PostgreSQL Database
+ command: |
+ docker run -d \
+ --name postgres-db \
+ -e POSTGRES_USER=postgres \
+ -e POSTGRES_PASSWORD=postgres \
+ -e POSTGRES_DB=circle_test \
+ -p 5432:5432 \
+ postgres:14
+ - run:
+ name: Wait for PostgreSQL to be ready
+ command: dockerize -wait tcp://localhost:5432 -timeout 1m
+ - attach_workspace:
+ at: ~/project
+ - run:
+ name: Load Docker Database Image
+ command: |
+ gunzip -c litellm-docker-database.tar.gz | docker load
+ docker images | grep litellm-docker-database
+ - run:
+ name: Run Docker container with test config
+ command: |
+ docker run -d \
+ -p 4000:4000 \
+ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
+ -e LITELLM_MASTER_KEY="sk-1234" \
+ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
+ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+ -e AWS_REGION_NAME="us-east-1" \
+ -e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \
+ --add-host host.docker.internal:host-gateway \
+ --name my-app \
+ -v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \
+ litellm-docker-database:ci \
+ --config /app/config.yaml \
+ --port 4000 \
+ --detailed_debug
+ - run:
+ name: Start outputting logs
+ command: docker logs -f my-app
+ background: true
+ - run:
+ name: Wait for app to be ready
+ command: dockerize -wait http://localhost:4000 -timeout 5m
+ - run:
+ name: Run Claude Agent SDK E2E Tests
+ command: |
+ export PATH="$HOME/miniconda/bin:$PATH"
+ source $HOME/miniconda/etc/profile.d/conda.sh
+ conda activate myenv
+ export LITELLM_PROXY_URL="http://localhost:4000"
+ export LITELLM_API_KEY="sk-1234"
+ pwd
+ ls
+ python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5
+ no_output_timeout: 120m
+
+ # Store test results
+ - store_test_results:
+ path: test-results
+
upload-coverage:
docker:
- image: cimg/python:3.9
@@ -3284,7 +3709,7 @@ jobs:
python -m venv venv
. venv/bin/activate
pip install coverage
- coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
+ coverage combine llm_translation_coverage realtime_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage litellm_router_coverage litellm_router_unit_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
coverage xml
- codecov/upload:
file: ./coverage.xml
@@ -3334,8 +3759,22 @@ jobs:
ls dist/
twine upload --verbose dist/*
else
- echo "Version ${VERSION} of package is already published on PyPI. Skipping PyPI publish."
- circleci step halt
+ echo "Version ${VERSION} of package is already published on PyPI."
+
+ # Check if corresponding Docker nightly image exists
+ NIGHTLY_TAG="v${VERSION}-nightly"
+ echo "Checking for Docker nightly image: litellm/litellm:${NIGHTLY_TAG}"
+
+ # Check Docker Hub for the nightly image
+ if curl -s "https://hub.docker.com/v2/repositories/litellm/litellm/tags/${NIGHTLY_TAG}" | grep -q "name"; then
+ echo "Docker nightly image ${NIGHTLY_TAG} exists. This release was already completed successfully."
+ echo "Skipping PyPI publish and continuing to ensure Docker images are up to date."
+ circleci step halt
+ else
+ echo "ERROR: PyPI package ${VERSION} exists but Docker nightly image ${NIGHTLY_TAG} does not exist!"
+ echo "This indicates an incomplete release. Please investigate."
+ exit 1
+ fi
fi
- run:
name: Trigger Github Action for new Docker Container + Trigger Load Testing
@@ -3344,11 +3783,21 @@ jobs:
python3 -m pip install toml
VERSION=$(python3 -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
echo "LiteLLM Version ${VERSION}"
+
+ # Determine which branch to use for Docker build
+ if [[ "$CIRCLE_BRANCH" =~ ^litellm_release_day_.* ]]; then
+ BUILD_BRANCH="$CIRCLE_BRANCH"
+ echo "Using release branch: $BUILD_BRANCH"
+ else
+ BUILD_BRANCH="main"
+ echo "Using default branch: $BUILD_BRANCH"
+ fi
+
curl -X POST \
-H "Accept: application/vnd.github.v3+json" \
-H "Authorization: Bearer $GITHUB_TOKEN" \
"https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \
- -d "{\"ref\":\"main\", \"inputs\":{\"tag\":\"v${VERSION}-nightly\", \"commit_hash\":\"$CIRCLE_SHA1\"}}"
+ -d "{\"ref\":\"${BUILD_BRANCH}\", \"inputs\":{\"tag\":\"v${VERSION}-nightly\", \"commit_hash\":\"$CIRCLE_SHA1\"}}"
echo "triggering load testing server for version ${VERSION} and commit ${CIRCLE_SHA1}"
curl -X POST "https://proxyloadtester-production.up.railway.app/start/load/test?version=${VERSION}&commit_hash=${CIRCLE_SHA1}&release_type=nightly"
@@ -3428,7 +3877,6 @@ jobs:
- run:
name: Get new version
command: |
- cd litellm-proxy-extras
NEW_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
echo "export NEW_VERSION=$NEW_VERSION" >> $BASH_ENV
@@ -3453,7 +3901,6 @@ jobs:
- run:
name: Publish to PyPI
command: |
- cd litellm-proxy-extras
echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc
python -m pip install --upgrade pip build twine setuptools wheel
rm -rf build dist
@@ -3482,6 +3929,9 @@ jobs:
cd ui/litellm-dashboard
+ # Remove node_modules and package-lock to ensure clean install (fixes dependency resolution issues)
+ rm -rf node_modules package-lock.json
+
# Install dependencies first
npm install
@@ -3557,6 +4007,9 @@ jobs:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
+ parameters:
+ browser:
+ type: string
steps:
- checkout
- setup_google_dns
@@ -3586,7 +4039,7 @@ jobs:
echo "Expires at: $EXPIRES_AT"
neon branches create \
--project-id $NEON_PROJECT_ID \
- --name preview/commit-${CIRCLE_SHA1:0:7} \
+ --name preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \
--expires-at $EXPIRES_AT \
--parent br-fancy-paper-ad1olsb3 \
--api-key $NEON_API_KEY || true
@@ -3596,7 +4049,7 @@ jobs:
E2E_UI_TEST_DATABASE_URL=$(neon connection-string \
--project-id $NEON_PROJECT_ID \
--api-key $NEON_API_KEY \
- --branch preview/commit-${CIRCLE_SHA1:0:7} \
+ --branch preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \
--database-name yuneng-trial-db \
--role neondb_owner)
echo $E2E_UI_TEST_DATABASE_URL
@@ -3608,7 +4061,7 @@ jobs:
-e UI_USERNAME="admin" \
-e UI_PASSWORD="gm" \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
- --name litellm-docker-database \
+ --name litellm-docker-database-<< parameters.browser >> \
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
@@ -3624,7 +4077,7 @@ jobs:
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start outputting logs
- command: docker logs -f litellm-docker-database
+ command: docker logs -f litellm-docker-database-<< parameters.browser >>
background: true
- run:
name: Wait for app to be ready
@@ -3633,6 +4086,7 @@ jobs:
name: Run Playwright Tests
command: |
npx playwright test \
+ --project << parameters.browser >> \
--config ui/litellm-dashboard/e2e_tests/playwright.config.ts \
--reporter=html \
--output=test-results
@@ -3739,7 +4193,19 @@ workflows:
only:
- main
- /litellm_.*/
- - local_testing:
+ - semgrep:
+ filters:
+ branches:
+ only:
+ - main
+ - /litellm_.*/
+ - local_testing_part1:
+ filters:
+ branches:
+ only:
+ - main
+ - /litellm_.*/
+ - local_testing_part2:
filters:
branches:
only:
@@ -3832,6 +4298,20 @@ workflows:
- main
- /litellm_.*/
- e2e_ui_testing:
+ name: e2e_ui_testing_chromium
+ browser: chromium
+ context: e2e_ui_tests
+ requires:
+ - ui_build
+ - build_docker_database_image
+ filters:
+ branches:
+ only:
+ - main
+ - /litellm_.*/
+ - e2e_ui_testing:
+ name: e2e_ui_testing_firefox
+ browser: firefox
context: e2e_ui_tests
requires:
- ui_build
@@ -3901,18 +4381,38 @@ workflows:
only:
- main
- /litellm_.*/
+ - proxy_e2e_anthropic_messages_tests:
+ requires:
+ - build_docker_database_image
+ filters:
+ branches:
+ only:
+ - main
+ - /litellm_.*/
- llm_translation_testing:
filters:
branches:
only:
- main
- /litellm_.*/
+ - realtime_translation_testing:
+ filters:
+ branches:
+ only:
+ - main
+ - /litellm_.*/
- mcp_testing:
filters:
branches:
only:
- main
- /litellm_.*/
+ - agent_testing:
+ filters:
+ branches:
+ only:
+ - main
+ - /litellm_.*/
- guardrails_testing:
filters:
branches:
@@ -3949,7 +4449,13 @@ workflows:
only:
- main
- /litellm_.*/
- - litellm_mapped_tests_proxy:
+ - litellm_mapped_tests_proxy_part1:
+ filters:
+ branches:
+ only:
+ - main
+ - /litellm_.*/
+ - litellm_mapped_tests_proxy_part2:
filters:
branches:
only:
@@ -3967,6 +4473,12 @@ workflows:
only:
- main
- /litellm_.*/
+ - litellm_mapped_tests_mcps:
+ filters:
+ branches:
+ only:
+ - main
+ - /litellm_.*/
- litellm_mapped_tests_integrations:
filters:
branches:
@@ -4018,15 +4530,19 @@ workflows:
- upload-coverage:
requires:
- llm_translation_testing
+ - realtime_translation_testing
- mcp_testing
+ - agent_testing
- google_generate_content_endpoint_testing
- guardrails_testing
- llm_responses_api_testing
- ocr_testing
- search_testing
- - litellm_mapped_tests_proxy
+ - litellm_mapped_tests_proxy_part1
+ - litellm_mapped_tests_proxy_part2
- litellm_mapped_tests_llms
- litellm_mapped_tests_core
+ - litellm_mapped_tests_mcps
- litellm_mapped_tests_integrations
- litellm_mapped_tests_litellm_core_utils
- litellm_mapped_enterprise_tests
@@ -4044,7 +4560,8 @@ workflows:
- litellm_proxy_unit_testing_part2
- litellm_security_tests
- langfuse_logging_unit_tests
- - local_testing
+ - local_testing_part1
+ - local_testing_part2
- litellm_assistants_api_testing
- auth_ui_unit_tests
- db_migration_disable_update_check:
@@ -4084,22 +4601,29 @@ workflows:
branches:
only:
- main
+ - /litellm_release_day_.*/
- publish_to_pypi:
requires:
- mypy_linting
- - local_testing
+ - semgrep
+ - local_testing_part1
+ - local_testing_part2
- build_and_test
- e2e_openai_endpoints
- test_bad_database_url
- llm_translation_testing
+ - realtime_translation_testing
- mcp_testing
+ - agent_testing
- google_generate_content_endpoint_testing
- llm_responses_api_testing
- ocr_testing
- search_testing
- - litellm_mapped_tests_proxy
+ - litellm_mapped_tests_proxy_part1
+ - litellm_mapped_tests_proxy_part2
- litellm_mapped_tests_llms
- litellm_mapped_tests_core
+ - litellm_mapped_tests_mcps
- litellm_mapped_tests_integrations
- litellm_mapped_tests_litellm_core_utils
- litellm_mapped_enterprise_tests
@@ -4116,7 +4640,8 @@ workflows:
- litellm_assistants_api_testing
- auth_ui_unit_tests
- db_migration_disable_update_check
- - e2e_ui_testing
+ - e2e_ui_testing_chromium
+ - e2e_ui_testing_firefox
- litellm_proxy_unit_testing_key_generation
- litellm_proxy_unit_testing_part1
- litellm_proxy_unit_testing_part2
diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt
index 8c44dc18305..a5ec74424fe 100644
--- a/.circleci/requirements.txt
+++ b/.circleci/requirements.txt
@@ -16,4 +16,5 @@ uvloop==0.21.0
mcp==1.25.0 # for MCP server
semantic_router==0.1.10 # for auto-routing with litellm
fastuuid==0.12.0
-responses==0.25.7 # for proxy client tests
\ No newline at end of file
+responses==0.25.7 # for proxy client tests
+pytest-retry==1.6.3 # for automatic test retries
\ No newline at end of file
diff --git a/.dockerignore b/.dockerignore
index 76e31546c2f..a487d2a859a 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -48,7 +48,7 @@ dist/
build/
*.egg-info/
.DS_Store
-node_modules/
+**/node_modules
*.log
.env
.env.local
diff --git a/.github/actions/helm-oci-chart-releaser/action.yml b/.github/actions/helm-oci-chart-releaser/action.yml
index 059277ed882..1823e262832 100644
--- a/.github/actions/helm-oci-chart-releaser/action.yml
+++ b/.github/actions/helm-oci-chart-releaser/action.yml
@@ -40,38 +40,33 @@ outputs:
runs:
using: composite
steps:
+ - name: Helm | Setup
+ uses: azure/setup-helm@v4
+ with:
+ version: v3.20.0
+
- name: Helm | Login
shell: bash
run: echo ${{ inputs.registry_password }} | helm registry login -u ${{ inputs.registry_username }} --password-stdin ${{ inputs.registry }}
- env:
- HELM_EXPERIMENTAL_OCI: '1'
-
+
- name: Helm | Dependency
if: inputs.update_dependencies == 'true'
shell: bash
run: helm dependency update ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
- env:
- HELM_EXPERIMENTAL_OCI: '1'
- name: Helm | Package
shell: bash
run: helm package ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} --version ${{ inputs.tag }} --app-version ${{ inputs.app_version }}
- env:
- HELM_EXPERIMENTAL_OCI: '1'
- name: Helm | Push
shell: bash
run: helm push ${{ inputs.name }}-${{ inputs.tag }}.tgz oci://${{ inputs.registry }}/${{ inputs.repository }}
- env:
- HELM_EXPERIMENTAL_OCI: '1'
- name: Helm | Logout
shell: bash
run: helm registry logout ${{ inputs.registry }}
- env:
- HELM_EXPERIMENTAL_OCI: '1'
- name: Helm | Output
id: output
shell: bash
- run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT
\ No newline at end of file
+ run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index b91b16c955c..f13039f4516 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -9,6 +9,7 @@
- [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
- [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
- [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
+- [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review
## CI (LiteLLM team)
diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml
index 35ebffeada3..7c5c269f899 100644
--- a/.github/workflows/test-linting.yml
+++ b/.github/workflows/test-linting.yml
@@ -73,4 +73,4 @@ jobs:
- name: Check import safety
run: |
- poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
\ No newline at end of file
+ poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml
new file mode 100644
index 00000000000..e57168dd551
--- /dev/null
+++ b/.github/workflows/test-litellm-matrix.yml
@@ -0,0 +1,118 @@
+name: LiteLLM Unit Tests (Matrix)
+
+on:
+ pull_request:
+ branches: [main]
+
+# Cancel in-progress runs for the same PR
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ timeout-minutes: 20 # Increased from 15 to 20
+ strategy:
+ fail-fast: false
+ matrix:
+ test-group:
+ # tests/test_litellm split by subdirectory (~560 files total)
+ # Vertex AI tests separated for better isolation (prevent auth/env pollution)
+ - name: "llms-vertex"
+ path: "tests/test_litellm/llms/vertex_ai"
+ workers: 1
+ reruns: 2
+ - name: "llms-other"
+ path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
+ workers: 2
+ reruns: 2
+ # tests/test_litellm/proxy split by subdirectory (~180 files total)
+ - name: "proxy-guardrails"
+ path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers"
+ workers: 2
+ reruns: 2
+ - name: "proxy-core"
+ path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine"
+ workers: 2
+ reruns: 2
+ - name: "proxy-misc"
+ path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py"
+ workers: 2
+ reruns: 2
+ - name: "integrations"
+ path: "tests/test_litellm/integrations"
+ workers: 2
+ reruns: 3 # Integration tests tend to be flakier
+ - name: "core-utils"
+ path: "tests/test_litellm/litellm_core_utils"
+ workers: 2
+ reruns: 1
+ - name: "other"
+ path: "tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types"
+ workers: 2
+ reruns: 2
+ - name: "root"
+ path: "tests/test_litellm/test_*.py"
+ workers: 2
+ reruns: 2
+ # tests/proxy_unit_tests split alphabetically (~48 files total)
+ - name: "proxy-unit-a"
+ path: "tests/proxy_unit_tests/test_[a-o]*.py"
+ workers: 2
+ reruns: 1
+ - name: "proxy-unit-b"
+ path: "tests/proxy_unit_tests/test_[p-z]*.py"
+ workers: 2
+ reruns: 1
+
+ name: test (${{ matrix.test-group.name }})
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+
+ - name: Cache Poetry dependencies
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/.cache/pypoetry
+ ~/.cache/pip
+ .venv
+ key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-poetry-
+
+ - name: Install dependencies
+ run: |
+ poetry config virtualenvs.in-project true
+ poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+ # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies
+ poetry run pip install google-genai==1.22.0 \
+ google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core
+
+ - name: Setup litellm-enterprise
+ run: |
+ poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+ - name: Generate Prisma client
+ run: |
+ poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+ - name: Run tests - ${{ matrix.test-group.name }}
+ run: |
+ poetry run pytest ${{ matrix.test-group.path }} \
+ --tb=short -vv \
+ --maxfail=10 \
+ -n ${{ matrix.test-group.workers }} \
+ --reruns ${{ matrix.test-group.reruns }} \
+ --reruns-delay 1 \
+ --dist=loadscope \
+ --durations=20
diff --git a/.github/workflows/test-litellm-ui-build.yml b/.github/workflows/test-litellm-ui-build.yml
new file mode 100644
index 00000000000..b0a8b648a44
--- /dev/null
+++ b/.github/workflows/test-litellm-ui-build.yml
@@ -0,0 +1,32 @@
+name: UI Build Check
+permissions:
+ contents: read
+
+on:
+ pull_request:
+ branches: [main]
+
+jobs:
+ build-ui:
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ defaults:
+ run:
+ working-directory: ui/litellm-dashboard
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Setup Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version: "20"
+ cache: "npm"
+ cache-dependency-path: ui/litellm-dashboard/package-lock.json
+
+ - name: Install dependencies
+ run: npm install
+
+ - name: Build
+ run: npm run build
diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml
index ba32dc1bf54..cf6928897be 100644
--- a/.github/workflows/test-litellm.yml
+++ b/.github/workflows/test-litellm.yml
@@ -1,8 +1,12 @@
name: LiteLLM Mock Tests (folder - tests/test_litellm)
+# DEPRECATED: This workflow is replaced by test-litellm-matrix.yml which runs
+# the same tests in parallel across 10 jobs for faster CI times.
+# Kept for manual debugging only.
on:
- pull_request:
- branches: [ main ]
+ workflow_dispatch: # Manual trigger only
+ # pull_request:
+ # branches: [ main ]
jobs:
test:
@@ -34,13 +38,11 @@ jobs:
poetry run pip install "google-genai==1.22.0"
poetry run pip install "google-cloud-aiplatform>=1.38"
poetry run pip install "fastapi-offline==1.7.3"
- poetry run pip install "python-multipart==0.0.18"
+ poetry run pip install "python-multipart==0.0.22"
poetry run pip install "openapi-core"
- name: Setup litellm-enterprise as local package
run: |
- cd enterprise
- poetry run pip install -e .
- cd ..
+ poetry run pip install --force-reinstall --no-deps -e enterprise/
- name: Run tests
run: |
poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50
diff --git a/.github/workflows/test-mcp.yml b/.github/workflows/test-mcp.yml
index e19e67c9c4f..2e32aae7680 100644
--- a/.github/workflows/test-mcp.yml
+++ b/.github/workflows/test-mcp.yml
@@ -40,9 +40,7 @@ jobs:
- name: Setup litellm-enterprise as local package
run: |
- cd enterprise
- python -m pip install -e .
- cd ..
+ poetry run pip install --force-reinstall --no-deps -e enterprise/
- name: Run MCP tests
run: |
diff --git a/.github/workflows/test-model-map.yaml b/.github/workflows/test-model-map.yaml
new file mode 100644
index 00000000000..ae5ac402e23
--- /dev/null
+++ b/.github/workflows/test-model-map.yaml
@@ -0,0 +1,15 @@
+name: Validate model_prices_and_context_window.json
+
+on:
+ pull_request:
+ branches: [ main ]
+
+jobs:
+ validate-model-prices-json:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Validate model_prices_and_context_window.json
+ run: |
+ jq empty model_prices_and_context_window.json
diff --git a/.github/workflows/test_server_root_path.yml b/.github/workflows/test_server_root_path.yml
new file mode 100644
index 00000000000..c359e38bff9
--- /dev/null
+++ b/.github/workflows/test_server_root_path.yml
@@ -0,0 +1,96 @@
+name: Test Proxy SERVER_ROOT_PATH Routing
+permissions:
+ contents: read
+
+on:
+ pull_request:
+ branches: [main]
+
+jobs:
+ test-server-root-path:
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+
+ strategy:
+ matrix:
+ root_path: ["/api/v1", "/llmproxy"]
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Build Docker image
+ uses: docker/build-push-action@v5
+ with:
+ context: .
+ file: ./docker/Dockerfile.non_root
+ tags: litellm-test:${{ github.sha }}
+ load: true
+ cache-from: type=gha
+ cache-to: type=gha,mode=max
+
+ - name: Start LiteLLM container with SERVER_ROOT_PATH
+ run: |
+ docker run -d \
+ --name litellm-test \
+ -p 4000:4000 \
+ -e SERVER_ROOT_PATH="${{ matrix.root_path }}" \
+ -e LITELLM_MASTER_KEY="sk-1234" \
+ litellm-test:${{ github.sha }} \
+ --detailed_debug
+
+ - name: Wait for container to be healthy
+ run: |
+ echo "Waiting for LiteLLM to start..."
+ max_attempts=30
+ attempt=0
+
+ while [ $attempt -lt $max_attempts ]; do
+ if docker logs litellm-test 2>&1 | grep -q "Uvicorn running"; then
+ echo "LiteLLM started successfully"
+ break
+ fi
+ attempt=$((attempt + 1))
+ echo "Attempt $attempt/$max_attempts - waiting for server to start..."
+ sleep 2
+ done
+
+ if [ $attempt -eq $max_attempts ]; then
+ echo "Server failed to start within timeout"
+ docker logs litellm-test
+ exit 1
+ fi
+
+ sleep 5
+
+ - name: Show container logs
+ if: always()
+ run: docker logs litellm-test
+
+ - name: Test UI endpoint with root path
+ run: |
+ ROOT_PATH="${{ matrix.root_path }}"
+ echo "Testing UI at: http://localhost:4000${ROOT_PATH}/ui/"
+
+ for i in 1 2 3; do
+ content=$(curl -sL --max-time 5 -H "Authorization: Bearer sk-1234" "http://localhost:4000${ROOT_PATH}/ui/")
+ if echo "$content" | grep -q -E "(html|` only creates a
+ # SEPARATE global package, it does NOT replace npm's internal copies.
+ # We must find and replace EVERY copy inside npm's directory.
+ GLOBAL="$(npm root -g)" && \
+ find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+ done && \
+ find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+ done && \
+ find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+ done && \
+ npm cache clean --force
WORKDIR /app
# Copy the current directory contents into the container at /app
@@ -61,16 +78,34 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
+# Replace the nodejs-wheel-binaries bundled node with the system node (fixes CVE-2025-55130)
+RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/null) && \
+ if [ -n "$NODEJS_WHEEL_NODE" ]; then cp /usr/bin/node "$NODEJS_WHEEL_NODE"; fi
+
# Remove test files and keys from dependencies
RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
find /usr/lib -type d -path "*/tornado/test" -delete
+# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
+# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
+# Patch every copy of tar, glob, and brace-expansion inside that tree.
+RUN GLOBAL="$(npm root -g)" && \
+ find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+ done && \
+ find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+ done && \
+ find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+ done
+
# Install semantic_router and aurelio-sdk using script
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
-# Generate prisma client
-RUN prisma generate
+# Generate prisma client using the correct schema
+RUN prisma generate --schema=./litellm/proxy/schema.prisma
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
diff --git a/Makefile b/Makefile
index 0da83c363cd..74031f418d6 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,12 @@
# LiteLLM Makefile
# Simple Makefile for running tests and basic development tasks
-.PHONY: help test test-unit test-integration test-unit-helm lint format install-dev install-proxy-dev install-test-deps install-helm-unittest check-circular-imports check-import-safety
+.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \
+ test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
+ test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
+ info lint lint-dev format \
+ install-dev install-proxy-dev install-test-deps \
+ install-helm-unittest check-circular-imports check-import-safety
# Default target
help:
@@ -22,9 +27,26 @@ help:
@echo " make check-import-safety - Check import safety"
@echo " make test - Run all tests"
@echo " make test-unit - Run unit tests (tests/test_litellm)"
+ @echo " make test-unit-llms - Run LLM provider tests (~225 files)"
+ @echo " make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)"
+ @echo " make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)"
+ @echo " make test-unit-proxy-misc - Run proxy misc tests (~77 files)"
+ @echo " make test-unit-integrations - Run integration tests (~60 files)"
+ @echo " make test-unit-core-utils - Run core utils tests (~32 files)"
+ @echo " make test-unit-other - Run other tests (caching, responses, etc., ~69 files)"
+ @echo " make test-unit-root - Run root-level tests (~34 files)"
+ @echo " make test-proxy-unit-a - Run proxy_unit_tests (a-o, ~20 files)"
+ @echo " make test-proxy-unit-b - Run proxy_unit_tests (p-z, ~28 files)"
@echo " make test-integration - Run integration tests"
@echo " make test-unit-helm - Run helm unit tests"
+# Keep PIP simple for edge cases:
+PIP := $(shell command -v pip > /dev/null 2>&1 && echo "pip" || echo "python3 -m pip")
+
+# Show info
+info:
+ @echo "PIP: $(PIP)"
+
# Installation targets
install-dev:
poetry install --with dev
@@ -34,19 +56,19 @@ install-proxy-dev:
# CI-compatible installations (matches GitHub workflows exactly)
install-dev-ci:
- pip install openai==2.8.0
+ $(PIP) install openai==2.8.0
poetry install --with dev
- pip install openai==2.8.0
+ $(PIP) install openai==2.8.0
install-proxy-dev-ci:
poetry install --with dev,proxy-dev --extras proxy
- pip install openai==2.8.0
+ $(PIP) install openai==2.8.0
install-test-deps: install-proxy-dev
- poetry run pip install "pytest-retry==1.6.3"
- poetry run pip install pytest-xdist
- poetry run pip install openapi-core
- cd enterprise && poetry run pip install -e . && cd ..
+ poetry run $(PIP) install "pytest-retry==1.6.3"
+ poetry run $(PIP) install pytest-xdist
+ poetry run $(PIP) install openapi-core
+ cd enterprise && poetry run $(PIP) install -e . && cd ..
install-helm-unittest:
helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
@@ -62,8 +84,40 @@ format-check: install-dev
lint-ruff: install-dev
cd litellm && poetry run ruff check . && cd ..
+# faster linter for developing ...
+# inspiration from:
+# https://github.com/astral-sh/ruff/discussions/10977
+# https://github.com/astral-sh/ruff/discussions/4049
+lint-format-changed: install-dev
+ @git diff origin/main --unified=0 --no-color -- '*.py' | \
+ perl -ne '\
+ if (/^diff --git a\/(.*) b\//) { $$file = $$1; } \
+ if (/^@@ .* \+(\d+)(?:,(\d+))? @@/) { \
+ $$start = $$1; $$count = $$2 || 1; $$end = $$start + $$count - 1; \
+ print "$$file:$$start:1-$$end:999\n"; \
+ }' | \
+ while read range; do \
+ file="$${range%%:*}"; \
+ lines="$${range#*:}"; \
+ echo "Formatting $$file (lines $$lines)"; \
+ poetry run ruff format --range "$$lines" "$$file"; \
+ done
+
+lint-ruff-dev: install-dev
+ @tmpfile=$$(mktemp /tmp/ruff-dev.XXXXXX) && \
+ cd litellm && \
+ (poetry run ruff check . --output-format=pylint || true) > "$$tmpfile" && \
+ poetry run diff-quality --violations=pylint "$$tmpfile" --compare-branch=origin/main && \
+ cd .. ; \
+ rm -f "$$tmpfile"
+
+lint-ruff-FULL-dev: install-dev
+ @files=$$(git diff --name-only origin/main -- '*.py'); \
+ if [ -n "$$files" ]; then echo "$$files" | xargs poetry run ruff check; \
+ else echo "No changed .py files to check."; fi
+
lint-mypy: install-dev
- poetry run pip install types-requests types-setuptools types-redis types-PyYAML
+ poetry run $(PIP) install types-requests types-setuptools types-redis types-PyYAML
cd litellm && poetry run mypy . --ignore-missing-imports && cd ..
lint-black: format-check
@@ -72,11 +126,14 @@ check-circular-imports: install-dev
cd litellm && poetry run python ../tests/documentation_tests/test_circular_imports.py && cd ..
check-import-safety: install-dev
- poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+ @poetry run python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
# Combined linting (matches test-linting.yml workflow)
lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety
+# Faster linting for local development (only checks changed code)
+lint-dev: lint-format-changed lint-mypy check-circular-imports check-import-safety
+
# Testing targets
test:
poetry run pytest tests/
@@ -84,6 +141,38 @@ test:
test-unit: install-test-deps
poetry run pytest tests/test_litellm -x -vv -n 4
+# Matrix test targets (matching CI workflow groups)
+test-unit-llms: install-test-deps
+ poetry run pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20
+
+test-unit-proxy-guardrails: install-test-deps
+ poetry run pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20
+
+test-unit-proxy-core: install-test-deps
+ poetry run pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20
+
+test-unit-proxy-misc: install-test-deps
+ poetry run pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20
+
+test-unit-integrations: install-test-deps
+ poetry run pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20
+
+test-unit-core-utils: install-test-deps
+ poetry run pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20
+
+test-unit-other: install-test-deps
+ poetry run pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20
+
+test-unit-root: install-test-deps
+ poetry run pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20
+
+# Proxy unit tests (tests/proxy_unit_tests split alphabetically)
+test-proxy-unit-a: install-test-deps
+ poetry run pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20
+
+test-proxy-unit-b: install-test-deps
+ poetry run pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20
+
test-integration:
poetry run pytest tests/ -k "not test_litellm"
diff --git a/README.md b/README.md
index 58ffa12c5e1..7790c67afd5 100644
--- a/README.md
+++ b/README.md
@@ -258,6 +258,19 @@ LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https:
Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
+## OSS Adopters
+
+
+
+  |
+  |
+  |
+  |
+ Netflix |
+  |
+
+
+
## Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers))
| Provider | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` |
@@ -296,7 +309,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
| [Deepgram (`deepgram`)](https://docs.litellm.ai/docs/providers/deepgram) | ✅ | ✅ | ✅ | | | ✅ | | | | |
| [DeepInfra (`deepinfra`)](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | | | | | | | |
| [Deepseek (`deepseek`)](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | | | | | | | |
-| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | | ✅ | | | |
+| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | |
| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | |
| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | |
diff --git a/ci_cd/.grype.yaml b/ci_cd/.grype.yaml
index 642e2dd9d03..b9bc9db58f5 100644
--- a/ci_cd/.grype.yaml
+++ b/ci_cd/.grype.yaml
@@ -1,3 +1,36 @@
ignore:
- vulnerability: CVE-2026-22184
reason: no fixed zlib package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists
+ # Wolfi base image: Python 3.13 and Node from apk have no fixed builds in Wolfi yet / not applicable
+ - vulnerability: CVE-2025-55130
+ reason: Node in Wolfi apk; only used for Admin UI build/prisma
+ - vulnerability: CVE-2025-59465
+ reason: Node in Wolfi apk; only used for Admin UI build/prisma
+ - vulnerability: CVE-2025-55131
+ reason: Node in Wolfi apk; only used for Admin UI build/prisma
+ - vulnerability: CVE-2025-59466
+ reason: Node in Wolfi apk; only used for Admin UI build/prisma
+ - vulnerability: CVE-2026-21637
+ reason: Node in Wolfi apk; only used for Admin UI build/prisma
+ - vulnerability: CVE-2025-55132
+ reason: Node in Wolfi apk; only used for Admin UI build/prisma
+ - vulnerability: GHSA-hx9q-6w63-j58v
+ reason: orjson dumps recursion; allowlisted
+ - vulnerability: GHSA-73rr-hh4g-fpgx
+ reason: diff npm transitive dep; override in package.json, allowlisted
+ - vulnerability: CVE-2026-0865
+ reason: Python 3.13 in Wolfi base; no fixed apk build yet
+ - vulnerability: CVE-2025-15282
+ reason: Python 3.13 in Wolfi base; no fixed apk build yet
+ - vulnerability: CVE-2026-0672
+ reason: Python 3.13 in Wolfi base; no fixed apk build yet
+ - vulnerability: CVE-2025-15366
+ reason: Python 3.13 in Wolfi base; no fixed apk build yet
+ - vulnerability: CVE-2025-15367
+ reason: Python 3.13 in Wolfi base; no fixed apk build yet
+ - vulnerability: CVE-2025-11468
+ reason: Python 3.13 in Wolfi base; no fixed apk build yet
+ - vulnerability: CVE-2025-12781
+ reason: Python 3.13 in Wolfi base; no fixed apk build yet
+ - vulnerability: CVE-2026-1299
+ reason: Python 3.13 in Wolfi base; no fixed apk build yet
diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh
index 04f3e27a944..2db72ae5c69 100755
--- a/ci_cd/security_scans.sh
+++ b/ci_cd/security_scans.sh
@@ -81,10 +81,10 @@ run_trivy_scans() {
echo "Running Trivy scans..."
echo "Scanning LiteLLM Docs..."
- trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
+ trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
echo "Scanning LiteLLM UI..."
- trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
+ trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
echo "Trivy scans completed successfully"
}
@@ -137,7 +137,27 @@ run_grype_scans() {
"CVE-2019-1010025" # glibc pthread heap address leak - awaiting patched Wolfi glibc build
"CVE-2026-22184" # zlib untgz buffer overflow - untgz unused + no fixed Wolfi build yet
"GHSA-58pv-8j8x-9vj2" # jaraco.context path traversal - setuptools vendored only (v5.3.0), not used in application code (using v6.1.0+)
+ "GHSA-34x7-hfp2-rc4v" # node-tar hardlink path traversal - not applicable, tar CLI not exposed in application code
"GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit
+ "GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel
+ "CVE-2025-59465" # Node only used for Admin UI build/prisma
+ "CVE-2025-55131" # Node only used for Admin UI build/prisma
+ "CVE-2025-59466" # Node only used for Admin UI build/prisma
+ "CVE-2025-55130" # Node only used for Admin UI build/prisma
+ "CVE-2025-59467" # Node only used for Admin UI build/prisma
+ "CVE-2026-21637" # Node only used for Admin UI build/prisma
+ "CVE-2025-55132" # Node only used for Admin UI build/prisma
+ "GHSA-hx9q-6w63-j58v" # orjson dumps recursion; allowlisted
+ "CVE-2025-15281" # No fix available yet
+ "CVE-2026-0865" # No fix available yet
+ "CVE-2025-15282" # No fix available yet
+ "CVE-2026-0672" # No fix available yet
+ "CVE-2025-15366" # No fix available yet
+ "CVE-2025-15367" # No fix available yet
+ "CVE-2025-12781" # No fix available yet
+ "CVE-2025-11468" # No fix available yet
+ "CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
+ "CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
)
# Build JSON array of allowlisted CVE IDs for jq
diff --git a/cookbook/anthropic_agent_sdk/README.md b/cookbook/anthropic_agent_sdk/README.md
new file mode 100644
index 00000000000..294d949e24e
--- /dev/null
+++ b/cookbook/anthropic_agent_sdk/README.md
@@ -0,0 +1,144 @@
+# Claude Agent SDK with LiteLLM Gateway
+
+A simple example showing how to use Claude's Agent SDK with LiteLLM as a proxy. This lets you use any LLM provider (OpenAI, Bedrock, Azure, etc.) through the Agent SDK.
+
+## Quick Start
+
+### 1. Install dependencies
+
+```bash
+pip install anthropic claude-agent-sdk litellm
+```
+
+### 2. Start LiteLLM proxy
+
+```bash
+# Simple start with Claude
+litellm --model claude-sonnet-4-20250514
+
+# Or with a config file
+litellm --config config.yaml
+```
+
+### 3. Run the chat
+
+**Basic Agent (no MCP):**
+
+```bash
+python main.py
+```
+
+**Agent with MCP (DeepWiki2 for research):**
+
+```bash
+python agent_with_mcp.py
+```
+
+If MCP connection fails, you can disable it:
+
+```bash
+USE_MCP=false python agent_with_mcp.py
+```
+
+That's it! You can now chat with the agent in your terminal.
+
+### Chat Commands
+
+While chatting, you can use these commands:
+- `models` - List all available models (fetched from your LiteLLM proxy)
+- `model` - Switch to a different model
+- `clear` - Start a new conversation
+- `quit` or `exit` - End the chat
+
+The chat automatically fetches available models from your LiteLLM proxy's `/models` endpoint, so you'll always see what's currently configured.
+
+## Configuration
+
+Set these environment variables if needed:
+
+```bash
+export LITELLM_PROXY_URL="http://localhost:4000"
+export LITELLM_API_KEY="sk-1234"
+export LITELLM_MODEL="bedrock-claude-sonnet-4.5"
+```
+
+Or just use the defaults - it'll connect to `http://localhost:4000` by default.
+
+## Files
+
+- `main.py` - Basic interactive agent without MCP
+- `agent_with_mcp.py` - Agent with MCP server integration (DeepWiki2)
+- `common.py` - Shared utilities and functions
+- `config.example.yaml` - Example LiteLLM configuration
+- `requirements.txt` - Python dependencies
+
+## Example Config File
+
+If you want to use multiple models, create a `config.yaml` (see `config.example.yaml`):
+
+```yaml
+model_list:
+ - model_name: bedrock-claude-sonnet-4
+ litellm_params:
+ model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
+ aws_region_name: "us-east-1"
+
+ - model_name: bedrock-claude-sonnet-4.5
+ litellm_params:
+ model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
+ aws_region_name: "us-east-1"
+```
+
+Then start LiteLLM with: `litellm --config config.yaml`
+
+## How It Works
+
+The key is pointing the Agent SDK to LiteLLM instead of directly to Anthropic:
+
+```python
+# Point to LiteLLM gateway (not Anthropic)
+os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
+os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM key
+
+# Use any model configured in LiteLLM
+options = ClaudeAgentOptions(
+ model="bedrock-claude-sonnet-4", # or gpt-4, or anything else
+ system_prompt="You are a helpful assistant.",
+ max_turns=50,
+)
+```
+
+Note: Don't add `/anthropic` to the base URL - LiteLLM handles the routing automatically.
+
+## Why Use This?
+
+- **Switch providers easily**: Use the same code with OpenAI, Bedrock, Azure, etc.
+- **Cost tracking**: LiteLLM tracks spending across all your agent conversations
+- **Rate limiting**: Set budgets and limits on your agent usage
+- **Load balancing**: Distribute requests across multiple API keys or regions
+- **Fallbacks**: Automatically retry with a different model if one fails
+
+## Troubleshooting
+
+**Connection errors?**
+- Make sure LiteLLM is running: `litellm --model your-model`
+- Check the URL is correct (default: `http://localhost:4000`)
+
+**Authentication errors?**
+- Verify your LiteLLM API key is correct
+- Make sure the model is configured in your LiteLLM setup
+
+**Model not found?**
+- Check the model name matches what's in your LiteLLM config
+- Run `litellm --model your-model` to test it works
+
+**Agent with MCP stuck or failing?**
+- The MCP server might not be available at `http://localhost:4000/mcp/deepwiki2`
+- Try disabling MCP: `USE_MCP=false python agent_with_mcp.py`
+- Or use the basic agent: `python main.py`
+
+## Learn More
+
+- [LiteLLM Docs](https://docs.litellm.ai/)
+- [Claude Agent SDK](https://github.com/anthropics/anthropic-agent-sdk)
+- [LiteLLM Proxy Guide](https://docs.litellm.ai/docs/proxy/quick_start)
diff --git a/cookbook/anthropic_agent_sdk/agent_with_mcp.py b/cookbook/anthropic_agent_sdk/agent_with_mcp.py
new file mode 100644
index 00000000000..ff25feb777f
--- /dev/null
+++ b/cookbook/anthropic_agent_sdk/agent_with_mcp.py
@@ -0,0 +1,140 @@
+"""
+Interactive Claude Agent SDK CLI with MCP Support
+
+This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy,
+with MCP (Model Context Protocol) server integration for enhanced capabilities.
+"""
+
+import asyncio
+import os
+from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
+from common import (
+ Config,
+ fetch_available_models,
+ setup_litellm_env,
+ print_header,
+ handle_model_list,
+ handle_model_switch,
+ stream_response,
+)
+
+
+async def interactive_chat_with_mcp():
+ """
+ Interactive CLI chat with the agent and MCP server
+ """
+ config = Config()
+
+ # Configure Anthropic SDK to point to LiteLLM gateway
+ litellm_base_url = setup_litellm_env(config)
+
+ # Fetch available models from proxy
+ available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
+
+ current_model = config.LITELLM_MODEL
+
+ # MCP server configuration
+ mcp_server_url = f"{litellm_base_url}/mcp/deepwiki2"
+ use_mcp = os.getenv("USE_MCP", "true").lower() == "true"
+
+ if not use_mcp:
+ print("⚠️ MCP disabled via USE_MCP=false")
+
+ print_header(litellm_base_url, current_model, has_mcp=use_mcp)
+
+ while True:
+ # Configure agent options
+ if use_mcp:
+ try:
+ # Try with MCP server (HTTP transport)
+ # Using McpHttpServerConfig format from Agent SDK
+ options = ClaudeAgentOptions(
+ system_prompt="You are a helpful AI assistant with access to DeepWiki for research. Be concise, accurate, and friendly.",
+ model=current_model,
+ max_turns=50,
+ mcp_servers={
+ "deepwiki2": {
+ "type": "http",
+ "url": mcp_server_url,
+ "headers": {
+ "Authorization": f"Bearer {config.LITELLM_API_KEY}"
+ }
+ }
+ },
+ )
+ except Exception as e:
+ print(f"⚠️ Warning: Could not configure MCP server: {e}")
+ print("Continuing without MCP...\n")
+ use_mcp = False
+ options = ClaudeAgentOptions(
+ system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
+ model=current_model,
+ max_turns=50,
+ )
+ else:
+ # Without MCP
+ options = ClaudeAgentOptions(
+ system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
+ model=current_model,
+ max_turns=50,
+ )
+
+ # Create agent client
+ try:
+ async with ClaudeSDKClient(options=options) as client:
+ conversation_active = True
+
+ while conversation_active:
+ # Get user input
+ try:
+ user_input = input("\n👤 You: ").strip()
+ except (EOFError, KeyboardInterrupt):
+ print("\n\n👋 Goodbye!")
+ return
+
+ # Handle commands
+ if user_input.lower() in ['quit', 'exit']:
+ print("\n👋 Goodbye!")
+ return
+
+ if user_input.lower() == 'clear':
+ print("\n🔄 Starting new conversation...\n")
+ conversation_active = False
+ continue
+
+ if user_input.lower() == 'models':
+ handle_model_list(available_models, current_model)
+ continue
+
+ if user_input.lower() == 'model':
+ new_model, should_restart = handle_model_switch(available_models, current_model)
+ if should_restart:
+ current_model = new_model
+ conversation_active = False
+ continue
+
+ if not user_input:
+ continue
+
+ # Stream response from agent
+ await stream_response(client, user_input)
+
+ except Exception as e:
+ print(f"\n❌ Error creating agent client: {e}")
+ print("This might be an MCP configuration issue. Try running without MCP:")
+ print(" USE_MCP=false python agent_with_mcp.py")
+ print("\nOr use the basic agent:")
+ print(" python main.py")
+ return
+
+
+def main():
+ """Run interactive chat with MCP"""
+ try:
+ asyncio.run(interactive_chat_with_mcp())
+ except KeyboardInterrupt:
+ print("\n\n👋 Goodbye!")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/cookbook/anthropic_agent_sdk/common.py b/cookbook/anthropic_agent_sdk/common.py
new file mode 100644
index 00000000000..d9ee65cb58d
--- /dev/null
+++ b/cookbook/anthropic_agent_sdk/common.py
@@ -0,0 +1,160 @@
+"""
+Common utilities for Claude Agent SDK examples
+"""
+
+import os
+import httpx
+
+
+class Config:
+ """Configuration for LiteLLM Gateway connection"""
+
+ # LiteLLM proxy URL (default to local instance)
+ LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
+
+ # LiteLLM API key (master key or virtual key)
+ LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
+
+ # Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.)
+ LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5")
+
+
+async def fetch_available_models(base_url: str, api_key: str) -> list[str]:
+ """
+ Fetch available models from LiteLLM proxy /models endpoint
+ """
+ try:
+ async with httpx.AsyncClient() as client:
+ response = await client.get(
+ f"{base_url}/models",
+ headers={"Authorization": f"Bearer {api_key}"},
+ timeout=10.0
+ )
+ response.raise_for_status()
+ data = response.json()
+ return [model["id"] for model in data.get("data", [])]
+ except Exception as e:
+ print(f"⚠️ Warning: Could not fetch models from proxy: {e}")
+ print("Using default model list...")
+ # Fallback to default models
+ return [
+ "bedrock-claude-sonnet-3.5",
+ "bedrock-claude-sonnet-4",
+ "bedrock-claude-sonnet-4.5",
+ "bedrock-claude-opus-4.5",
+ "bedrock-nova-premier",
+ ]
+
+
+def setup_litellm_env(config: Config):
+ """
+ Configure environment variables to point Agent SDK to LiteLLM
+ """
+ litellm_base_url = config.LITELLM_PROXY_URL.rstrip('/')
+ os.environ["ANTHROPIC_BASE_URL"] = litellm_base_url
+ os.environ["ANTHROPIC_API_KEY"] = config.LITELLM_API_KEY
+ return litellm_base_url
+
+
+def print_header(base_url: str, current_model: str, has_mcp: bool = False):
+ """
+ Print the chat header
+ """
+ mcp_indicator = " + MCP" if has_mcp else ""
+ print("=" * 70)
+ print(f"🤖 Claude Agent SDK with LiteLLM Gateway{mcp_indicator} - Interactive Chat")
+ print("=" * 70)
+ print(f"🚀 Connected to: {base_url}")
+ print(f"📦 Current model: {current_model}")
+ if has_mcp:
+ print("🔌 MCP: deepwiki2 enabled")
+ print("\nType your messages below. Commands:")
+ print(" - 'quit' or 'exit' to end the conversation")
+ print(" - 'clear' to start a new conversation")
+ print(" - 'model' to switch models")
+ print(" - 'models' to list available models")
+ print("=" * 70)
+ print()
+
+
+def handle_model_list(available_models: list[str], current_model: str):
+ """
+ Display available models
+ """
+ print("\n📋 Available models:")
+ for i, model in enumerate(available_models, 1):
+ marker = "✓" if model == current_model else " "
+ print(f" {marker} {i}. {model}")
+
+
+def handle_model_switch(available_models: list[str], current_model: str) -> tuple[str, bool]:
+ """
+ Handle model switching
+
+ Returns:
+ tuple: (new_model, should_restart_conversation)
+ """
+ print("\n📋 Select a model:")
+ for i, model in enumerate(available_models, 1):
+ marker = "✓" if model == current_model else " "
+ print(f" {marker} {i}. {model}")
+
+ try:
+ choice = input("\nEnter number (or press Enter to cancel): ").strip()
+ if choice:
+ idx = int(choice) - 1
+ if 0 <= idx < len(available_models):
+ new_model = available_models[idx]
+ print(f"\n✅ Switched to: {new_model}")
+ print("🔄 Starting new conversation with new model...\n")
+ return new_model, True
+ else:
+ print("❌ Invalid choice")
+ except (ValueError, IndexError):
+ print("❌ Invalid input")
+
+ return current_model, False
+
+
+async def stream_response(client, user_input: str):
+ """
+ Stream response from the agent
+ """
+ print("\n🤖 Assistant: ", end='', flush=True)
+
+ try:
+ await client.query(user_input)
+
+ # Show loading indicator
+ print("⏳ thinking...", end='', flush=True)
+
+ # Stream the response
+ first_chunk = True
+ async for msg in client.receive_response():
+ # Clear loading indicator on first message
+ if first_chunk:
+ print("\r🤖 Assistant: ", end='', flush=True)
+ first_chunk = False
+
+ # Handle different message types
+ if hasattr(msg, 'type'):
+ if msg.type == 'content_block_delta':
+ # Streaming text delta
+ if hasattr(msg, 'delta') and hasattr(msg.delta, 'text'):
+ print(msg.delta.text, end='', flush=True)
+ elif msg.type == 'content_block_start':
+ # Start of content block
+ if hasattr(msg, 'content_block') and hasattr(msg.content_block, 'text'):
+ print(msg.content_block.text, end='', flush=True)
+
+ # Fallback to original content handling
+ if hasattr(msg, 'content'):
+ for content_block in msg.content:
+ if hasattr(content_block, 'text'):
+ print(content_block.text, end='', flush=True)
+
+ print() # New line after response
+
+ except Exception as e:
+ print(f"\r\n❌ Error: {e}")
+ print("Please check your LiteLLM gateway is running and configured correctly.")
diff --git a/cookbook/anthropic_agent_sdk/config.example.yaml b/cookbook/anthropic_agent_sdk/config.example.yaml
new file mode 100644
index 00000000000..eb1984fc4ea
--- /dev/null
+++ b/cookbook/anthropic_agent_sdk/config.example.yaml
@@ -0,0 +1,25 @@
+model_list:
+ - model_name: bedrock-claude-sonnet-3.5
+ litellm_params:
+ model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
+ aws_region_name: "us-east-1"
+
+ - model_name: bedrock-claude-sonnet-4
+ litellm_params:
+ model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
+ aws_region_name: "us-east-1"
+
+ - model_name: bedrock-claude-sonnet-4.5
+ litellm_params:
+ model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
+ aws_region_name: "us-east-1"
+
+ - model_name: bedrock-claude-opus-4.5
+ litellm_params:
+ model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0"
+ aws_region_name: "us-east-1"
+
+ - model_name: bedrock-nova-premier
+ litellm_params:
+ model: "bedrock/amazon.nova-premier-v1:0"
+ aws_region_name: "us-east-1"
diff --git a/cookbook/anthropic_agent_sdk/main.py b/cookbook/anthropic_agent_sdk/main.py
new file mode 100644
index 00000000000..231b57ca97b
--- /dev/null
+++ b/cookbook/anthropic_agent_sdk/main.py
@@ -0,0 +1,95 @@
+"""
+Simple Interactive Claude Agent SDK CLI using LiteLLM Gateway
+
+This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy.
+LiteLLM acts as a unified interface, allowing you to use any LLM provider (OpenAI, Azure, Bedrock, etc.)
+through the Claude Agent SDK by pointing it to the LiteLLM gateway.
+"""
+
+import asyncio
+from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
+from common import (
+ Config,
+ fetch_available_models,
+ setup_litellm_env,
+ print_header,
+ handle_model_list,
+ handle_model_switch,
+ stream_response,
+)
+
+
+async def interactive_chat():
+ """
+ Interactive CLI chat with the agent
+ """
+ config = Config()
+
+ # Configure Anthropic SDK to point to LiteLLM gateway
+ litellm_base_url = setup_litellm_env(config)
+
+ # Fetch available models from proxy
+ available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
+
+ current_model = config.LITELLM_MODEL
+
+ print_header(litellm_base_url, current_model)
+
+ while True:
+ # Configure agent options for each conversation
+ options = ClaudeAgentOptions(
+ system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
+ model=current_model,
+ max_turns=50,
+ )
+
+ # Create agent client
+ async with ClaudeSDKClient(options=options) as client:
+ conversation_active = True
+
+ while conversation_active:
+ # Get user input
+ try:
+ user_input = input("\n👤 You: ").strip()
+ except (EOFError, KeyboardInterrupt):
+ print("\n\n👋 Goodbye!")
+ return
+
+ # Handle commands
+ if user_input.lower() in ['quit', 'exit']:
+ print("\n👋 Goodbye!")
+ return
+
+ if user_input.lower() == 'clear':
+ print("\n🔄 Starting new conversation...\n")
+ conversation_active = False
+ continue
+
+ if user_input.lower() == 'models':
+ handle_model_list(available_models, current_model)
+ continue
+
+ if user_input.lower() == 'model':
+ new_model, should_restart = handle_model_switch(available_models, current_model)
+ if should_restart:
+ current_model = new_model
+ conversation_active = False
+ continue
+
+ if not user_input:
+ continue
+
+ # Stream response from agent
+ await stream_response(client, user_input)
+
+
+def main():
+ """Run interactive chat"""
+ try:
+ asyncio.run(interactive_chat())
+ except KeyboardInterrupt:
+ print("\n\n👋 Goodbye!")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/cookbook/anthropic_agent_sdk/requirements.txt b/cookbook/anthropic_agent_sdk/requirements.txt
new file mode 100644
index 00000000000..1e810bb7d99
--- /dev/null
+++ b/cookbook/anthropic_agent_sdk/requirements.txt
@@ -0,0 +1,2 @@
+claude-agent-sdk
+httpx>=0.27.0
diff --git a/cookbook/livekit_agent_sdk/README.md b/cookbook/livekit_agent_sdk/README.md
new file mode 100644
index 00000000000..1c3f0bf9564
--- /dev/null
+++ b/cookbook/livekit_agent_sdk/README.md
@@ -0,0 +1,114 @@
+# LiveKit Voice Agent with LiteLLM Gateway
+
+Simple example showing how to use LiveKit's xAI realtime plugin with LiteLLM as a proxy. This lets you switch between xAI, OpenAI, and Azure realtime APIs without changing your code.
+
+## Quick Start
+
+### 1. Install dependencies
+
+```bash
+pip install livekit-agents[xai] websockets
+```
+
+### 2. Start LiteLLM proxy
+
+```bash
+# With xAI
+export XAI_API_KEY="your-xai-key"
+litellm --config config.yaml --port 4000
+```
+
+### 3. Run the voice agent
+
+```bash
+python main.py
+```
+
+Type your message and get a voice response from Grok!
+
+## Configuration
+
+Set these environment variables if needed:
+
+```bash
+export LITELLM_PROXY_URL="http://localhost:4000"
+export LITELLM_API_KEY="sk-1234"
+export LITELLM_MODEL="grok-voice-agent"
+```
+
+Or use the defaults - connects to `http://localhost:4000` by default.
+
+## Example Config File
+
+Create a `config.yaml` with your realtime models:
+
+```yaml
+model_list:
+ - model_name: grok-voice-agent
+ litellm_params:
+ model: xai/grok-2-vision-1212
+ api_key: os.environ/XAI_API_KEY
+ model_info:
+ mode: realtime
+
+ - model_name: openai-voice-agent
+ litellm_params:
+ model: gpt-4o-realtime-preview
+ api_key: os.environ/OPENAI_API_KEY
+ model_info:
+ mode: realtime
+
+general_settings:
+ master_key: sk-1234
+```
+
+Then start: `litellm --config config.yaml --port 4000`
+
+## How It Works
+
+LiveKit's xAI plugin connects through LiteLLM proxy by setting `base_url`:
+
+```python
+from livekit.plugins import xai
+
+model = xai.realtime.RealtimeModel(
+ voice="ara",
+ api_key="sk-1234", # LiteLLM proxy key
+ base_url="http://localhost:4000", # Point to LiteLLM
+)
+```
+
+## Switching Providers
+
+Just change the model in your config - no code changes needed:
+
+**xAI Grok:**
+```yaml
+model: xai/grok-2-vision-1212
+```
+
+**OpenAI:**
+```yaml
+model: gpt-4o-realtime-preview
+```
+
+**Azure OpenAI:**
+```yaml
+model: azure/gpt-4o-realtime-preview
+api_base: https://your-endpoint.openai.azure.com/
+```
+
+## Why Use LiteLLM?
+
+- ✅ **Switch providers** without changing agent code
+- ✅ **Cost tracking** across all voice sessions
+- ✅ **Rate limiting** and budgets
+- ✅ **Load balancing** across multiple API keys
+- ✅ **Fallbacks** to backup models
+
+## Learn More
+
+- [LiveKit xAI Realtime Tutorial](/docs/tutorials/livekit_xai_realtime)
+- [xAI Realtime Docs](/docs/providers/xai_realtime)
+- [LiveKit Agents Documentation](https://docs.livekit.io/agents/)
+- [LiteLLM Realtime API](/docs/realtime)
diff --git a/cookbook/livekit_agent_sdk/config.example.yaml b/cookbook/livekit_agent_sdk/config.example.yaml
new file mode 100644
index 00000000000..1361f36af34
--- /dev/null
+++ b/cookbook/livekit_agent_sdk/config.example.yaml
@@ -0,0 +1,21 @@
+model_list:
+ - model_name: grok-voice-agent
+ litellm_params:
+ model: xai/grok-2-vision-1212
+ api_key: os.environ/XAI_API_KEY
+ model_info:
+ mode: realtime
+
+ - model_name: openai-voice-agent
+ litellm_params:
+ model: gpt-4o-realtime-preview
+ api_key: os.environ/OPENAI_API_KEY
+ model_info:
+ mode: realtime
+
+litellm_settings:
+ drop_params: True
+ telemetry: False
+
+general_settings:
+ master_key: sk-1234 # Change this to a secure key
diff --git a/cookbook/livekit_agent_sdk/main.py b/cookbook/livekit_agent_sdk/main.py
new file mode 100644
index 00000000000..0e2d7ebdfaf
--- /dev/null
+++ b/cookbook/livekit_agent_sdk/main.py
@@ -0,0 +1,112 @@
+"""
+Simple xAI Voice Agent using LiveKit SDK with LiteLLM Gateway
+
+This example shows how to use LiveKit's xAI realtime plugin through LiteLLM proxy.
+LiteLLM acts as a unified interface, allowing you to switch between xAI, OpenAI,
+and Azure realtime APIs without changing your agent code.
+"""
+import asyncio
+import json
+import os
+import websockets
+
+# Configuration
+PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
+API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
+MODEL = os.getenv("LITELLM_MODEL", "grok-voice-agent")
+
+
+async def run_voice_agent():
+ """
+ Simple voice agent that:
+ 1. Connects to xAI realtime API through LiteLLM proxy
+ 2. Sends a user message
+ 3. Streams back the response
+ """
+
+ url = f"ws://{PROXY_URL.replace('http://', '').replace('https://', '')}/v1/realtime?model={MODEL}"
+ headers = {"Authorization": f"Bearer {API_KEY}"}
+
+ print(f"🎙️ Connecting to voice agent...")
+ print(f" Model: {MODEL}")
+ print(f" Proxy: {PROXY_URL}")
+ print()
+
+ async with websockets.connect(url, additional_headers=headers) as ws:
+ # Receive initial connection event
+ initial = json.loads(await ws.recv())
+ print(f"✅ Connected! Event: {initial['type']}\n")
+
+ # Get user input
+ user_message = input("💬 Your message: ").strip()
+ if not user_message:
+ user_message = "Tell me a fun fact about AI!"
+
+ print(f"\n🤖 Sending to {MODEL}...\n")
+
+ # Send user message
+ await ws.send(json.dumps({
+ "type": "conversation.item.create",
+ "item": {
+ "type": "message",
+ "role": "user",
+ "content": [{"type": "input_text", "text": user_message}]
+ }
+ }))
+
+ # Request response
+ await ws.send(json.dumps({
+ "type": "response.create",
+ "response": {"modalities": ["text", "audio"]}
+ }))
+
+ # Stream response
+ print("🎤 Response: ", end='', flush=True)
+ transcript = []
+
+ try:
+ while True:
+ msg = await asyncio.wait_for(ws.recv(), timeout=15.0)
+ event = json.loads(msg)
+
+ # Capture transcript deltas
+ if event['type'] == 'response.output_audio_transcript.delta':
+ delta = event.get('delta', '')
+ if delta:
+ print(delta, end='', flush=True)
+ transcript.append(delta)
+
+ # Done when response completes
+ elif event['type'] == 'response.done':
+ break
+
+ except asyncio.TimeoutError:
+ pass
+
+ print("\n")
+
+ if transcript:
+ print(f"✅ Complete response: {''.join(transcript)}")
+
+ await ws.close()
+
+
+def main():
+ """Run the voice agent"""
+ print("=" * 70)
+ print("LiveKit xAI Voice Agent via LiteLLM Proxy")
+ print("=" * 70)
+ print()
+
+ try:
+ asyncio.run(run_voice_agent())
+ except KeyboardInterrupt:
+ print("\n\n👋 Goodbye!")
+ except Exception as e:
+ print(f"\n❌ Error: {e}")
+ print("\nMake sure LiteLLM proxy is running:")
+ print(f" litellm --config config.yaml --port 4000")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/cookbook/livekit_agent_sdk/requirements.txt b/cookbook/livekit_agent_sdk/requirements.txt
new file mode 100644
index 00000000000..9e3542fac27
--- /dev/null
+++ b/cookbook/livekit_agent_sdk/requirements.txt
@@ -0,0 +1,2 @@
+livekit-agents[xai]>=1.3.12
+websockets>=15.0.1
diff --git a/cookbook/mock_prompt_management_server/README.md b/cookbook/mock_prompt_management_server/README.md
new file mode 100644
index 00000000000..9ec76baacf7
--- /dev/null
+++ b/cookbook/mock_prompt_management_server/README.md
@@ -0,0 +1,293 @@
+# Mock Prompt Management Server
+
+A reference implementation of the [LiteLLM Generic Prompt Management API](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api).
+
+This FastAPI server demonstrates how to build a prompt management API that integrates with LiteLLM without requiring a PR to the LiteLLM repository.
+
+## Quick Start
+
+### 1. Install Dependencies
+
+```bash
+pip install fastapi uvicorn pydantic
+```
+
+### 2. Start the Server
+
+```bash
+python mock_prompt_management_server.py
+```
+
+The server will start on `http://localhost:8080`
+
+### 3. Test the Endpoint
+
+```bash
+# Get a prompt
+curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"
+
+# Get a prompt with authentication
+curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt" \
+ -H "Authorization: Bearer test-token-12345"
+
+# List all prompts
+curl "http://localhost:8080/prompts"
+
+# Get prompt variables
+curl "http://localhost:8080/prompts/hello-world-prompt/variables"
+```
+
+## Using with LiteLLM
+
+### Configuration
+
+Create a `config.yaml` file:
+
+```yaml
+model_list:
+ - model_name: gpt-3.5-turbo
+ litellm_params:
+ model: openai/gpt-3.5-turbo
+ api_key: os.environ/OPENAI_API_KEY
+
+prompts:
+ - prompt_id: "hello-world-prompt"
+ litellm_params:
+ prompt_integration: "generic_prompt_management"
+ api_base: http://localhost:8080
+ api_key: test-token-12345
+```
+
+### Start LiteLLM Proxy
+
+```bash
+litellm --config config.yaml
+```
+
+### Make a Request
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer sk-1234" \
+ -d '{
+ "model": "gpt-3.5-turbo",
+ "prompt_id": "hello-world-prompt",
+ "prompt_variables": {
+ "domain": "data science",
+ "task": "analyzing customer behavior"
+ },
+ "messages": [
+ {"role": "user", "content": "Please help me get started"}
+ ]
+ }'
+```
+
+## Available Prompts
+
+The server includes several example prompts:
+
+| Prompt ID | Description | Variables |
+|-----------|-------------|-----------|
+| `hello-world-prompt` | Basic helpful assistant | `domain`, `task` |
+| `code-review-prompt` | Code review assistant | `years_experience`, `language`, `code` |
+| `customer-support-prompt` | Customer support agent | `company_name`, `customer_message` |
+| `data-analysis-prompt` | Data analysis expert | `analysis_type`, `dataset_name`, `data` |
+| `creative-writing-prompt` | Creative writing assistant | `genre`, `length`, `topic` |
+
+## Authentication
+
+The server supports optional Bearer token authentication. Valid tokens for testing:
+
+- `test-token-12345`
+- `dev-token-67890`
+- `prod-token-abcdef`
+
+If no `Authorization` header is provided, requests are allowed (for testing purposes).
+
+## API Endpoints
+
+### LiteLLM Spec Endpoints
+
+#### `GET /beta/litellm_prompt_management`
+
+Get a prompt by ID (required by LiteLLM).
+
+**Query Parameters:**
+- `prompt_id` (required): The prompt ID
+- `project_name` (optional): Project filter
+- `slug` (optional): Slug filter
+- `version` (optional): Version filter
+
+**Response:**
+```json
+{
+ "prompt_id": "hello-world-prompt",
+ "prompt_template": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant specialized in {domain}."
+ },
+ {
+ "role": "user",
+ "content": "Help me with: {task}"
+ }
+ ],
+ "prompt_template_model": "gpt-4",
+ "prompt_template_optional_params": {
+ "temperature": 0.7,
+ "max_tokens": 500
+ }
+}
+```
+
+### Convenience Endpoints (Not in LiteLLM Spec)
+
+#### `GET /health`
+
+Health check endpoint.
+
+#### `GET /prompts`
+
+List all available prompts.
+
+#### `GET /prompts/{prompt_id}/variables`
+
+Get all variables used in a prompt template.
+
+#### `POST /prompts`
+
+Create a new prompt (in-memory only, for testing).
+
+## Example: Full Integration Test
+
+### 1. Start the Mock Server
+
+```bash
+python mock_prompt_management_server.py
+```
+
+### 2. Test with Python
+
+```python
+from litellm import completion
+
+# The completion will:
+# 1. Fetch the prompt from your API
+# 2. Replace {domain} with "machine learning"
+# 3. Replace {task} with "building a recommendation system"
+# 4. Merge with your messages
+# 5. Use the model and params from the prompt
+
+response = completion(
+ model="gpt-4",
+ prompt_id="hello-world-prompt",
+ prompt_variables={
+ "domain": "machine learning",
+ "task": "building a recommendation system"
+ },
+ messages=[
+ {"role": "user", "content": "I have user behavior data from the past year."}
+ ],
+ # Configure the generic prompt manager
+ generic_prompt_config={
+ "api_base": "http://localhost:8080",
+ "api_key": "test-token-12345",
+ }
+)
+
+print(response.choices[0].message.content)
+```
+
+## Customization
+
+### Adding New Prompts
+
+Edit the `PROMPTS_DB` dictionary in `mock_prompt_management_server.py`:
+
+```python
+PROMPTS_DB = {
+ "my-custom-prompt": {
+ "prompt_id": "my-custom-prompt",
+ "prompt_template": [
+ {
+ "role": "system",
+ "content": "You are a {role}."
+ },
+ {
+ "role": "user",
+ "content": "{user_input}"
+ }
+ ],
+ "prompt_template_model": "gpt-4",
+ "prompt_template_optional_params": {
+ "temperature": 0.8,
+ "max_tokens": 1000
+ }
+ }
+}
+```
+
+### Using a Database
+
+Replace the `PROMPTS_DB` dictionary with database queries:
+
+```python
+@app.get("/beta/litellm_prompt_management")
+async def get_prompt(prompt_id: str):
+ # Fetch from database
+ prompt = await db.prompts.find_one({"prompt_id": prompt_id})
+
+ if not prompt:
+ raise HTTPException(status_code=404, detail="Prompt not found")
+
+ return PromptResponse(**prompt)
+```
+
+### Adding Access Control
+
+Use the custom query parameters for access control:
+
+```python
+@app.get("/beta/litellm_prompt_management")
+async def get_prompt(
+ prompt_id: str,
+ project_name: Optional[str] = None,
+ user_id: Optional[str] = None,
+ authorization: Optional[str] = Header(None)
+):
+ token = verify_api_key(authorization)
+
+ # Check if user has access to this project
+ if not has_project_access(token, project_name):
+ raise HTTPException(status_code=403, detail="Access denied")
+
+ # Fetch and return prompt
+ ...
+```
+
+## Production Considerations
+
+Before deploying to production:
+
+1. **Use a real database** instead of in-memory storage
+2. **Implement proper authentication** with JWT tokens or API keys
+3. **Add rate limiting** to prevent abuse
+4. **Use HTTPS** for encrypted communication
+5. **Add logging and monitoring** for observability
+6. **Implement caching** for frequently accessed prompts
+7. **Add versioning** for prompt management
+8. **Implement access control** based on teams/users
+9. **Add input validation** for all parameters
+10. **Use environment variables** for configuration
+
+## Related Documentation
+
+- [Generic Prompt Management API Documentation](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api)
+- [LiteLLM Prompt Management](https://docs.litellm.ai/docs/proxy/prompt_management)
+- [Generic Guardrail API](https://docs.litellm.ai/docs/adding_provider/generic_guardrail_api)
+
+## Questions?
+
+This is a reference implementation for the LiteLLM Generic Prompt Management API. For questions or issues, please open an issue on the [LiteLLM GitHub repository](https://github.com/BerriAI/litellm).
+
diff --git a/cookbook/mock_prompt_management_server/mock_prompt_management_server.py b/cookbook/mock_prompt_management_server/mock_prompt_management_server.py
new file mode 100644
index 00000000000..295a96e12a9
--- /dev/null
+++ b/cookbook/mock_prompt_management_server/mock_prompt_management_server.py
@@ -0,0 +1,390 @@
+#!/usr/bin/env python3
+"""
+Mock Prompt Management API Server
+
+This is a FastAPI server that implements the LiteLLM Generic Prompt Management API
+for testing and demonstration purposes.
+
+Usage:
+ python mock_prompt_management_server.py
+
+The server will start on http://localhost:8080
+
+Test the endpoint:
+ curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"
+"""
+
+import os
+import json
+from typing import Any, Dict, List, Optional
+
+from fastapi import FastAPI, HTTPException, Header, Query, status
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field
+
+# ============================================================================
+# Response Models
+# ============================================================================
+
+
+class MessageContent(BaseModel):
+ """A single message in the prompt template"""
+
+ role: str = Field(..., description="Message role (system, user, assistant)")
+ content: str = Field(
+ ..., description="Message content with optional {variable} placeholders"
+ )
+
+
+class PromptResponse(BaseModel):
+ """Response format for the prompt management API"""
+
+ prompt_id: str = Field(..., description="The ID of the prompt")
+ prompt_template: List[MessageContent] = Field(
+ ..., description="Array of messages in OpenAI format"
+ )
+ prompt_template_model: Optional[str] = Field(
+ None, description="Optional model to use for this prompt"
+ )
+ prompt_template_optional_params: Optional[Dict[str, Any]] = Field(
+ None, description="Optional parameters like temperature, max_tokens, etc."
+ )
+
+
+# ============================================================================
+# Mock Prompt Database
+# ============================================================================
+
+PROMPTS_DB = {
+ "hello-world-prompt": {
+ "prompt_id": "hello-world-prompt",
+ "prompt_template": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant specialized in {domain}.",
+ },
+ {"role": "user", "content": "Help me with: {task}"},
+ ],
+ "prompt_template_model": "gpt-4",
+ "prompt_template_optional_params": {"temperature": 0.7, "max_tokens": 500},
+ },
+ "code-review-prompt": {
+ "prompt_id": "code-review-prompt",
+ "prompt_template": [
+ {
+ "role": "system",
+ "content": "You are an expert code reviewer with {years_experience} years of experience in {language}.",
+ },
+ {
+ "role": "user",
+ "content": "Please review the following code for bugs, security issues, and best practices:\n\n{code}",
+ },
+ ],
+ "prompt_template_model": "gpt-4-turbo",
+ "prompt_template_optional_params": {
+ "temperature": 0.3,
+ "max_tokens": 1500,
+ },
+ },
+ "customer-support-prompt": {
+ "prompt_id": "customer-support-prompt",
+ "prompt_template": [
+ {
+ "role": "system",
+ "content": "You are a friendly customer support agent for {company_name}. Always be professional, empathetic, and solution-oriented.",
+ },
+ {
+ "role": "user",
+ "content": "Customer inquiry: {customer_message}",
+ },
+ ],
+ "prompt_template_model": "gpt-3.5-turbo",
+ "prompt_template_optional_params": {
+ "temperature": 0.8,
+ "max_tokens": 800,
+ "top_p": 0.9,
+ },
+ },
+ "data-analysis-prompt": {
+ "prompt_id": "data-analysis-prompt",
+ "prompt_template": [
+ {
+ "role": "system",
+ "content": "You are a data scientist expert in {analysis_type} analysis.",
+ },
+ {
+ "role": "user",
+ "content": "Analyze the following data and provide insights:\n\nDataset: {dataset_name}\nData: {data}",
+ },
+ ],
+ "prompt_template_model": "gpt-4",
+ "prompt_template_optional_params": {
+ "temperature": 0.5,
+ "max_tokens": 2000,
+ },
+ },
+ "creative-writing-prompt": {
+ "prompt_id": "creative-writing-prompt",
+ "prompt_template": [
+ {
+ "role": "system",
+ "content": "You are a creative writer specializing in {genre} fiction.",
+ },
+ {
+ "role": "user",
+ "content": "Write a {length} story about: {topic}",
+ },
+ ],
+ "prompt_template_model": "gpt-4",
+ "prompt_template_optional_params": {
+ "temperature": 0.9,
+ "max_tokens": 3000,
+ "top_p": 0.95,
+ },
+ },
+}
+
+# Valid API tokens for authentication (in production, use a secure token store)
+VALID_API_TOKENS = {
+ "test-token-12345",
+ "dev-token-67890",
+ "prod-token-abcdef",
+}
+
+# ============================================================================
+# FastAPI App
+# ============================================================================
+
+app = FastAPI(
+ title="Mock Prompt Management API",
+ description="A mock server implementing the LiteLLM Generic Prompt Management API",
+ version="1.0.0",
+)
+
+
+def verify_api_key(authorization: Optional[str] = Header(None)) -> bool:
+ """
+ Verify the API key from the Authorization header.
+
+ Args:
+ authorization: Authorization header (Bearer token)
+
+ Returns:
+ True if valid, raises HTTPException if invalid
+ """
+ if authorization is None:
+ # Allow requests without authentication for testing
+ return True
+
+ # Extract token from "Bearer "
+ if not authorization.startswith("Bearer "):
+ raise HTTPException(
+ status_code=status.HTTP_401_UNAUTHORIZED,
+ detail="Invalid authorization header format. Expected 'Bearer '",
+ )
+
+ token = authorization.replace("Bearer ", "").strip()
+
+ if token not in VALID_API_TOKENS:
+ raise HTTPException(
+ status_code=status.HTTP_401_UNAUTHORIZED,
+ detail="Invalid API key",
+ )
+
+ return True
+
+
+@app.get("/beta/litellm_prompt_management", response_model=PromptResponse)
+async def get_prompt(
+ prompt_id: str = Query(..., description="The ID of the prompt to fetch"),
+ project_name: Optional[str] = Query(
+ None, description="Optional project name filter"
+ ),
+ slug: Optional[str] = Query(None, description="Optional slug filter"),
+ version: Optional[str] = Query(None, description="Optional version filter"),
+ authorization: Optional[str] = Header(None),
+) -> PromptResponse:
+ """
+ Get a prompt by ID with optional filtering.
+
+ This endpoint implements the LiteLLM Generic Prompt Management API specification.
+
+ Args:
+ prompt_id: The ID of the prompt to fetch
+ project_name: Optional project name for filtering
+ slug: Optional slug for filtering
+ version: Optional version for filtering
+ authorization: Optional Bearer token for authentication
+
+ Returns:
+ PromptResponse with the prompt template and configuration
+
+ Raises:
+ HTTPException: 401 if authentication fails, 404 if prompt not found
+ """
+ # Verify authentication
+ verify_api_key(authorization)
+
+ # Log the request parameters (useful for debugging)
+ print(f"Fetching prompt: {prompt_id}")
+ if project_name:
+ print(f" Project: {project_name}")
+ if slug:
+ print(f" Slug: {slug}")
+ if version:
+ print(f" Version: {version}")
+
+ # Check if prompt exists
+ if prompt_id not in PROMPTS_DB:
+ raise HTTPException(
+ status_code=status.HTTP_404_NOT_FOUND,
+ detail=f"Prompt '{prompt_id}' not found. Available prompts: {list(PROMPTS_DB.keys())}",
+ )
+
+ # Get the prompt from the database
+ prompt_data = PROMPTS_DB[prompt_id]
+
+ # Optional: Apply filtering based on project_name, slug, or version
+ # In a real implementation, you might use these to filter prompts by access control
+ # or to fetch specific versions from your database
+
+ return PromptResponse(**prompt_data)
+
+
+@app.get("/health")
+async def health_check():
+ """Health check endpoint"""
+ return {
+ "status": "healthy",
+ "service": "mock-prompt-management-api",
+ "version": "1.0.0",
+ }
+
+
+@app.get("/prompts")
+async def list_prompts(authorization: Optional[str] = Header(None)):
+ """
+ List all available prompts.
+
+ This is a convenience endpoint (not part of the LiteLLM spec) for
+ discovering available prompts.
+ """
+ # Verify authentication
+ verify_api_key(authorization)
+
+ prompts_list = [
+ {
+ "prompt_id": pid,
+ "model": p.get("prompt_template_model"),
+ "has_variables": any(
+ "{" in msg.get("content", "") for msg in p.get("prompt_template", [])
+ ),
+ }
+ for pid, p in PROMPTS_DB.items()
+ ]
+
+ return {"prompts": prompts_list, "total": len(prompts_list)}
+
+
+@app.get("/prompts/{prompt_id}/variables")
+async def get_prompt_variables(
+ prompt_id: str, authorization: Optional[str] = Header(None)
+):
+ """
+ Get all variables in a prompt template.
+
+ This is a convenience endpoint (not part of the LiteLLM spec) for
+ discovering what variables a prompt expects.
+ """
+ # Verify authentication
+ verify_api_key(authorization)
+
+ if prompt_id not in PROMPTS_DB:
+ raise HTTPException(
+ status_code=status.HTTP_404_NOT_FOUND,
+ detail=f"Prompt '{prompt_id}' not found",
+ )
+
+ prompt_data = PROMPTS_DB[prompt_id]
+ variables = set()
+
+ # Extract variables from the prompt template
+ import re
+
+ for message in prompt_data["prompt_template"]:
+ content = message.get("content", "")
+ # Find all {variable} patterns
+ found_vars = re.findall(r"\{(\w+)\}", content)
+ variables.update(found_vars)
+
+ return {
+ "prompt_id": prompt_id,
+ "variables": sorted(list(variables)),
+ "example_usage": {
+ "prompt_id": prompt_id,
+ "prompt_variables": {var: f"<{var}_value>" for var in variables},
+ },
+ }
+
+
+@app.post("/prompts")
+async def create_prompt(
+ prompt: PromptResponse, authorization: Optional[str] = Header(None)
+):
+ """
+ Create a new prompt (convenience endpoint for testing).
+
+ This is NOT part of the LiteLLM spec - it's just for testing purposes.
+ """
+ # Verify authentication
+ verify_api_key(authorization)
+
+ if prompt.prompt_id in PROMPTS_DB:
+ raise HTTPException(
+ status_code=status.HTTP_409_CONFLICT,
+ detail=f"Prompt '{prompt.prompt_id}' already exists",
+ )
+
+ PROMPTS_DB[prompt.prompt_id] = prompt.dict()
+
+ return {
+ "status": "created",
+ "prompt_id": prompt.prompt_id,
+ "message": "Prompt created successfully (in-memory only)",
+ }
+
+
+# ============================================================================
+# Main
+# ============================================================================
+
+if __name__ == "__main__":
+ import uvicorn
+
+ print("=" * 70)
+ print("Mock Prompt Management API Server")
+ print("=" * 70)
+ print(f"\nStarting server on http://localhost:8080")
+ print(f"\nAvailable prompts: {len(PROMPTS_DB)}")
+ for prompt_id in PROMPTS_DB.keys():
+ print(f" - {prompt_id}")
+ print(f"\nValid API tokens: {len(VALID_API_TOKENS)}")
+ print(" - test-token-12345")
+ print(" - dev-token-67890")
+ print(" - prod-token-abcdef")
+ print("\nEndpoints:")
+ print(" GET /beta/litellm_prompt_management?prompt_id= (LiteLLM spec)")
+ print(" GET /health (health check)")
+ print(" GET /prompts (list all prompts)")
+ print(
+ " GET /prompts/{id}/variables (get prompt variables)"
+ )
+ print(" POST /prompts (create prompt)")
+ print("\nExample usage:")
+ print(
+ ' curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"'
+ )
+ print("\nPress CTRL+C to stop the server")
+ print("=" * 70)
+
+ uvicorn.run(app, host="0.0.0.0", port=8080, log_level="info")
diff --git a/cookbook/nova_sonic_realtime.py b/cookbook/nova_sonic_realtime.py
new file mode 100644
index 00000000000..c7a73c1d00f
--- /dev/null
+++ b/cookbook/nova_sonic_realtime.py
@@ -0,0 +1,288 @@
+"""
+Client script to test Nova Sonic realtime API through LiteLLM proxy.
+
+This script connects to LiteLLM proxy's realtime endpoint and enables
+speech-to-speech conversation with Bedrock Nova Sonic.
+
+Prerequisites:
+- LiteLLM proxy running with Bedrock configured
+- pyaudio installed: pip install pyaudio
+- websockets installed: pip install websockets
+
+Usage:
+ python nova_sonic_realtime.py
+"""
+
+import asyncio
+import base64
+import json
+import os
+import pyaudio
+import websockets
+from typing import Optional
+
+# Bounded queue size for audio chunks (configurable via env to avoid unbounded memory)
+AUDIO_QUEUE_MAXSIZE = int(os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 10_000))
+
+# Audio configuration (matching Nova Sonic requirements)
+INPUT_SAMPLE_RATE = 16000 # Nova Sonic expects 16kHz input
+OUTPUT_SAMPLE_RATE = 24000 # Nova Sonic outputs 24kHz
+CHANNELS = 1
+FORMAT = pyaudio.paInt16
+CHUNK_SIZE = 1024
+
+# LiteLLM proxy configuration
+LITELLM_PROXY_URL = "ws://localhost:4000/v1/realtime?model=bedrock-sonic"
+LITELLM_API_KEY = "sk-12345" # Your LiteLLM API key
+
+
+class RealtimeClient:
+ """Client for LiteLLM realtime API with audio support."""
+
+ def __init__(self, url: str, api_key: str):
+ self.url = url
+ self.api_key = api_key
+ self.ws: Optional[websockets.WebSocketClientProtocol] = None
+ self.is_active = False
+ self.audio_queue = asyncio.Queue(maxsize=AUDIO_QUEUE_MAXSIZE)
+ self.pyaudio = pyaudio.PyAudio()
+ self.input_stream = None
+ self.output_stream = None
+
+ async def connect(self):
+ """Connect to LiteLLM proxy realtime endpoint."""
+ print(f"Connecting to {self.url}...")
+
+ headers = {}
+ if self.api_key:
+ headers["Authorization"] = f"Bearer {self.api_key}"
+
+ self.ws = await websockets.connect(
+ self.url,
+ additional_headers=headers,
+ max_size=10 * 1024 * 1024, # 10MB max message size
+ )
+ self.is_active = True
+ print("✓ Connected to LiteLLM proxy")
+
+ async def send_session_update(self):
+ """Send session configuration."""
+ session_update = {
+ "type": "session.update",
+ "session": {
+ "instructions": "You are a friendly assistant. Keep your responses short and conversational.",
+ "voice": "matthew",
+ "temperature": 0.8,
+ "max_response_output_tokens": 1024,
+ "modalities": ["text", "audio"],
+ "input_audio_format": "pcm16",
+ "output_audio_format": "pcm16",
+ "turn_detection": {
+ "type": "server_vad",
+ "threshold": 0.5,
+ "prefix_padding_ms": 300,
+ "silence_duration_ms": 500,
+ },
+ },
+ }
+ await self.ws.send(json.dumps(session_update))
+ print("✓ Session configuration sent")
+
+ async def receive_messages(self):
+ """Receive and process messages from the server."""
+ try:
+ async for message in self.ws:
+ if not self.is_active:
+ break
+
+ try:
+ data = json.loads(message)
+ event_type = data.get("type")
+
+ if event_type == "session.created":
+ print(f"✓ Session created: {data.get('session', {}).get('id')}")
+
+ elif event_type == "response.created":
+ print("🤖 Assistant is responding...")
+
+ elif event_type == "response.text.delta":
+ # Print text transcription
+ delta = data.get("delta", "")
+ print(delta, end="", flush=True)
+
+ elif event_type == "response.audio.delta":
+ # Queue audio for playback
+ audio_b64 = data.get("delta", "")
+ if audio_b64:
+ audio_bytes = base64.b64decode(audio_b64)
+ await self.audio_queue.put(audio_bytes)
+
+ elif event_type == "response.text.done":
+ print() # New line after text
+
+ elif event_type == "response.done":
+ print("✓ Response complete")
+
+ elif event_type == "error":
+ print(f"❌ Error: {data.get('error', {})}")
+
+ else:
+ # Debug: print other event types
+ print(f"[{event_type}]", end=" ")
+
+ except json.JSONDecodeError:
+ print(f"Failed to parse message: {message[:100]}")
+
+ except websockets.exceptions.ConnectionClosed:
+ print("\n✗ Connection closed")
+ except Exception as e:
+ print(f"\n✗ Error receiving messages: {e}")
+ finally:
+ self.is_active = False
+
+ async def send_audio_chunk(self, audio_bytes: bytes):
+ """Send audio chunk to server."""
+ if not self.is_active or not self.ws:
+ return
+
+ audio_b64 = base64.b64encode(audio_bytes).decode("utf-8")
+ message = {
+ "type": "input_audio_buffer.append",
+ "audio": audio_b64,
+ }
+ await self.ws.send(json.dumps(message))
+
+ async def commit_audio_buffer(self):
+ """Commit the audio buffer to trigger processing."""
+ if not self.is_active or not self.ws:
+ return
+
+ message = {"type": "input_audio_buffer.commit"}
+ await self.ws.send(json.dumps(message))
+
+ async def capture_audio(self):
+ """Capture audio from microphone and send to server."""
+ print("\n🎤 Starting audio capture...")
+ print("Speak into your microphone. Press Ctrl+C to stop.\n")
+
+ self.input_stream = self.pyaudio.open(
+ format=FORMAT,
+ channels=CHANNELS,
+ rate=INPUT_SAMPLE_RATE,
+ input=True,
+ frames_per_buffer=CHUNK_SIZE,
+ )
+
+ try:
+ while self.is_active:
+ audio_data = self.input_stream.read(CHUNK_SIZE, exception_on_overflow=False)
+ await self.send_audio_chunk(audio_data)
+ await asyncio.sleep(0.01) # Small delay to prevent overwhelming
+ except Exception as e:
+ print(f"Error capturing audio: {e}")
+ finally:
+ if self.input_stream:
+ self.input_stream.stop_stream()
+ self.input_stream.close()
+
+ async def play_audio(self):
+ """Play audio responses from the server."""
+ print("🔊 Starting audio playback...")
+
+ self.output_stream = self.pyaudio.open(
+ format=FORMAT,
+ channels=CHANNELS,
+ rate=OUTPUT_SAMPLE_RATE,
+ output=True,
+ frames_per_buffer=CHUNK_SIZE,
+ )
+
+ try:
+ while self.is_active:
+ try:
+ audio_data = await asyncio.wait_for(
+ self.audio_queue.get(), timeout=0.1
+ )
+ if audio_data:
+ self.output_stream.write(audio_data)
+ except asyncio.TimeoutError:
+ continue
+ except Exception as e:
+ print(f"Error playing audio: {e}")
+ finally:
+ if self.output_stream:
+ self.output_stream.stop_stream()
+ self.output_stream.close()
+
+ async def close(self):
+ """Close the connection and cleanup."""
+ self.is_active = False
+
+ if self.ws:
+ await self.ws.close()
+
+ if self.input_stream:
+ self.input_stream.stop_stream()
+ self.input_stream.close()
+
+ if self.output_stream:
+ self.output_stream.stop_stream()
+ self.output_stream.close()
+
+ self.pyaudio.terminate()
+ print("\n✓ Connection closed")
+
+
+async def main():
+ """Main function to run the realtime client."""
+ print("=" * 80)
+ print("Bedrock Nova Sonic Realtime Client")
+ print("=" * 80)
+ print()
+
+ client = RealtimeClient(LITELLM_PROXY_URL, LITELLM_API_KEY)
+
+ try:
+ # Connect to server
+ await client.connect()
+
+ # Send session configuration
+ await client.send_session_update()
+
+ # Wait a moment for session to be established
+ await asyncio.sleep(0.5)
+
+ # Start tasks
+ receive_task = asyncio.create_task(client.receive_messages())
+ capture_task = asyncio.create_task(client.capture_audio())
+ playback_task = asyncio.create_task(client.play_audio())
+
+ # Wait for user to interrupt
+ await asyncio.gather(
+ receive_task,
+ capture_task,
+ playback_task,
+ return_exceptions=True,
+ )
+
+ except KeyboardInterrupt:
+ print("\n\n⚠ Interrupted by user")
+ except Exception as e:
+ print(f"\n❌ Error: {e}")
+ import traceback
+ traceback.print_exc()
+ finally:
+ await client.close()
+
+
+if __name__ == "__main__":
+ print("\nMake sure:")
+ print("1. LiteLLM proxy is running on port 4000")
+ print("2. Bedrock is configured in proxy_server_config.yaml")
+ print("3. AWS credentials are set")
+ print()
+
+ try:
+ asyncio.run(main())
+ except KeyboardInterrupt:
+ print("\n\nGoodbye!")
diff --git a/deploy/charts/litellm-helm/Chart.yaml b/deploy/charts/litellm-helm/Chart.yaml
index 8a08f0b4e29..0f6db331e50 100644
--- a/deploy/charts/litellm-helm/Chart.yaml
+++ b/deploy/charts/litellm-helm/Chart.yaml
@@ -26,6 +26,10 @@ version: 1.1.0
# It is recommended to use it with quotes.
appVersion: v1.80.12
+annotations:
+ org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
+ org.opencontainers.image.url: "https://docs.litellm.ai/"
+
dependencies:
- name: "postgresql"
version: ">=13.3.0"
diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml
index c3e0055e380..4ac5582d060 100644
--- a/deploy/charts/litellm-helm/templates/deployment.yaml
+++ b/deploy/charts/litellm-helm/templates/deployment.yaml
@@ -38,6 +38,10 @@ spec:
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
+ {{- with .Values.extraInitContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
containers:
- name: {{ include "litellm.name" . }}
securityContext:
diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml
index f8893a47afe..3459fa12d1c 100644
--- a/deploy/charts/litellm-helm/templates/migrations-job.yaml
+++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml
@@ -35,6 +35,10 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
+ {{- with .Values.migrationJob.extraInitContainers }}
+ initContainers:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
containers:
- name: prisma-migrations
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (printf "main-%s" .Chart.AppVersion) }}"
diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml
index 54271756998..cea25974bb0 100644
--- a/deploy/charts/litellm-helm/values.yaml
+++ b/deploy/charts/litellm-helm/values.yaml
@@ -234,6 +234,14 @@ db:
# instance. See the "postgresql" top level key for additional configuration.
deployStandalone: true
+# Lifecycle hooks for the LiteLLM container
+# Example:
+# lifecycle:
+# preStop:
+# exec:
+# command: ["/bin/sh", "-c", "sleep 10"]
+lifecycle: {}
+
# Settings for Bitnami postgresql chart (if db.deployStandalone is true, ignored
# otherwise)
postgresql:
@@ -273,6 +281,7 @@ migrationJob:
# cpu: 100m
# memory: 100Mi
extraContainers: []
+ extraInitContainers: []
# Hook configuration
hooks:
diff --git a/docker/Dockerfile.custom_ui b/docker/Dockerfile.custom_ui
index c437929a27e..177d7b7b12a 100644
--- a/docker/Dockerfile.custom_ui
+++ b/docker/Dockerfile.custom_ui
@@ -5,7 +5,19 @@ FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
WORKDIR /app
# Install Node.js and npm (adjust version as needed)
-RUN apt-get update && apt-get install -y nodejs npm
+RUN apt-get update && apt-get install -y nodejs npm && \
+ npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
+ GLOBAL="$(npm root -g)" && \
+ find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+ done && \
+ find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+ done && \
+ find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+ done && \
+ npm cache clean --force
# Copy the UI source into the container
COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard
diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database
index 49655129506..a6fcd98ab6d 100644
--- a/docker/Dockerfile.database
+++ b/docker/Dockerfile.database
@@ -49,7 +49,19 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
# Install runtime dependencies
-RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
+ npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
+ GLOBAL="$(npm root -g)" && \
+ find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+ done && \
+ find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+ done && \
+ find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+ done && \
+ npm cache clean --force
WORKDIR /app
# Copy the current directory contents into the container at /app
@@ -63,6 +75,20 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
+# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
+# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
+# Patch every copy of tar, glob, and brace-expansion inside that tree.
+RUN GLOBAL="$(npm root -g)" && \
+ find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+ done && \
+ find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+ done && \
+ find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+ done
+
# Install semantic_router and aurelio-sdk using script
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev
index 67966f9c739..bc1d22d5e05 100644
--- a/docker/Dockerfile.dev
+++ b/docker/Dockerfile.dev
@@ -61,7 +61,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
libatomic1 \
nodejs \
npm \
- && rm -rf /var/lib/apt/lists/*
+ && rm -rf /var/lib/apt/lists/* \
+ && npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \
+ && GLOBAL="$(npm root -g)" \
+ && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+ done \
+ && find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+ done \
+ && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+ done \
+ && npm cache clean --force
WORKDIR /app
@@ -79,6 +91,20 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/
rm -f *.whl && \
rm -rf /wheels
+# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
+# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
+# Patch every copy of tar, glob, and brace-expansion inside that tree.
+RUN GLOBAL="$(npm root -g)" && \
+ find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+ done && \
+ find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+ done && \
+ find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+ done
+
# Generate prisma client and set permissions
# Convert Windows line endings to Unix for entrypoint scripts
RUN prisma generate && \
diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root
index 8c795f3b17f..004377e19b3 100644
--- a/docker/Dockerfile.non_root
+++ b/docker/Dockerfile.non_root
@@ -47,7 +47,6 @@ RUN mkdir -p /var/lib/litellm/ui && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
fi && \
- rm -f package-lock.json && \
npm install --legacy-peer-deps && \
npm run build && \
cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \
@@ -60,7 +59,8 @@ RUN mkdir -p /var/lib/litellm/ui && \
mkdir -p "$folder_name" && \
mv "$html_file" "$folder_name/index.html"; \
fi; \
- done ) && \
+ done && \
+ touch .litellm_ui_ready ) && \
cd /app/ui/litellm-dashboard && rm -rf ./out
# Build litellm wheel and place it in wheels dir (replace any PyPI wheels)
@@ -104,7 +104,19 @@ RUN for i in 1 2 3; do \
done \
&& for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
- done
+ done \
+ && npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \
+ && GLOBAL="$(npm root -g)" \
+ && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+ done \
+ && find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+ done \
+ && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+ done \
+ && npm cache clean --force
# Copy artifacts from builder
COPY --from=builder /app/requirements.txt /app/requirements.txt
@@ -146,6 +158,20 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
fi; \
fi
+# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
+# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
+# Patch every copy of tar, glob, and brace-expansion inside that tree.
+RUN GLOBAL="$(npm root -g)" && \
+ find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+ done && \
+ find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+ done && \
+ find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
+ rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+ done
+
# Permissions, cleanup, and Prisma prep
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && \
@@ -170,12 +196,14 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g+rX $PRISMA_PATH && \
chmod -R g+rX /app/.cache && \
- mkdir -p /tmp/.npm /nonexistent /.npm && \
- prisma generate
+ mkdir -p /tmp/.npm /nonexistent /.npm
# Switch to non-root user for runtime
USER nobody
+# Generate Prisma client as nobody user to ensure correct file ownership
+RUN prisma generate
+
# Prisma runtime knobs for offline containers
ENV PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
PRISMA_HIDE_UPDATE_MESSAGE=1 \
diff --git a/docker/README.md b/docker/README.md
index 6d81276bb4b..7027a30fdd7 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -70,9 +70,12 @@ docker compose -f docker-compose.yml -f docker-compose.hardened.yml up -d
This setup:
- Builds from `docker/Dockerfile.non_root` with Prisma engines and Node toolchain baked into the image.
-- Runs the proxy as a non-root user with a read-only rootfs and only two writable tmpfs mounts:
+- Runs the proxy as a non-root user with a read-only rootfs and only writable tmpfs mounts:
- `/app/cache` (Prisma/NPM cache; backing `PRISMA_BINARY_CACHE_DIR`, `NPM_CONFIG_CACHE`, `XDG_CACHE_HOME`)
- `/app/migrations` (Prisma migration workspace; backing `LITELLM_MIGRATION_DIR`)
+- Pre-builds and serves the admin UI from read-only paths:
+ - `/var/lib/litellm/ui` (pre-restructured Next.js UI with `.litellm_ui_ready` marker)
+ - `/var/lib/litellm/assets` (UI logos and assets)
- Routes all outbound traffic through a local Squid proxy that denies egress, so Prisma migrations must use the cached CLI and engines.
You should also verify offline Prisma behaviour with:
diff --git a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md
index 7015918e924..8a54426dfb0 100644
--- a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md
+++ b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md
@@ -15,6 +15,7 @@ authors:
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+description: "Guide to Claude Opus 4.5 and advanced features in LiteLLM: Tool Search, Programmatic Tool Calling, and Effort Parameter."
tags: [anthropic, claude, tool search, programmatic tool calling, effort, advanced features]
hide_table_of_contents: false
---
diff --git a/docs/my-website/blog/claude_code_beta_headers/index.md b/docs/my-website/blog/claude_code_beta_headers/index.md
new file mode 100644
index 00000000000..44567f616aa
--- /dev/null
+++ b/docs/my-website/blog/claude_code_beta_headers/index.md
@@ -0,0 +1,177 @@
+---
+slug: claude-code-beta-headers-incident
+title: "Incident Report: Invalid beta headers with Claude Code"
+date: 2026-02-16T10:00:00
+authors:
+ - name: Sameer Kankute
+ title: SWE @ LiteLLM (LLM Translation)
+ url: https://www.linkedin.com/in/sameer-kankute/
+ image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+ - name: Ishaan Jaff
+ title: "CTO, LiteLLM"
+ url: https://www.linkedin.com/in/reffajnaahsi/
+ image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+ - name: Krrish Dholakia
+ title: "CEO, LiteLLM"
+ url: https://www.linkedin.com/in/krish-d/
+ image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+tags: [incident-report, anthropic, stability]
+hide_table_of_contents: false
+---
+
+**Date:** February 13, 2026
+**Duration:** ~3 hours
+**Severity:** High
+**Status:** Resolved
+
+> **Note:** This fix will be available starting from `v1.81.13-nightly` or higher of LiteLLM.
+
+## Summary
+
+Claude Code began sending unsupported Anthropic beta headers to non-Anthropic providers (Bedrock, Azure AI, Vertex AI), causing `invalid beta flag` errors. LiteLLM was forwarding all beta headers without provider-specific validation. Users experienced request failures when routing Claude Code requests through LiteLLM to these providers.
+
+- **LLM calls to Anthropic:** No impact.
+- **LLM calls to Bedrock/Azure/Vertex:** Failed with `invalid beta flag` errors when unsupported headers were present.
+- **Cost tracking and routing:** No impact.
+
+{/* truncate */}
+
+---
+
+## Background
+
+Anthropic uses beta headers to enable experimental features in Claude. When Claude Code makes API requests, it includes headers like `anthropic-beta: prompt-caching-scope-2026-01-05,advanced-tool-use-2025-11-20`. However, not all providers support all Anthropic beta features.
+
+Before this incident, LiteLLM forwarded all beta headers to all providers without validation:
+
+```mermaid
+sequenceDiagram
+ participant CC as Claude Code
+ participant LP as LiteLLM (old behavior)
+ participant Provider as Provider (Bedrock/Azure/Vertex)
+
+ CC->>LP: Request with beta headers
+ Note over CC,LP: anthropic-beta: header1,header2,header3
+
+ LP->>Provider: Forward ALL headers (no validation)
+ Note over LP,Provider: anthropic-beta: header1,header2,header3
+
+ Provider-->>LP: ❌ Error: invalid beta flag
+ LP-->>CC: Request fails
+```
+
+Requests succeeded for Anthropic (native support) but failed for other providers when Claude Code sent headers those providers didn't support.
+
+---
+
+## Root cause
+
+LiteLLM lacked provider-specific beta header validation. When Claude Code introduced new beta features or sent headers that specific providers didn't support, those headers were blindly forwarded, causing provider API errors.
+
+---
+
+## Remediation
+
+| # | Action | Status | Code |
+|---|---|---|---|
+| 1 | Create `anthropic_beta_headers_config.json` with provider-specific mappings | ✅ Done | [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) |
+| 2 | Implement strict validation: headers must be explicitly mapped to be forwarded | ✅ Done | [`litellm_logging.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/litellm_logging.py) |
+| 3 | Add `/reload/anthropic_beta_headers` endpoint for dynamic config updates | ✅ Done | Proxy management endpoints |
+| 4 | Add `/schedule/anthropic_beta_headers_reload` for automatic periodic updates | ✅ Done | Proxy management endpoints |
+| 5 | Support `LITELLM_ANTHROPIC_BETA_HEADERS_URL` for custom config sources | ✅ Done | Environment configuration |
+| 6 | Support `LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS` for air-gapped deployments | ✅ Done | Environment configuration |
+
+Now LiteLLM validates and transforms headers per-provider:
+
+```mermaid
+sequenceDiagram
+ participant CC as Claude Code
+ participant LP as LiteLLM (new behavior)
+ participant Config as Beta Headers Config
+ participant Provider as Provider (Bedrock/Azure/Vertex)
+
+ CC->>LP: Request with beta headers
+ Note over CC,LP: anthropic-beta: header1,header2,header3
+
+ LP->>Config: Load header mapping for provider
+ Config-->>LP: Returns mapping (header→value or null)
+
+ Note over LP: Validate & Transform:
1. Check if header exists in mapping
2. Filter out null values
3. Map to provider-specific names
+
+ LP->>Provider: Request with filtered & mapped headers
+ Note over LP,Provider: anthropic-beta: mapped-header2
(header1, header3 filtered out)
+
+ Provider-->>LP: ✅ Success response
+ LP-->>CC: Response
+```
+
+---
+
+## Dynamic configuration updates
+
+A key improvement is zero-downtime configuration updates. When Anthropic releases new beta features, users can update their configuration without restarting:
+
+```bash
+# Manually trigger reload (no restart needed)
+curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
+ -H "Authorization: Bearer YOUR_ADMIN_TOKEN"
+
+# Or schedule automatic reloads every 24 hours
+curl -X POST "https://your-proxy-url/schedule/anthropic_beta_headers_reload?hours=24" \
+ -H "Authorization: Bearer YOUR_ADMIN_TOKEN"
+```
+
+This prevents future incidents where Claude Code introduces new headers before LiteLLM configuration is updated.
+
+---
+
+## Configuration format
+
+The `anthropic_beta_headers_config.json` file maps input headers to provider-specific output headers:
+
+```json
+{
+ "description": "Mapping of Anthropic beta headers for each provider.",
+ "anthropic": {
+ "advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
+ "computer-use-2025-01-24": "computer-use-2025-01-24"
+ },
+ "bedrock_converse": {
+ "advanced-tool-use-2025-11-20": null,
+ "computer-use-2025-01-24": "computer-use-2025-01-24"
+ },
+ "azure_ai": {
+ "advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
+ "computer-use-2025-01-24": "computer-use-2025-01-24"
+ }
+}
+```
+
+**Validation rules:**
+1. Headers must exist in the mapping for the target provider
+2. Headers with `null` values are filtered out (unsupported)
+3. Header names can be transformed per-provider (e.g., Bedrock uses different names for some features)
+
+---
+
+## Resolution steps for users
+
+For users still experiencing issues, update to the latest LiteLLM version if < v1.81.11-nightly:
+
+```bash
+pip install --upgrade litellm
+```
+
+Or manually reload the configuration without restarting:
+
+```bash
+curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
+ -H "Authorization: Bearer YOUR_ADMIN_TOKEN"
+```
+
+---
+
+## Related documentation
+
+- [Managing Anthropic Beta Headers](../proxy/sync_anthropic_beta_headers.md) - Complete configuration guide
+- [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) - Current configuration file
diff --git a/docs/my-website/blog/claude_opus_4_6/index.md b/docs/my-website/blog/claude_opus_4_6/index.md
new file mode 100644
index 00000000000..e44420bd570
--- /dev/null
+++ b/docs/my-website/blog/claude_opus_4_6/index.md
@@ -0,0 +1,730 @@
+---
+slug: claude_opus_4_6
+title: "Day 0 Support: Claude Opus 4.6"
+date: 2026-02-05T10:00:00
+authors:
+ - name: Sameer Kankute
+ title: SWE @ LiteLLM (LLM Translation)
+ url: https://www.linkedin.com/in/sameer-kankute/
+ image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+ - name: Ishaan Jaff
+ title: "CTO, LiteLLM"
+ url: https://www.linkedin.com/in/reffajnaahsi/
+ image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+ - name: Krrish Dholakia
+ title: "CEO, LiteLLM"
+ url: https://www.linkedin.com/in/krish-d/
+ image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+description: "Day 0 support for Claude Opus 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
+tags: [anthropic, claude, opus 4.6]
+hide_table_of_contents: false
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+LiteLLM now supports Claude Opus 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
+
+## Docker Image
+
+```bash
+docker pull ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6
+```
+
+## Usage - Anthropic
+
+
+
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+ - model_name: claude-opus-4-6
+ litellm_params:
+ model: anthropic/claude-opus-4-6
+ api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+ -p 4000:4000 \
+ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
+ -v $(pwd)/config.yaml:/app/config.yaml \
+ ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6 \
+ --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "claude-opus-4-6",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ]
+}'
+```
+
+
+
+
+## Usage - Azure
+
+
+
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+ - model_name: claude-opus-4-6
+ litellm_params:
+ model: azure_ai/claude-opus-4-6
+ api_key: os.environ/AZURE_AI_API_KEY
+ api_base: os.environ/AZURE_AI_API_BASE # https://.services.ai.azure.com
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+ -p 4000:4000 \
+ -e AZURE_AI_API_KEY=$AZURE_AI_API_KEY \
+ -e AZURE_AI_API_BASE=$AZURE_AI_API_BASE \
+ -v $(pwd)/config.yaml:/app/config.yaml \
+ ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6 \
+ --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "claude-opus-4-6",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ]
+}'
+```
+
+
+
+
+## Usage - Vertex AI
+
+
+
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+ - model_name: claude-opus-4-6
+ litellm_params:
+ model: vertex_ai/claude-opus-4-6
+ vertex_project: os.environ/VERTEX_PROJECT
+ vertex_location: us-east5
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+ -p 4000:4000 \
+ -e VERTEX_PROJECT=$VERTEX_PROJECT \
+ -e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json \
+ -v $(pwd)/config.yaml:/app/config.yaml \
+ -v $(pwd)/credentials.json:/app/credentials.json \
+ ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6 \
+ --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "claude-opus-4-6",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ]
+}'
+```
+
+
+
+
+## Usage - Bedrock
+
+
+
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+ - model_name: claude-opus-4-6
+ litellm_params:
+ model: bedrock/anthropic.claude-opus-4-6-v1
+ aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+ aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+ aws_region_name: us-east-1
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+ -p 4000:4000 \
+ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+ -v $(pwd)/config.yaml:/app/config.yaml \
+ ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6 \
+ --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "claude-opus-4-6",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ]
+}'
+```
+
+
+
+
+## Advanced Features
+
+### Compaction
+
+
+
+
+Litellm supports enabling compaction for the new claude-opus-4-6.
+
+**Enabling Compaction**
+
+To enable compaction, add the `context_management` parameter with the `compact_20260112` edit type:
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "claude-opus-4-6",
+ "messages": [
+ {
+ "role": "user",
+ "content": "What is the weather in San Francisco?"
+ }
+ ],
+ "context_management": {
+ "edits": [
+ {
+ "type": "compact_20260112"
+ }
+ ]
+ },
+ "max_tokens": 100
+}'
+```
+All the parameters supported for context_management by anthropic are supported and can be directly added. Litellm automatically adds the `compact-2026-01-12` beta header in the request.
+
+
+
+
+Enable compaction to reduce context size while preserving key information. LiteLLM automatically adds the `compact-2026-01-12` beta header when compaction is enabled.
+
+:::info
+**Provider Support:** Compaction is supported on Anthropic, Azure AI, and Vertex AI. It is **not supported** on Bedrock (Invoke or Converse APIs).
+:::
+
+```bash
+curl --location 'http://0.0.0.0:4000/v1/messages' \
+--header 'x-api-key: sk-12345' \
+--header 'content-type: application/json' \
+--data '{
+ "model": "claude-opus-4-6",
+ "max_tokens": 4096,
+ "messages": [
+ {
+ "role": "user",
+ "content": "Hi"
+ }
+ ],
+ "context_management": {
+ "edits": [
+ {
+ "type": "compact_20260112"
+ }
+ ]
+ }
+}'
+```
+
+
+
+
+
+**Response with Compaction Block**
+
+The response will include the compaction summary in `provider_specific_fields.compaction_blocks`:
+
+```json
+{
+ "id": "chatcmpl-a6c105a3-4b25-419e-9551-c800633b6cb2",
+ "created": 1770357619,
+ "model": "claude-opus-4-6",
+ "object": "chat.completion",
+ "choices": [
+ {
+ "finish_reason": "length",
+ "index": 0,
+ "message": {
+ "content": "I don't have access to real-time data, so I can't provide the current weather in San Francisco. To get up-to-date weather information, I'd recommend checking:\n\n- **Weather websites** like weather.com, accuweather.com, or wunderground.com\n- **Search engines** – just Google \"San Francisco weather\"\n- **Weather apps** on your phone (e.g., Apple Weather, Google Weather)\n- **National",
+ "role": "assistant",
+ "provider_specific_fields": {
+ "compaction_blocks": [
+ {
+ "type": "compaction",
+ "content": "Summary of the conversation: The user requested help building a web scraper..."
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "usage": {
+ "completion_tokens": 100,
+ "prompt_tokens": 86,
+ "total_tokens": 186
+ }
+}
+```
+
+**Using Compaction Blocks in Follow-up Requests**
+
+To continue the conversation with compaction, include the compaction block in the assistant message's `provider_specific_fields`:
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "claude-opus-4-6",
+ "messages": [
+ {
+ "role": "user",
+ "content": "How can I build a web scraper?"
+ },
+ {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Certainly! To build a basic web scraper, you'll typically use a programming language like Python along with libraries such as `requests` (for fetching web pages) and `BeautifulSoup` (for parsing HTML). Here's a basic example:\n\n```python\nimport requests\nfrom bs4 import BeautifulSoup\n\nurl = 'https://example.com'\nresponse = requests.get(url)\nsoup = BeautifulSoup(response.text, 'html.parser')\n\n# Extract and print all text\ntext = soup.get_text()\nprint(text)\n```\n\nLet me know what you're interested in scraping or if you need help with a specific website!"
+ }
+ ],
+ "provider_specific_fields": {
+ "compaction_blocks": [
+ {
+ "type": "compaction",
+ "content": "Summary of the conversation: The user asked how to build a web scraper, and the assistant gave an overview using Python with requests and BeautifulSoup."
+ }
+ ]
+ }
+ },
+ {
+ "role": "user",
+ "content": "How do I use it to scrape product prices?"
+ }
+ ],
+ "context_management": {
+ "edits": [
+ {
+ "type": "compact_20260112"
+ }
+ ]
+ },
+ "max_tokens": 100
+}'
+```
+
+**Streaming Support**
+
+Compaction blocks are also supported in streaming mode. You'll receive:
+- `compaction_start` event when a compaction block begins
+- `compaction_delta` events with the compaction content
+- The accumulated `compaction_blocks` in `provider_specific_fields`
+
+### Adaptive Thinking
+
+:::note
+When using `reasoning_effort` with Claude Opus 4.6, all values (`low`, `medium`, `high`) are mapped to `thinking: {type: "adaptive"}`. To use explicit thinking budgets with `type: "enabled"`, pass the native `thinking` parameter directly (see "Native thinking param" tab below).
+:::
+
+
+
+
+LiteLLM supports adaptive thinking through the `reasoning_effort` parameter:
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "claude-opus-4-6",
+ "messages": [
+ {
+ "role": "user",
+ "content": "Solve this complex problem: What is the optimal strategy for..."
+ }
+ ],
+ "reasoning_effort": "high"
+}'
+```
+
+
+
+
+Use the `thinking` parameter with `type: "adaptive"` to enable adaptive thinking mode:
+
+```bash
+curl --location 'http://0.0.0.0:4000/v1/messages' \
+--header 'x-api-key: sk-12345' \
+--header 'content-type: application/json' \
+--data '{
+ "model": "claude-opus-4-6",
+ "max_tokens": 16000,
+ "thinking": {
+ "type": "adaptive"
+ },
+ "messages": [
+ {
+ "role": "user",
+ "content": "Explain why the sum of two even numbers is always even."
+ }
+ ]
+}'
+```
+
+
+
+
+Use the `thinking` parameter directly for adaptive thinking via the SDK:
+
+```python
+import litellm
+
+response = litellm.completion(
+ model="anthropic/claude-opus-4-6",
+ messages=[{"role": "user", "content": "Solve this complex problem: What is the optimal strategy for..."}],
+ thinking={"type": "adaptive"},
+)
+```
+
+
+
+
+### Effort Levels
+
+
+
+
+Four effort levels available: `low`, `medium`, `high` (default), and `max`. Pass directly via the `output_config` parameter:
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "claude-opus-4-6",
+ "messages": [
+ {
+ "role": "user",
+ "content": "Explain quantum computing"
+ }
+ ],
+ "output_config": {
+ "effort": "medium"
+ }
+}'
+```
+
+You can use reasoning effort plus output_config to have more control on the model.
+
+
+
+
+Four effort levels available: `low`, `medium`, `high` (default), and `max`. Pass directly via the `output_config` parameter:
+
+```bash
+curl --location 'http://0.0.0.0:4000/v1/messages' \
+--header 'x-api-key: sk-12345' \
+--header 'content-type: application/json' \
+--data '{
+ "model": "claude-opus-4-6",
+ "max_tokens": 4096,
+ "messages": [
+ {
+ "role": "user",
+ "content": "Explain quantum computing"
+ }
+ ],
+ "output_config": {
+ "effort": "medium"
+ }
+}'
+```
+
+
+
+
+### 1M Token Context (Beta)
+
+Opus 4.6 supports 1M token context. Premium pricing applies for prompts exceeding 200k tokens ($10/$37.50 per million input/output tokens). LiteLLM supports cost calculations for 1M token contexts.
+
+
+
+
+To use the 1M token context window, you need to forward the `anthropic-beta` header from your client to the LLM provider.
+
+**Step 1: Enable header forwarding in your config**
+
+```yaml
+general_settings:
+ forward_client_headers_to_llm_api: true
+```
+
+**Step 2: Send requests with the beta header**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--header 'anthropic-beta: context-1m-2025-08-07' \
+--data '{
+ "model": "claude-opus-4-6",
+ "messages": [
+ {
+ "role": "user",
+ "content": "Analyze this large document..."
+ }
+ ]
+}'
+```
+
+
+
+
+To use the 1M token context window, you need to forward the `anthropic-beta` header from your client to the LLM provider.
+
+**Step 1: Enable header forwarding in your config**
+
+```yaml
+general_settings:
+ forward_client_headers_to_llm_api: true
+```
+
+**Step 2: Send requests with the beta header**
+
+```bash
+curl --location 'http://0.0.0.0:4000/v1/messages' \
+--header 'x-api-key: sk-12345' \
+--header 'anthropic-beta: context-1m-2025-08-07' \
+--header 'content-type: application/json' \
+--data '{
+ "model": "claude-opus-4-6",
+ "max_tokens": 16000,
+ "messages": [
+ {
+ "role": "user",
+ "content": "Analyze this large document..."
+ }
+ ]
+}'
+```
+
+:::tip
+You can combine multiple beta headers by separating them with commas:
+```bash
+--header 'anthropic-beta: context-1m-2025-08-07,compact-2026-01-12'
+```
+:::
+
+
+
+
+### US-Only Inference
+
+Available at 1.1× token pricing. LiteLLM automatically tracks costs for US-only inference.
+
+
+
+
+Use the `inference_geo` parameter to specify US-only inference:
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "claude-opus-4-6",
+ "messages": [
+ {
+ "role": "user",
+ "content": "What is the capital of France?"
+ }
+ ],
+ "inference_geo": "us"
+}'
+```
+
+LiteLLM will automatically apply the 1.1× pricing multiplier for US-only inference in cost tracking.
+
+
+
+
+Use the `inference_geo` parameter to specify US-only inference:
+
+```bash
+curl --location 'http://0.0.0.0:4000/v1/messages' \
+--header 'x-api-key: sk-12345' \
+--header 'content-type: application/json' \
+--data '{
+ "model": "claude-opus-4-6",
+ "max_tokens": 4096,
+ "messages": [
+ {
+ "role": "user",
+ "content": "What is the capital of France?"
+ }
+ ],
+ "inference_geo": "us"
+}'
+```
+
+LiteLLM will automatically apply the 1.1× pricing multiplier for US-only inference in cost tracking.
+
+
+
+
+### Fast Mode
+
+:::info
+Fast mode is **only supported on the Anthropic provider** (`anthropic/claude-opus-4-6`). It is not available on Azure AI, Vertex AI, or Bedrock.
+:::
+
+**Pricing:**
+- Standard: $5 input / $25 output per MTok
+- Fast: $30 input / $150 output per MTok (6× premium)
+
+
+
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "claude-opus-4-6",
+ "messages": [
+ {
+ "role": "user",
+ "content": "Refactor this module..."
+ }
+ ],
+ "max_tokens": 4096,
+ "speed": "fast"
+}'
+```
+
+**Using OpenAI SDK:**
+
+```python
+import openai
+
+client = openai.OpenAI(
+ api_key="your-litellm-key",
+ base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+ model="claude-opus-4-6",
+ messages=[{"role": "user", "content": "Refactor this module..."}],
+ max_tokens=4096,
+ extra_body={"speed": "fast"}
+)
+```
+
+**Using LiteLLM SDK:**
+
+```python
+from litellm import completion
+
+response = completion(
+ model="anthropic/claude-opus-4-6",
+ messages=[{"role": "user", "content": "Refactor this module..."}],
+ max_tokens=4096,
+ speed="fast"
+)
+```
+
+LiteLLM automatically tracks the higher costs for fast mode in usage and cost calculations.
+
+
+
+
+```bash
+curl --location 'http://0.0.0.0:4000/v1/messages' \
+--header 'x-api-key: sk-12345' \
+--header 'content-type: application/json' \
+--data '{
+ "model": "claude-opus-4-6",
+ "max_tokens": 4096,
+ "speed": "fast",
+ "messages": [
+ {
+ "role": "user",
+ "content": "Refactor this module..."
+ }
+ ]
+}'
+```
+
+LiteLLM automatically:
+- Adds the `fast-mode-2026-02-01` beta header
+- Tracks the 6× premium pricing in cost calculations
+
+
+
diff --git a/docs/my-website/blog/claude_sonnet_4_6/index.md b/docs/my-website/blog/claude_sonnet_4_6/index.md
new file mode 100644
index 00000000000..df54fa09792
--- /dev/null
+++ b/docs/my-website/blog/claude_sonnet_4_6/index.md
@@ -0,0 +1,283 @@
+---
+slug: claude_sonnet_4_6
+title: "Day 0 Support: Claude Sonnet 4.6"
+date: 2026-02-17T10:00:00
+authors:
+ - name: Ishaan Jaff
+ title: "CTO, LiteLLM"
+ url: https://www.linkedin.com/in/reffajnaahsi/
+ image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+ - name: Krrish Dholakia
+ title: "CEO, LiteLLM"
+ url: https://www.linkedin.com/in/krish-d/
+ image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+description: "Day 0 support for Claude Sonnet 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
+tags: [anthropic, claude, sonnet 4.6]
+hide_table_of_contents: false
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+LiteLLM now supports Claude Sonnet 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
+
+## Docker Image
+
+```bash
+docker pull ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6
+```
+
+## Usage - Anthropic
+
+
+
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+ - model_name: claude-sonnet-4-6
+ litellm_params:
+ model: anthropic/claude-sonnet-4-6
+ api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+ -p 4000:4000 \
+ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
+ -v $(pwd)/config.yaml:/app/config.yaml \
+ ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
+ --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "claude-sonnet-4-6",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ]
+}'
+```
+
+
+
+
+
+```python
+from litellm import completion
+
+response = completion(
+ model="anthropic/claude-sonnet-4-6",
+ messages=[{"role": "user", "content": "what llm are you"}]
+)
+print(response.choices[0].message.content)
+```
+
+
+
+
+## Usage - Azure
+
+
+
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+ - model_name: claude-sonnet-4-6
+ litellm_params:
+ model: azure_ai/claude-sonnet-4-6
+ api_key: os.environ/AZURE_AI_API_KEY
+ api_base: os.environ/AZURE_AI_API_BASE # https://.services.ai.azure.com
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+ -p 4000:4000 \
+ -e AZURE_AI_API_KEY=$AZURE_AI_API_KEY \
+ -e AZURE_AI_API_BASE=$AZURE_AI_API_BASE \
+ -v $(pwd)/config.yaml:/app/config.yaml \
+ ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
+ --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "claude-sonnet-4-6",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ]
+}'
+```
+
+
+
+
+
+```python
+from litellm import completion
+
+response = completion(
+ model="azure_ai/claude-sonnet-4-6",
+ api_key="your-azure-api-key",
+ api_base="https://.services.ai.azure.com",
+ messages=[{"role": "user", "content": "what llm are you"}]
+)
+print(response.choices[0].message.content)
+```
+
+
+
+
+## Usage - Vertex AI
+
+
+
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+ - model_name: claude-sonnet-4-6
+ litellm_params:
+ model: vertex_ai/claude-sonnet-4-6
+ vertex_project: os.environ/VERTEX_PROJECT
+ vertex_location: us-east5
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+ -p 4000:4000 \
+ -e VERTEX_PROJECT=$VERTEX_PROJECT \
+ -e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json \
+ -v $(pwd)/config.yaml:/app/config.yaml \
+ -v $(pwd)/credentials.json:/app/credentials.json \
+ ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
+ --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "claude-sonnet-4-6",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ]
+}'
+```
+
+
+
+
+
+```python
+from litellm import completion
+
+response = completion(
+ model="vertex_ai/claude-sonnet-4-6",
+ vertex_project="your-project-id",
+ vertex_location="us-east5",
+ messages=[{"role": "user", "content": "what llm are you"}]
+)
+print(response.choices[0].message.content)
+```
+
+
+
+
+## Usage - Bedrock
+
+
+
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+ - model_name: claude-sonnet-4-6
+ litellm_params:
+ model: bedrock/anthropic.claude-sonnet-4-6-v1
+ aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+ aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+ aws_region_name: us-east-1
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+ -p 4000:4000 \
+ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+ -v $(pwd)/config.yaml:/app/config.yaml \
+ ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
+ --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "claude-sonnet-4-6",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ]
+}'
+```
+
+
+
+
+
+```python
+from litellm import completion
+
+response = completion(
+ model="bedrock/anthropic.claude-sonnet-4-6-v1",
+ aws_access_key_id="your-access-key",
+ aws_secret_access_key="your-secret-key",
+ aws_region_name="us-east-1",
+ messages=[{"role": "user", "content": "what llm are you"}]
+)
+print(response.choices[0].message.content)
+```
+
+
+
diff --git a/docs/my-website/blog/fastapi_middleware_performance/index.mdx b/docs/my-website/blog/fastapi_middleware_performance/index.mdx
new file mode 100644
index 00000000000..b0c5ba13634
--- /dev/null
+++ b/docs/my-website/blog/fastapi_middleware_performance/index.mdx
@@ -0,0 +1,220 @@
+---
+slug: fastapi-middleware-performance
+title: "Your Middleware Could Be a Bottleneck"
+date: 2026-02-07T10:00:00
+authors:
+ - name: Krrish Dholakia
+ title: "CEO, LiteLLM"
+ url: https://www.linkedin.com/in/krish-d/
+ image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+ - name: Ishaan Jaff
+ title: "CTO, LiteLLM"
+ url: https://www.linkedin.com/in/reffajnaahsi/
+ image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+ - name: Ryan Crabbe
+ title: "Performance Engineer, LiteLLM"
+ url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
+ image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M
+description: "How we improved LiteLLM proxy latency and throughput by replacing a single middleware base class"
+tags: [performance, fastapi, middleware]
+hide_table_of_contents: false
+---
+
+import { BaseHTTPMiddlewareAnimation, PureASGIAnimation, BenchmarkVisualization } from '@site/src/components/MiddlewareDiagrams';
+
+> How we improved LiteLLM proxy latency and throughput by replacing a single, simple middleware base class
+
+---
+
+## Our Setup
+
+The LiteLLM proxy server has two middleware layers. The first is Starlette's `CORSMiddleware` (re-exported by FastAPI), which is a pure ASGI middleware. Then we have a simple BaseHTTPMiddleware called PrometheusAuthMiddleware.
+
+The job of `PrometheusAuthMiddleware` is to authenticate requests to the `/metrics` endpoint. It's not on by default, you enable it with a flag in your proxy config:
+
+
+Proxy config flag
+
+```yaml
+litellm_settings:
+ require_auth_for_metrics_endpoint: true
+```
+
+
+
+The middleware checks two things: is the request hitting `/metrics`, and is auth even enabled? If both checks fail, which they do for the vast majority of requests, it just passes the request through unchanged.
+
+
+PrometheusAuthMiddleware source
+
+```python
+class PrometheusAuthMiddleware(BaseHTTPMiddleware):
+ async def dispatch(self, request: Request, call_next):
+ if self._is_prometheus_metrics_endpoint(request):
+ if self._should_run_auth_on_metrics_endpoint() is True:
+ try:
+ await user_api_key_auth(request=request, api_key=...)
+ except Exception as e:
+ return JSONResponse(status_code=401, content=...)
+ response = await call_next(request)
+ return response
+
+ @staticmethod
+ def _is_prometheus_metrics_endpoint(request: Request):
+ if "/metrics" in request.url.path:
+ return True
+ return False
+```
+
+
+
+Looks harmless. Subclass `BaseHTTPMiddleware`, implement `dispatch()`, done. This is what you will see in Starlette's documentation[1](#footnote-1).
+
+{/* truncate */}
+
+---
+
+## What BaseHTTPMiddleware Actually Does
+
+When you write a `dispatch()` method, you'd expect the request to flow straight through your function and out the other side. What actually happens is much more involved.
+
+On every request, even a pure passthrough (meaning nothing happens), `BaseHTTPMiddleware` creates **7 intermediate objects and tasks**:
+
+
+
+It wraps the request in a new object to track body state, creates a synchronization event, allocates an in-memory channel to pass messages between your middleware and the inner app, sets up a task group to manage the lifecycle, and then runs your actual route handler in a *separate background task* when you call `call_next()`. The response body then flows back through that in-memory channel, gets re-wrapped in a streaming response object, and finally reaches the caller. That's a lot.
+
+For a middleware that for us, does nothing on 99.9% of requests, paying this cost doesn't make sense.
+
+Compare that to a pure ASGI middleware, which we can have just check the request path and continue along.
+
+
+
+Our middleware is doing something really simple. For the vast majority of requests it doesn't need to do anything at all but just let the request pass through. It doesn't need task groups, memory streams, or cancel scopes. It needs a function call.
+
+---
+
+## Comparing Both
+
+We replaced the `BaseHTTPMiddleware` subclass with a pure ASGI middleware. To benchmark the difference, we used Apache Bench[2](#footnote-2) to compare both configurations of LiteLLM's middleware stack: the old setup (1 pure ASGI + 1 `BaseHTTPMiddleware`) against the new setup (2 pure ASGI).
+
+A minimal FastAPI app serves `GET /health` → `PlainTextResponse("ok")`. The endpoint does zero work to isolate the middleware overhead: any difference between configs is purely the cost of the middleware plumbing itself. Both middlewares are just calling the next layer. Same work, different base class.
+
+Apache Bench (`ab`) fires requests at the server with 1,000 concurrent connections and a single uvicorn worker. One worker means one event loop, so the benchmark directly measures how each middleware design handles concurrent load on a single thread.
+
+
+
+
+Try it yourself
+
+Save the script below as `benchmark_middleware.py`, then run:
+
+```bash
+# Terminal 1 — start the "before" server (1 ASGI + 1 BaseHTTPMiddleware)
+python benchmark_middleware.py --middleware mixed
+
+# Terminal 2 — benchmark it
+ab -n 50000 -c 1000 http://localhost:8000/health
+
+# Stop the server, then start the "after" server (2x pure ASGI)
+python benchmark_middleware.py --middleware asgi
+
+# Terminal 2 — benchmark again
+ab -n 50000 -c 1000 http://localhost:8000/health
+```
+
+```python
+import argparse
+import uvicorn
+from fastapi import FastAPI
+from fastapi.responses import PlainTextResponse
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.types import ASGIApp, Receive, Scope, Send
+
+
+class NoOpBaseHTTPMiddleware(BaseHTTPMiddleware):
+ async def dispatch(self, request: Request, call_next):
+ return await call_next(request)
+
+
+class NoOpPureASGIMiddleware:
+ def __init__(self, app: ASGIApp) -> None:
+ self.app = app
+
+ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+ await self.app(scope, receive, send)
+
+
+def create_app(middleware_type: str | None = None, layers: int = 2) -> FastAPI:
+ app = FastAPI()
+
+ @app.get("/health")
+ async def health():
+ return PlainTextResponse("ok")
+
+ if middleware_type == "mixed":
+ app.add_middleware(NoOpBaseHTTPMiddleware)
+ app.add_middleware(NoOpPureASGIMiddleware)
+ elif middleware_type == "asgi":
+ for _ in range(layers):
+ app.add_middleware(NoOpPureASGIMiddleware)
+
+ return app
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--middleware", choices=["asgi", "mixed"], default=None)
+ parser.add_argument("--layers", type=int, default=2)
+ parser.add_argument("--port", type=int, default=8000)
+ args = parser.parse_args()
+
+ app = create_app(middleware_type=args.middleware, layers=args.layers)
+ uvicorn.run(app, host="0.0.0.0", port=args.port, workers=1, log_level="warning")
+```
+
+
+
+---
+
+## Our Change
+
+Here's what we replaced it with:
+
+```python
+class PrometheusAuthMiddleware:
+ def __init__(self, app: ASGIApp) -> None:
+ self.app = app
+
+ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+ if scope["type"] != "http" or "/metrics" not in scope.get("path", ""):
+ await self.app(scope, receive, send)
+ return
+
+ if litellm.require_auth_for_metrics_endpoint is True:
+ request = Request(scope, receive)
+ api_key = request.headers.get("Authorization") or ""
+ try:
+ await user_api_key_auth(request=request, api_key=api_key)
+ except Exception as e:
+ # send 401 directly via ASGI protocol
+ ...
+ return
+
+ await self.app(scope, receive, send)
+```
+
+For the 99.9% of requests that aren't hitting `/metrics`, the middleware is now one dict lookup, one string check, and one function call. No objects allocated, no tasks spawned.
+
+It's important to evaluate if the tools you're using are the right fit for the job as your software grows and handles more responsiblity. We're now putting in a static analysis check to prevent this from happening again with any newly introduced middlewares. If we find the use case is necessary then that's okay and we'll reevalute but for everything LiteLLM needs to do at the moment it's not.
+
+This middleware change was one part of a broader optimization effort on the LiteLLM proxy. Across all optimizations combined, we've measured about a **30% reduction in proxy overhead** over the past two weeks.
+
+---
+
+
+1 [Starlette Middleware — BaseHTTPMiddleware](https://starlette.dev/middleware/#basehttpmiddleware)
+
+
+2 [Apache HTTP server benchmarking tool (`ab`)](https://httpd.apache.org/docs/2.4/programs/ab.html)
diff --git a/docs/my-website/blog/gemini_3/index.md b/docs/my-website/blog/gemini_3/index.md
index 26dbc2d02b5..7263acc12c9 100644
--- a/docs/my-website/blog/gemini_3/index.md
+++ b/docs/my-website/blog/gemini_3/index.md
@@ -15,6 +15,7 @@ authors:
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+description: "Common questions and best practices for using gemini-3-pro-preview with LiteLLM Proxy and SDK."
tags: [gemini, day 0 support, llms]
hide_table_of_contents: false
---
diff --git a/docs/my-website/blog/gemini_3_flash/index.md b/docs/my-website/blog/gemini_3_flash/index.md
index 6cb8ddad992..830c21e5f66 100644
--- a/docs/my-website/blog/gemini_3_flash/index.md
+++ b/docs/my-website/blog/gemini_3_flash/index.md
@@ -15,6 +15,7 @@ authors:
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+description: "Guide to using Gemini 3 Flash on LiteLLM Proxy and SDK with day 0 support."
tags: [gemini, day 0 support, llms]
hide_table_of_contents: false
---
diff --git a/docs/my-website/blog/litellm_observatory/index.md b/docs/my-website/blog/litellm_observatory/index.md
new file mode 100644
index 00000000000..4554f77fb85
--- /dev/null
+++ b/docs/my-website/blog/litellm_observatory/index.md
@@ -0,0 +1,136 @@
+---
+slug: litellm-observatory
+title: "Improve release stability with 24 hour load tests"
+date: 2026-02-06T10:00:00
+authors:
+ - name: Alexsander Hamir
+ title: "Performance Engineer, LiteLLM"
+ url: https://www.linkedin.com/in/alexsander-baptista/
+ image_url: https://github.com/AlexsanderHamir.png
+ - name: Krrish Dholakia
+ title: "CEO, LiteLLM"
+ url: https://www.linkedin.com/in/krish-d/
+ image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+ - name: Ishaan Jaff
+ title: "CTO, LiteLLM"
+ url: https://www.linkedin.com/in/reffajnaahsi/
+ image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+description: "How we built a long-running, release-validation system to catch regressions before they reach users."
+tags: [testing, observability, reliability, releases]
+hide_table_of_contents: false
+---
+
+
+
+# Improve release stability with 24 hour load tests
+
+As LiteLLM adoption has grown, so have expectations around reliability, performance, and operational safety. Meeting those expectations requires more than correctness-focused tests, it requires validating how the system behaves over time, under real-world conditions.
+
+This post introduces **LiteLLM Observatory**, a long-running release-validation system we built to catch regressions before they reach users.
+
+---
+
+## Why We Built the Observatory
+
+LiteLLM operates at the intersection of external providers, long-lived network connections, and high-throughput workloads. While our unit and integration tests do an excellent job validating correctness, they are not designed to surface issues that only appear after extended operation.
+
+A subtle lifecycle edge case discovered in v1.81.3 reinforced the need for stronger release validation in this area.
+
+---
+
+## A Real-World Lifecycle Edge Case
+
+In v1.81.3, we shipped a fix for an HTTP client memory leak. The change passed unit and integration tests and behaved correctly in short-lived runs.
+
+The issue that surfaced was not caused by a single incorrect line of logic, but by how multiple components interacted over time:
+
+- A cached `httpx` client was configured with a 1-hour TTL
+- When the cache expired, the underlying HTTP connection was closed as expected
+- A higher-level client continued to hold a reference to that connection
+- Subsequent requests failed with:
+
+```
+Cannot send a request, as the client has been closed
+```
+
+**Before (with bug):**
+
+| Provider | Requests | Success | Failures | Fail % |
+|----------|----------|---------|----------|--------|
+| OpenAI | 720,000 | 432,000 | 288,000 | 40% |
+| Azure | 692,000 | 415,200 | 276,800 | 40% |
+
+**After (fixed):**
+
+| Provider | Requests | Success | Failures | Fail % |
+|----------|------------|-----------|----------|---------|
+| OpenAI | 1,200,000 | 1,199,988 | 12 | 0.001% |
+| Azure | 1,150,000 | 1,149,982 | 18 | 0.002% |
+
+Our focus moving forward is on being the first to detect issues, even when they aren’t covered by unit tests. LiteLLM Observatory is designed to surface latency regressions, OOMs, and failure modes that only appear under real traffic patterns in **our own production deployments** during release validation.
+
+
+---
+
+### How the Observatory Works
+
+[LiteLLM Observatory](https://github.com/BerriAI/litellm-observatory) is a testing service that runs long-running tests against our LiteLLM deployments. We trigger tests by sending API requests, and results are automatically sent to Slack when tests complete.
+
+#### How Tests Run
+
+1. **Start a Test**: We send a request to the Observatory API with:
+ - Which LiteLLM deployment to test (URL and API key)
+ - Which test to run (e.g., `TestOAIAzureRelease`)
+ - Test settings (which models to test, how long to run, failure thresholds)
+
+2. **Smart Queueing**:
+ - The system checks whether we are attempting to run the exact same test more than once
+ - If a duplicate test is already running or queued, we receive an error to avoid wasting resources
+ - Otherwise, the test is added to a queue and runs when capacity is available (up to 5 tests can run concurrently by default)
+
+3. **Instant Response**: The API responds immediately—we do not wait for the test to finish. Tests may run for hours, but the request itself completes in milliseconds.
+
+4. **Background Execution**:
+ - The test runs in the background, issuing requests against our LiteLLM deployment
+ - It tracks request success and failure rates over time
+ - When the test completes, results are automatically posted to our Slack channel
+
+#### Example: The OpenAI / Azure Reliability Test
+
+The `TestOAIAzureRelease` test is designed to catch a class of bugs that only surface after sustained runtime:
+
+- **Duration**: Runs continuously for 3 hours
+- **Behavior**: Cycles through specified models (such as `gpt-4` and `gpt-3.5-turbo`), issuing requests continuously
+- **Why 3 Hours**: This helps catch issues where HTTP clients degrade or fail after extended use (for example, a bug observed in LiteLLM v1.81.3)
+- **Pass / Fail Criteria**: The test passes if fewer than 1% of requests fail. If the failure rate exceeds 1%, the test fails and we are notified in Slack
+- **Key Detail**: The same HTTP client is reused for the entire run, allowing us to detect lifecycle-related bugs that only appear under prolonged reuse
+
+#### When We Use It
+
+- **Before Deployments**: Run tests before promoting a new LiteLLM version to production
+- **Routine Validation**: Schedule regular runs (daily or weekly) to catch regressions early
+- **Issue Investigation**: Run tests on demand when we suspect a deployment issue
+- **Long-Running Failure Detection**: Identify bugs that only appear under sustained load, beyond what short smoke tests can reveal
+
+
+### Complementing Unit Tests
+
+Unit tests remain a foundational part of our development process. They are fast and precise, but they don’t cover:
+
+- Real provider behavior
+- Long-lived network interactions
+- Resource lifecycle edge cases
+- Time-dependent regressions
+
+LiteLLM Observatory complements unit tests by validating the system as it actually runs in production-like environments.
+
+---
+
+### Looking Ahead
+
+Reliability is an ongoing investment.
+
+LiteLLM Observatory is one of several systems we’re building to continuously raise the bar on release quality and operational safety. As LiteLLM evolves, so will our validation tooling, informed by real-world usage and lessons learned.
+
+We’ll continue to share those improvements openly as we go.
+
diff --git a/docs/my-website/blog/minimax_m2_5/index.md b/docs/my-website/blog/minimax_m2_5/index.md
new file mode 100644
index 00000000000..50084fcc1e5
--- /dev/null
+++ b/docs/my-website/blog/minimax_m2_5/index.md
@@ -0,0 +1,394 @@
+---
+slug: minimax_m2_5
+title: "Day 0 Support: MiniMax-M2.5"
+date: 2026-02-12T10:00:00
+authors:
+ - name: Sameer Kankute
+ title: SWE @ LiteLLM (LLM Translation)
+ url: https://www.linkedin.com/in/sameer-kankute/
+ image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+ - name: Krrish Dholakia
+ title: "CEO, LiteLLM"
+ url: https://www.linkedin.com/in/krish-d/
+ image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+ - name: Ishaan Jaff
+ title: "CTO, LiteLLM"
+ url: https://www.linkedin.com/in/reffajnaahsi/
+ image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+description: "Day 0 support for MiniMax-M2.5 on LiteLLM"
+tags: [minimax, M2.5, llm]
+hide_table_of_contents: false
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+LiteLLM now supports MiniMax-M2.5 on Day 0. Use it across OpenAI-compatible and Anthropic-compatible APIs through the LiteLLM AI Gateway.
+
+## Supported Models
+
+LiteLLM supports the following MiniMax models:
+
+| Model | Description | Input Cost | Output Cost | Context Window |
+|-------|-------------|------------|-------------|----------------|
+| **MiniMax-M2.5** | Advanced reasoning, Agentic capabilities | $0.3/M tokens | $1.2/M tokens | 1M tokens |
+| **MiniMax-M2.5-lightning** | Faster and More Agile (~100 tps) | $0.3/M tokens | $2.4/M tokens | 1M tokens |
+
+## Features Supported
+
+- **Prompt Caching**: Reduce costs with cached prompts ($0.03/M tokens for cache read, $0.375/M tokens for cache write)
+- **Function Calling**: Built-in tool calling support
+- **Reasoning**: Advanced reasoning capabilities with thinking support
+- **System Messages**: Full system message support
+- **Cost Tracking**: Automatic cost calculation for all requests
+
+## Docker Image
+
+```bash
+docker pull litellm/litellm:v1.81.3-stable
+```
+
+## Usage - OpenAI Compatible API (/v1/chat/completions)
+
+
+
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+ - model_name: minimax-m2-5
+ litellm_params:
+ model: minimax/MiniMax-M2.5
+ api_key: os.environ/MINIMAX_API_KEY
+ api_base: https://api.minimax.io/v1
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+ -p 4000:4000 \
+ -e MINIMAX_API_KEY=$MINIMAX_API_KEY \
+ -v $(pwd)/config.yaml:/app/config.yaml \
+ ghcr.io/berriai/litellm:v1.81.3-stable \
+ --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "minimax-m2-5",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ]
+}'
+```
+
+
+
+
+### With Reasoning Split
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "minimax-m2-5",
+ "messages": [
+ {
+ "role": "user",
+ "content": "Solve: 2+2=?"
+ }
+ ],
+ "extra_body": {
+ "reasoning_split": true
+ }
+}'
+```
+
+## Usage - Anthropic Compatible API (/v1/messages)
+
+
+
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+ - model_name: minimax-m2-5
+ litellm_params:
+ model: minimax/MiniMax-M2.5
+ api_key: os.environ/MINIMAX_API_KEY
+ api_base: https://api.minimax.io/anthropic/v1/messages
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+ -p 4000:4000 \
+ -e MINIMAX_API_KEY=$MINIMAX_API_KEY \
+ -v $(pwd)/config.yaml:/app/config.yaml \
+ ghcr.io/berriai/litellm:v1.81.3-stable \
+ --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/v1/messages' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "minimax-m2-5",
+ "max_tokens": 1000,
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ]
+}'
+```
+
+
+
+
+### With Thinking
+
+```bash
+curl --location 'http://0.0.0.0:4000/v1/messages' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+ "model": "minimax-m2-5",
+ "max_tokens": 1000,
+ "thinking": {
+ "type": "enabled",
+ "budget_tokens": 1000
+ },
+ "messages": [
+ {
+ "role": "user",
+ "content": "Solve: 2+2=?"
+ }
+ ]
+}'
+```
+
+## Usage - LiteLLM SDK
+
+### OpenAI-compatible API
+
+```python
+import litellm
+
+response = litellm.completion(
+ model="minimax/MiniMax-M2.5",
+ messages=[
+ {"role": "user", "content": "Hello, how are you?"}
+ ],
+ api_key="your-minimax-api-key",
+ api_base="https://api.minimax.io/v1"
+)
+
+print(response.choices[0].message.content)
+```
+
+### Anthropic-compatible API
+
+```python
+import litellm
+
+response = litellm.anthropic.messages.acreate(
+ model="minimax/MiniMax-M2.5",
+ messages=[{"role": "user", "content": "Hello, how are you?"}],
+ api_key="your-minimax-api-key",
+ api_base="https://api.minimax.io/anthropic/v1/messages",
+ max_tokens=1000
+)
+
+print(response.choices[0].message.content)
+```
+
+### With Thinking
+
+```python
+response = litellm.anthropic.messages.acreate(
+ model="minimax/MiniMax-M2.5",
+ messages=[{"role": "user", "content": "Solve: 2+2=?"}],
+ thinking={"type": "enabled", "budget_tokens": 1000},
+ api_key="your-minimax-api-key"
+)
+
+# Access thinking content
+for block in response.choices[0].message.content:
+ if hasattr(block, 'type') and block.type == 'thinking':
+ print(f"Thinking: {block.thinking}")
+```
+
+### With Reasoning Split (OpenAI API)
+
+```python
+response = litellm.completion(
+ model="minimax/MiniMax-M2.5",
+ messages=[
+ {"role": "user", "content": "Solve: 2+2=?"}
+ ],
+ extra_body={"reasoning_split": True},
+ api_key="your-minimax-api-key",
+ api_base="https://api.minimax.io/v1"
+)
+
+# Access thinking and response
+if hasattr(response.choices[0].message, 'reasoning_details'):
+ print(f"Thinking: {response.choices[0].message.reasoning_details}")
+print(f"Response: {response.choices[0].message.content}")
+```
+
+## Cost Tracking
+
+LiteLLM automatically tracks costs for MiniMax-M2.5 requests. The pricing is:
+
+- **Input**: $0.3 per 1M tokens
+- **Output**: $1.2 per 1M tokens
+- **Cache Read**: $0.03 per 1M tokens
+- **Cache Write**: $0.375 per 1M tokens
+
+### Accessing Cost Information
+
+```python
+response = litellm.completion(
+ model="minimax/MiniMax-M2.5",
+ messages=[{"role": "user", "content": "Hello!"}],
+ api_key="your-minimax-api-key"
+)
+
+# Access cost information
+print(f"Cost: ${response._hidden_params.get('response_cost', 0)}")
+```
+
+## Streaming Support
+
+### OpenAI API
+
+```python
+response = litellm.completion(
+ model="minimax/MiniMax-M2.5",
+ messages=[{"role": "user", "content": "Tell me a story"}],
+ stream=True,
+ api_key="your-minimax-api-key",
+ api_base="https://api.minimax.io/v1"
+)
+
+for chunk in response:
+ if chunk.choices[0].delta.content:
+ print(chunk.choices[0].delta.content, end="")
+```
+
+### Streaming with Reasoning Split
+
+```python
+stream = litellm.completion(
+ model="minimax/MiniMax-M2.5",
+ messages=[
+ {"role": "user", "content": "Tell me a story"},
+ ],
+ extra_body={"reasoning_split": True},
+ stream=True,
+ api_key="your-minimax-api-key",
+ api_base="https://api.minimax.io/v1"
+)
+
+reasoning_buffer = ""
+text_buffer = ""
+
+for chunk in stream:
+ if hasattr(chunk.choices[0].delta, "reasoning_details") and chunk.choices[0].delta.reasoning_details:
+ for detail in chunk.choices[0].delta.reasoning_details:
+ if "text" in detail:
+ reasoning_text = detail["text"]
+ new_reasoning = reasoning_text[len(reasoning_buffer):]
+ if new_reasoning:
+ print(new_reasoning, end="", flush=True)
+ reasoning_buffer = reasoning_text
+
+ if chunk.choices[0].delta.content:
+ content_text = chunk.choices[0].delta.content
+ new_text = content_text[len(text_buffer):] if text_buffer else content_text
+ if new_text:
+ print(new_text, end="", flush=True)
+ text_buffer = content_text
+```
+
+## Using with Native SDKs
+
+### Anthropic SDK via LiteLLM Proxy
+
+```python
+import os
+os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
+os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM proxy key
+
+import anthropic
+
+client = anthropic.Anthropic()
+
+message = client.messages.create(
+ model="minimax-m2-5",
+ max_tokens=1000,
+ system="You are a helpful assistant.",
+ messages=[
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Hi, how are you?"
+ }
+ ]
+ }
+ ]
+)
+
+for block in message.content:
+ if block.type == "thinking":
+ print(f"Thinking:\n{block.thinking}\n")
+ elif block.type == "text":
+ print(f"Text:\n{block.text}\n")
+```
+
+### OpenAI SDK via LiteLLM Proxy
+
+```python
+import os
+os.environ["OPENAI_BASE_URL"] = "http://localhost:4000"
+os.environ["OPENAI_API_KEY"] = "sk-1234" # Your LiteLLM proxy key
+
+from openai import OpenAI
+
+client = OpenAI()
+
+response = client.chat.completions.create(
+ model="minimax-m2-5",
+ messages=[
+ {"role": "system", "content": "You are a helpful assistant."},
+ {"role": "user", "content": "Hi, how are you?"},
+ ],
+ extra_body={"reasoning_split": True},
+)
+
+# Access thinking and response
+if hasattr(response.choices[0].message, 'reasoning_details'):
+ print(f"Thinking:\n{response.choices[0].message.reasoning_details[0]['text']}\n")
+print(f"Text:\n{response.choices[0].message.content}\n")
+```
diff --git a/docs/my-website/blog/model_cost_map_incident/index.md b/docs/my-website/blog/model_cost_map_incident/index.md
new file mode 100644
index 00000000000..b9ff20e4128
--- /dev/null
+++ b/docs/my-website/blog/model_cost_map_incident/index.md
@@ -0,0 +1,95 @@
+---
+slug: model-cost-map-incident
+title: "Incident Report: Invalid model cost map on main"
+date: 2026-02-10T10:00:00
+authors:
+ - name: Ishaan Jaffer
+ title: "CTO, LiteLLM"
+ url: https://www.linkedin.com/in/ishaanjaffer/
+ image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+tags: [incident-report, stability]
+hide_table_of_contents: false
+---
+
+**Date:** January 27, 2026
+**Duration:** ~20 minutes
+**Severity:** Low
+**Status:** Resolved
+
+## Summary
+
+A malformed JSON entry in `model_prices_and_context_window.json` was merged to `main` ([`562f0a0`](https://github.com/BerriAI/litellm/commit/562f0a028251750e3d75386bee0e630d9796d0df)). This caused LiteLLM to silently fall back to a stale local copy of the model cost map. Users on older package versions lost cost tracking for newer models only (e.g. `azure/gpt-5.2`). No LLM calls were blocked.
+
+- **LLM calls and proxy routing:** No impact.
+- **Cost tracking:** Impacted for newer models not present in the local backup. Older models were unaffected. The incident lasted ~20 minutes until the commit was reverted.
+
+{/* truncate */}
+
+---
+
+## Background
+
+The model cost map is not in the request path. It is used after the LLM response comes back, inside a try/catch, to calculate spend. A missing entry never blocks a call.
+
+```mermaid
+flowchart TD
+ A["1. litellm.completion() receives request
+ litellm/main.py"] --> B["2. Route to provider
+ litellm/litellm_core_utils/get_llm_provider_logic.py"]
+ B --> C["3. LLM returns response
+ litellm/main.py"]
+ C --> D["4. Post-call: look up model in cost map
+ litellm/cost_calculator.py"]
+ D -->|"found"| E["5a. Attach cost to response"]
+ D -->|"not found (try/catch)"| F["5b. Log warning, set cost=0"]
+ E --> G["6. Return response to caller"]
+ F --> G
+
+ style D fill:#fff3cd,stroke:#ffc107
+ style F fill:#fff3cd,stroke:#ffc107
+ style E fill:#d4edda,stroke:#28a745
+ style G fill:#d4edda,stroke:#28a745
+```
+
+Both paths return a response to the caller. When the cost map lookup fails, the only difference is `cost=0` on that request.
+
+---
+
+## Root cause
+
+LiteLLM fetches the model cost map from GitHub `main` at import time. If the fetch fails, it falls back to a local backup bundled with the package. Before this incident, the fallback was completely silent -- no warning was logged.
+
+A contributor PR introduced an extra `{` bracket, producing invalid JSON. The remote fetch failed with `JSONDecodeError`, triggering the silent fallback. Users on older package versions had backup files missing newer models.
+
+**Timeline:**
+
+1. Malformed JSON merged to `main`
+2. LiteLLM installations fall back to local backup on next import
+3. Users report `"This model isn't mapped yet"` for newer models
+4. Bad commit identified and reverted (~20 minutes)
+
+---
+
+## Remediation
+
+| # | Action | Status | Code |
+|---|---|---|---|
+| 1 | CI validation on `model_prices_and_context_window.json` | ✅ Done | [`test-model-map.yaml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test-model-map.yaml) |
+| 2 | Warning log on fallback to local backup | ✅ Done | [`get_model_cost_map.py#L57-L68`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py#L57-L68) |
+| 3 | `GetModelCostMap` class with integrity validation helpers | ✅ Done | [`get_model_cost_map.py#L24-L149`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py#L24-L149) |
+| 4 | Resilience test suite (bad hosted map, fallback, completion) | ✅ Done | [`test_model_cost_map_resilience.py#L150-L291`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py#L150-L291) |
+| 5 | Test that backup model cost map always exists and contains common models | ✅ Done | [`test_model_cost_map_resilience.py#L213-L228`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py#L213-L228) |
+
+Enterprises that require zero external dependencies at import time can set `LITELLM_LOCAL_MODEL_COST_MAP=True` to skip the GitHub fetch entirely.
+
+---
+
+## Other dependencies on external resources
+
+| Dependency | Impact if unavailable | Fallback |
+|---|---|---|
+| Model cost map (GitHub) | Cost tracking for newer models | Local backup (now with warning) |
+| JWT public keys (IDP/SSO) | Auth fails | None |
+| OIDC UserInfo (IDP/SSO) | Auth fails | None |
+| HuggingFace model API | HF provider calls fail | None |
+| Ollama tags (localhost) | Ollama model list stale | Static list |
diff --git a/docs/my-website/blog/sub_millisecond_proxy_overhead/index.md b/docs/my-website/blog/sub_millisecond_proxy_overhead/index.md
new file mode 100644
index 00000000000..1857383363c
--- /dev/null
+++ b/docs/my-website/blog/sub_millisecond_proxy_overhead/index.md
@@ -0,0 +1,92 @@
+---
+slug: sub-millisecond-proxy-overhead
+title: "Achieving Sub-Millisecond Proxy Overhead"
+date: 2026-02-02T10:00:00
+authors:
+ - name: Alexsander Hamir
+ title: "Performance Engineer, LiteLLM"
+ url: https://www.linkedin.com/in/alexsander-baptista/
+ image_url: https://github.com/AlexsanderHamir.png
+ - name: Krrish Dholakia
+ title: "CEO, LiteLLM"
+ url: https://www.linkedin.com/in/krish-d/
+ image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+ - name: Ishaan Jaff
+ title: "CTO, LiteLLM"
+ url: https://www.linkedin.com/in/reffajnaahsi/
+ image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+description: "Our Q1 performance target and architectural direction for achieving sub-millisecond proxy overhead on modest hardware."
+tags: [performance, architecture]
+hide_table_of_contents: false
+---
+
+
+
+# Achieving Sub-Millisecond Proxy Overhead
+
+## Introduction
+
+Our Q1 performance target is to aggressively move toward sub-millisecond proxy overhead on a single instance with 4 CPUs and 8 GB of RAM, and to continue pushing that boundary over time. Our broader goal is to make LiteLLM inexpensive to deploy, lightweight, and fast. This post outlines the architectural direction behind that effort.
+
+Proxy overhead refers to the latency introduced by LiteLLM itself, independent of the upstream provider.
+
+To measure it, we run the same workload directly against the provider and through LiteLLM at identical QPS (for example, 1,000 QPS) and compare the latency delta. To reduce noise, the load generator, LiteLLM, and a mock LLM endpoint all run on the same machine, ensuring the difference reflects proxy overhead rather than network latency.
+
+---
+
+## Where We're Coming From
+
+Under the same benchmark originally conducted by [TensorZero](https://www.tensorzero.com/docs/gateway/benchmarks), LiteLLM previously failed at around 1,000 QPS.
+
+That is no longer the case. Today, LiteLLM can be stress-tested at 1,000 QPS with no failures and can scale up to 5,000 QPS without failures on a 4-CPU, 8-GB RAM single instance setup.
+
+This establishes a more up to date baseline and provides useful context as we continue working on proxy overhead and overall performance.
+
+---
+
+## Design Choice
+
+Achieving sub-millisecond proxy overhead with a Python-based system requires being deliberate about where work happens.
+
+Python is a strong fit for flexibility and extensibility: provider abstraction, configuration-driven routing, and a rich callback ecosystem. These are areas where development velocity and correctness matter more than raw throughput.
+
+At higher request rates, however, certain classes of work become expensive when executed inside the Python process on every request. Rather than rewriting LiteLLM or introducing complex deployment requirements, we adopt an optional **sidecar architecture**.
+
+This architectural change is how we intend to make LiteLLM **permanently fast**. While it supports our near-term performance targets, it is a long-term investment.
+
+Python continues to own:
+
+- Request validation and normalization
+- Model and provider selection
+- Callbacks and integrations
+
+The sidecar owns **performance-critical execution**, such as:
+
+- Efficient request forwarding
+- Connection reuse and pooling
+- Enforcing timeouts and limits
+- Aggregating high-frequency metrics
+
+This separation allows each component to focus on what it does best: Python acts as the control plane, while the sidecar handles the hot path.
+
+---
+
+### Why the Sidecar Is Optional
+
+The sidecar is intentionally **optional**.
+
+This allows us to ship it incrementally, validate it under real-world workloads, and avoid making it a hard dependency before it is fully battle-tested across all LiteLLM features.
+
+Just as importantly, this ensures that self-hosting LiteLLM remains simple. The sidecar is bundled and started automatically, requires no additional infrastructure, and can be disabled entirely. From a user's perspective, LiteLLM continues to behave like a single service.
+
+As of today, the sidecar is an optimization, not a requirement.
+
+---
+
+## Conclusion
+
+Sub-millisecond proxy overhead is not achieved through a single optimization, but through architectural changes.
+
+By keeping Python focused on orchestration and extensibility, and offloading performance-critical execution to a sidecar, we establish a foundation for making LiteLLM **permanently fast over time**—even on modest hardware such as a 1-CPU, 2-GB RAM instance, while keeping deployment and self-hosting simple.
+
+This work extends beyond Q1, and we will continue sharing benchmarks and updates as the architecture evolves.
diff --git a/docs/my-website/blog/vllm_embeddings_incident/index.md b/docs/my-website/blog/vllm_embeddings_incident/index.md
new file mode 100644
index 00000000000..a1ce8152857
--- /dev/null
+++ b/docs/my-website/blog/vllm_embeddings_incident/index.md
@@ -0,0 +1,117 @@
+---
+slug: vllm-embeddings-incident
+title: "Incident Report: vLLM Embeddings Broken by encoding_format Parameter"
+date: 2026-02-18T10:00:00
+authors:
+ - name: Sameer Kankute
+ title: SWE @ LiteLLM (LLM Translation)
+ url: https://www.linkedin.com/in/sameer-kankute/
+ image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+ - name: Krrish Dholakia
+ title: "CEO, LiteLLM"
+ url: https://www.linkedin.com/in/krish-d/
+ image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+ - name: Ishaan Jaff
+ title: "CTO, LiteLLM"
+ url: https://www.linkedin.com/in/reffajnaahsi/
+ image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+tags: [incident-report, embeddings, vllm]
+hide_table_of_contents: false
+---
+
+**Date:** Feb 16, 2026
+**Duration:** ~3 hours
+**Severity:** High (for vLLM embedding users)
+**Status:** Resolved
+
+## Summary
+
+A commit ([`dbcae4a`](https://github.com/BerriAI/litellm/commit/dbcae4aca5836770d0e9cd43abab0333c3d61ab2)) intended to fix OpenAI SDK behavior broke vLLM embeddings by explicitly passing `encoding_format=None` in API requests. vLLM rejects this with error: `"unknown variant \`\`, expected float or base64"`.
+
+- **vLLM embedding calls:** Complete failure - all requests rejected
+- **Other providers:** No impact - OpenAI and other providers functioned normally
+- **Other vLLM functionality:** No impact - only embeddings were affected
+
+{/* truncate */}
+
+---
+
+## Background
+
+The `encoding_format` parameter for embeddings specifies whether vectors should be returned as `float` arrays or `base64` encoded strings. Different providers have different expectations:
+
+- **OpenAI SDK:** If `encoding_format` is omitted, the SDK adds a default value of `"float"`
+- **vLLM:** Strictly validates `encoding_format` - only accepts `"float"`, `"base64"`, or complete omission. Rejects `None` or empty string values.
+
+```mermaid
+flowchart TD
+ A["1. User calls litellm.embedding()
+ litellm/main.py"] --> B["2. Transform request for provider
+ litellm/llms/openai_like/embedding/handler.py"]
+ B --> C["3. Send request to vLLM endpoint"]
+ C -->|"encoding_format omitted"| D["4a. ✅ vLLM processes request"]
+ C -->|"encoding_format='float' or 'base64'"| D
+ C -->|"encoding_format=None or ''"| E["4b. ❌ vLLM rejects with error:
+ 'unknown variant, expected float or base64'"]
+
+ style D fill:#d4edda,stroke:#28a745
+ style E fill:#f8d7da,stroke:#dc3545
+ style B fill:#fff3cd,stroke:#ffc107
+```
+
+---
+
+## Root cause
+
+A well-intentioned fix for OpenAI SDK behavior inadvertently broke vLLM embeddings:
+
+**The Breaking Change ([`dbcae4a`](https://github.com/BerriAI/litellm/commit/dbcae4aca5836770d0e9cd43abab0333c3d61ab2)):**
+
+In `litellm/main.py`, the code was changed to explicitly set `encoding_format=None` instead of omitting it:
+
+```python
+# Added in dbcae4a
+if encoding_format is not None:
+ optional_params["encoding_format"] = encoding_format
+else:
+ # Omitting causes openai sdk to add default value of "float"
+ optional_params["encoding_format"] = None
+```
+
+This fix worked correctly for OpenAI - explicitly passing `None` prevented the SDK from adding its default value. However, vLLM's strict parameter validation rejected `None` values, causing all embedding requests to fail.
+
+---
+
+## The Fix
+
+Fix deployed ([`55348dd`](https://github.com/BerriAI/litellm/commit/55348dd9c51b5b028f676d25ad023b8f052fc071)). The solution filters out `None` and empty string values from `optional_params` before sending requests to OpenAI-like providers (including vLLM).
+
+**In `litellm/llms/openai_like/embedding/handler.py`:**
+
+```python
+# Before (broken)
+data = {"model": model, "input": input, **optional_params}
+
+# After (fixed)
+filtered_optional_params = {k: v for k, v in optional_params.items() if v not in (None, '')}
+data = {"model": model, "input": input, **filtered_optional_params}
+```
+
+This ensures:
+- Valid values (`"float"`, `"base64"`) are preserved and sent
+- `None` and empty string values are filtered out (parameter omitted entirely)
+- OpenAI SDK no longer adds defaults because liteLLM handles the parameter upstream
+
+---
+
+## Remediation
+
+| # | Action | Status | Code |
+|---|---|---|---|
+| 1 | Filter `None` and empty string values in OpenAI-like embedding handler | ✅ Done | [`handler.py#L108`](https://github.com/BerriAI/litellm/blob/main/litellm/llms/openai_like/embedding/handler.py#L108) |
+| 2 | Unit tests for parameter filtering (None, empty string, valid values) | ✅ Done | [`test_openai_like_embedding.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/openai_like/embedding/test_openai_like_embedding.py) |
+| 3 | Transformation tests for hosted_vllm embedding config | ✅ Done | [`test_hosted_vllm_embedding_transformation.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_transformation.py) |
+| 4 | E2E tests with actual vLLM endpoint | ✅ Done | [`test_hosted_vllm_embedding_e2e.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_e2e.py) |
+| 5 | Validate JSON payload structure matches vLLM expectations | ✅ Done | Tests verify exact JSON sent to endpoint |
+
+---
diff --git a/docs/my-website/docs/a2a.md b/docs/my-website/docs/a2a.md
index d7145e4b83c..b1166a7809c 100644
--- a/docs/my-website/docs/a2a.md
+++ b/docs/my-website/docs/a2a.md
@@ -68,116 +68,9 @@ Follow [this guide, to add your pydantic ai agent to LiteLLM Agent Gateway](./pr
## Invoking your Agents
-Use the [A2A Python SDK](https://pypi.org/project/a2a/) to invoke agents through LiteLLM.
-
-This example shows how to:
-1. **List available agents** - Query `/v1/agents` to see which agents your key can access
-2. **Select an agent** - Pick an agent from the list
-3. **Invoke via A2A** - Use the A2A protocol to send messages to the agent
-
-```python showLineNumbers title="invoke_a2a_agent.py"
-from uuid import uuid4
-import httpx
-import asyncio
-from a2a.client import A2ACardResolver, A2AClient
-from a2a.types import MessageSendParams, SendMessageRequest
-
-# === CONFIGURE THESE ===
-LITELLM_BASE_URL = "http://localhost:4000" # Your LiteLLM proxy URL
-LITELLM_VIRTUAL_KEY = "sk-1234" # Your LiteLLM Virtual Key
-# =======================
-
-async def main():
- headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"}
-
- async with httpx.AsyncClient(headers=headers) as client:
- # Step 1: List available agents
- response = await client.get(f"{LITELLM_BASE_URL}/v1/agents")
- agents = response.json()
-
- print("Available agents:")
- for agent in agents:
- print(f" - {agent['agent_name']} (ID: {agent['agent_id']})")
-
- if not agents:
- print("No agents available for this key")
- return
-
- # Step 2: Select an agent and invoke it
- selected_agent = agents[0]
- agent_id = selected_agent["agent_id"]
- agent_name = selected_agent["agent_name"]
- print(f"\nInvoking: {agent_name}")
-
- # Step 3: Use A2A protocol to invoke the agent
- base_url = f"{LITELLM_BASE_URL}/a2a/{agent_id}"
- resolver = A2ACardResolver(httpx_client=client, base_url=base_url)
- agent_card = await resolver.get_agent_card()
- a2a_client = A2AClient(httpx_client=client, agent_card=agent_card)
-
- request = SendMessageRequest(
- id=str(uuid4()),
- params=MessageSendParams(
- message={
- "role": "user",
- "parts": [{"kind": "text", "text": "Hello, what can you do?"}],
- "messageId": uuid4().hex,
- }
- ),
- )
- response = await a2a_client.send_message(request)
- print(f"Response: {response.model_dump(mode='json', exclude_none=True, indent=4)}")
-
-if __name__ == "__main__":
- asyncio.run(main())
-```
-
-### Streaming Responses
-
-For streaming responses, use `send_message_streaming`:
-
-```python showLineNumbers title="invoke_a2a_agent_streaming.py"
-from uuid import uuid4
-import httpx
-import asyncio
-from a2a.client import A2ACardResolver, A2AClient
-from a2a.types import MessageSendParams, SendStreamingMessageRequest
-
-# === CONFIGURE THESE ===
-LITELLM_BASE_URL = "http://localhost:4000" # Your LiteLLM proxy URL
-LITELLM_VIRTUAL_KEY = "sk-1234" # Your LiteLLM Virtual Key
-LITELLM_AGENT_NAME = "ij-local" # Agent name registered in LiteLLM
-# =======================
-
-async def main():
- base_url = f"{LITELLM_BASE_URL}/a2a/{LITELLM_AGENT_NAME}"
- headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"}
-
- async with httpx.AsyncClient(headers=headers) as httpx_client:
- # Resolve agent card and create client
- resolver = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
- agent_card = await resolver.get_agent_card()
- client = A2AClient(httpx_client=httpx_client, agent_card=agent_card)
-
- # Send a streaming message
- request = SendStreamingMessageRequest(
- id=str(uuid4()),
- params=MessageSendParams(
- message={
- "role": "user",
- "parts": [{"kind": "text", "text": "Hello, what can you do?"}],
- "messageId": uuid4().hex,
- }
- ),
- )
-
- # Stream the response
- async for chunk in client.send_message_streaming(request):
- print(chunk.model_dump(mode="json", exclude_none=True))
-
-if __name__ == "__main__":
- asyncio.run(main())
-```
+See the [Invoking A2A Agents](./a2a_invoking_agents) guide to learn how to call your agents using:
+- **A2A SDK** - Native A2A protocol with full support for tasks and artifacts
+- **OpenAI SDK** - Familiar `/chat/completions` interface with `a2a/` model prefix
## Tracking Agent Logs
@@ -193,6 +86,120 @@ The logs show:
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
+
+## Forwarding LiteLLM Context Headers
+
+When LiteLLM invokes your A2A agent, it sends special headers that enable:
+- **Trace Grouping**: All LLM calls from the same agent execution appear under one trace
+- **Agent Spend Tracking**: Costs are attributed to the specific agent
+
+| Header | Purpose |
+|--------|---------|
+| `X-LiteLLM-Trace-Id` | Links all LLM calls to the same execution flow |
+| `X-LiteLLM-Agent-Id` | Attributes spend to the correct agent |
+
+
+To enable these features, your A2A server must **forward these headers** to any LLM calls it makes back to LiteLLM.
+
+### Implementation Steps
+
+**Step 1: Extract headers from incoming A2A request**
+```python def get_litellm_headers(request) -> dict:
+ """Extract X-LiteLLM-* headers from incoming A2A request."""
+ all_headers = request.call_context.state.get('headers', {})
+ return {
+ k: v for k, v in all_headers.items()
+ if k.lower().startswith('x-litellm-')
+ }
+```
+
+**Step 2: Forward headers to your LLM calls**
+Pass the extracted headers when making calls back to LiteLLM:
+
+
+
+```python from openai import OpenAI
+
+headers = get_litellm_headers(request)
+
+client = OpenAI(
+ api_key="sk-your-litellm-key",
+ base_url="http://localhost:4000",
+ default_headers=headers, # Forward headers
+)
+
+response = client.chat.completions.create(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "Hello"}]
+)
+```
+
+
+
+
+```python
+from langchain_openai import ChatOpenAI
+
+headers = get_litellm_headers(request)
+
+llm = ChatOpenAI(
+ model="gpt-4o",
+ openai_api_key="sk-your-litellm-key",
+ base_url="http://localhost:4000",
+ default_headers=headers, # Forward headers
+)
+```
+
+
+
+```python
+import litellm
+
+headers = get_litellm_headers(request)
+
+response = litellm.completion(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "Hello"}],
+ api_base="http://localhost:4000",
+ extra_headers=headers, # Forward headers
+)
+```
+
+
+
+```python
+import httpx
+
+headers = get_litellm_headers(request)
+headers["Authorization"] = "Bearer sk-your-litellm-key"
+
+response = httpx.post(
+ "http://localhost:4000/v1/chat/completions",
+ headers=headers,
+ json={"model": "gpt-4o", "messages": [{"role": "user", "content": "Hello"}]}
+)
+```
+
+
+
+### Result
+
+With header forwarding enabled, you'll see:
+
+**Trace Grouping in Langfuse:**
+
+
+
+**Agent Spend Attribution:**
+
+
+
## API Reference
### Endpoint
diff --git a/docs/my-website/docs/a2a_invoking_agents.md b/docs/my-website/docs/a2a_invoking_agents.md
new file mode 100644
index 00000000000..3bb248e4561
--- /dev/null
+++ b/docs/my-website/docs/a2a_invoking_agents.md
@@ -0,0 +1,280 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Invoking A2A Agents
+
+Learn how to invoke A2A agents through LiteLLM using different methods.
+
+:::tip Deploy Your Own A2A Agent
+
+Want to test with your own agent? Deploy this template A2A agent powered by Google Gemini:
+
+[**shin-bot-litellm/a2a-gemini-agent**](https://github.com/shin-bot-litellm/a2a-gemini-agent) - Simple deployable A2A agent with streaming support
+
+:::
+
+## A2A SDK
+
+Use the [A2A Python SDK](https://pypi.org/project/a2a-sdk) to invoke agents through LiteLLM using the A2A protocol.
+
+### Non-Streaming
+
+This example shows how to:
+1. **List available agents** - Query `/v1/agents` to see which agents your key can access
+2. **Select an agent** - Pick an agent from the list
+3. **Invoke via A2A** - Use the A2A protocol to send messages to the agent
+
+```python showLineNumbers title="invoke_a2a_agent.py"
+from uuid import uuid4
+import httpx
+import asyncio
+from a2a.client import A2ACardResolver, A2AClient
+from a2a.types import MessageSendParams, SendMessageRequest
+
+# === CONFIGURE THESE ===
+LITELLM_BASE_URL = "http://localhost:4000" # Your LiteLLM proxy URL
+LITELLM_VIRTUAL_KEY = "sk-1234" # Your LiteLLM Virtual Key
+# =======================
+
+async def main():
+ headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"}
+
+ async with httpx.AsyncClient(headers=headers) as client:
+ # Step 1: List available agents
+ response = await client.get(f"{LITELLM_BASE_URL}/v1/agents")
+ agents = response.json()
+
+ print("Available agents:")
+ for agent in agents:
+ print(f" - {agent['agent_name']} (ID: {agent['agent_id']})")
+
+ if not agents:
+ print("No agents available for this key")
+ return
+
+ # Step 2: Select an agent and invoke it
+ selected_agent = agents[0]
+ agent_id = selected_agent["agent_id"]
+ agent_name = selected_agent["agent_name"]
+ print(f"\nInvoking: {agent_name}")
+
+ # Step 3: Use A2A protocol to invoke the agent
+ base_url = f"{LITELLM_BASE_URL}/a2a/{agent_id}"
+ resolver = A2ACardResolver(httpx_client=client, base_url=base_url)
+ agent_card = await resolver.get_agent_card()
+ a2a_client = A2AClient(httpx_client=client, agent_card=agent_card)
+
+ request = SendMessageRequest(
+ id=str(uuid4()),
+ params=MessageSendParams(
+ message={
+ "role": "user",
+ "parts": [{"kind": "text", "text": "Hello, what can you do?"}],
+ "messageId": uuid4().hex,
+ }
+ ),
+ )
+ response = await a2a_client.send_message(request)
+ print(f"Response: {response.model_dump(mode='json', exclude_none=True, indent=4)}")
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+### Streaming
+
+For streaming responses, use `send_message_streaming`:
+
+```python showLineNumbers title="invoke_a2a_agent_streaming.py"
+from uuid import uuid4
+import httpx
+import asyncio
+from a2a.client import A2ACardResolver, A2AClient
+from a2a.types import MessageSendParams, SendStreamingMessageRequest
+
+# === CONFIGURE THESE ===
+LITELLM_BASE_URL = "http://localhost:4000" # Your LiteLLM proxy URL
+LITELLM_VIRTUAL_KEY = "sk-1234" # Your LiteLLM Virtual Key
+LITELLM_AGENT_NAME = "ij-local" # Agent name registered in LiteLLM
+# =======================
+
+async def main():
+ base_url = f"{LITELLM_BASE_URL}/a2a/{LITELLM_AGENT_NAME}"
+ headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"}
+
+ async with httpx.AsyncClient(headers=headers) as httpx_client:
+ # Resolve agent card and create client
+ resolver = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
+ agent_card = await resolver.get_agent_card()
+ client = A2AClient(httpx_client=httpx_client, agent_card=agent_card)
+
+ # Send a streaming message
+ request = SendStreamingMessageRequest(
+ id=str(uuid4()),
+ params=MessageSendParams(
+ message={
+ "role": "user",
+ "parts": [{"kind": "text", "text": "Tell me a long story"}],
+ "messageId": uuid4().hex,
+ }
+ ),
+ )
+
+ # Stream the response
+ async for chunk in client.send_message_streaming(request):
+ print(chunk.model_dump(mode="json", exclude_none=True))
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+## /chat/completions API (OpenAI SDK)
+
+You can also invoke A2A agents using the familiar OpenAI SDK by using the `a2a/` model prefix.
+
+### Non-Streaming
+
+
+
+
+```python showLineNumbers title="openai_non_streaming.py"
+import openai
+
+client = openai.OpenAI(
+ api_key="sk-1234", # Your LiteLLM Virtual Key
+ base_url="http://localhost:4000" # Your LiteLLM proxy URL
+)
+
+response = client.chat.completions.create(
+ model="a2a/my-agent", # Use a2a/ prefix with your agent name
+ messages=[
+ {"role": "user", "content": "Hello, what can you do?"}
+ ]
+)
+
+print(response.choices[0].message.content)
+```
+
+
+
+
+```typescript showLineNumbers title="openai_non_streaming.ts"
+import OpenAI from 'openai';
+
+const client = new OpenAI({
+ apiKey: 'sk-1234', // Your LiteLLM Virtual Key
+ baseURL: 'http://localhost:4000' // Your LiteLLM proxy URL
+});
+
+const response = await client.chat.completions.create({
+ model: 'a2a/my-agent', // Use a2a/ prefix with your agent name
+ messages: [
+ { role: 'user', content: 'Hello, what can you do?' }
+ ]
+});
+
+console.log(response.choices[0].message.content);
+```
+
+
+
+
+```bash showLineNumbers title="curl_non_streaming.sh"
+curl -X POST http://localhost:4000/v1/chat/completions \
+ -H "Authorization: Bearer sk-1234" \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "a2a/my-agent",
+ "messages": [
+ {"role": "user", "content": "Hello, what can you do?"}
+ ]
+ }'
+```
+
+
+
+
+### Streaming
+
+
+
+
+```python showLineNumbers title="openai_streaming.py"
+import openai
+
+client = openai.OpenAI(
+ api_key="sk-1234", # Your LiteLLM Virtual Key
+ base_url="http://localhost:4000" # Your LiteLLM proxy URL
+)
+
+stream = client.chat.completions.create(
+ model="a2a/my-agent", # Use a2a/ prefix with your agent name
+ messages=[
+ {"role": "user", "content": "Tell me a long story"}
+ ],
+ stream=True
+)
+
+for chunk in stream:
+ if chunk.choices[0].delta.content:
+ print(chunk.choices[0].delta.content, end="", flush=True)
+```
+
+
+
+
+```typescript showLineNumbers title="openai_streaming.ts"
+import OpenAI from 'openai';
+
+const client = new OpenAI({
+ apiKey: 'sk-1234', // Your LiteLLM Virtual Key
+ baseURL: 'http://localhost:4000' // Your LiteLLM proxy URL
+});
+
+const stream = await client.chat.completions.create({
+ model: 'a2a/my-agent', // Use a2a/ prefix with your agent name
+ messages: [
+ { role: 'user', content: 'Tell me a long story' }
+ ],
+ stream: true
+});
+
+for await (const chunk of stream) {
+ const content = chunk.choices[0]?.delta?.content;
+ if (content) {
+ process.stdout.write(content);
+ }
+}
+```
+
+
+
+
+```bash showLineNumbers title="curl_streaming.sh"
+curl -X POST http://localhost:4000/v1/chat/completions \
+ -H "Authorization: Bearer sk-1234" \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "a2a/my-agent",
+ "messages": [
+ {"role": "user", "content": "Tell me a long story"}
+ ],
+ "stream": true
+ }'
+```
+
+
+
+
+## Key Differences
+
+| Method | Use Case | Advantages |
+|--------|----------|------------|
+| **A2A SDK** | Native A2A protocol integration | • Full A2A protocol support
• Access to task states and artifacts
• Context management |
+| **OpenAI SDK** | Familiar OpenAI-style interface | • Drop-in replacement for OpenAI calls
• Easier migration from LLM to agent workflows
• Works with existing OpenAI tooling |
+
+:::tip Model Prefix
+
+When using the OpenAI SDK, always prefix your agent name with `a2a/` (e.g., `a2a/my-agent`) to route requests to the A2A agent instead of an LLM provider.
+
+:::
diff --git a/docs/my-website/docs/adding_provider/generic_guardrail_api.md b/docs/my-website/docs/adding_provider/generic_guardrail_api.md
index 482dedaa8a9..eb567a69fcb 100644
--- a/docs/my-website/docs/adding_provider/generic_guardrail_api.md
+++ b/docs/my-website/docs/adding_provider/generic_guardrail_api.md
@@ -93,6 +93,12 @@ Implement `POST /beta/litellm_basic_guardrail_api`
"user_api_key_end_user_id": "end user id associated with the litellm virtual key used",
"user_api_key_org_id": "org id associated with the litellm virtual key used"
},
+ "request_headers": { // optional: inbound request headers (allowlist). Allowed headers show their value; all others show "[present]" to indicate the header existed.
+ "User-Agent": "OpenAI/Python 2.17.0",
+ "Content-Type": "application/json",
+ "X-Request-Id": "[present]"
+ },
+ "litellm_version": "1.x.y", // optional: LiteLLM library version running this proxy
"input_type": "request", // "request" or "response"
"litellm_call_id": "unique_call_id", // the call id of the individual LLM call
"litellm_trace_id": "trace_id", // the trace id of the LLM call - useful if there are multiple LLM calls for the same conversation
@@ -231,6 +237,7 @@ litellm_settings:
mode: pre_call # or post_call, during_call
api_base: https://your-guardrail-api.com
api_key: os.environ/YOUR_GUARDRAIL_API_KEY # optional
+ unreachable_fallback: fail_closed # default: fail_closed. Set to fail_open to proceed if the guardrail endpoint is unreachable (network errors, or HTTP 502/503/504 from an upstream proxy/LB).
additional_provider_specific_params:
# your custom parameters
threshold: 0.8
diff --git a/docs/my-website/docs/adding_provider/generic_prompt_management_api.md b/docs/my-website/docs/adding_provider/generic_prompt_management_api.md
new file mode 100644
index 00000000000..d1b119d94c5
--- /dev/null
+++ b/docs/my-website/docs/adding_provider/generic_prompt_management_api.md
@@ -0,0 +1,576 @@
+# [BETA] Generic Prompt Management API - Integrate Without a PR
+
+## The Problem
+
+As a prompt management provider, integrating with LiteLLM traditionally requires:
+- Making a PR to the LiteLLM repository
+- Waiting for review and merge
+- Maintaining provider-specific code in LiteLLM's codebase
+- Updating the integration for changes to your API
+
+## The Solution
+
+The **Generic Prompt Management API** lets you integrate with LiteLLM **instantly** by implementing a simple API endpoint. No PR required.
+
+### Key Benefits
+
+1. **No PR Needed** - Deploy and integrate immediately
+3. **Simple Contract** - One GET endpoint, standard JSON response
+4. **Variable Substitution** - Support for prompt variables with `{variable}` syntax
+5. **Custom Parameters** - Pass provider-specific query params via config
+6. **Full Control** - You own and maintain your prompt management API
+7. **Model & Parameters Override** - Optionally override model and parameters from your prompts
+
+## Get Started in 3 Steps
+
+### Step 1: Configure LiteLLM
+
+Add to your `config.yaml`:
+
+```yaml
+prompts:
+ - prompt_id: "simple_prompt"
+ litellm_params:
+ prompt_integration: "generic_prompt_management"
+ api_base: http://localhost:8080
+ api_key: os.environ/YOUR_API_KEY
+```
+
+### Step 2: Implement Your API Endpoint
+
+```python
+from fastapi import FastAPI
+from pydantic import BaseModel
+
+app = FastAPI()
+
+@app.get("/beta/litellm_prompt_management")
+async def get_prompt(prompt_id: str):
+ return {
+ "prompt_id": prompt_id,
+ "prompt_template": [
+ {"role": "system", "content": "You are a helpful assistant."},
+ {"role": "user", "content": "Help me with {task}"}
+ ],
+ "prompt_template_model": "gpt-4",
+ "prompt_template_optional_params": {"temperature": 0.7}
+ }
+```
+
+### Step 3: Use in Your App
+
+```python
+from litellm import completion
+
+response = completion(
+ model="gpt-4",
+ prompt_id="simple_prompt",
+ prompt_variables={"task": "data analysis"},
+ messages=[{"role": "user", "content": "I have sales data"}]
+)
+```
+
+That's it! LiteLLM fetches your prompt, applies variables, and makes the request
+
+## API Contract
+
+### Endpoint
+
+Implement `GET /beta/litellm_prompt_management`
+
+### Request Format
+
+Your endpoint will receive a GET request with query parameters:
+
+```
+GET /beta/litellm_prompt_management?prompt_id={prompt_id}&{custom_params}
+```
+
+**Query Parameters:**
+- `prompt_id` (required): The ID of the prompt to fetch
+- Custom parameters: Any additional parameters you configured in `provider_specific_query_params`
+
+**Example:**
+```
+GET /beta/litellm_prompt_management?prompt_id=hello-world-prompt-2bac&project_name=litellm&slug=hello-world-prompt-2bac
+```
+
+### Response Format
+
+```json
+{
+ "prompt_id": "hello-world-prompt-2bac",
+ "prompt_template": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant specialized in {domain}."
+ },
+ {
+ "role": "user",
+ "content": "Help me with {task}"
+ }
+ ],
+ "prompt_template_model": "gpt-4",
+ "prompt_template_optional_params": {
+ "temperature": 0.7,
+ "max_tokens": 500,
+ "top_p": 0.9
+ }
+}
+```
+
+**Response Fields:**
+- `prompt_id` (string, required): The ID of the prompt
+- `prompt_template` (array, required): Array of OpenAI-format messages with optional `{variable}` placeholders
+- `prompt_template_model` (string, optional): Model to use for this prompt (overrides client model unless `ignore_prompt_manager_model: true`)
+- `prompt_template_optional_params` (object, optional): Additional parameters like temperature, max_tokens, etc. (merged with client params unless `ignore_prompt_manager_optional_params: true`)
+
+## LiteLLM Configuration
+
+Add to `config.yaml`:
+
+```yaml
+model_list:
+ - model_name: gpt-3.5-turbo
+ litellm_params:
+ model: openai/gpt-3.5-turbo
+ api_key: os.environ/OPENAI_API_KEY
+
+prompts:
+ - prompt_id: "simple_prompt"
+ litellm_params:
+ prompt_integration: "generic_prompt_management"
+ provider_specific_query_params:
+ project_name: litellm
+ slug: hello-world-prompt-2bac
+ api_base: http://localhost:8080
+ api_key: os.environ/YOUR_PROMPT_API_KEY # optional
+ ignore_prompt_manager_model: true # optional, keep client's model
+ ignore_prompt_manager_optional_params: true # optional, don't merge prompt manager's params (e.g. temperature, max_tokens, etc.)
+```
+
+### Configuration Parameters
+
+- `prompt_integration`: Must be `"generic_prompt_management"`
+- `provider_specific_query_params`: Custom query parameters sent to your API (optional)
+- `api_base`: Base URL of your prompt management API
+- `api_key`: Optional API key for authentication (sent as `Bearer` token)
+- `ignore_prompt_manager_model`: If `true`, use the model specified by client instead of prompt's model (default: `false`)
+- `ignore_prompt_manager_optional_params`: If `true`, don't merge prompt's optional params with client params (default: `false`)
+
+## Usage
+
+### Using with LiteLLM SDK
+
+**Basic usage with prompt ID:**
+
+```python
+from litellm import completion
+
+response = completion(
+ model="gpt-4",
+ prompt_id="simple_prompt",
+ messages=[{"role": "user", "content": "Additional message"}]
+)
+```
+
+**With prompt variables:**
+
+```python
+response = completion(
+ model="gpt-4",
+ prompt_id="simple_prompt",
+ prompt_variables={
+ "domain": "data science",
+ "task": "analyzing customer churn"
+ },
+ messages=[{"role": "user", "content": "Please provide a detailed analysis"}]
+)
+```
+
+The prompt template will have `{domain}` replaced with "data science" and `{task}` replaced with "analyzing customer churn".
+
+### Using with LiteLLM Proxy
+
+**1. Start the proxy with your config:**
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+**2. Make requests with prompt_id:**
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer sk-1234" \
+ -d '{
+ "model": "gpt-4",
+ "prompt_id": "simple_prompt",
+ "prompt_variables": {
+ "domain": "healthcare",
+ "task": "patient risk assessment"
+ },
+ "messages": [
+ {"role": "user", "content": "Analyze the following data..."}
+ ]
+ }'
+```
+
+**3. Using with OpenAI SDK:**
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+ base_url="http://0.0.0.0:4000",
+ api_key="sk-1234"
+)
+
+response = client.chat.completions.create(
+ model="gpt-4",
+ messages=[
+ {"role": "user", "content": "Analyze the data"}
+ ],
+ extra_body={
+ "prompt_id": "simple_prompt",
+ "prompt_variables": {
+ "domain": "finance",
+ "task": "fraud detection"
+ }
+ }
+)
+```
+
+## Implementation Example
+
+See [mock_prompt_management_server.py](https://github.com/BerriAI/litellm/blob/main/cookbook/mock_prompt_management_server/mock_prompt_management_server.py) for a complete reference implementation with multiple example prompts, authentication, and convenience endpoints.
+
+**Minimal FastAPI example:**
+
+```python
+from fastapi import FastAPI, HTTPException, Header
+from typing import Optional, Dict, Any, List
+from pydantic import BaseModel
+
+app = FastAPI()
+
+# In-memory prompt storage (replace with your database)
+PROMPTS = {
+ "hello-world-prompt": {
+ "prompt_id": "hello-world-prompt",
+ "prompt_template": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant specialized in {domain}."
+ },
+ {
+ "role": "user",
+ "content": "Help me with: {task}"
+ }
+ ],
+ "prompt_template_model": "gpt-4",
+ "prompt_template_optional_params": {
+ "temperature": 0.7,
+ "max_tokens": 500
+ }
+ },
+ "code-review-prompt": {
+ "prompt_id": "code-review-prompt",
+ "prompt_template": [
+ {
+ "role": "system",
+ "content": "You are an expert code reviewer. Review code for {language}."
+ },
+ {
+ "role": "user",
+ "content": "Review the following code:\n\n{code}"
+ }
+ ],
+ "prompt_template_model": "gpt-4-turbo",
+ "prompt_template_optional_params": {
+ "temperature": 0.3,
+ "max_tokens": 1000
+ }
+ }
+}
+
+class PromptResponse(BaseModel):
+ prompt_id: str
+ prompt_template: List[Dict[str, str]]
+ prompt_template_model: Optional[str] = None
+ prompt_template_optional_params: Optional[Dict[str, Any]] = None
+
+@app.get("/beta/litellm_prompt_management", response_model=PromptResponse)
+async def get_prompt(
+ prompt_id: str,
+ authorization: Optional[str] = Header(None),
+ project_name: Optional[str] = None,
+ slug: Optional[str] = None,
+):
+ """
+ Get a prompt by ID with optional filtering by project_name and slug.
+
+ Args:
+ prompt_id: The ID of the prompt to fetch
+ authorization: Optional Bearer token for authentication
+ project_name: Optional project name filter
+ slug: Optional slug filter
+ """
+
+ # Optional: Validate authorization
+ if authorization:
+ token = authorization.replace("Bearer ", "")
+ # Validate your token here
+ if not is_valid_token(token):
+ raise HTTPException(status_code=401, detail="Invalid API key")
+
+ # Optional: Apply additional filtering based on custom params
+ if project_name or slug:
+ # You can use these parameters to filter or validate access
+ # For example, check if the user has access to this project
+ pass
+
+ # Fetch the prompt from your storage
+ if prompt_id not in PROMPTS:
+ raise HTTPException(
+ status_code=404,
+ detail=f"Prompt '{prompt_id}' not found"
+ )
+
+ prompt_data = PROMPTS[prompt_id]
+
+ return PromptResponse(**prompt_data)
+
+def is_valid_token(token: str) -> bool:
+ """Validate API token - implement your logic here"""
+ # Example: Check against your database or secret store
+ valid_tokens = ["your-secret-token", "another-valid-token"]
+ return token in valid_tokens
+
+# Optional: Health check endpoint
+@app.get("/health")
+async def health_check():
+ return {"status": "healthy"}
+
+# Optional: List all prompts endpoint
+@app.get("/prompts")
+async def list_prompts(authorization: Optional[str] = Header(None)):
+ """List all available prompts"""
+ if authorization:
+ token = authorization.replace("Bearer ", "")
+ if not is_valid_token(token):
+ raise HTTPException(status_code=401, detail="Invalid API key")
+
+ return {
+ "prompts": [
+ {"prompt_id": pid, "model": p.get("prompt_template_model")}
+ for pid, p in PROMPTS.items()
+ ]
+ }
+
+if __name__ == "__main__":
+ import uvicorn
+ uvicorn.run(app, host="0.0.0.0", port=8080)
+```
+
+### Running the Example Server
+
+1. Install dependencies:
+```bash
+pip install fastapi uvicorn
+```
+
+2. Save the code above to `prompt_server.py`
+
+3. Run the server:
+```bash
+python prompt_server.py
+```
+
+4. Test the endpoint:
+```bash
+curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt&project_name=litellm&slug=hello-world-prompt-2bac"
+```
+
+Expected response:
+```json
+{
+ "prompt_id": "hello-world-prompt",
+ "prompt_template": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant specialized in {domain}."
+ },
+ {
+ "role": "user",
+ "content": "Help me with: {task}"
+ }
+ ],
+ "prompt_template_model": "gpt-4",
+ "prompt_template_optional_params": {
+ "temperature": 0.7,
+ "max_tokens": 500
+ }
+}
+```
+
+## Advanced Features
+
+### Variable Substitution
+
+LiteLLM automatically substitutes variables in your prompt templates using the `{variable}` syntax. Both `{variable}` and `{{variable}}` formats are supported.
+
+**Example prompt template:**
+```json
+{
+ "prompt_template": [
+ {
+ "role": "system",
+ "content": "You are an expert in {domain} with {years} years of experience."
+ }
+ ]
+}
+```
+
+**Client request:**
+```python
+completion(
+ model="gpt-4",
+ prompt_id="expert_prompt",
+ prompt_variables={
+ "domain": "machine learning",
+ "years": "10"
+ }
+)
+```
+
+**Result:**
+```
+"You are an expert in machine learning with 10 years of experience."
+```
+
+### Caching
+
+LiteLLM automatically caches fetched prompts in memory. The cache key includes:
+- `prompt_id`
+- `prompt_label` (if provided)
+- `prompt_version` (if provided)
+
+This means your API endpoint is only called once per unique prompt configuration.
+
+### Model Override Behavior
+
+**Default behavior (without `ignore_prompt_manager_model`):**
+```yaml
+prompts:
+ - prompt_id: "my_prompt"
+ litellm_params:
+ prompt_integration: "generic_prompt_management"
+ api_base: http://localhost:8080
+```
+
+If your API returns `"prompt_template_model": "gpt-4"`, LiteLLM will use `gpt-4` regardless of what the client specified.
+
+**With `ignore_prompt_manager_model: true`:**
+```yaml
+prompts:
+ - prompt_id: "my_prompt"
+ litellm_params:
+ prompt_integration: "generic_prompt_management"
+ api_base: http://localhost:8080
+ ignore_prompt_manager_model: true
+```
+
+LiteLLM will use the model specified by the client, ignoring the prompt's model.
+
+### Parameter Merging Behavior
+
+**Default behavior (without `ignore_prompt_manager_optional_params`):**
+
+Client params are merged with prompt params, with prompt params taking precedence:
+```python
+# Prompt returns: {"temperature": 0.7, "max_tokens": 500}
+# Client sends: {"temperature": 0.9, "top_p": 0.95}
+# Final params: {"temperature": 0.7, "max_tokens": 500, "top_p": 0.95}
+```
+
+**With `ignore_prompt_manager_optional_params: true`:**
+
+Only client params are used:
+```python
+# Prompt returns: {"temperature": 0.7, "max_tokens": 500}
+# Client sends: {"temperature": 0.9, "top_p": 0.95}
+# Final params: {"temperature": 0.9, "top_p": 0.95}
+```
+
+## Security Considerations
+
+1. **Authentication**: Use the `api_key` parameter to secure your prompt management API
+2. **Authorization**: Implement team/user-based access control using the custom query parameters
+3. **Rate Limiting**: Add rate limiting to prevent abuse of your API
+4. **Input Validation**: Validate all query parameters before processing
+5. **HTTPS**: Always use HTTPS in production for encrypted communication
+6. **Secrets**: Store API keys in environment variables, not in config files
+
+## Use Cases
+
+✅ **Use Generic Prompt Management API when:**
+- You want instant integration without waiting for PRs
+- You maintain your own prompt management service
+- You need full control over prompt versioning and updates
+- You want to build custom prompt management features
+- You need to integrate with your internal systems
+
+✅ **Common scenarios:**
+- Internal prompt management system for your organization
+- Multi-tenant prompt management with team-based access control
+- A/B testing different prompt versions
+- Prompt experimentation and analytics
+- Integration with existing prompt engineering workflows
+
+## When to Use This
+
+✅ **Use Generic Prompt Management API when:**
+- You want instant integration without waiting for PRs
+- You maintain your own prompt management service
+- You need full control over updates and features
+- You want custom prompt storage and versioning logic
+
+❌ **Make a PR when:**
+- You want deeper integration with LiteLLM internals
+- Your integration requires complex LiteLLM-specific logic
+- You want to be featured as a built-in provider
+- You're building a reusable integration for the community
+
+## Troubleshooting
+
+### Prompt not found
+- Verify the `prompt_id` matches exactly (case-sensitive)
+- Check that your API endpoint is accessible from LiteLLM
+- Verify authentication if using `api_key`
+
+### Variables not substituted
+- Ensure variables use `{variable}` or `{{variable}}` syntax
+- Check that variable names in `prompt_variables` match template exactly
+- Variables are case-sensitive
+
+### Model not being overridden
+- Check if `ignore_prompt_manager_model: true` is set in config
+- Verify your API is returning `prompt_template_model` in the response
+
+### Parameters not being applied
+- Check if `ignore_prompt_manager_optional_params: true` is set
+- Verify your API is returning `prompt_template_optional_params`
+- Ensure parameter names match OpenAI's parameter names
+
+## Questions?
+
+This is a **beta API**. We're actively improving it based on feedback. Open an issue or PR if you need additional capabilities.
+
+## Related Documentation
+
+- [Prompt Management Overview](../proxy/prompt_management.md)
+- [Generic Guardrail API](./generic_guardrail_api.md)
+- [LiteLLM Proxy Setup](../proxy/quick_start.md)
+
diff --git a/docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md b/docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md
index 9c654cd1560..884a7397bde 100644
--- a/docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md
+++ b/docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md
@@ -101,12 +101,11 @@ model_list:
- model_name: gpt-4
litellm_params:
model: gpt-4
- api_key: os.environ/OPENAI_API_KEY
+ api_key: os.environ/OPENAI_API_KEY
-litellm_settings:
- guardrails:
+guardrails:
- guardrail_name: my_guardrail
- litellm_params:
+ litellm_params:
guardrail: my_guardrail
mode: during_call
api_key: os.environ/MY_GUARDRAIL_API_KEY
diff --git a/docs/my-website/docs/benchmarks.md b/docs/my-website/docs/benchmarks.md
index 640212808bd..1f818cef498 100644
--- a/docs/my-website/docs/benchmarks.md
+++ b/docs/my-website/docs/benchmarks.md
@@ -5,6 +5,13 @@ import Image from '@theme/IdealImage';
Benchmarks for LiteLLM Gateway (Proxy Server) tested against a fake OpenAI endpoint.
+## Setting Up a Fake OpenAI Endpoint
+
+For load testing and benchmarking, you can use a fake OpenAI proxy server. LiteLLM provides:
+
+1. **Hosted endpoint**: Use our free hosted fake endpoint at `https://exampleopenaiendpoint-production.up.railway.app/`
+2. **Self-hosted**: Set up your own fake OpenAI proxy server using [github.com/BerriAI/example_openai_endpoint](https://github.com/BerriAI/example_openai_endpoint)
+
Use this config for testing:
```yaml
@@ -12,7 +19,7 @@ model_list:
- model_name: "fake-openai-endpoint"
litellm_params:
model: openai/any
- api_base: https://your-fake-openai-endpoint.com/chat/completions
+ api_base: https://exampleopenaiendpoint-production.up.railway.app/ # or your self-hosted endpoint
api_key: "test"
```
@@ -48,6 +55,28 @@ In these tests the baseline latency characteristics are measured against a fake-
- High-percentile latencies drop significantly: P95 630 ms → 150 ms, P99 1,200 ms → 240 ms.
- Setting workers equal to CPU count gives optimal performance.
+## `/realtime` API Benchmarks
+
+End-to-end latency benchmarks for the `/realtime` endpoint tested against a fake realtime endpoint.
+
+### Performance Metrics
+
+| Metric | Value |
+| --------------- | ---------- |
+| Median latency | 59 ms |
+| p95 latency | 67 ms |
+| p99 latency | 99 ms |
+| Average latency | 63 ms |
+| RPS | 1,207 |
+
+### Test Setup
+
+| Category | Specification |
+|----------|---------------|
+| **Load Testing** | Locust: 1,000 concurrent users, 500 ramp-up |
+| **System** | 4 vCPUs, 8 GB RAM, 4 workers, 4 instances |
+| **Database** | PostgreSQL (Redis unused) |
+
## Machine Spec used for testing
Each machine deploying LiteLLM had the following specs:
diff --git a/docs/my-website/docs/completion/message_sanitization.md b/docs/my-website/docs/completion/message_sanitization.md
new file mode 100644
index 00000000000..17482c59339
--- /dev/null
+++ b/docs/my-website/docs/completion/message_sanitization.md
@@ -0,0 +1,465 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Message Sanitization for Tool Calling for anthropic models
+
+**Automatically fix common message formatting issues when using tool calling with `modify_params=True`**
+
+LiteLLM can automatically sanitize messages to handle common issues that occur during tool calling workflows, especially when using OpenAI-compatible clients with providers that have strict message format requirements (like Anthropic Claude).
+
+## Overview
+
+When `litellm.modify_params = True` is enabled, LiteLLM automatically sanitizes messages to fix three common issues:
+
+1. **Orphaned Tool Calls** - Assistant messages with tool_calls but missing tool results
+2. **Orphaned Tool Results** - Tool messages that reference non-existent tool_call_ids
+3. **Empty Message Content** - Messages with empty or whitespace-only text content
+
+This ensures your tool calling workflows work seamlessly across different LLM providers without manual message validation.
+
+## Why Message Sanitization?
+
+Different LLM providers have varying requirements for message formats, especially during tool calling:
+
+- **Anthropic Claude** requires every tool_call to have a corresponding tool result
+- Some providers reject messages with empty content
+- OpenAI-compatible clients may not always maintain perfect message consistency
+
+Without sanitization, these issues cause API errors that interrupt your workflows. With `modify_params=True`, LiteLLM handles these edge cases automatically.
+
+## Quick Start
+
+
+
+
+```python
+import litellm
+
+# Enable automatic message sanitization
+litellm.modify_params = True
+
+# This will work even if messages have formatting issues
+response = litellm.completion(
+ model="anthropic/claude-3-5-sonnet-20241022",
+ messages=[
+ {"role": "user", "content": "What's the weather in Boston?"},
+ {
+ "role": "assistant",
+ "tool_calls": [
+ {
+ "id": "call_123",
+ "type": "function",
+ "function": {"name": "get_weather", "arguments": '{"city": "Boston"}'}
+ }
+ ]
+ # Missing tool result - LiteLLM will add a dummy result automatically
+ },
+ {"role": "user", "content": "Thanks!"}
+ ],
+ tools=[{
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Get weather for a city",
+ "parameters": {
+ "type": "object",
+ "properties": {"city": {"type": "string"}},
+ "required": ["city"]
+ }
+ }
+ }]
+)
+```
+
+
+
+
+```yaml
+litellm_settings:
+ modify_params: true # Enable automatic message sanitization
+
+model_list:
+ - model_name: claude-3-5-sonnet
+ litellm_params:
+ model: anthropic/claude-3-5-sonnet-20241022
+```
+
+
+
+
+## Sanitization Cases
+
+### Case A: Orphaned Tool Calls (Missing Tool Results)
+
+**Problem:** An assistant message contains `tool_calls`, but no corresponding tool result messages follow.
+
+**Solution:** LiteLLM automatically adds dummy tool result messages for any missing tool results.
+
+**Example:**
+
+```python
+import litellm
+litellm.modify_params = True
+
+# Messages with orphaned tool calls
+messages = [
+ {"role": "user", "content": "Search for Python tutorials"},
+ {
+ "role": "assistant",
+ "tool_calls": [
+ {
+ "id": "call_abc123",
+ "type": "function",
+ "function": {"name": "web_search", "arguments": '{"query": "Python tutorials"}'}
+ }
+ ]
+ },
+ # Missing tool result here!
+ {"role": "user", "content": "What about JavaScript?"}
+]
+
+# LiteLLM automatically adds:
+# {
+# "role": "tool",
+# "tool_call_id": "call_abc123",
+# "content": "[System: Tool execution skipped/interrupted by user. No result provided for tool 'web_search'.]"
+# }
+
+response = litellm.completion(
+ model="anthropic/claude-3-5-sonnet-20241022",
+ messages=messages,
+ tools=[...]
+)
+```
+
+**When this happens:**
+- User interrupts tool execution
+- Client loses tool results due to network issues
+- Conversation flow changes before tool completes
+- Multi-turn conversations where tools are optional
+
+### Case B: Orphaned Tool Results (Invalid tool_call_id)
+
+**Problem:** A tool message references a `tool_call_id` that doesn't exist in any previous assistant message.
+
+**Solution:** LiteLLM automatically removes these orphaned tool result messages.
+
+**Example:**
+
+```python
+import litellm
+litellm.modify_params = True
+
+# Messages with orphaned tool result
+messages = [
+ {"role": "user", "content": "Hello"},
+ {"role": "assistant", "content": "Hi! How can I help?"},
+ {
+ "role": "tool",
+ "tool_call_id": "call_nonexistent", # This tool_call_id doesn't exist!
+ "content": "Some result"
+ }
+]
+
+# LiteLLM automatically removes the orphaned tool message
+
+response = litellm.completion(
+ model="anthropic/claude-3-5-sonnet-20241022",
+ messages=messages
+)
+```
+
+**When this happens:**
+- Message history is manually edited
+- Tool results are duplicated or mismatched
+- Conversation state is restored incorrectly
+- Messages are merged from different conversations
+
+### Case C: Empty Message Content
+
+**Problem:** User or assistant messages have empty or whitespace-only content.
+
+**Solution:** LiteLLM replaces empty content with a system placeholder message.
+
+**Example:**
+
+```python
+import litellm
+litellm.modify_params = True
+
+# Messages with empty content
+messages = [
+ {"role": "user", "content": ""}, # Empty content
+ {"role": "assistant", "content": " "}, # Whitespace only
+]
+
+# LiteLLM automatically replaces with:
+# {"role": "user", "content": "[System: Empty message content sanitised to satisfy protocol]"}
+# {"role": "assistant", "content": "[System: Empty message content sanitised to satisfy protocol]"}
+
+response = litellm.completion(
+ model="anthropic/claude-3-5-sonnet-20241022",
+ messages=messages
+)
+```
+
+**When this happens:**
+- UI sends empty messages
+- Content is stripped during preprocessing
+- Placeholder messages in conversation history
+- Edge cases in message construction
+
+## Configuration
+
+### Enable Globally
+
+
+
+
+```python
+import litellm
+
+# Enable for all completion calls
+litellm.modify_params = True
+```
+
+
+
+
+```yaml
+litellm_settings:
+ modify_params: true
+```
+
+
+
+
+```bash
+export LITELLM_MODIFY_PARAMS=True
+```
+
+
+
+
+### Enable Per-Request
+
+```python
+import litellm
+
+# Enable only for specific requests
+response = litellm.completion(
+ model="anthropic/claude-3-5-sonnet-20241022",
+ messages=messages,
+ modify_params=True # Override global setting
+)
+```
+
+## Supported Providers
+
+Message sanitization currently works with:
+
+- ✅ Anthropic (Claude)
+
+**Note:** While the sanitization logic is provider-agnostic, it is currently only applied in the Anthropic message transformation pipeline. Support for additional providers may be added in future releases.
+
+## Implementation Details
+
+### How It Works
+
+The message sanitization process runs **before** messages are converted to provider-specific formats:
+
+1. **Input:** OpenAI-format messages with potential issues
+2. **Sanitization:** Three helper functions process the messages:
+ - `_sanitize_empty_text_content()` - Fixes empty content
+ - `_add_missing_tool_results()` - Adds dummy tool results
+ - `_is_orphaned_tool_result()` - Identifies orphaned results
+3. **Output:** Clean, provider-compatible messages
+
+### Code Reference
+
+The sanitization logic is implemented in:
+- `litellm/litellm_core_utils/prompt_templates/factory.py`
+- Function: `sanitize_messages_for_tool_calling()`
+
+### Logging
+
+When sanitization occurs, LiteLLM logs debug messages:
+
+```python
+import litellm
+litellm.set_verbose = True # Enable debug logging
+
+# You'll see logs like:
+# "_add_missing_tool_results: Found 1 orphaned tool calls. Adding dummy tool results."
+# "_is_orphaned_tool_result: Found orphaned tool result with tool_call_id=call_123"
+# "_sanitize_empty_text_content: Replaced empty text content in user message"
+```
+
+## Best Practices
+
+### 1. Enable for Production Workflows
+
+```python
+# Recommended for production
+litellm.modify_params = True
+
+# Ensures robust handling of edge cases
+response = litellm.completion(
+ model="anthropic/claude-3-5-sonnet-20241022",
+ messages=messages,
+ tools=tools
+)
+```
+
+### 2. Preserve Tool Results When Possible
+
+While sanitization handles missing tool results, it's better to provide actual results:
+
+```python
+# Good: Provide actual tool results
+messages = [
+ {"role": "user", "content": "Search for Python"},
+ {"role": "assistant", "tool_calls": [...]},
+ {"role": "tool", "tool_call_id": "call_123", "content": "Actual search results"}
+]
+
+# Fallback: Sanitization adds dummy result if missing
+messages = [
+ {"role": "user", "content": "Search for Python"},
+ {"role": "assistant", "tool_calls": [...]},
+ # Missing tool result - sanitization adds dummy
+]
+```
+
+### 3. Monitor Sanitization Events
+
+Use logging to track when sanitization occurs:
+
+```python
+import litellm
+import logging
+
+# Enable debug logging
+litellm.set_verbose = True
+logging.basicConfig(level=logging.DEBUG)
+
+# Track sanitization events in your application
+response = litellm.completion(
+ model="anthropic/claude-3-5-sonnet-20241022",
+ messages=messages
+)
+```
+
+### 4. Test Edge Cases
+
+Ensure your application handles sanitized messages correctly:
+
+```python
+import litellm
+litellm.modify_params = True
+
+# Test orphaned tool calls
+test_messages = [
+ {"role": "user", "content": "Test"},
+ {"role": "assistant", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "test", "arguments": "{}"}}]},
+ {"role": "user", "content": "Continue"} # No tool result
+]
+
+response = litellm.completion(
+ model="anthropic/claude-3-5-sonnet-20241022",
+ messages=test_messages,
+ tools=[...]
+)
+
+# Verify the response handles the dummy tool result appropriately
+```
+
+## Related Features
+
+- **[Drop Params](./drop_params.md)** - Drop unsupported parameters for specific providers
+- **[Message Trimming](./message_trimming.md)** - Trim messages to fit token limits
+- **[Function Calling](./function_call.md)** - Complete guide to tool/function calling
+- **[Reasoning Content](../reasoning_content.md)** - Extended thinking with tool calling
+
+## Troubleshooting
+
+### Sanitization Not Working
+
+**Issue:** Messages still cause errors despite `modify_params=True`
+
+**Solution:**
+1. Verify `modify_params` is enabled:
+ ```python
+ import litellm
+ print(litellm.modify_params) # Should be True
+ ```
+
+2. Check if the issue is provider-specific:
+ ```python
+ litellm.set_verbose = True # Enable debug logging
+ ```
+
+3. Ensure you're using a recent version of LiteLLM:
+ ```bash
+ pip install --upgrade litellm
+ ```
+
+### Unexpected Dummy Tool Results
+
+**Issue:** Dummy tool results appear when you expect actual results
+
+**Cause:** Tool result messages are missing or have incorrect `tool_call_id`
+
+**Solution:**
+1. Verify tool result messages have correct `tool_call_id`:
+ ```python
+ # Correct
+ {"role": "tool", "tool_call_id": "call_123", "content": "result"}
+
+ # Incorrect - will be treated as orphaned
+ {"role": "tool", "tool_call_id": "wrong_id", "content": "result"}
+ ```
+
+2. Ensure tool results immediately follow assistant messages with tool_calls
+
+### Performance Impact
+
+**Issue:** Concerned about performance overhead
+
+**Details:** Message sanitization has minimal performance impact:
+- Runs in O(n) time where n = number of messages
+- Only processes messages when `modify_params=True`
+- Typically adds < 1ms to request processing time
+
+## FAQ
+
+**Q: Does sanitization modify my original messages?**
+
+A: No, sanitization creates a new list of messages. Your original messages remain unchanged.
+
+**Q: Can I disable specific sanitization cases?**
+
+A: Currently, all three cases are handled together when `modify_params=True`. To disable sanitization entirely, set `modify_params=False`.
+
+**Q: What happens to the dummy tool results?**
+
+A: Dummy tool results are sent to the LLM provider along with other messages. The model sees them as regular tool results with informative error messages.
+
+**Q: Does this work with streaming?**
+
+A: Yes, message sanitization works with both streaming and non-streaming requests.
+
+**Q: Is this related to `drop_params`?**
+
+A: No, they're separate features:
+- `modify_params` - Modifies/fixes message content and structure
+- `drop_params` - Removes unsupported API parameters
+
+Both can be enabled simultaneously.
+
+## See Also
+
+- [Reasoning Content with Tool Calling](../reasoning_content.md)
+- [Function Calling Guide](./function_call.md)
+- [Bedrock Provider Documentation](../providers/bedrock.md)
+- [Anthropic Provider Documentation](../providers/anthropic.md)
diff --git a/docs/my-website/docs/completion/web_search.md b/docs/my-website/docs/completion/web_search.md
index db50c7b5bc5..1f5ba2dee4e 100644
--- a/docs/my-website/docs/completion/web_search.md
+++ b/docs/my-website/docs/completion/web_search.md
@@ -18,16 +18,46 @@ Each provider uses their own search backend:
| Provider | Search Engine | Notes |
|----------|---------------|-------|
-| **OpenAI** (`gpt-4o-search-preview`) | OpenAI's internal search | Real-time web data |
+| **OpenAI** (`gpt-5-search-api`, `gpt-4o-search-preview`, `gpt-4o-mini-search-preview`) | OpenAI's internal search | Real-time web data |
| **xAI** (`grok-3`) | xAI's search + X/Twitter | Real-time social media data |
| **Google AI/Vertex** (`gemini-2.0-flash`) | **Google Search** | Uses actual Google search results |
| **Anthropic** (`claude-3-5-sonnet`) | Anthropic's web search | Real-time web data |
| **Perplexity** | Perplexity's search engine | AI-powered search and reasoning |
+:::warning Important: Only Search Models Support `web_search_options`
+For OpenAI, only dedicated search models support the `web_search_options` parameter:
+- `gpt-4o-search-preview`
+- `gpt-4o-mini-search-preview`
+- `gpt-5-search-api`
+
+**Regular models like `gpt-5`, `gpt-4.1`, `gpt-4o` do not support `web_search_options`**
+:::
+
+:::tip The `web_search_options` parameter is optional
+Search models (like `gpt-4o-search-preview`) **automatically search the web** even without the `web_search_options` parameter.
+
+Use `web_search_options` when you need to:
+- Adjust `search_context_size` (`"low"`, `"medium"`, `"high"`)
+- Specify `user_location` for localized results
+:::
+
:::info
**Anthropic Web Search Models**: Claude models that support web search: `claude-3-5-sonnet-latest`, `claude-3-5-sonnet-20241022`, `claude-3-5-haiku-latest`, `claude-3-5-haiku-20241022`, `claude-3-7-sonnet-20250219`
:::
+## OpenAI Web Search: Two Approaches
+
+OpenAI offers two distinct ways to use web search depending on the endpoint and model:
+
+| Approach | Endpoint | Models | How to enable |
+|----------|----------|--------|---------------|
+| **Search Models** | `/chat/completions` | `gpt-5-search-api`, `gpt-4o-search-preview`, `gpt-4o-mini-search-preview` | Pass `web_search_options` parameter |
+| **Web Search Tool** | `/responses` | `gpt-5`, `gpt-4.1`, `gpt-4o`, and other regular models | Pass `web_search_preview` tool |
+
+:::tip Search models search automatically
+Search models like `gpt-5-search-api` **automatically search the web** even without the `web_search_options` parameter. Use `web_search_options` to set `search_context_size` (`"low"`, `"medium"`, `"high"`) or specify `user_location` for localized results.
+:::
+
## `/chat/completions` (litellm.completion)
### Quick Start
@@ -39,7 +69,7 @@ Each provider uses their own search backend:
from litellm import completion
response = completion(
- model="openai/gpt-4o-search-preview",
+ model="openai/gpt-5-search-api",
messages=[
{
"role": "user",
@@ -59,31 +89,36 @@ response = completion(
```yaml
model_list:
- # OpenAI
+ # OpenAI search models
+ - model_name: gpt-5-search-api
+ litellm_params:
+ model: openai/gpt-5-search-api
+ api_key: os.environ/OPENAI_API_KEY
+
- model_name: gpt-4o-search-preview
litellm_params:
model: openai/gpt-4o-search-preview
api_key: os.environ/OPENAI_API_KEY
-
+
# xAI
- model_name: grok-3
litellm_params:
model: xai/grok-3
api_key: os.environ/XAI_API_KEY
-
+
# Anthropic
- model_name: claude-3-5-sonnet-latest
litellm_params:
model: anthropic/claude-3-5-sonnet-latest
api_key: os.environ/ANTHROPIC_API_KEY
-
+
# VertexAI
- model_name: gemini-2-flash
litellm_params:
model: gemini-2.0-flash
vertex_project: your-project-id
vertex_location: us-central1
-
+
# Google AI Studio
- model_name: gemini-2-flash-studio
litellm_params:
@@ -91,13 +126,13 @@ model_list:
api_key: os.environ/GOOGLE_API_KEY
```
-2. Start the proxy
+2. Start the proxy
```bash
litellm --config /path/to/config.yaml
```
-3. Test it!
+3. Test it!
```python showLineNumbers
from openai import OpenAI
@@ -109,13 +144,18 @@ client = OpenAI(
)
response = client.chat.completions.create(
- model="grok-3", # or any other web search enabled model
+ model="gpt-5-search-api", # or any other web search enabled model
messages=[
{
"role": "user",
"content": "What was a positive news story from today?"
}
- ]
+ ],
+ extra_body={
+ "web_search_options": {
+ "search_context_size": "medium"
+ }
+ }
)
```
@@ -132,7 +172,7 @@ from litellm import completion
# Customize search context size
response = completion(
- model="openai/gpt-4o-search-preview",
+ model="openai/gpt-5-search-api",
messages=[
{
"role": "user",
@@ -240,6 +280,12 @@ response = client.chat.completions.create(
## `/responses` (litellm.responses)
+Use the `web_search_preview` tool with models like `gpt-5`, `gpt-4.1`, `gpt-4o`, etc.
+
+:::info
+Search-dedicated models like `gpt-5-search-api` and `gpt-4o-search-preview` do **not** support the `/responses` endpoint. Use them with `/chat/completions` + `web_search_options` instead (see above).
+:::
+
### Quick Start
@@ -249,18 +295,14 @@ response = client.chat.completions.create(
from litellm import responses
response = responses(
- model="openai/gpt-4o",
- input=[
- {
- "role": "user",
- "content": "What was a positive news story from today?"
- }
- ],
+ model="openai/gpt-5",
+ input="What is the capital of France?",
tools=[{
"type": "web_search_preview" # enables web search with default medium context size
}]
)
```
+
@@ -268,19 +310,24 @@ response = responses(
```yaml
model_list:
- - model_name: gpt-4o
+ - model_name: gpt-5
litellm_params:
- model: openai/gpt-4o
+ model: openai/gpt-5
+ api_key: os.environ/OPENAI_API_KEY
+
+ - model_name: gpt-4.1
+ litellm_params:
+ model: openai/gpt-4.1
api_key: os.environ/OPENAI_API_KEY
```
-2. Start the proxy
+2. Start the proxy
```bash
litellm --config /path/to/config.yaml
```
-3. Test it!
+3. Test it!
```python showLineNumbers
from openai import OpenAI
@@ -292,11 +339,11 @@ client = OpenAI(
)
response = client.responses.create(
- model="gpt-4o",
+ model="gpt-5",
tools=[{
"type": "web_search_preview"
}],
- input="What was a positive news story from today?",
+ input="What is the capital of France?",
)
print(response.output_text)
@@ -314,13 +361,8 @@ from litellm import responses
# Customize search context size
response = responses(
- model="openai/gpt-4o",
- input=[
- {
- "role": "user",
- "content": "What was a positive news story from today?"
- }
- ],
+ model="openai/gpt-5",
+ input="What is the capital of France?",
tools=[{
"type": "web_search_preview",
"search_context_size": "low" # Options: "low", "medium" (default), "high"
@@ -341,12 +383,12 @@ client = OpenAI(
# Customize search context size
response = client.responses.create(
- model="gpt-4o",
+ model="gpt-5",
tools=[{
"type": "web_search_preview",
"search_context_size": "low" # Options: "low", "medium" (default), "high"
}],
- input="What was a positive news story from today?",
+ input="What is the capital of France?",
)
print(response.output_text)
@@ -400,14 +442,14 @@ model_list:
web_search_options:
search_context_size: "high" # Options: "low", "medium", "high"
- # Different context size for different models
- - model_name: gpt-4o-search-preview
+ # OpenAI search model with custom context size
+ - model_name: gpt-5-search-api
litellm_params:
- model: openai/gpt-4o-search-preview
+ model: openai/gpt-5-search-api
api_key: os.environ/OPENAI_API_KEY
web_search_options:
search_context_size: "low"
-
+
# Gemini with medium context (default)
- model_name: gemini-2-flash
litellm_params:
@@ -432,6 +474,7 @@ Use `litellm.supports_web_search(model="model_name")` -> returns `True` if model
```python showLineNumbers
# Check OpenAI models
+assert litellm.supports_web_search(model="openai/gpt-5-search-api") == True
assert litellm.supports_web_search(model="openai/gpt-4o-search-preview") == True
# Check xAI models
@@ -455,13 +498,20 @@ assert litellm.supports_web_search(model="gemini/gemini-2.0-flash") == True
```yaml
model_list:
# OpenAI
+ - model_name: gpt-5-search-api
+ litellm_params:
+ model: openai/gpt-5-search-api
+ api_key: os.environ/OPENAI_API_KEY
+ model_info:
+ supports_web_search: True
+
- model_name: gpt-4o-search-preview
litellm_params:
model: openai/gpt-4o-search-preview
api_key: os.environ/OPENAI_API_KEY
model_info:
supports_web_search: True
-
+
# xAI
- model_name: grok-3
litellm_params:
@@ -516,6 +566,12 @@ Expected Response
```json showLineNumbers
{
"data": [
+ {
+ "model_group": "gpt-5-search-api",
+ "providers": ["openai"],
+ "max_tokens": 128000,
+ "supports_web_search": true
+ },
{
"model_group": "gpt-4o-search-preview",
"providers": ["openai"],
diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md
index 2eed0f53e59..0a1b47f0621 100644
--- a/docs/my-website/docs/enterprise.md
+++ b/docs/my-website/docs/enterprise.md
@@ -74,6 +74,18 @@ You can find [supported data regions litellm here](../docs/data_security#support
## Frequently Asked Questions
+### How to set up and verify your Enterprise License
+
+1. Add your license key to the environment:
+
+```env
+LITELLM_LICENSE="eyJ..."
+```
+
+2. Restart LiteLLM Proxy.
+
+3. Open `http://:/` — the Swagger page should show **"Enterprise Edition"** in the description. If it doesn't, check that the key is correct, unexpired, and that the proxy was fully restarted.
+
### SLA's + Professional Support
Professional Support can assist with LLM/Provider integrations, deployment, upgrade management, and LLM Provider troubleshooting. We can’t solve your own infrastructure-related issues but we will guide you to fix them.
diff --git a/docs/my-website/docs/evals_api.md b/docs/my-website/docs/evals_api.md
new file mode 100644
index 00000000000..bb66e9fdc0a
--- /dev/null
+++ b/docs/my-website/docs/evals_api.md
@@ -0,0 +1,441 @@
+# /evals
+
+LiteLLM Proxy supports OpenAI's Evaluations (Evals) API, allowing you to create, manage, and run evaluations to measure model performance against defined testing criteria.
+
+## What are Evals?
+
+OpenAI Evals API provides a structured way to:
+- **Create Evaluations**: Define testing criteria and data sources for evaluating model outputs
+- **Run Evaluations**: Execute evaluations against specific models and datasets
+- **Track Results**: Monitor evaluation progress and review detailed results
+
+## Quick Start
+
+### Setup LiteLLM Proxy
+
+First, start your LiteLLM Proxy server:
+
+```bash
+litellm --config config.yaml
+
+# Proxy will run on http://localhost:4000
+```
+
+### Initialize OpenAI Client
+
+```python
+from openai import OpenAI
+
+# Point to your LiteLLM Proxy
+client = OpenAI(
+ api_key="sk-1234", # Your LiteLLM proxy API key
+ base_url="http://localhost:4000" # Your proxy URL
+)
+```
+
+
+For async operations:
+
+```python
+from openai import AsyncOpenAI
+
+client = AsyncOpenAI(
+ api_key="sk-1234",
+ base_url="http://localhost:4000"
+)
+```
+
+---
+
+## Evaluation Management
+
+### Create an Evaluation
+
+Create an evaluation with testing criteria and data source configuration.
+
+#### Example: Sentiment Classification Eval
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+ api_key="sk-1234",
+ base_url="http://localhost:4000"
+)
+
+# Create evaluation with label model grader
+eval_obj = client.evals.create(
+ name="Sentiment Classification",
+ data_source_config={
+ "type": "stored_completions",
+ "metadata": {"usecase": "chatbot"}
+ },
+ testing_criteria=[
+ {
+ "type": "label_model",
+ "model": "gpt-4o-mini",
+ "input": [
+ {
+ "role": "developer",
+ "content": "Classify the sentiment of the following statement as one of 'positive', 'neutral', or 'negative'"
+ },
+ {
+ "role": "user",
+ "content": "Statement: {{item.input}}"
+ }
+ ],
+ "passing_labels": ["positive"],
+ "labels": ["positive", "neutral", "negative"],
+ "name": "Sentiment Grader"
+ }
+ ]
+)
+
+# Note: If you want to use model-specific credentials for this evaluation, you can specify the model name in the extra body parameters.
+
+print(f"Created eval: {eval_obj.id}")
+print(f"Eval name: {eval_obj.name}")
+```
+
+#### Example: Push Notifications Summarizer Monitoring
+
+This example shows how to monitor prompt changes for regressions in a push notifications summarizer:
+
+```python
+from openai import AsyncOpenAI
+
+client = AsyncOpenAI(
+ api_key="sk-1234",
+ base_url="http://localhost:4000"
+)
+
+# Define data source for stored completions
+data_source_config = {
+ "type": "stored_completions",
+ "metadata": {
+ "usecase": "push_notifications_summarizer"
+ }
+}
+
+# Define grader criteria
+GRADER_DEVELOPER_PROMPT = """
+Label the following push notification summary as either correct or incorrect.
+The push notification and the summary will be provided below.
+A good push notification summary is concise and snappy.
+If it is good, then label it as correct, if not, then incorrect.
+"""
+
+GRADER_TEMPLATE_PROMPT = """
+Push notifications: {{item.input}}
+Summary: {{sample.output_text}}
+"""
+
+push_notification_grader = {
+ "name": "Push Notification Summary Grader",
+ "type": "label_model",
+ "model": "gpt-4o-mini",
+ "input": [
+ {
+ "role": "developer",
+ "content": GRADER_DEVELOPER_PROMPT,
+ },
+ {
+ "role": "user",
+ "content": GRADER_TEMPLATE_PROMPT,
+ },
+ ],
+ "passing_labels": ["correct"],
+ "labels": ["correct", "incorrect"],
+}
+
+# Create the evaluation
+eval_result = await client.evals.create(
+ name="Push Notification Completion Monitoring",
+ metadata={"description": "This eval monitors completions"},
+ data_source_config=data_source_config,
+ testing_criteria=[push_notification_grader],
+)
+
+eval_id = eval_result.id
+print(f"Created eval: {eval_id}")
+```
+
+### List Evaluations
+
+Retrieve a list of all your evaluations with pagination support.
+
+```python
+# List all evaluations
+evals_response = client.evals.list(
+ limit=20,
+ order="desc"
+)
+
+for eval in evals_response.data:
+ print(f"Eval ID: {eval.id}, Name: {eval.name}")
+
+# Check if there are more evals
+if evals_response.has_more:
+ # Fetch next page
+ next_evals = client.evals.list(
+ after=evals_response.last_id,
+ limit=20
+ )
+```
+
+### Get a Specific Evaluation
+
+Retrieve details of a specific evaluation by ID.
+
+```python
+eval = client.evals.retrieve(
+ eval_id="eval_abc123"
+)
+
+print(f"Eval ID: {eval.id}")
+print(f"Name: {eval.name}")
+print(f"Data Source: {eval.data_source_config}")
+print(f"Testing Criteria: {eval.testing_criteria}")
+```
+
+### Update an Evaluation
+
+Update evaluation metadata or name.
+
+```python
+updated_eval = client.evals.update(
+ eval_id="eval_abc123",
+ name="Updated Evaluation Name",
+ metadata={
+ "version": "2.0",
+ "updated_by": "user@example.com"
+ }
+)
+
+print(f"Updated eval: {updated_eval.name}")
+```
+
+### Delete an Evaluation
+
+Permanently delete an evaluation.
+
+```python
+delete_response = client.evals.delete(
+ eval_id="eval_abc123"
+)
+
+print(f"Deleted: {delete_response.deleted}") # True
+```
+
+---
+
+## Evaluation Runs
+
+### Create a Run
+
+Execute an evaluation by creating a run. The run processes your data through the model and applies testing criteria.
+
+#### Using Stored Completions
+
+First, generate some test data by making chat completions with metadata:
+
+```python
+from openai import AsyncOpenAI
+import asyncio
+
+client = AsyncOpenAI(
+ api_key="sk-1234",
+ base_url="http://localhost:4000"
+)
+
+# Generate test data with different prompt versions
+push_notification_data = [
+ """
+- New message from Sarah: "Can you call me later?"
+- Your package has been delivered!
+- Flash sale: 20% off electronics for the next 2 hours!
+""",
+ """
+- Weather alert: Thunderstorm expected in your area.
+- Reminder: Doctor's appointment at 3 PM.
+- John liked your photo on Instagram.
+"""
+]
+
+PROMPTS = [
+ (
+ """
+ You are a helpful assistant that summarizes push notifications.
+ You are given a list of push notifications and you need to collapse them into a single one.
+ Output only the final summary, nothing else.
+ """,
+ "v1"
+ ),
+ (
+ """
+ You are a helpful assistant that summarizes push notifications.
+ You are given a list of push notifications and you need to collapse them into a single one.
+ The summary should be longer than it needs to be and include more information than is necessary.
+ Output only the final summary, nothing else.
+ """,
+ "v2"
+ )
+]
+
+# Create completions with metadata for tracking
+tasks = []
+for notifications in push_notification_data:
+ for (prompt, version) in PROMPTS:
+ tasks.append(client.chat.completions.create(
+ model="gpt-4o-mini",
+ messages=[
+ {"role": "developer", "content": prompt},
+ {"role": "user", "content": notifications},
+ ],
+ metadata={
+ "prompt_version": version,
+ "usecase": "push_notifications_summarizer"
+ }
+ ))
+
+await asyncio.gather(*tasks)
+```
+
+Now create runs to evaluate different prompt versions:
+
+```python
+# Grade prompt_version=v1
+eval_run_result = await client.evals.runs.create(
+ eval_id=eval_id,
+ name="v1-run",
+ data_source={
+ "type": "completions",
+ "source": {
+ "type": "stored_completions",
+ "metadata": {
+ "prompt_version": "v1",
+ }
+ }
+ }
+)
+
+print(f"Run ID: {eval_run_result.id}")
+print(f"Status: {eval_run_result.status}")
+print(f"Report URL: {eval_run_result.report_url}")
+
+# Grade prompt_version=v2
+eval_run_result_v2 = await client.evals.runs.create(
+ eval_id=eval_id,
+ name="v2-run",
+ data_source={
+ "type": "completions",
+ "source": {
+ "type": "stored_completions",
+ "metadata": {
+ "prompt_version": "v2",
+ }
+ }
+ }
+)
+
+print(f"Run ID: {eval_run_result_v2.id}")
+print(f"Report URL: {eval_run_result_v2.report_url}")
+```
+
+#### Using Completions with Different Models
+
+Test how different models perform on the same inputs:
+
+```python
+# Test with GPT-4o using stored completions as input
+tasks = []
+for prompt_version in ["v1", "v2"]:
+ tasks.append(client.evals.runs.create(
+ eval_id=eval_id,
+ name=f"gpt-4o-run-{prompt_version}",
+ data_source={
+ "type": "completions",
+ "input_messages": {
+ "type": "item_reference",
+ "item_reference": "item.input",
+ },
+ "model": "gpt-4o",
+ "source": {
+ "type": "stored_completions",
+ "metadata": {
+ "prompt_version": prompt_version,
+ }
+ }
+ }
+ ))
+
+results = await asyncio.gather(*tasks)
+for run in results:
+ print(f"Report URL: {run.report_url}")
+```
+
+### List Runs
+
+Get all runs for a specific evaluation.
+
+```python
+# List all runs for an evaluation
+runs_response = client.evals.runs.list(
+ eval_id="eval_abc123",
+ limit=20,
+ order="desc"
+)
+
+for run in runs_response.data:
+ print(f"Run ID: {run.id}")
+ print(f"Status: {run.status}")
+ print(f"Name: {run.name}")
+ if run.result_counts:
+ print(f"Results: {run.result_counts.passed}/{run.result_counts.total} passed")
+```
+
+### Get Run Details
+
+Retrieve detailed information about a specific run, including results.
+
+```python
+run = client.evals.runs.retrieve(
+ eval_id="eval_abc123",
+ run_id="run_def456"
+)
+
+print(f"Run ID: {run.id}")
+print(f"Status: {run.status}")
+print(f"Started: {run.started_at}")
+print(f"Completed: {run.completed_at}")
+
+# Check results
+if run.result_counts:
+ print(f"\nOverall Results:")
+ print(f"Total: {run.result_counts.total}")
+ print(f"Passed: {run.result_counts.passed}")
+ print(f"Failed: {run.result_counts.failed}")
+ print(f"Error: {run.result_counts.errored}")
+
+# Per-criteria results
+if run.per_testing_criteria_results:
+ for criteria_result in run.per_testing_criteria_results:
+ print(f"\nCriteria {criteria_result.testing_criteria_index}:")
+ print(f" Passed: {criteria_result.result_counts.passed}")
+ print(f" Average Score: {criteria_result.average_score}")
+```
+
+### Delete a Run
+
+Permanently delete a run and its results.
+
+```python
+delete_response = await client.evals.runs.delete(
+ eval_id="eval_abc123",
+ run_id="run_def456"
+)
+
+print(f"Deleted: {delete_response.deleted}") # True
+print(f"Run ID: {delete_response.run_id}")
+```
+
diff --git a/docs/my-website/docs/extras/contributing_code.md b/docs/my-website/docs/extras/contributing_code.md
index 930a47eec7e..673a83aca05 100644
--- a/docs/my-website/docs/extras/contributing_code.md
+++ b/docs/my-website/docs/extras/contributing_code.md
@@ -1,27 +1,36 @@
# Contributing Code
-## **Checklist before submitting a PR**
+## Checklist before submitting a PR
-Here are the core requirements for any PR submitted to LiteLLM
+Here are the core requirements for any PR submitted to LiteLLM:
-- [ ] Sign the Contributor License Agreement (CLA) - [see details](#contributor-license-agreement-cla)
-- [ ] Add testing, **Adding at least 1 test is a hard requirement** - [see details](#2-adding-testing-to-your-pr)
-- [ ] Ensure your PR passes the following tests:
- - [ ] [Unit Tests](#3-running-unit-tests)
- - [ ] [Formatting / Linting Tests](#35-running-linting-tests)
-- [ ] Keep scope as isolated as possible. As a general rule, your changes should address 1 specific problem at a time
+- [ ] Sign the [Contributor License Agreement (CLA)](#contributor-license-agreement-cla)
+- [ ] Keep scope as isolated as possible — your changes should address **one specific problem** at a time
-## **Contributor License Agreement (CLA)**
+### Proxy (Backend) PRs
+
+- [ ] Add testing — **at least 1 test is a hard requirement** ([details](#2-adding-tests))
+- [ ] Ensure your PR passes:
+ - [ ] [Unit Tests](#3-running-unit-tests) — `make test-unit`
+ - [ ] [Formatting / Linting Tests](#4-running-linting-tests) — `make lint`
+
+### UI PRs
+
+- [ ] Ensure the UI builds successfully — `npm run build`
+- [ ] Ensure all UI unit tests pass — `npm run test`
+- [ ] If you are adding a **new component** or **new logic**, add corresponding tests
+
+## Contributor License Agreement (CLA)
Before contributing code to LiteLLM, you must sign our [Contributor License Agreement (CLA)](https://cla-assistant.io/BerriAI/litellm). This is a legal requirement for all contributions to be merged into the main repository. The CLA helps protect both you and the project by clearly defining the terms under which your contributions are made.
-**Important:** We strongly recommend reviewing and signing the CLA before starting work on your contribution to avoid any delays in the PR process. You can find the CLA [here](https://cla-assistant.io/BerriAI/litellm) and sign it through our CLA management system when you submit your first PR.
+**Important:** We strongly recommend signing the CLA **before** starting work on your contribution to avoid delays in the review process. You can find and sign the CLA [here](https://cla-assistant.io/BerriAI/litellm).
-## Quick start
+---
-## 1. Setup your local dev environment
+## Proxy (Backend)
-Here's how to modify the repo locally:
+### 1. Setting up your local dev environment
Step 1: Clone the repo
@@ -29,56 +38,53 @@ Step 1: Clone the repo
git clone https://github.com/BerriAI/litellm.git
```
-Step 2: Install dev dependencies:
+Step 2: Install dev dependencies
```shell
poetry install --with dev --extras proxy
```
-That's it, your local dev environment is ready!
+### 2. Adding tests
-## 2. Adding Testing to your PR
+- Add your tests to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/litellm).
+- This directory mirrors the `litellm/` directory 1:1 and should **only** contain mocked tests.
+- **Do not** add real LLM API calls to this directory.
-- Add your test to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/litellm)
+#### File naming convention for `tests/test_litellm/`
-- This directory 1:1 maps the the `litellm/` directory, and can only contain mocked tests.
-- Do not add real llm api calls to this directory.
+The test directory follows the same structure as `litellm/`:
-### 2.1 File Naming Convention for `tests/test_litellm/`
-
-The `tests/test_litellm/` directory follows the same directory structure as `litellm/`.
-
-- `litellm/proxy/test_caching_routes.py` maps to `litellm/proxy/caching_routes.py`
- `test_{filename}.py` maps to `litellm/{filename}.py`
+- `litellm/proxy/test_caching_routes.py` maps to `litellm/proxy/caching_routes.py`
-## 3. Running Unit Tests
+### 3. Running unit tests
-run the following command on the root of the litellm directory
+Run the following command from the root of the `litellm` directory:
```shell
make test-unit
```
-## 3.5 Running Linting Tests
+### 4. Running linting tests
-run the following command on the root of the litellm directory
+Run the following command from the root of the `litellm` directory:
```shell
make lint
```
-LiteLLM uses mypy for linting. On ci/cd we also run `black` for formatting.
+LiteLLM uses `mypy` for type checking. CI/CD also runs `black` for formatting.
-## 4. Submit a PR with your changes!
+### 5. Submit a PR
-- push your fork to your GitHub repo
-- submit a PR from there
+- Push your changes to your fork on GitHub
+- Open a Pull Request from your fork
-## Advanced
+---
-### Building LiteLLM Docker Image
+## UI
-Some people might want to build the LiteLLM docker image themselves. Follow these instructions if you want to build / run the LiteLLM Docker Image yourself.
+### 1. Setting up your local dev environment
Step 1: Clone the repo
@@ -86,17 +92,72 @@ Step 1: Clone the repo
git clone https://github.com/BerriAI/litellm.git
```
-Step 2: Build the Docker Image
+Step 2: Navigate to the UI dashboard directory
-Build using Dockerfile.non_root
+```shell
+cd ui/litellm-dashboard
+```
+
+Step 3: Install dependencies
+
+```shell
+npm install
+```
+
+Step 4: Start the development server
+
+```shell
+npm run dev
+```
+
+### 2. Adding tests
+
+If you are adding a **new component** or **new logic**, you must add corresponding tests.
+
+### 3. Running UI unit tests
+
+```shell
+npm run test
+```
+
+### 4. Building the UI
+
+Ensure the UI builds successfully before submitting your PR:
+
+```shell
+npm run build
+```
+
+### 5. Submit a PR
+
+- Push your changes to your fork on GitHub
+- Open a Pull Request from your fork
+
+---
+
+## Advanced
+
+### Building the LiteLLM Docker Image
+
+Follow these instructions if you want to build and run the LiteLLM Docker image yourself.
+
+Step 1: Clone the repo
+
+```shell
+git clone https://github.com/BerriAI/litellm.git
+```
+
+Step 2: Build the Docker image
+
+Build using `Dockerfile.non_root`:
```shell
docker build -f docker/Dockerfile.non_root -t litellm_test_image .
```
-Step 3: Run the Docker Image
+Step 3: Run the Docker image
-Make sure config.yaml is present in the root directory. This is your litellm proxy config file.
+Make sure `config.yaml` is present in the root directory. This is your LiteLLM proxy config file.
```shell
docker run \
@@ -107,18 +168,19 @@ docker run \
litellm_test_image \
--config /app/config.yaml --detailed_debug
```
-### Running LiteLLM Proxy Locally
-1. cd into the `proxy/` directory
+### Running the LiteLLM Proxy Locally
-```
+1. Navigate to the `proxy/` directory:
+
+```shell
cd litellm/litellm/proxy
```
-2. Run the proxy
+2. Run the proxy:
```shell
python3 proxy_cli.py --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
-```
\ No newline at end of file
+```
diff --git a/docs/my-website/docs/integrations/websearch_interception.md b/docs/my-website/docs/integrations/websearch_interception.md
new file mode 100644
index 00000000000..0c5d8927013
--- /dev/null
+++ b/docs/my-website/docs/integrations/websearch_interception.md
@@ -0,0 +1,411 @@
+# Web Search Integration
+
+Enable transparent server-side web search execution for any LLM provider. LiteLLM automatically intercepts web search tool calls and executes them using your configured search provider (Perplexity, Tavily, etc.).
+
+## Quick Start
+
+### 1. Configure Web Search Interception
+
+Add to your `config.yaml`:
+
+```yaml
+model_list:
+ - model_name: gpt-4o
+ litellm_params:
+ model: openai/gpt-4o
+ api_key: os.environ/OPENAI_API_KEY
+
+litellm_settings:
+ callbacks:
+ - websearch_interception:
+ enabled_providers:
+ - openai
+ - minimax
+ - anthropic
+ search_tool_name: perplexity-search # Optional
+
+search_tools:
+ - search_tool_name: perplexity-search
+ litellm_params:
+ search_provider: perplexity
+ api_key: os.environ/PERPLEXITY_API_KEY
+```
+
+### 2. Use with Any Provider
+
+```python
+import litellm
+
+response = await litellm.acompletion(
+ model="gpt-4o",
+ messages=[
+ {"role": "user", "content": "What's the weather in San Francisco today?"}
+ ],
+ tools=[
+ {
+ "type": "function",
+ "function": {
+ "name": "litellm_web_search",
+ "description": "Search the web for information",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "query": {"type": "string", "description": "Search query"}
+ },
+ "required": ["query"]
+ }
+ }
+ }
+ ]
+)
+
+# Response includes search results automatically!
+print(response.choices[0].message.content)
+```
+
+## How It Works
+
+When a model makes a web search tool call, LiteLLM:
+
+1. **Detects** the `litellm_web_search` tool call in the response
+2. **Executes** the search using your configured search provider
+3. **Makes a follow-up request** with the search results
+4. **Returns** the final answer to the user
+
+```mermaid
+sequenceDiagram
+ participant User
+ participant LiteLLM
+ participant LLM as LLM Provider
+ participant Search as Search Provider
+
+ User->>LiteLLM: Request with web_search tool
+ LiteLLM->>LLM: Forward request
+ LLM-->>LiteLLM: Response with tool_call
+ Note over LiteLLM: Detect web search
tool call
+ LiteLLM->>Search: Execute search
+ Search-->>LiteLLM: Search results
+ LiteLLM->>LLM: Follow-up with results
+ LLM-->>LiteLLM: Final answer
+ LiteLLM-->>User: Final answer with search results
+```
+
+**Result**: One API call from user → Complete answer with search results
+
+## Supported Providers
+
+Web search integration works with **all providers** that use:
+- ✅ **Base HTTP Handler** (`BaseLLMHTTPHandler`)
+- ✅ **OpenAI Completion Handler** (`OpenAIChatCompletion`)
+
+### Providers Using Base HTTP Handler
+
+| Provider | Status | Notes |
+|----------|--------|-------|
+| **OpenAI** | ✅ Supported | GPT-4, GPT-3.5, etc. |
+| **Anthropic** | ✅ Supported | Claude models via HTTP handler |
+| **MiniMax** | ✅ Supported | All MiniMax models |
+| **Mistral** | ✅ Supported | Mistral AI models |
+| **Cohere** | ✅ Supported | Command models |
+| **Fireworks AI** | ✅ Supported | All Fireworks models |
+| **Together AI** | ✅ Supported | All Together AI models |
+| **Groq** | ✅ Supported | All Groq models |
+| **Perplexity** | ✅ Supported | Perplexity models |
+| **DeepSeek** | ✅ Supported | DeepSeek models |
+| **xAI** | ✅ Supported | Grok models |
+| **Hugging Face** | ✅ Supported | Inference API models |
+| **OCI** | ✅ Supported | Oracle Cloud models |
+| **Vertex AI** | ✅ Supported | Google Vertex AI models |
+| **Bedrock** | ✅ Supported | AWS Bedrock models (converse_like route) |
+| **Azure OpenAI** | ✅ Supported | Azure-hosted OpenAI models |
+| **Sagemaker** | ✅ Supported | AWS Sagemaker models |
+| **Databricks** | ✅ Supported | Databricks models |
+| **DataRobot** | ✅ Supported | DataRobot models |
+| **Hosted VLLM** | ✅ Supported | Self-hosted VLLM |
+| **Heroku** | ✅ Supported | Heroku-hosted models |
+| **RAGFlow** | ✅ Supported | RAGFlow models |
+| **Compactif** | ✅ Supported | Compactif models |
+| **Cometapi** | ✅ Supported | Comet API models |
+| **A2A** | ✅ Supported | Agent-to-Agent models |
+| **Bytez** | ✅ Supported | Bytez models |
+
+### Providers Using OpenAI Handler
+
+| Provider | Status | Notes |
+|----------|--------|-------|
+| **OpenAI** | ✅ Supported | Native OpenAI API |
+| **Azure OpenAI** | ✅ Supported | Azure-hosted OpenAI |
+| **OpenAI-Compatible** | ✅ Supported | Any OpenAI-compatible API |
+
+## Configuration
+
+### WebSearch Interception Parameters
+
+| Parameter | Type | Required | Description | Example |
+|-----------|------|----------|-------------|---------|
+| `enabled_providers` | List[String] | Yes | List of providers to enable web search for | `[openai, minimax, anthropic]` |
+| `search_tool_name` | String | No | Specific search tool from `search_tools` config. If not set, uses first available. | `perplexity-search` |
+
+### Provider Values
+
+Use these values in `enabled_providers`:
+
+| Provider | Value | Provider | Value |
+|----------|-------|----------|-------|
+| OpenAI | `openai` | Anthropic | `anthropic` |
+| MiniMax | `minimax` | Mistral | `mistral` |
+| Cohere | `cohere` | Fireworks AI | `fireworks_ai` |
+| Together AI | `together_ai` | Groq | `groq` |
+| Perplexity | `perplexity` | DeepSeek | `deepseek` |
+| xAI | `xai` | Hugging Face | `huggingface` |
+| OCI | `oci` | Vertex AI | `vertex_ai` |
+| Bedrock | `bedrock` | Azure | `azure` |
+| Sagemaker | `sagemaker_chat` | Databricks | `databricks` |
+| DataRobot | `datarobot` | VLLM | `hosted_vllm` |
+| Heroku | `heroku` | RAGFlow | `ragflow` |
+| Compactif | `compactif` | Cometapi | `cometapi` |
+| A2A | `a2a` | Bytez | `bytez` |
+
+## Search Providers
+
+Configure which search provider to use. LiteLLM supports multiple search providers:
+
+| Provider | `search_provider` Value | Environment Variable |
+|----------|------------------------|----------------------|
+| **Perplexity AI** | `perplexity` | `PERPLEXITYAI_API_KEY` |
+| **Tavily** | `tavily` | `TAVILY_API_KEY` |
+| **Exa AI** | `exa_ai` | `EXA_API_KEY` |
+| **Parallel AI** | `parallel_ai` | `PARALLEL_AI_API_KEY` |
+| **Google PSE** | `google_pse` | `GOOGLE_PSE_API_KEY`, `GOOGLE_PSE_ENGINE_ID` |
+| **DataForSEO** | `dataforseo` | `DATAFORSEO_LOGIN`, `DATAFORSEO_PASSWORD` |
+| **Firecrawl** | `firecrawl` | `FIRECRAWL_API_KEY` |
+| **SearXNG** | `searxng` | `SEARXNG_API_BASE` (required) |
+| **Linkup** | `linkup` | `LINKUP_API_KEY` |
+
+See [Search Providers Documentation](../search/index.md) for detailed setup instructions.
+
+## Complete Configuration Example
+
+```yaml
+model_list:
+ # OpenAI
+ - model_name: gpt-4o
+ litellm_params:
+ model: openai/gpt-4o
+ api_key: os.environ/OPENAI_API_KEY
+
+ # MiniMax
+ - model_name: minimax
+ litellm_params:
+ model: minimax/MiniMax-M2.1
+ api_key: os.environ/MINIMAX_API_KEY
+
+ # Anthropic
+ - model_name: claude
+ litellm_params:
+ model: anthropic/claude-sonnet-4-5
+ api_key: os.environ/ANTHROPIC_API_KEY
+
+ # Azure OpenAI
+ - model_name: azure-gpt4
+ litellm_params:
+ model: azure/gpt-4
+ api_base: https://my-azure.openai.azure.com
+ api_key: os.environ/AZURE_API_KEY
+
+litellm_settings:
+ callbacks:
+ - websearch_interception:
+ enabled_providers:
+ - openai
+ - minimax
+ - anthropic
+ - azure
+ search_tool_name: perplexity-search
+
+search_tools:
+ - search_tool_name: perplexity-search
+ litellm_params:
+ search_provider: perplexity
+ api_key: os.environ/PERPLEXITY_API_KEY
+
+ - search_tool_name: tavily-search
+ litellm_params:
+ search_provider: tavily
+ api_key: os.environ/TAVILY_API_KEY
+```
+
+## Usage Examples
+
+### Python SDK
+
+```python
+import litellm
+
+# Configure callbacks
+litellm.callbacks = ["websearch_interception"]
+
+# Make completion with web search tool
+response = await litellm.acompletion(
+ model="gpt-4o",
+ messages=[
+ {"role": "user", "content": "What are the latest AI news?"}
+ ],
+ tools=[
+ {
+ "type": "function",
+ "function": {
+ "name": "litellm_web_search",
+ "description": "Search the web for current information",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "query": {
+ "type": "string",
+ "description": "Search query"
+ }
+ },
+ "required": ["query"]
+ }
+ }
+ }
+ ]
+)
+
+print(response.choices[0].message.content)
+```
+
+### Proxy Server
+
+```bash
+# Start proxy with config
+litellm --config config.yaml
+
+# Make request
+curl http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer sk-1234" \
+ -d '{
+ "model": "gpt-4o",
+ "messages": [
+ {"role": "user", "content": "What is the weather in San Francisco?"}
+ ],
+ "tools": [
+ {
+ "type": "function",
+ "function": {
+ "name": "litellm_web_search",
+ "description": "Search the web",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "query": {"type": "string"}
+ },
+ "required": ["query"]
+ }
+ }
+ }
+ ]
+ }'
+```
+
+## How Search Tool Selection Works
+
+1. **If `search_tool_name` is specified** → Uses that specific search tool
+2. **If `search_tool_name` is not specified** → Uses first search tool in `search_tools` list
+
+```yaml
+search_tools:
+ - search_tool_name: perplexity-search # ← This will be used if no search_tool_name specified
+ litellm_params:
+ search_provider: perplexity
+ api_key: os.environ/PERPLEXITY_API_KEY
+
+ - search_tool_name: tavily-search
+ litellm_params:
+ search_provider: tavily
+ api_key: os.environ/TAVILY_API_KEY
+```
+
+## Troubleshooting
+
+### Web Search Not Working
+
+1. **Check provider is enabled**:
+ ```yaml
+ enabled_providers:
+ - openai # Make sure your provider is in this list
+ ```
+
+2. **Verify search tool is configured**:
+ ```yaml
+ search_tools:
+ - search_tool_name: perplexity-search
+ litellm_params:
+ search_provider: perplexity
+ api_key: os.environ/PERPLEXITY_API_KEY
+ ```
+
+3. **Check API keys are set**:
+ ```bash
+ export PERPLEXITY_API_KEY=your-key
+ ```
+
+4. **Enable debug logging**:
+ ```python
+ litellm.set_verbose = True
+ ```
+
+### Common Issues
+
+**Issue**: Model returns tool_calls instead of final answer
+- **Cause**: Provider not in `enabled_providers` list
+- **Solution**: Add provider to `enabled_providers`
+
+**Issue**: "No search tool configured" error
+- **Cause**: No search tools in `search_tools` config
+- **Solution**: Add at least one search tool configuration
+
+**Issue**: "Invalid function arguments json string" error (MiniMax)
+- **Cause**: Fixed in latest version - arguments weren't properly JSON serialized
+- **Solution**: Update to latest LiteLLM version
+
+## Related Documentation
+
+- [Search Providers](../search/index.md) - Detailed search provider setup
+- [Claude Code WebSearch](../tutorials/claude_code_websearch.md) - Using with Claude Code
+- [Tool Calling](../completion/function_call.md) - General tool calling documentation
+- [Callbacks](./custom_callback.md) - Custom callback documentation
+
+## Technical Details
+
+### Architecture
+
+Web search integration is implemented as a custom callback (`WebSearchInterceptionLogger`) that:
+
+1. **Pre-request Hook**: Converts native web search tools to LiteLLM standard format
+2. **Post-response Hook**: Detects web search tool calls in responses
+3. **Agentic Loop**: Executes searches and makes follow-up requests automatically
+
+### Supported APIs
+
+- ✅ **Chat Completions API** (OpenAI format)
+- ✅ **Anthropic Messages API** (Anthropic format)
+- ✅ **Streaming** (automatically converted)
+- ✅ **Non-streaming**
+
+### Response Format Detection
+
+The handler automatically detects response format:
+- **OpenAI format**: `tool_calls` in assistant message
+- **Anthropic format**: `tool_use` blocks in content
+
+### Performance
+
+- **Latency**: Adds one additional LLM call (follow-up request with search results)
+- **Caching**: Search results can be cached (depends on search provider)
+- **Parallel Searches**: Multiple search queries executed in parallel
+
+## Contributing
+
+Found a bug or want to add support for a new provider? See our [Contributing Guide](https://github.com/BerriAI/litellm/blob/main/CONTRIBUTING.md).
diff --git a/docs/my-website/docs/load_test.md b/docs/my-website/docs/load_test.md
index 4641a70366c..071b097904b 100644
--- a/docs/my-website/docs/load_test.md
+++ b/docs/my-website/docs/load_test.md
@@ -4,8 +4,9 @@ import Image from '@theme/IdealImage';
## Locust Load Test LiteLLM Proxy
-1. Add `fake-openai-endpoint` to your proxy config.yaml and start your litellm proxy
-litellm provides a free hosted `fake-openai-endpoint` you can load test against
+1. Add `fake-openai-endpoint` to your proxy config.yaml and start your litellm proxy.
+
+LiteLLM provides a free hosted `fake-openai-endpoint` you can load test against. You can also self-host your own fake OpenAI proxy server using [github.com/BerriAI/example_openai_endpoint](https://github.com/BerriAI/example_openai_endpoint).
```yaml
model_list:
diff --git a/docs/my-website/docs/load_test_advanced.md b/docs/my-website/docs/load_test_advanced.md
index 3171bc33594..d35b5f74784 100644
--- a/docs/my-website/docs/load_test_advanced.md
+++ b/docs/my-website/docs/load_test_advanced.md
@@ -29,12 +29,16 @@ Tutorial on how to get to 1K+ RPS with LiteLLM Proxy on locust
**Note:** we're currently migrating to aiohttp which has 10x higher throughput. We recommend using the `openai/` provider for load testing.
+:::tip Setting Up a Fake OpenAI Endpoint
+You can use our hosted fake endpoint or self-host your own using [github.com/BerriAI/example_openai_endpoint](https://github.com/BerriAI/example_openai_endpoint).
+:::
+
```yaml
model_list:
- model_name: "fake-openai-endpoint"
litellm_params:
model: openai/any
- api_base: https://your-fake-openai-endpoint.com/chat/completions
+ api_base: https://exampleopenaiendpoint-production.up.railway.app/ # or your self-hosted endpoint
api_key: "test"
```
diff --git a/docs/my-website/docs/mcp.md b/docs/my-website/docs/mcp.md
index d63b55ee29e..84d10c25931 100644
--- a/docs/my-website/docs/mcp.md
+++ b/docs/my-website/docs/mcp.md
@@ -506,7 +506,14 @@ Your OpenAPI specification should follow standard OpenAPI/Swagger conventions:
- **Operation IDs**: Each operation should have a unique `operationId` (this becomes the tool name)
- **Parameters**: Request parameters should be properly documented with types and descriptions
-## MCP Oauth
+## MCP OAuth
+
+LiteLLM supports OAuth 2.0 for MCP servers -- both interactive (PKCE) flows for user-facing clients and machine-to-machine (M2M) `client_credentials` for backend services.
+
+See the **[MCP OAuth guide](./mcp_oauth.md)** for setup instructions, sequence diagrams, and a test server.
+
+
+Detailed OAuth reference (click to expand)
LiteLLM v 1.77.6 added support for OAuth 2.0 Client Credentials for MCP servers.
@@ -588,6 +595,8 @@ sequenceDiagram
See the official [MCP Authorization Flow](https://modelcontextprotocol.io/specification/2025-06-18/basic/authorization#authorization-flow-steps) for additional reference.
+
+
## Forwarding Custom Headers to MCP Servers
@@ -1486,7 +1495,7 @@ async with stdio_client(server_params) as (read, write):
**Q: How do I use OAuth2 client_credentials (machine-to-machine) with MCP servers behind LiteLLM?**
-At the moment LiteLLM only forwards whatever `Authorization` header/value you configure for the MCP server; it does not issue OAuth2 tokens by itself. If your MCP requires the Client Credentials grant, obtain the access token directly from the authorization server and set that bearer token as the MCP server’s Authorization header value. LiteLLM does not yet fetch or refresh those machine-to-machine tokens on your behalf, but we plan to add first-class client_credentials support in a future release so the proxy can manage those tokens automatically.
+LiteLLM supports automatic token management for the `client_credentials` grant. Configure `client_id`, `client_secret`, and `token_url` on your MCP server and LiteLLM will fetch, cache, and refresh tokens automatically. See the [MCP OAuth M2M guide](./mcp_oauth.md#machine-to-machine-m2m-auth) for setup instructions.
**Q: When I fetch an OAuth token from the LiteLLM UI, where is it stored?**
diff --git a/docs/my-website/docs/mcp_oauth.md b/docs/my-website/docs/mcp_oauth.md
new file mode 100644
index 00000000000..5c4b70cc5b3
--- /dev/null
+++ b/docs/my-website/docs/mcp_oauth.md
@@ -0,0 +1,337 @@
+# MCP OAuth
+
+LiteLLM supports two OAuth 2.0 flows for MCP servers:
+
+| Flow | Use Case | How It Works |
+|------|----------|--------------|
+| **Interactive (PKCE)** | User-facing apps (Claude Code, Cursor) | Browser-based consent, per-user tokens |
+| **Machine-to-Machine (M2M)** | Backend services, CI/CD, automated agents | `client_credentials` grant, proxy-managed tokens |
+
+## Interactive OAuth (PKCE)
+
+For user-facing MCP clients (Claude Code, Cursor), LiteLLM supports the full OAuth 2.0 authorization code flow with PKCE.
+
+### Setup
+
+```yaml title="config.yaml" showLineNumbers
+mcp_servers:
+ github_mcp:
+ url: "https://api.githubcopilot.com/mcp"
+ auth_type: oauth2
+ client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
+ client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
+```
+
+[**See Claude Code Tutorial**](./tutorials/claude_responses_api#connecting-mcp-servers)
+
+### How It Works
+
+```mermaid
+sequenceDiagram
+ participant Browser as User-Agent (Browser)
+ participant Client as Client
+ participant LiteLLM as LiteLLM Proxy
+ participant MCP as MCP Server (Resource Server)
+ participant Auth as Authorization Server
+
+ Note over Client,LiteLLM: Step 1 – Resource discovery
+ Client->>LiteLLM: GET /.well-known/oauth-protected-resource/{mcp_server_name}/mcp
+ LiteLLM->>Client: Return resource metadata
+
+ Note over Client,LiteLLM: Step 2 – Authorization server discovery
+ Client->>LiteLLM: GET /.well-known/oauth-authorization-server/{mcp_server_name}
+ LiteLLM->>Client: Return authorization server metadata
+
+ Note over Client,Auth: Step 3 – Dynamic client registration
+ Client->>LiteLLM: POST /{mcp_server_name}/register
+ LiteLLM->>Auth: Forward registration request
+ Auth->>LiteLLM: Issue client credentials
+ LiteLLM->>Client: Return client credentials
+
+ Note over Client,Browser: Step 4 – User authorization (PKCE)
+ Client->>Browser: Open authorization URL + code_challenge + resource
+ Browser->>Auth: Authorization request
+ Note over Auth: User authorizes
+ Auth->>Browser: Redirect with authorization code
+ Browser->>LiteLLM: Callback to LiteLLM with code
+ LiteLLM->>Browser: Redirect back with authorization code
+ Browser->>Client: Callback with authorization code
+
+ Note over Client,Auth: Step 5 – Token exchange
+ Client->>LiteLLM: Token request + code_verifier + resource
+ LiteLLM->>Auth: Forward token request
+ Auth->>LiteLLM: Access (and refresh) token
+ LiteLLM->>Client: Return tokens
+
+ Note over Client,MCP: Step 6 – Authenticated MCP call
+ Client->>LiteLLM: MCP request with access token + LiteLLM API key
+ LiteLLM->>MCP: MCP request with Bearer token
+ MCP-->>LiteLLM: MCP response
+ LiteLLM-->>Client: Return MCP response
+```
+
+**Participants**
+
+- **Client** -- The MCP-capable AI agent (e.g., Claude Code, Cursor, or another IDE/agent) that initiates OAuth discovery, authorization, and tool invocations on behalf of the user.
+- **LiteLLM Proxy** -- Mediates all OAuth discovery, registration, token exchange, and MCP traffic while protecting stored credentials.
+- **Authorization Server** -- Issues OAuth 2.0 tokens via dynamic client registration, PKCE authorization, and token endpoints.
+- **MCP Server (Resource Server)** -- The protected MCP endpoint that receives LiteLLM's authenticated JSON-RPC requests.
+- **User-Agent (Browser)** -- Temporarily involved so the end user can grant consent during the authorization step.
+
+**Flow Steps**
+
+1. **Resource Discovery**: The client fetches MCP resource metadata from LiteLLM's `.well-known/oauth-protected-resource` endpoint to understand scopes and capabilities.
+2. **Authorization Server Discovery**: The client retrieves the OAuth server metadata (token endpoint, authorization endpoint, supported PKCE methods) through LiteLLM's `.well-known/oauth-authorization-server` endpoint.
+3. **Dynamic Client Registration**: The client registers through LiteLLM, which forwards the request to the authorization server (RFC 7591). If the provider doesn't support dynamic registration, you can pre-store `client_id`/`client_secret` in LiteLLM (e.g., GitHub MCP) and the flow proceeds the same way.
+4. **User Authorization**: The client launches a browser session (with code challenge and resource hints). The user approves access, the authorization server sends the code through LiteLLM back to the client.
+5. **Token Exchange**: The client calls LiteLLM with the authorization code, code verifier, and resource. LiteLLM exchanges them with the authorization server and returns the issued access/refresh tokens.
+6. **MCP Invocation**: With a valid token, the client sends the MCP JSON-RPC request (plus LiteLLM API key) to LiteLLM, which forwards it to the MCP server and relays the tool response.
+
+See the official [MCP Authorization Flow](https://modelcontextprotocol.io/specification/2025-06-18/basic/authorization#authorization-flow-steps) for additional reference.
+
+## Machine-to-Machine (M2M) Auth
+
+LiteLLM automatically fetches, caches, and refreshes OAuth2 tokens using the `client_credentials` grant. No manual token management required.
+
+### Setup
+
+You can configure M2M OAuth via the LiteLLM UI or `config.yaml`.
+
+### UI Setup
+
+Navigate to the **MCP Servers** page and click **+ Add New MCP Server**.
+
+
+
+Enter a name for your server and select **HTTP** as the transport type.
+
+
+
+Paste the MCP server URL.
+
+
+
+Under **Authentication**, select **OAuth**.
+
+
+
+
+
+Choose **Machine-to-Machine (M2M)** as the OAuth flow type. This is for server-to-server authentication using the `client_credentials` grant — no browser interaction required.
+
+
+
+
+
+Fill in the **Client ID** and **Client Secret** provided by your OAuth provider.
+
+
+
+Enter the **Token URL** — this is the endpoint LiteLLM will call to fetch access tokens using `client_credentials`.
+
+
+
+
+
+Scroll down and review the server URL and all fields, then click **Create MCP Server**.
+
+
+
+
+
+
+
+Once created, open the server and navigate to the **MCP Tools** tab to verify that LiteLLM can connect and list available tools.
+
+
+
+
+
+Select a tool (e.g. **echo**) to test it. Fill in the required parameters and click **Call Tool**.
+
+
+
+
+
+
+
+LiteLLM automatically fetches an OAuth token behind the scenes and calls the tool. The result confirms the M2M OAuth flow is working end-to-end.
+
+
+
+### Config.yaml Setup
+
+```yaml title="config.yaml" showLineNumbers
+mcp_servers:
+ my_mcp_server:
+ url: "https://my-mcp-server.com/mcp"
+ auth_type: oauth2
+ client_id: os.environ/MCP_CLIENT_ID
+ client_secret: os.environ/MCP_CLIENT_SECRET
+ token_url: "https://auth.example.com/oauth/token"
+ scopes: ["mcp:read", "mcp:write"] # optional
+```
+
+### How It Works
+
+1. On first MCP request, LiteLLM POSTs to `token_url` with `grant_type=client_credentials`
+2. The access token is cached in-memory with TTL = `expires_in - 60s`
+3. Subsequent requests reuse the cached token
+4. When the token expires, LiteLLM fetches a new one automatically
+
+```mermaid
+sequenceDiagram
+ participant Client as Client
+ participant LiteLLM as LiteLLM Proxy
+ participant Auth as Authorization Server
+ participant MCP as MCP Server
+
+ Client->>LiteLLM: MCP request + LiteLLM API key
+ LiteLLM->>Auth: POST /oauth/token (client_credentials)
+ Auth->>LiteLLM: access_token (expires_in: 3600)
+ LiteLLM->>MCP: MCP request + Bearer token
+ MCP-->>LiteLLM: MCP response
+ LiteLLM-->>Client: MCP response
+
+ Note over LiteLLM: Token cached for subsequent requests
+ Client->>LiteLLM: Next MCP request
+ LiteLLM->>MCP: MCP request + cached Bearer token
+ MCP-->>LiteLLM: MCP response
+ LiteLLM-->>Client: MCP response
+```
+
+### Test with Mock Server
+
+Use [BerriAI/mock-oauth2-mcp-server](https://github.com/BerriAI/mock-oauth2-mcp-server) to test locally:
+
+```bash title="Terminal 1 - Start mock server" showLineNumbers
+pip install fastapi uvicorn
+python mock_oauth2_mcp_server.py # starts on :8765
+```
+
+```yaml title="config.yaml" showLineNumbers
+mcp_servers:
+ test_oauth2:
+ url: "http://localhost:8765/mcp"
+ auth_type: oauth2
+ client_id: "test-client"
+ client_secret: "test-secret"
+ token_url: "http://localhost:8765/oauth/token"
+```
+
+```bash title="Terminal 2 - Start proxy and test" showLineNumbers
+litellm --config config.yaml --port 4000
+
+# List tools
+curl http://localhost:4000/mcp-rest/tools/list \
+ -H "Authorization: Bearer sk-1234"
+
+# Call a tool
+curl http://localhost:4000/mcp-rest/tools/call \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer sk-1234" \
+ -d '{"name": "echo", "arguments": {"message": "hello"}}'
+```
+
+### Config Reference
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| `auth_type` | Yes | Must be `oauth2` |
+| `client_id` | Yes | OAuth2 client ID. Supports `os.environ/VAR_NAME` |
+| `client_secret` | Yes | OAuth2 client secret. Supports `os.environ/VAR_NAME` |
+| `token_url` | Yes | Token endpoint URL |
+| `scopes` | No | List of scopes to request |
+
+## Debugging OAuth
+
+When the LiteLLM proxy is hosted remotely and you cannot access server logs, enable **debug headers** to get masked authentication diagnostics in the HTTP response.
+
+### Enable Debug Mode
+
+Add the `x-litellm-mcp-debug: true` header to your MCP client request.
+
+**Claude Code:**
+
+```bash
+claude mcp add --transport http litellm_proxy http://proxy.example.com/atlassian_mcp/mcp \
+ --header "x-litellm-api-key: Bearer sk-..." \
+ --header "x-litellm-mcp-debug: true"
+```
+
+**curl:**
+
+```bash
+curl -X POST http://localhost:4000/atlassian_mcp/mcp \
+ -H "Content-Type: application/json" \
+ -H "x-litellm-api-key: Bearer sk-..." \
+ -H "x-litellm-mcp-debug: true" \
+ -d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}'
+```
+
+### Reading the Debug Response Headers
+
+The response includes these headers (all sensitive values are masked):
+
+| Header | Description |
+|--------|-------------|
+| `x-mcp-debug-inbound-auth` | Which inbound auth headers were present. |
+| `x-mcp-debug-oauth2-token` | The OAuth2 token (masked). Shows `SAME_AS_LITELLM_KEY` if the LiteLLM key is leaking. |
+| `x-mcp-debug-auth-resolution` | Which auth method was used: `oauth2-passthrough`, `m2m-client-credentials`, `per-request-header`, `static-token`, or `no-auth`. |
+| `x-mcp-debug-outbound-url` | The upstream MCP server URL. |
+| `x-mcp-debug-server-auth-type` | The `auth_type` configured on the server. |
+
+**Example — healthy OAuth2 passthrough:**
+
+```
+x-mcp-debug-inbound-auth: x-litellm-api-key=Bearer****1234; authorization=Bearer****ef01
+x-mcp-debug-oauth2-token: Bearer****ef01
+x-mcp-debug-auth-resolution: oauth2-passthrough
+x-mcp-debug-outbound-url: https://mcp.atlassian.com/v1/mcp
+x-mcp-debug-server-auth-type: oauth2
+```
+
+**Example — LiteLLM key leaking (misconfigured):**
+
+```
+x-mcp-debug-inbound-auth: authorization=Bearer****1234
+x-mcp-debug-oauth2-token: Bearer****1234 (SAME_AS_LITELLM_KEY - likely misconfigured)
+x-mcp-debug-auth-resolution: oauth2-passthrough
+x-mcp-debug-outbound-url: https://mcp.atlassian.com/v1/mcp
+x-mcp-debug-server-auth-type: oauth2
+```
+
+### Common Issues
+
+#### LiteLLM API key leaking to the MCP server
+
+**Symptom:** `x-mcp-debug-oauth2-token` shows `SAME_AS_LITELLM_KEY`.
+
+The `Authorization` header carries the LiteLLM API key instead of an OAuth2 token. The OAuth2 flow never ran because the client already had an `Authorization` header set.
+
+**Fix:** Move the LiteLLM key to `x-litellm-api-key`:
+
+```bash
+# WRONG — blocks OAuth2 discovery
+claude mcp add --transport http my_server http://proxy/mcp/server \
+ --header "Authorization: Bearer sk-..."
+
+# CORRECT — LiteLLM key in dedicated header, Authorization free for OAuth2
+claude mcp add --transport http my_server http://proxy/mcp/server \
+ --header "x-litellm-api-key: Bearer sk-..."
+```
+
+#### No OAuth2 token present
+
+**Symptom:** `x-mcp-debug-oauth2-token` shows `(none)` and `x-mcp-debug-auth-resolution` shows `no-auth`.
+
+Check that:
+1. The `Authorization` header is NOT set as a static header in the client config.
+2. The MCP server in LiteLLM config has `auth_type: oauth2`.
+3. The `.well-known/oauth-protected-resource` endpoint returns valid metadata.
+
+#### M2M token used instead of user token
+
+**Symptom:** `x-mcp-debug-auth-resolution` shows `m2m-client-credentials`.
+
+The server has `client_id`/`client_secret`/`token_url` configured so LiteLLM is fetching a machine-to-machine token instead of using the per-user OAuth2 token. To use per-user tokens, remove the client credentials from the server config.
diff --git a/docs/my-website/docs/mcp_public_internet.md b/docs/my-website/docs/mcp_public_internet.md
new file mode 100644
index 00000000000..69dd7464657
--- /dev/null
+++ b/docs/my-website/docs/mcp_public_internet.md
@@ -0,0 +1,251 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Exposing MCPs on the Public Internet
+
+Control which MCP servers are visible to external callers (e.g., ChatGPT, Claude Desktop) vs. internal-only callers. This is useful when you want a subset of your MCP servers available publicly while keeping sensitive servers restricted to your private network.
+
+## Overview
+
+| Property | Details |
+|-------|-------|
+| Description | IP-based access control for MCP servers — external callers only see servers marked as public |
+| Setting | `available_on_public_internet` on each MCP server |
+| Network Config | `mcp_internal_ip_ranges` in `general_settings` |
+| Supported Clients | ChatGPT, Claude Desktop, Cursor, OpenAI API, or any MCP client |
+
+## How It Works
+
+When a request arrives at LiteLLM's MCP endpoints, LiteLLM checks the caller's IP address to determine whether they are an **internal** or **external** caller:
+
+1. **Extract the client IP** from the incoming request (supports `X-Forwarded-For` when configured behind a reverse proxy).
+2. **Classify the IP** as internal or external by checking it against the configured private IP ranges (defaults to RFC 1918: `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `127.0.0.0/8`).
+3. **Filter the server list**:
+ - **Internal callers** see all MCP servers (public and private).
+ - **External callers** only see servers with `available_on_public_internet: true`.
+
+This filtering is applied at every MCP access point: the MCP registry, tool listing, tool calling, dynamic server routes, and OAuth discovery endpoints.
+
+```mermaid
+flowchart TD
+ A[Incoming MCP Request] --> B[Extract Client IP Address]
+ B --> C{Is IP in private ranges?}
+ C -->|Yes - Internal caller| D[Return ALL MCP servers]
+ C -->|No - External caller| E[Return ONLY servers with
available_on_public_internet = true]
+```
+
+## Walkthrough
+
+This walkthrough covers two flows:
+1. **Adding a public MCP server** (DeepWiki) and connecting to it from ChatGPT
+2. **Making an existing server private** (Exa) and verifying ChatGPT no longer sees it
+
+### Flow 1: Add a Public MCP Server (DeepWiki)
+
+DeepWiki is a free MCP server — a good candidate to expose publicly so AI gateway users can access it from ChatGPT.
+
+#### Step 1: Create the MCP Server
+
+Navigate to the MCP Servers page and click **"+ Add New MCP Server"**.
+
+
+
+The create dialog opens. Enter **"DeepWiki"** as the server name.
+
+
+
+For the transport type dropdown, select **HTTP** since DeepWiki uses the Streamable HTTP transport.
+
+
+
+Now scroll down to the MCP Server URL field.
+
+
+
+Enter the DeepWiki MCP URL: `https://mcp.deepwiki.com/mcp`.
+
+
+
+With the name, transport, and URL filled in, the basic server configuration is complete.
+
+
+
+#### Step 2: Enable "Available on Public Internet"
+
+Before creating, scroll down and expand the **Permission Management / Access Control** section. This is where you control who can see this server.
+
+
+
+Toggle **"Available on Public Internet"** on. This is the key setting — it tells LiteLLM that external callers (like ChatGPT connecting from the public internet) should be able to discover and use this server.
+
+
+
+With the toggle enabled, click **"Create"** to save the server.
+
+
+
+#### Step 3: Connect from ChatGPT
+
+Now let's verify it works. Open ChatGPT and look for the MCP server icon to add a new connection. The endpoint to use is `/mcp`.
+
+
+
+In the dropdown, select **"Add an MCP server"** to configure a new connection.
+
+
+
+ChatGPT asks for a server label. Give it a recognizable name like "LiteLLM".
+
+
+
+Next, enter the Server URL. This should be your LiteLLM proxy's MCP endpoint — `/mcp`.
+
+
+
+Paste your LiteLLM URL and confirm it looks correct.
+
+
+
+ChatGPT also needs authentication. Enter your LiteLLM API key in the authentication field so it can connect to the proxy.
+
+
+
+Click **"Connect"** to establish the connection.
+
+
+
+ChatGPT connects and shows the available tools. Since both DeepWiki and Exa are currently marked as public, ChatGPT can see tools from both servers.
+
+
+
+---
+
+### Flow 2: Make an Existing Server Private (Exa)
+
+Now let's do the reverse — take an existing MCP server (Exa) that's currently public and restrict it to internal access only. After this change, ChatGPT should no longer see Exa's tools.
+
+#### Step 1: Edit the Server
+
+Go to the MCP Servers table and click on the Exa server to open its detail view.
+
+
+
+Switch to the **"Settings"** tab to access the edit form.
+
+
+
+The edit form loads with Exa's current configuration.
+
+
+
+#### Step 2: Toggle Off "Available on Public Internet"
+
+Scroll down and expand the **Permission Management / Access Control** section to find the public internet toggle.
+
+
+
+Toggle **"Available on Public Internet"** off. This will hide Exa from any caller outside your private network.
+
+
+
+Click **"Save Changes"** to apply. The change takes effect immediately — no proxy restart needed.
+
+
+
+#### Step 3: Verify in ChatGPT
+
+Go back to ChatGPT to confirm Exa is no longer visible. You'll need to reconnect for ChatGPT to re-fetch the tool list.
+
+
+
+Open the MCP server settings and select to add or reconnect a server.
+
+
+
+Enter the same LiteLLM MCP URL as before.
+
+
+
+Set the server label.
+
+
+
+Enter your API key for authentication.
+
+
+
+Click **"Connect"** to re-establish the connection.
+
+
+
+This time, only DeepWiki's tools appear — Exa is gone. LiteLLM detected that ChatGPT is calling from a public IP and filtered out Exa since it's no longer marked as public. Internal users on your private network would still see both servers.
+
+
+
+## Configuration Reference
+
+### Per-Server Setting
+
+
+
+
+Toggle **"Available on Public Internet"** in the Permission Management section when creating or editing an MCP server.
+
+
+
+
+```yaml title="config.yaml" showLineNumbers
+mcp_servers:
+ deepwiki:
+ url: https://mcp.deepwiki.com/mcp
+ available_on_public_internet: true # visible to external callers
+
+ exa:
+ url: https://exa.ai/mcp
+ auth_type: api_key
+ auth_value: os.environ/EXA_API_KEY
+ available_on_public_internet: false # internal only (default)
+```
+
+
+
+
+```bash title="Create a public MCP server" showLineNumbers
+curl -X POST /v1/mcp/server \
+ -H "Authorization: Bearer sk-..." \
+ -H "Content-Type: application/json" \
+ -d '{
+ "server_name": "DeepWiki",
+ "url": "https://mcp.deepwiki.com/mcp",
+ "transport": "http",
+ "available_on_public_internet": true
+ }'
+```
+
+```bash title="Update an existing server" showLineNumbers
+curl -X PUT /v1/mcp/server \
+ -H "Authorization: Bearer sk-..." \
+ -H "Content-Type: application/json" \
+ -d '{
+ "server_id": "",
+ "available_on_public_internet": false
+ }'
+```
+
+
+
+
+### Custom Private IP Ranges
+
+By default, LiteLLM treats RFC 1918 private ranges as internal. You can customize this in the **Network Settings** tab under MCP Servers, or via config:
+
+```yaml title="config.yaml" showLineNumbers
+general_settings:
+ mcp_internal_ip_ranges:
+ - "10.0.0.0/8"
+ - "172.16.0.0/12"
+ - "192.168.0.0/16"
+ - "100.64.0.0/10" # Add your VPN/Tailscale range
+```
+
+When empty, the standard private ranges are used (`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `127.0.0.0/8`).
diff --git a/docs/my-website/docs/mcp_semantic_filter.md b/docs/my-website/docs/mcp_semantic_filter.md
new file mode 100644
index 00000000000..c58be80a680
--- /dev/null
+++ b/docs/my-website/docs/mcp_semantic_filter.md
@@ -0,0 +1,158 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# MCP Semantic Tool Filter
+
+Automatically filter MCP tools by semantic relevance. When you have many MCP tools registered, LiteLLM semantically matches the user's query against tool descriptions and sends only the most relevant tools to the LLM.
+
+## How It Works
+
+Tool search shifts tool selection from a prompt-engineering problem to a retrieval problem. Instead of injecting a large static list of tools into every prompt, the semantic filter:
+
+1. Builds a semantic index of all available MCP tools on startup
+2. On each request, semantically matches the user's query against tool descriptions
+3. Returns only the top-K most relevant tools to the LLM
+
+This approach improves context efficiency, increases reliability by reducing tool confusion, and enables scalability to ecosystems with hundreds or thousands of MCP tools.
+
+```mermaid
+sequenceDiagram
+ participant Client
+ participant LiteLLM as LiteLLM Proxy
+ participant SemanticFilter as Semantic Filter
+ participant MCP as MCP Registry
+ participant LLM as LLM Provider
+
+ Note over LiteLLM,MCP: Startup: Build Semantic Index
+ LiteLLM->>MCP: Fetch all registered MCP tools
+ MCP->>LiteLLM: Return all tools (e.g., 50 tools)
+ LiteLLM->>SemanticFilter: Build semantic router with embeddings
+ SemanticFilter->>LLM: Generate embeddings for tool descriptions
+ LLM->>SemanticFilter: Return embeddings
+ Note over SemanticFilter: Index ready for fast lookup
+
+ Note over Client,LLM: Request: Semantic Tool Filtering
+ Client->>LiteLLM: POST /v1/responses with MCP tools
+ LiteLLM->>SemanticFilter: Expand MCP references (50 tools available)
+ SemanticFilter->>SemanticFilter: Extract user query from request
+ SemanticFilter->>LLM: Generate query embedding
+ LLM->>SemanticFilter: Return query embedding
+ SemanticFilter->>SemanticFilter: Match query against tool embeddings
+ SemanticFilter->>LiteLLM: Return top-K tools (e.g., 3 most relevant)
+ LiteLLM->>LLM: Forward request with filtered tools (3 tools)
+ LLM->>LiteLLM: Return response
+ LiteLLM->>Client: Response with headers
x-litellm-semantic-filter: 50->3
x-litellm-semantic-filter-tools: tool1,tool2,tool3
+```
+
+## Configuration
+
+Enable semantic filtering in your LiteLLM config:
+
+```yaml title="config.yaml" showLineNumbers
+litellm_settings:
+ mcp_semantic_tool_filter:
+ enabled: true
+ embedding_model: "text-embedding-3-small" # Model for semantic matching
+ top_k: 5 # Max tools to return
+ similarity_threshold: 0.3 # Min similarity score
+```
+
+**Configuration Options:**
+- `enabled` - Enable/disable semantic filtering (default: `false`)
+- `embedding_model` - Model for generating embeddings (default: `"text-embedding-3-small"`)
+- `top_k` - Maximum number of tools to return (default: `10`)
+- `similarity_threshold` - Minimum similarity score for matches (default: `0.3`)
+
+## Usage
+
+Use MCP tools normally with the Responses API or Chat Completions. The semantic filter runs automatically:
+
+
+
+
+```bash title="Responses API with Semantic Filtering" showLineNumbers
+curl --location 'http://localhost:4000/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer sk-1234" \
+--data '{
+ "model": "gpt-4o",
+ "input": [
+ {
+ "role": "user",
+ "content": "give me TLDR of what BerriAI/litellm repo is about",
+ "type": "message"
+ }
+ ],
+ "tools": [
+ {
+ "type": "mcp",
+ "server_url": "litellm_proxy",
+ "require_approval": "never"
+ }
+ ],
+ "tool_choice": "required"
+}'
+```
+
+
+
+
+```bash title="Chat Completions with Semantic Filtering" showLineNumbers
+curl --location 'http://localhost:4000/v1/chat/completions' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer sk-1234" \
+--data '{
+ "model": "gpt-4o",
+ "messages": [
+ {"role": "user", "content": "Search Wikipedia for LiteLLM"}
+ ],
+ "tools": [
+ {
+ "type": "mcp",
+ "server_url": "litellm_proxy"
+ }
+ ]
+}'
+```
+
+
+
+
+## Response Headers
+
+The semantic filter adds diagnostic headers to every response:
+
+```
+x-litellm-semantic-filter: 10->3
+x-litellm-semantic-filter-tools: wikipedia-fetch,github-search,slack-post
+```
+
+- **`x-litellm-semantic-filter`** - Shows before→after tool count (e.g., `10->3` means 10 tools were filtered down to 3)
+- **`x-litellm-semantic-filter-tools`** - CSV list of the filtered tool names (max 150 chars, clipped with `...` if longer)
+
+These headers help you understand which tools were selected for each request and verify the filter is working correctly.
+
+## Example
+
+If you have 50 MCP tools registered and make a request asking about Wikipedia, the semantic filter will:
+
+1. Semantically match your query `"Search Wikipedia for LiteLLM"` against all 50 tool descriptions
+2. Select the top 5 most relevant tools (e.g., `wikipedia-fetch`, `wikipedia-search`, etc.)
+3. Pass only those 5 tools to the LLM
+4. Add headers showing `x-litellm-semantic-filter: 50->5`
+
+This dramatically reduces prompt size while ensuring the LLM has access to the right tools for the task.
+
+## Performance
+
+The semantic filter is optimized for production:
+- Router builds once on startup (no per-request overhead)
+- Semantic matching typically takes under 50ms
+- Fails gracefully - returns all tools if filtering fails
+- No impact on latency for requests without MCP tools
+
+## Related
+
+- [MCP Overview](./mcp.md) - Learn about MCP in LiteLLM
+- [MCP Permission Management](./mcp_control.md) - Control tool access by key/team
+- [Using MCP](./mcp_usage.md) - Complete MCP usage guide
diff --git a/docs/my-website/docs/mcp_troubleshoot.md b/docs/my-website/docs/mcp_troubleshoot.md
index 27ba0e4d787..57e7bfa674d 100644
--- a/docs/my-website/docs/mcp_troubleshoot.md
+++ b/docs/my-website/docs/mcp_troubleshoot.md
@@ -6,6 +6,39 @@ When LiteLLM acts as an MCP proxy, traffic normally flows `Client → LiteLLM Pr
For provisioning steps, transport options, and configuration fields, refer to [mcp.md](./mcp.md).
+## Quick Start: Debug with One Command
+
+The fastest way to debug MCP issues is to enable **debug headers**. Run this curl against your LiteLLM proxy and check the response headers:
+
+```bash
+curl -si -X POST http://localhost:4000/{your_mcp_server}/mcp \
+ -H "Content-Type: application/json" \
+ -H "x-litellm-api-key: Bearer sk-YOUR_KEY" \
+ -H "x-litellm-mcp-debug: true" \
+ -d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' \
+ 2>&1 | grep -i "x-mcp-debug"
+```
+
+This returns masked diagnostic headers that tell you exactly what's happening with authentication:
+
+```
+x-mcp-debug-inbound-auth: x-litellm-api-key=Bearer****1234
+x-mcp-debug-oauth2-token: Bearer****ef01
+x-mcp-debug-auth-resolution: oauth2-passthrough
+x-mcp-debug-outbound-url: https://mcp.atlassian.com/v1/mcp
+x-mcp-debug-server-auth-type: oauth2
+```
+
+If you see `SAME_AS_LITELLM_KEY` in `x-mcp-debug-oauth2-token`, your LiteLLM API key is leaking to the MCP server instead of an OAuth2 token. See [Debugging OAuth](./mcp_oauth#debugging-oauth) for the fix and other common issues.
+
+For Claude Code, add the debug header to your MCP config:
+
+```bash
+claude mcp add --transport http my_server http://localhost:4000/my_mcp/mcp \
+ --header "x-litellm-api-key: Bearer sk-..." \
+ --header "x-litellm-mcp-debug: true"
+```
+
## Locate the Error Source
Pin down where the failure occurs before adjusting settings so you do not mix symptoms from separate hops.
@@ -13,7 +46,7 @@ Pin down where the failure occurs before adjusting settings so you do not mix sy
### LiteLLM UI / Playground Errors (LiteLLM → MCP)
Failures shown on the MCP creation form or within the MCP Tool Testing Playground mean the LiteLLM proxy cannot reach the MCP server. Typical causes are misconfiguration (transport, headers, credentials), MCP/server outages, network/firewall blocks, or inaccessible OAuth metadata.
-
@@ -22,7 +55,7 @@ Failures shown on the MCP creation form or within the MCP Tool Testing Playgroun
**Actions**
- Capture LiteLLM proxy logs alongside MCP-server logs (see [Error Log Example](./mcp_troubleshoot#error-log-example-failed-mcp-call)) to inspect the request/response pair and stack traces.
-- From the LiteLLM server, run Method 2 ([`curl` smoke test](./mcp_troubleshoot#curl-smoke-test)) against the MCP endpoint to confirm basic connectivity.
+- From the LiteLLM server, run a [`curl` smoke test](./mcp_troubleshoot#curl-smoke-test) against the MCP endpoint to confirm basic connectivity.
### Client Traffic Issues (Client → LiteLLM)
If only real client requests fail, determine whether LiteLLM ever reaches the MCP hop.
@@ -43,7 +76,7 @@ During `/responses` or `/chat/completions`, LiteLLM may trigger MCP tool calls m
- Validate MCP connectivity with the [MCP Inspector](./mcp_troubleshoot#mcp-inspector) to ensure the server responds.
- Reproduce the same MCP call via the LiteLLM Playground to confirm LiteLLM can complete the MCP hop independently.
-
@@ -55,6 +88,10 @@ LiteLLM performs metadata discovery per the MCP spec ([section 2.3](https://mode
- Use `curl ` (or similar) from the LiteLLM host to ensure the discovery document is reachable and contains the expected authorization/token endpoints.
- Record the exact metadata URL, requested scopes, and any static client credentials so support can replay the discovery step if needed.
+## Debugging OAuth
+
+For detailed OAuth2 debugging — including debug header reference, common misconfigurations, and example output — see [Debugging OAuth](./mcp_oauth#debugging-oauth).
+
## Verify Connectivity
Run lightweight validations before impacting production traffic.
@@ -66,7 +103,7 @@ Use the MCP Inspector when you need to test both `Client → LiteLLM` and `Clien
2. Configure and connect:
- **Transport Type:** choose the transport the client uses (Streamable HTTP for LiteLLM).
- **URL:** the endpoint under test (LiteLLM MCP URL for `Client → LiteLLM`, or the MCP server URL for `Client → MCP`).
- - **Custom Headers:** e.g., `Authorization: Bearer `.
+ - **Custom Headers:** e.g., `x-litellm-api-key: Bearer `.
3. Open the **Tools** tab and click **List Tools** to verify the MCP alias responds.
### `curl` Smoke Test
@@ -79,7 +116,7 @@ curl -X POST https://your-target-domain.example.com/mcp \
-d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}'
```
-Add `-H "Authorization: Bearer "` when the target is a LiteLLM endpoint that requires authentication. Adjust the headers, or payload to target other MCP methods. Matching failures between `curl` and LiteLLM confirm that the MCP server or network/OAuth layer is the culprit.
+Add `-H "x-litellm-api-key: Bearer "` when the target is a LiteLLM endpoint that requires authentication. Adjust the headers or payload to target other MCP methods. Matching failures between `curl` and LiteLLM confirm that the MCP server or network/OAuth layer is the culprit.
## Review Logs
diff --git a/docs/my-website/docs/observability/datadog.md b/docs/my-website/docs/observability/datadog.md
index 6f785be1013..9385b0020cf 100644
--- a/docs/my-website/docs/observability/datadog.md
+++ b/docs/my-website/docs/observability/datadog.md
@@ -253,3 +253,12 @@ LiteLLM supports customizing the following Datadog environment variables
\* **Required when using Direct API** (default): `DD_API_KEY` and `DD_SITE` are required
\* **Optional when using DataDog Agent**: Set `LITELLM_DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required for **Datadog Logs**. (**Note: `DD_API_KEY` IS REQUIRED for Datadog LLM Observability**)
+## Automatic Tags
+
+LiteLLM automatically adds the following tags to your Datadog logs and metrics if the information is available in the request:
+
+| Tag | Description | Source |
+|-----|-------------|--------|
+| `team` | The team alias or ID associated with the API Key | `user_api_key_team_alias`, `team_alias`, `user_api_key_team_id`, or `team_id` in metadata |
+| `request_tag` | Custom tags passed in the request | `request_tags` in logging payload |
+
diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md
index a81336c5bc6..d3c5a44d481 100644
--- a/docs/my-website/docs/observability/langfuse_integration.md
+++ b/docs/my-website/docs/observability/langfuse_integration.md
@@ -215,6 +215,66 @@ The following parameters can be updated on a continuation of a trace by passing
Any other key value pairs passed into the metadata not listed in the above spec for a `litellm` completion will be added as a metadata key value pair for the generation.
+#### Multiple Langfuse Projects (Per-Request Credentials)
+
+You can send traces to different Langfuse projects per request by passing credentials directly to `completion()` or `acompletion()`. This works alongside (or instead of) the global env vars and is useful when different teams or business processes use different Langfuse projects.
+
+Pass **`langfuse_public_key`**, **`langfuse_secret_key`** (or **`langfuse_secret`**), and optionally **`langfuse_host`** as keyword arguments:
+
+```python
+import litellm
+from litellm import completion
+
+# Optional: set a default via env for requests that don't pass credentials
+# os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-default..."
+# os.environ["LANGFUSE_SECRET_KEY"] = "sk-default..."
+
+litellm.success_callback = ["langfuse"]
+litellm.failure_callback = ["langfuse"]
+
+# Request 1 → Langfuse Project A
+response_a = completion(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Hello from team A"}],
+ langfuse_public_key="pk-lf-project-a...",
+ langfuse_secret_key="sk-lf-project-a...",
+ langfuse_host="https://us.cloud.langfuse.com", # optional
+)
+
+# Request 2 → Langfuse Project B (different project)
+response_b = completion(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Hello from team B"}],
+ langfuse_public_key="pk-lf-project-b...",
+ langfuse_secret_key="sk-lf-project-b...",
+ langfuse_host="https://eu.cloud.langfuse.com", # optional, can differ per project
+)
+```
+
+Async usage with per-request credentials:
+
+```python
+import litellm
+from litellm import acompletion
+
+litellm.success_callback = ["langfuse"]
+litellm.failure_callback = ["langfuse"]
+
+response = await acompletion(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Hi"}],
+ langfuse_public_key="pk-lf-...",
+ langfuse_secret_key="sk-lf-...",
+ langfuse_host="https://us.cloud.langfuse.com", # optional
+)
+```
+
+- **`langfuse_public_key`** – Langfuse project public key (required for per-request override).
+- **`langfuse_secret_key`** or **`langfuse_secret`** – Langfuse secret key (either name is accepted).
+- **`langfuse_host`** – Langfuse host URL (e.g. `https://us.cloud.langfuse.com`); optional, defaults to env or Langfuse cloud.
+
+When these are passed, that request uses this project (and host) for the Langfuse callback; when omitted, the callback uses the global Langfuse client (from env vars if set). LiteLLM caches a Langfuse client per credential set to avoid creating a new client on every request.
+
#### Disable Logging - Specific Calls
To disable logging for specific calls use the `no-log` flag.
diff --git a/docs/my-website/docs/pass_through/bedrock.md b/docs/my-website/docs/pass_through/bedrock.md
index b8d20d77da0..65c5d8caadc 100644
--- a/docs/my-website/docs/pass_through/bedrock.md
+++ b/docs/my-website/docs/pass_through/bedrock.md
@@ -556,3 +556,147 @@ for event in response.get("completion"):
print(completion)
```
+
+## Using LangChain AWS SDK with LiteLLM
+
+You can use the [LangChain AWS SDK](https://python.langchain.com/docs/integrations/chat/bedrock/) with LiteLLM Proxy to get cost tracking, load balancing, and other LiteLLM features.
+
+### Quick Start
+
+**1. Install LangChain AWS**:
+
+```bash showLineNumbers
+pip install langchain-aws
+```
+
+**2. Setup LiteLLM Proxy**:
+
+Create a `config.yaml`:
+
+```yaml showLineNumbers
+model_list:
+ - model_name: claude-sonnet
+ litellm_params:
+ model: bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0
+ aws_region_name: us-east-1
+ custom_llm_provider: bedrock
+```
+
+Start the proxy:
+
+```bash showLineNumbers
+export AWS_ACCESS_KEY_ID="your-access-key"
+export AWS_SECRET_ACCESS_KEY="your-secret-key"
+
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+**3. Use LangChain with LiteLLM**:
+
+```python showLineNumbers
+from langchain_aws import ChatBedrockConverse
+from langchain_core.messages import HumanMessage
+
+# Your LiteLLM API key
+API_KEY = "Bearer sk-1234"
+
+# Initialize ChatBedrockConverse pointing to LiteLLM proxy
+llm = ChatBedrockConverse(
+ model_id="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+ endpoint_url="http://localhost:4000/bedrock",
+ region_name="us-east-1",
+ aws_access_key_id=API_KEY,
+ aws_secret_access_key="bedrock" # Any non-empty value works
+)
+
+# Invoke the model
+messages = [HumanMessage(content="Hello, how are you?")]
+response = llm.invoke(messages)
+
+print(response.content)
+```
+
+### Advanced Example: PDF Document Processing with Citations
+
+LangChain AWS SDK supports Bedrock's document processing features. Here's how to use it with LiteLLM:
+
+```python showLineNumbers
+import os
+import json
+from langchain_aws import ChatBedrockConverse
+from langchain_core.messages import HumanMessage
+
+# Your LiteLLM API key
+API_KEY = "Bearer sk-1234"
+
+def get_llm() -> ChatBedrockConverse:
+ """Initialize LLM pointing to LiteLLM proxy"""
+ llm = ChatBedrockConverse(
+ model_id="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+ base_model_id="anthropic.claude-3-7-sonnet-20250219-v1:0",
+ endpoint_url="http://localhost:4000/bedrock",
+ region_name="us-east-1",
+ aws_access_key_id=API_KEY,
+ aws_secret_access_key="bedrock"
+ )
+ return llm
+
+if __name__ == "__main__":
+ # Initialize the LLM
+ llm = get_llm()
+
+ # Read PDF file as bytes (Converse API requires raw bytes)
+ with open("your-document.pdf", "rb") as file:
+ file_bytes = file.read()
+
+ # Prepare messages with document attachment
+ messages = [
+ HumanMessage(content=[
+ {"text": "What is the policy number in this document?"},
+ {
+ "document": {
+ "format": "pdf",
+ "name": "PolicyDocument",
+ "source": {"bytes": file_bytes},
+ "citations": {"enabled": True}
+ }
+ }
+ ])
+ ]
+
+ # Invoke the LLM
+ response = llm.invoke(messages)
+
+ # Print response with citations
+ print(json.dumps(response.content, indent=4))
+```
+
+### Supported LangChain Features
+
+All LangChain AWS features work with LiteLLM:
+
+| Feature | Supported | Notes |
+|---------|-----------|-------|
+| Text Generation | ✅ | Full support |
+| Streaming | ✅ | Use `stream()` method |
+| Document Processing | ✅ | PDF, images, etc. |
+| Citations | ✅ | Enable in document config |
+| Tool Use | ✅ | Function calling support |
+| Multi-modal | ✅ | Text + images + documents |
+
+### Troubleshooting
+
+**Issue**: `UnknownOperationException` error
+
+**Solution**: Make sure you're using the correct endpoint URL format:
+- ✅ Correct: `http://localhost:4000/bedrock`
+- ❌ Wrong: `http://localhost:4000/bedrock/v2`
+
+**Issue**: Authentication errors
+
+**Solution**: Ensure your API key is in the correct format:
+```python
+aws_access_key_id="Bearer sk-1234" # Include "Bearer " prefix
+```
diff --git a/docs/my-website/docs/pass_through/openai_passthrough.md b/docs/my-website/docs/pass_through/openai_passthrough.md
index d7c98eba7b3..49026f8aa2d 100644
--- a/docs/my-website/docs/pass_through/openai_passthrough.md
+++ b/docs/my-website/docs/pass_through/openai_passthrough.md
@@ -1,6 +1,6 @@
# OpenAI Passthrough
-Pass-through endpoints for `/openai`
+Pass-through endpoints for direct OpenAI API access
## Overview
@@ -10,12 +10,27 @@ Pass-through endpoints for `/openai`
| Logging | ✅ | Works across all integrations |
| Streaming | ✅ | Fully supported |
-### When to use this?
+## Available Endpoints
+
+### `/openai_passthrough` - Recommended
+Dedicated passthrough endpoint that guarantees direct routing to OpenAI without conflicts.
+
+**Use this for:**
+- OpenAI Responses API (`/v1/responses`)
+- Any endpoint where you need guaranteed passthrough
+- When `/openai` routes are conflicting with LiteLLM's native implementations
+
+### `/openai` - Legacy
+Standard passthrough endpoint that may conflict with LiteLLM's native implementations.
+
+**Note:** Some endpoints like `/openai/v1/responses` will be routed to LiteLLM's native implementation instead of OpenAI.
+
+## When to use this?
- For 90% of your use cases, you should use the [native LiteLLM OpenAI Integration](https://docs.litellm.ai/docs/providers/openai) (`/chat/completions`, `/embeddings`, `/completions`, `/images`, `/batches`, etc.)
-- Use this passthrough to call less popular or newer OpenAI endpoints that LiteLLM doesn't fully support yet, such as `/assistants`, `/threads`, `/vector_stores`
+- Use `/openai_passthrough` to call less popular or newer OpenAI endpoints that LiteLLM doesn't fully support yet, such as `/assistants`, `/threads`, `/vector_stores`, `/responses`
-Simply replace `https://api.openai.com` with `LITELLM_PROXY_BASE_URL/openai`
+Simply replace `https://api.openai.com` with `LITELLM_PROXY_BASE_URL/openai_passthrough`
## Usage Examples
@@ -34,7 +49,7 @@ Make sure you do the following:
import openai
client = openai.OpenAI(
- base_url="http://0.0.0.0:4000/openai", # /openai
+ base_url="http://0.0.0.0:4000/openai_passthrough", # /openai_passthrough
api_key="sk-anything" #
)
```
diff --git a/docs/my-website/docs/projects/openai-agents.md b/docs/my-website/docs/projects/openai-agents.md
index 95a2191b883..86983e7e510 100644
--- a/docs/my-website/docs/projects/openai-agents.md
+++ b/docs/my-website/docs/projects/openai-agents.md
@@ -1,22 +1,121 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
# OpenAI Agents SDK
-The [OpenAI Agents SDK](https://github.com/openai/openai-agents-python) is a lightweight framework for building multi-agent workflows.
-It includes an official LiteLLM extension that lets you use any of the 100+ supported providers (Anthropic, Gemini, Mistral, Bedrock, etc.)
+Use OpenAI Agents SDK with any LLM provider through LiteLLM Proxy.
+
+The [OpenAI Agents SDK](https://github.com/openai/openai-agents-python) is a lightweight framework for building multi-agent workflows. It includes an official LiteLLM extension that lets you use any of the 100+ supported providers.
+
+## Quick Start
+
+### 1. Install Dependencies
+
+```bash
+pip install "openai-agents[litellm]"
+```
+
+### 2. Add Model to Config
+
+```yaml title="config.yaml"
+model_list:
+ - model_name: gpt-4o
+ litellm_params:
+ model: "openai/gpt-4o"
+ api_key: "os.environ/OPENAI_API_KEY"
+
+ - model_name: claude-sonnet
+ litellm_params:
+ model: "anthropic/claude-3-5-sonnet-20241022"
+ api_key: "os.environ/ANTHROPIC_API_KEY"
+
+ - model_name: gemini-pro
+ litellm_params:
+ model: "gemini/gemini-2.0-flash-exp"
+ api_key: "os.environ/GEMINI_API_KEY"
+```
+
+### 3. Start LiteLLM Proxy
+
+```bash
+litellm --config config.yaml
+```
+
+### 4. Use with Proxy
+
+
+
```python
from agents import Agent, Runner
from agents.extensions.models.litellm_model import LitellmModel
+# Point to LiteLLM proxy
agent = Agent(
name="Assistant",
instructions="You are a helpful assistant.",
- model=LitellmModel(model="provider/model-name")
+ model=LitellmModel(
+ model="claude-sonnet", # Model from config.yaml
+ api_key="sk-1234", # LiteLLM API key
+ base_url="http://localhost:4000"
+ )
)
-result = Runner.run_sync(agent, "your_prompt_here")
-print("Result:", result.final_output)
+result = await Runner.run(agent, "What is LiteLLM?")
+print(result.final_output)
```
-- [GitHub](https://github.com/openai/openai-agents-python)
-- [LiteLLM Extension Docs](https://openai.github.io/openai-agents-python/ref/extensions/litellm/)
+
+
+
+```python
+from agents import Agent, Runner
+from agents.extensions.models.litellm_model import LitellmModel
+
+# Use any provider directly
+agent = Agent(
+ name="Assistant",
+ instructions="You are a helpful assistant.",
+ model=LitellmModel(
+ model="anthropic/claude-3-5-sonnet-20241022",
+ api_key="your-anthropic-key"
+ )
+)
+
+result = await Runner.run(agent, "What is LiteLLM?")
+print(result.final_output)
+```
+
+
+
+
+## Track Usage
+
+Enable usage tracking to monitor token consumption:
+
+```python
+from agents import Agent, ModelSettings
+from agents.extensions.models.litellm_model import LitellmModel
+
+agent = Agent(
+ name="Assistant",
+ model=LitellmModel(model="claude-sonnet", api_key="sk-1234"),
+ model_settings=ModelSettings(include_usage=True)
+)
+
+result = await Runner.run(agent, "Hello")
+print(result.context_wrapper.usage) # Token counts
+```
+
+## Environment Variables
+
+| Variable | Value | Description |
+|----------|-------|-------------|
+| `LITELLM_BASE_URL` | `http://localhost:4000` | LiteLLM proxy URL |
+| `LITELLM_API_KEY` | `sk-1234` | Your LiteLLM API key |
+
+## Related Resources
+
+- [OpenAI Agents SDK Documentation](https://openai.github.io/openai-agents-python/)
+- [LiteLLM Extension Docs](https://openai.github.io/openai-agents-python/models/litellm/)
+- [LiteLLM Proxy Quick Start](../proxy/quick_start)
diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md
index 446d663c5ac..de5a4dc610c 100644
--- a/docs/my-website/docs/providers/anthropic.md
+++ b/docs/my-website/docs/providers/anthropic.md
@@ -1473,6 +1473,20 @@ LiteLLM translates OpenAI's `reasoning_effort` to Anthropic's `thinking` paramet
| "medium" | "budget_tokens": 2048 |
| "high" | "budget_tokens": 4096 |
+:::note
+For Claude Opus 4.6, all `reasoning_effort` values (`low`, `medium`, `high`) are mapped to `thinking: {type: "adaptive"}`. To use explicit thinking budgets, pass the native `thinking` parameter directly:
+
+```python
+from litellm import completion
+
+resp = completion(
+ model="anthropic/claude-opus-4-6",
+ messages=[{"role": "user", "content": "What is the capital of France?"}],
+ thinking={"type": "enabled", "budget_tokens": 1024},
+)
+```
+:::
+
@@ -1614,8 +1628,65 @@ curl http://0.0.0.0:4000/v1/chat/completions \
+#### Adaptive Thinking (Claude Opus 4.6)
+
+
+```python
+response = litellm.completion(
+ model="anthropic/claude-opus-4-6",
+ messages=[{"role": "user", "content": "What is the optimal strategy for solving this problem?"}],
+ thinking={"type": "adaptive"},
+)
+```
+
+
+
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer $LITELLM_KEY" \
+ -d '{
+ "model": "anthropic/claude-opus-4-6",
+ "messages": [{"role": "user", "content": "What is the optimal strategy for solving this problem?"}],
+ "thinking": {"type": "adaptive"}
+ }'
+```
+
+
+
+
+#### Enabled Thinking with Budget
+
+
+
+
+```python
+response = litellm.completion(
+ model="anthropic/claude-opus-4-6",
+ messages=[{"role": "user", "content": "What is the capital of France?"}],
+ thinking={"type": "enabled", "budget_tokens": 5000},
+)
+```
+
+
+
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer $LITELLM_KEY" \
+ -d '{
+ "model": "anthropic/claude-opus-4-6",
+ "messages": [{"role": "user", "content": "What is the capital of France?"}],
+ "thinking": {"type": "enabled", "budget_tokens": 5000}
+ }'
+```
+
+
+
## **Passing Extra Headers to Anthropic API**
diff --git a/docs/my-website/docs/providers/anthropic_tool_search.md b/docs/my-website/docs/providers/anthropic_tool_search.md
index 28ce5688eeb..203a2947ebc 100644
--- a/docs/my-website/docs/providers/anthropic_tool_search.md
+++ b/docs/my-website/docs/providers/anthropic_tool_search.md
@@ -1,43 +1,46 @@
-# Anthropic Tool Search
+# Tool Search
Tool search enables Claude to dynamically discover and load tools on-demand from large tool catalogs (10,000+ tools). Instead of loading all tool definitions into the context window upfront, Claude searches your tool catalog and loads only the tools it needs.
+## Supported Providers
+
+| Provider | Chat Completions API | Messages API |
+|----------|---------------------|--------------|
+| **Anthropic API** | ✅ | ✅ |
+| **Azure Anthropic** (Microsoft Foundry) | ✅ | ✅ |
+| **Google Cloud Vertex AI** | ✅ | ✅ |
+| **Amazon Bedrock** | ✅ (Invoke API only, Opus 4.5 only) | ✅ (Invoke API only, Opus 4.5 only) |
+
+
## Benefits
- **Context efficiency**: Avoid consuming massive portions of your context window with tool definitions
- **Better tool selection**: Claude's tool selection accuracy degrades with more than 30-50 tools. Tool search maintains accuracy even with thousands of tools
- **On-demand loading**: Tools are only loaded when Claude needs them
-## Supported Models
-
-Tool search is available on:
-- Claude Opus 4.5
-- Claude Sonnet 4.5
-
-## Supported Platforms
-
-- Anthropic API (direct)
-- Azure Anthropic (Microsoft Foundry)
-- Google Cloud Vertex AI
-- Amazon Bedrock (invoke API only, not converse API)
-
## Tool Search Variants
LiteLLM supports both tool search variants:
### 1. Regex Tool Search (`tool_search_tool_regex_20251119`)
-Claude constructs regex patterns to search for tools.
+Claude constructs regex patterns to search for tools. Best for exact pattern matching (faster).
### 2. BM25 Tool Search (`tool_search_tool_bm25_20251119`)
-Claude uses natural language queries to search for tools using the BM25 algorithm.
+Claude uses natural language queries to search for tools using the BM25 algorithm. Best for natural language semantic search.
-## Quick Start
+**Note**: BM25 variant is not supported on Bedrock.
-### Basic Example with Regex Tool Search
+---
-```python
+## Chat Completions API
+
+### SDK Usage
+
+#### Basic Example with Regex Tool Search
+
+```python showLineNumbers title="Basic Tool Search Example"
import litellm
response = litellm.completion(
@@ -70,26 +73,6 @@ response = litellm.completion(
}
},
"defer_loading": True # Mark for deferred loading
- },
- # Another deferred tool
- {
- "type": "function",
- "function": {
- "name": "search_files",
- "description": "Search through files in the workspace",
- "parameters": {
- "type": "object",
- "properties": {
- "query": {"type": "string"},
- "file_types": {
- "type": "array",
- "items": {"type": "string"}
- }
- },
- "required": ["query"]
- }
- },
- "defer_loading": True
}
]
)
@@ -97,9 +80,9 @@ response = litellm.completion(
print(response.choices[0].message.content)
```
-### BM25 Tool Search Example
+#### BM25 Tool Search Example
-```python
+```python showLineNumbers title="BM25 Tool Search"
import litellm
response = litellm.completion(
@@ -134,9 +117,9 @@ response = litellm.completion(
)
```
-## Using with Azure Anthropic
+#### Azure Anthropic Example
-```python
+```python showLineNumbers title="Azure Anthropic Tool Search"
import litellm
response = litellm.completion(
@@ -170,9 +153,9 @@ response = litellm.completion(
)
```
-## Using with Vertex AI
+#### Vertex AI Example
-```python
+```python showLineNumbers title="Vertex AI Tool Search"
import litellm
response = litellm.completion(
@@ -192,11 +175,9 @@ response = litellm.completion(
)
```
-## Streaming Support
+#### Streaming Support
-Tool search works with streaming:
-
-```python
+```python showLineNumbers title="Streaming with Tool Search"
import litellm
response = litellm.completion(
@@ -233,13 +214,13 @@ for chunk in response:
print(chunk.choices[0].delta.content, end="")
```
-## LiteLLM Proxy
+### AI Gateway Usage
-Tool search works automatically through the LiteLLM proxy:
+Tool search works automatically through the LiteLLM proxy.
-### Proxy Config
+#### Proxy Configuration
-```yaml
+```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: claude-sonnet
litellm_params:
@@ -247,18 +228,19 @@ model_list:
api_key: os.environ/ANTHROPIC_API_KEY
```
-### Client Request
+#### Client Request
-```python
-import openai
+```python showLineNumbers title="Client Request via Proxy"
+from anthropic import Anthropic
-client = openai.OpenAI(
+client = Anthropic(
api_key="your-litellm-proxy-key",
base_url="http://0.0.0.0:4000"
)
-response = client.chat.completions.create(
+response = client.messages.create(
model="claude-sonnet",
+ max_tokens=1024,
messages=[
{"role": "user", "content": "What's the weather?"}
],
@@ -268,17 +250,14 @@ response = client.chat.completions.create(
"name": "tool_search_tool_regex"
},
{
- "type": "function",
- "function": {
- "name": "get_weather",
- "description": "Get weather information",
- "parameters": {
- "type": "object",
- "properties": {
- "location": {"type": "string"}
- },
- "required": ["location"]
- }
+ "name": "get_weather",
+ "description": "Get weather information",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "location": {"type": "string"}
+ },
+ "required": ["location"]
},
"defer_loading": True
}
@@ -286,127 +265,278 @@ response = client.chat.completions.create(
)
```
-## Important Notes
+---
-### Beta Header
+## Messages API
-LiteLLM automatically detects tool search tools and adds the appropriate beta header based on your provider:
+The Messages API provides native Anthropic-style tool search support via the `litellm.anthropic.messages` interface.
-- **Anthropic API & Microsoft Foundry**: `advanced-tool-use-2025-11-20`
-- **Google Cloud Vertex AI**: `tool-search-tool-2025-10-19`
-- **Amazon Bedrock** (Invoke API, Opus 4.5 only): `tool-search-tool-2025-10-19`
+### SDK Usage
-You don't need to manually specify beta headers—LiteLLM handles this automatically.
+#### Basic Example
-### Deferred Loading
+```python showLineNumbers title="Messages API - Basic Tool Search"
+import litellm
-- Tools with `defer_loading: true` are only loaded when Claude discovers them via search
-- At least one tool must be non-deferred (the tool search tool itself)
-- Keep your 3-5 most frequently used tools as non-deferred for optimal performance
-
-### Tool Descriptions
-
-Write clear, descriptive tool names and descriptions that match how users describe tasks. The search algorithm uses:
-- Tool names
-- Tool descriptions
-- Argument names
-- Argument descriptions
-
-### Usage Tracking
-
-Tool search requests are tracked in the usage object:
-
-```python
-response = litellm.completion(
- model="anthropic/claude-sonnet-4-5-20250929",
- messages=[{"role": "user", "content": "Search for tools"}],
- tools=[...]
+response = await litellm.anthropic.messages.acreate(
+ model="anthropic/claude-sonnet-4-20250514",
+ messages=[
+ {
+ "role": "user",
+ "content": "What's the weather in San Francisco?"
+ }
+ ],
+ tools=[
+ {
+ "type": "tool_search_tool_regex_20251119",
+ "name": "tool_search_tool_regex"
+ },
+ {
+ "name": "get_weather",
+ "description": "Get the current weather for a location",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "location": {
+ "type": "string",
+ "description": "The city and state, e.g. San Francisco, CA"
+ }
+ },
+ "required": ["location"]
+ },
+ "defer_loading": True
+ }
+ ],
+ max_tokens=1024,
+ extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
)
-# Check tool search usage
-if response.usage.server_tool_use:
- print(f"Tool search requests: {response.usage.server_tool_use.tool_search_requests}")
+print(response)
```
-## Error Handling
+#### Azure Anthropic Messages Example
-### All Tools Deferred
+```python showLineNumbers title="Azure Anthropic Messages API"
+import litellm
-```python
-# ❌ This will fail - at least one tool must be non-deferred
-tools = [
- {
- "type": "function",
- "function": {...},
- "defer_loading": True
- }
-]
-
-# ✅ Correct - tool search tool is non-deferred
-tools = [
- {
- "type": "tool_search_tool_regex_20251119",
- "name": "tool_search_tool_regex"
- },
- {
- "type": "function",
- "function": {...},
- "defer_loading": True
- }
-]
+response = await litellm.anthropic.messages.acreate(
+ model="azure_anthropic/claude-sonnet-4-20250514",
+ messages=[
+ {
+ "role": "user",
+ "content": "What's the stock price of Apple?"
+ }
+ ],
+ tools=[
+ {
+ "type": "tool_search_tool_regex_20251119",
+ "name": "tool_search_tool_regex"
+ },
+ {
+ "name": "get_stock_price",
+ "description": "Get the current stock price for a ticker symbol",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "ticker": {
+ "type": "string",
+ "description": "The stock ticker symbol, e.g. AAPL"
+ }
+ },
+ "required": ["ticker"]
+ },
+ "defer_loading": True
+ }
+ ],
+ max_tokens=1024,
+ extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
+)
```
-### Missing Tool Definition
+#### Vertex AI Messages Example
-If Claude references a tool that isn't in your deferred tools list, you'll get an error. Make sure all tools that might be discovered are included in the tools parameter with `defer_loading: true`.
+```python showLineNumbers title="Vertex AI Messages API"
+import litellm
-## Best Practices
+response = await litellm.anthropic.messages.acreate(
+ model="vertex_ai/claude-sonnet-4@20250514",
+ messages=[
+ {
+ "role": "user",
+ "content": "Search the web for information about AI"
+ }
+ ],
+ tools=[
+ {
+ "type": "tool_search_tool_bm25_20251119",
+ "name": "tool_search_tool_bm25"
+ },
+ {
+ "name": "search_web",
+ "description": "Search the web for information",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "query": {
+ "type": "string",
+ "description": "The search query"
+ }
+ },
+ "required": ["query"]
+ },
+ "defer_loading": True
+ }
+ ],
+ max_tokens=1024,
+ extra_headers={"anthropic-beta": "tool-search-tool-2025-10-19"}
+)
+```
-1. **Keep frequently used tools non-deferred**: Your 3-5 most common tools should not have `defer_loading: true`
+#### Bedrock Messages Example
-2. **Use semantic descriptions**: Tool descriptions should use natural language that matches user queries
+```python showLineNumbers title="Bedrock Messages API (Invoke)"
+import litellm
-3. **Choose the right variant**:
- - Use **regex** for exact pattern matching (faster)
- - Use **BM25** for natural language semantic search
+response = await litellm.anthropic.messages.acreate(
+ model="bedrock/invoke/anthropic.claude-opus-4-20250514-v1:0",
+ messages=[
+ {
+ "role": "user",
+ "content": "What's the weather?"
+ }
+ ],
+ tools=[
+ {
+ "type": "tool_search_tool_regex_20251119",
+ "name": "tool_search_tool_regex"
+ },
+ {
+ "name": "get_weather",
+ "description": "Get weather information",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "location": {"type": "string"}
+ },
+ "required": ["location"]
+ },
+ "defer_loading": True
+ }
+ ],
+ max_tokens=1024,
+ extra_headers={"anthropic-beta": "tool-search-tool-2025-10-19"}
+)
+```
-4. **Monitor usage**: Track `tool_search_requests` in the usage object to understand search patterns
+#### Streaming Support
-5. **Optimize tool catalog**: Remove unused tools and consolidate similar functionality
+```python showLineNumbers title="Messages API - Streaming"
+import litellm
+import json
-## When to Use Tool Search
+response = await litellm.anthropic.messages.acreate(
+ model="anthropic/claude-sonnet-4-20250514",
+ messages=[
+ {
+ "role": "user",
+ "content": "What's the weather in Tokyo?"
+ }
+ ],
+ tools=[
+ {
+ "type": "tool_search_tool_regex_20251119",
+ "name": "tool_search_tool_regex"
+ },
+ {
+ "name": "get_weather",
+ "description": "Get weather information",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "location": {"type": "string"}
+ },
+ "required": ["location"]
+ },
+ "defer_loading": True
+ }
+ ],
+ max_tokens=1024,
+ stream=True,
+ extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
+)
-**Good use cases:**
-- 10+ tools available in your system
-- Tool definitions consuming >10K tokens
-- Experiencing tool selection accuracy issues
-- Building systems with multiple tool categories
-- Tool library growing over time
+async for chunk in response:
+ if isinstance(chunk, bytes):
+ chunk_str = chunk.decode("utf-8")
+ for line in chunk_str.split("\n"):
+ if line.startswith("data: "):
+ try:
+ json_data = json.loads(line[6:])
+ print(json_data)
+ except json.JSONDecodeError:
+ pass
+```
-**When traditional tool calling is better:**
-- Less than 10 tools total
-- All tools are frequently used
-- Very small tool definitions (\<100 tokens total)
+### AI Gateway Usage
-## Limitations
+Configure the proxy to use Messages API endpoints.
-- Not compatible with tool use examples
-- Requires Claude Opus 4.5 or Sonnet 4.5
-- On Bedrock, only available via invoke API (not converse API)
-- On Bedrock, only supported for Claude Opus 4.5 (not Sonnet 4.5)
-- BM25 variant (`tool_search_tool_bm25_20251119`) is not supported on Bedrock
-- Maximum 10,000 tools in catalog
-- Returns 3-5 most relevant tools per search
+#### Proxy Configuration
-### Bedrock-Specific Notes
+```yaml showLineNumbers title="config.yaml"
+model_list:
+ - model_name: claude-sonnet-messages
+ litellm_params:
+ model: anthropic/claude-sonnet-4-20250514
+ api_key: os.environ/ANTHROPIC_API_KEY
+```
-When using Bedrock's Invoke API:
-- The regex variant (`tool_search_tool_regex_20251119`) is automatically normalized to `tool_search_tool_regex`
-- The BM25 variant (`tool_search_tool_bm25_20251119`) is automatically filtered out as it's not supported
-- Tool search is only available for Claude Opus 4.5 models
+#### Client Request
+
+```python showLineNumbers title="Client Request via Proxy (Messages API)"
+from anthropic import Anthropic
+
+client = Anthropic(
+ api_key="your-litellm-proxy-key",
+ base_url="http://0.0.0.0:4000"
+)
+
+response = client.messages.create(
+ model="claude-sonnet-messages",
+ max_tokens=1024,
+ messages=[
+ {
+ "role": "user",
+ "content": "What's the weather?"
+ }
+ ],
+ tools=[
+ {
+ "type": "tool_search_tool_regex_20251119",
+ "name": "tool_search_tool_regex"
+ },
+ {
+ "name": "get_weather",
+ "description": "Get weather information",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "location": {"type": "string"}
+ },
+ "required": ["location"]
+ },
+ "defer_loading": True
+ }
+ ],
+ extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
+)
+
+print(response)
+```
+
+---
## Additional Resources
- [Anthropic Tool Search Documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/tool-search)
- [LiteLLM Tool Calling Guide](https://docs.litellm.ai/docs/completion/function_call)
-
diff --git a/docs/my-website/docs/providers/azure_ai/azure_model_router.md b/docs/my-website/docs/providers/azure_ai/azure_model_router.md
index 5e14c7283f6..16bc1afb70e 100644
--- a/docs/my-website/docs/providers/azure_ai/azure_model_router.md
+++ b/docs/my-website/docs/providers/azure_ai/azure_model_router.md
@@ -5,19 +5,38 @@ Azure Model Router is a feature in Azure AI Foundry that automatically routes yo
## Key Features
- **Automatic Model Selection**: Azure Model Router dynamically selects the best model for your request
-- **Cost Tracking**: LiteLLM automatically tracks costs based on the actual model used (e.g., `gpt-4.1-nano`), not the router endpoint
+- **Cost Tracking**: LiteLLM automatically tracks costs based on the actual model used (e.g., `gpt-4.1-nano`), plus the Model Router infrastructure fee
- **Streaming Support**: Full support for streaming responses with accurate cost calculation
+- **Simple Configuration**: Easy to set up via UI or config file
+
+## Model Naming Pattern
+
+Use the pattern: `azure_ai/model_router/`
+
+**Components:**
+- `azure_ai` - The provider identifier
+- `model_router` - Indicates this is a Model Router deployment
+- `` - Your actual deployment name from Azure AI Foundry (e.g., `azure-model-router`)
+
+**Example:** `azure_ai/model_router/azure-model-router`
+
+**How it works:**
+- LiteLLM automatically strips the `model_router/` prefix when sending requests to Azure
+- Only your deployment name (e.g., `azure-model-router`) is sent to the Azure API
+- The full path is preserved in responses and logs for proper cost tracking
## LiteLLM Python SDK
### Basic Usage
+Use the pattern `azure_ai/model_router/` where `` is your Azure deployment name:
+
```python
import litellm
import os
response = litellm.completion(
- model="azure_ai/azure-model-router",
+ model="azure_ai/model_router/azure-model-router", # Use your deployment name
messages=[{"role": "user", "content": "Hello!"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"),
@@ -26,6 +45,13 @@ response = litellm.completion(
print(response)
```
+**Pattern Explanation:**
+- `azure_ai` - The provider
+- `model_router` - Indicates this is a model router deployment
+- `azure-model-router` - Your actual deployment name from Azure AI Foundry
+
+LiteLLM will automatically strip the `model_router/` prefix when sending the request to Azure, so only `azure-model-router` is sent to the API.
+
### Streaming with Usage Tracking
```python
@@ -33,7 +59,7 @@ import litellm
import os
response = await litellm.acompletion(
- model="azure_ai/azure-model-router",
+ model="azure_ai/model_router/azure-model-router", # Use your deployment name
messages=[{"role": "user", "content": "hi"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"),
@@ -51,13 +77,15 @@ async for chunk in response:
```yaml
model_list:
- - model_name: azure-model-router
+ - model_name: azure-model-router # Public name for your users
litellm_params:
- model: azure_ai/azure-model-router
+ model: azure_ai/model_router/azure-model-router # Use your deployment name
api_base: https://your-endpoint.cognitiveservices.azure.com/openai/v1/
api_key: os.environ/AZURE_MODEL_ROUTER_API_KEY
```
+**Note:** Replace `azure-model-router` in the model path with your actual deployment name from Azure AI Foundry.
+
### Start Proxy
```bash
@@ -80,49 +108,42 @@ curl -X POST http://localhost:4000/chat/completions \
This walkthrough shows how to add an Azure Model Router endpoint to LiteLLM using the Admin Dashboard.
-### Select Provider
+### Quick Start
+
+1. Navigate to the **Models** page in the LiteLLM UI
+2. Select **"Azure AI Foundry (Studio)"** as the provider
+3. Enter your deployment name (e.g., `azure-model-router`)
+4. LiteLLM will automatically format it as `azure_ai/model_router/azure-model-router`
+5. Add your API base URL and API key
+6. Test and save
+
+### Detailed Walkthrough
+
+#### Step 1: Select Provider
Navigate to the Models page and select "Azure AI Foundry (Studio)" as the provider.
-#### Navigate to Models Page
+##### Navigate to Models Page
-#### Click Provider Dropdown
+##### Click Provider Dropdown
-#### Choose Azure AI Foundry
+##### Choose Azure AI Foundry
-### Configure Model Name
+#### Step 2: Enter Deployment Name
-Set up the model name by entering `azure_ai/` followed by your model router deployment name from Azure.
+**New Simplified Method:** Just enter your deployment name directly in the text field. If your deployment name contains "model-router" or "model_router", LiteLLM will automatically format it as `azure_ai/model_router/`.
-#### Click Model Name Field
+**Example:**
+- Enter: `azure-model-router`
+- LiteLLM creates: `azure_ai/model_router/azure-model-router`
-
-
-#### Select Custom Model Name
-
-
-
-#### Enter LiteLLM Model Name
-
-
-
-#### Click Custom Model Name Field
-
-
-
-#### Type Model Prefix
-
-Type `azure_ai/` as the prefix.
-
-
-
-#### Copy Model Name from Azure Portal
+##### Copy Deployment Name from Azure Portal
Switch to Azure AI Foundry and copy your model router deployment name.
@@ -130,73 +151,79 @@ Switch to Azure AI Foundry and copy your model router deployment name.
-#### Paste Model Name
+##### Enter Deployment Name in LiteLLM
-Paste to get `azure_ai/azure-model-router`.
+Paste your deployment name (e.g., `azure-model-router`) directly into the text field.
-
+
-### Configure API Base and Key
+**What happens behind the scenes:**
+- You enter: `azure-model-router`
+- LiteLLM automatically detects this is a model router deployment
+- The full model path becomes: `azure_ai/model_router/azure-model-router`
+- When making API calls, only `azure-model-router` is sent to Azure
+
+#### Step 3: Configure API Base and Key
Copy the endpoint URL and API key from Azure portal.
-#### Copy API Base URL from Azure
+##### Copy API Base URL from Azure
-#### Enter API Base in LiteLLM
+##### Enter API Base in LiteLLM
-#### Copy API Key from Azure
+##### Copy API Key from Azure
-#### Enter API Key in LiteLLM
+##### Enter API Key in LiteLLM
-### Test and Add Model
+#### Step 4: Test and Add Model
Verify your configuration works and save the model.
-#### Test Connection
+##### Test Connection
-#### Close Test Dialog
+##### Close Test Dialog
-#### Add Model
+##### Add Model
-### Verify in Playground
+#### Step 5: Verify in Playground
Test your model and verify cost tracking is working.
-#### Open Playground
+##### Open Playground
-#### Select Model
+##### Select Model
-#### Send Test Message
+##### Send Test Message
-#### View Logs
+##### View Logs
-#### Verify Cost Tracking
+##### Verify Cost Tracking
-Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`).
+Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`), plus a flat infrastructure cost of $0.14 per million input tokens for using the Model Router.
@@ -205,28 +232,50 @@ Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`).
LiteLLM automatically handles cost tracking for Azure Model Router by:
1. **Detecting the actual model**: When Azure Model Router routes your request to a specific model (e.g., `gpt-4.1-nano-2025-04-14`), LiteLLM extracts this from the response
-2. **Calculating accurate costs**: Costs are calculated based on the actual model used, not the router endpoint name
+2. **Calculating accurate costs**: Costs are calculated based on:
+ - The actual model used (e.g., `gpt-4.1-nano` token costs)
+ - Plus a flat infrastructure cost of **$0.14 per million input tokens** for using the Model Router
3. **Streaming support**: Cost tracking works correctly for both streaming and non-streaming requests
+### Cost Breakdown
+
+When you use Azure Model Router, the total cost includes:
+
+- **Model Cost**: Based on the actual model that handled your request (e.g., `gpt-4.1-nano`)
+- **Router Flat Cost**: $0.14 per million input tokens (Azure AI Foundry infrastructure fee)
+
### Example Response with Cost
```python
import litellm
response = litellm.completion(
- model="azure_ai/azure-model-router",
+ model="azure_ai/model_router/azure-model-router",
messages=[{"role": "user", "content": "Hello!"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key="your-api-key",
)
# The response will show the actual model used
-print(f"Model used: {response.model}") # e.g., "gpt-4.1-nano-2025-04-14"
+print(f"Model used: {response.model}") # e.g., "azure_ai/gpt-4.1-nano-2025-04-14"
-# Get cost
+# Get cost (includes both model cost and router flat cost)
from litellm import completion_cost
cost = completion_cost(completion_response=response)
-print(f"Cost: ${cost}")
+print(f"Total cost: ${cost}")
+
+# Access detailed cost breakdown
+if hasattr(response, '_hidden_params') and 'response_cost' in response._hidden_params:
+ print(f"Response cost: ${response._hidden_params['response_cost']}")
```
+### Viewing Cost Breakdown in UI
+
+When viewing logs in the LiteLLM UI, you'll see:
+- **Model Cost**: The cost for the actual model used
+- **Azure Model Router Flat Cost**: The $0.14/M input tokens infrastructure fee
+- **Total Cost**: Sum of both costs
+
+This breakdown helps you understand exactly what you're paying for when using the Model Router.
+
diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md
index 487212ad655..e546ed97656 100644
--- a/docs/my-website/docs/providers/bedrock.md
+++ b/docs/my-website/docs/providers/bedrock.md
@@ -9,7 +9,7 @@ ALL Bedrock models (Anthropic, Meta, Deepseek, Mistral, Amazon, etc.) are Suppor
| Description | Amazon Bedrock is a fully managed service that offers a choice of high-performing foundation models (FMs). |
| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models), [`bedrock/qwen2/`](./bedrock_imported.md#qwen2-imported-models), [`bedrock/openai/`](./bedrock_imported.md#openai-compatible-imported-models-qwen-25-vl-etc), [`bedrock/moonshot`](./bedrock_imported.md#moonshot-kimi-k2-thinking) |
| Provider Doc | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) |
-| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings`, `/images/generations` |
+| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings`, `/images/generations`, `/v1/realtime`|
| Rerank Endpoint | `/rerank` |
| Pass-through Endpoint | [Supported](../pass_through/bedrock.md) |
diff --git a/docs/my-website/docs/providers/bedrock_realtime_with_audio.md b/docs/my-website/docs/providers/bedrock_realtime_with_audio.md
new file mode 100644
index 00000000000..a2d9813ffd9
--- /dev/null
+++ b/docs/my-website/docs/providers/bedrock_realtime_with_audio.md
@@ -0,0 +1,362 @@
+# Bedrock Realtime API
+
+## Overview
+
+Amazon Bedrock's Nova Sonic model supports real-time bidirectional audio streaming for voice conversations. This tutorial shows how to use it through LiteLLM Proxy.
+
+## Setup
+
+### 1. Configure LiteLLM Proxy
+
+Create a `config.yaml` file:
+
+```yaml
+model_list:
+ - model_name: "bedrock-sonic"
+ litellm_params:
+ model: bedrock/amazon.nova-sonic-v1:0
+ aws_region_name: us-east-1 # or your preferred region
+ model_info:
+ mode: realtime
+```
+
+### 2. Start LiteLLM Proxy
+
+```bash
+litellm --config config.yaml
+```
+
+## Basic Text Interaction
+
+```python
+import asyncio
+import websockets
+import json
+
+LITELLM_API_KEY = "sk-1234" # Your LiteLLM API key
+LITELLM_URL = 'ws://localhost:4000/v1/realtime?model=bedrock-sonic'
+
+async def test_text_conversation():
+ async with websockets.connect(
+ LITELLM_URL,
+ additional_headers={
+ "Authorization": f"Bearer {LITELLM_API_KEY}"
+ }
+ ) as ws:
+ # Wait for session.created
+ response = await ws.recv()
+ print(f"Connected: {json.loads(response)['type']}")
+
+ # Configure session
+ session_update = {
+ "type": "session.update",
+ "session": {
+ "instructions": "You are a helpful assistant.",
+ "modalities": ["text"],
+ "temperature": 0.8
+ }
+ }
+ await ws.send(json.dumps(session_update))
+
+ # Send a message
+ message = {
+ "type": "conversation.item.create",
+ "item": {
+ "type": "message",
+ "role": "user",
+ "content": [{"type": "input_text", "text": "Hello!"}]
+ }
+ }
+ await ws.send(json.dumps(message))
+
+ # Trigger response
+ await ws.send(json.dumps({"type": "response.create"}))
+
+ # Listen for response
+ while True:
+ response = await ws.recv()
+ event = json.loads(response)
+
+ if event['type'] == 'response.text.delta':
+ print(event['delta'], end='', flush=True)
+ elif event['type'] == 'response.done':
+ print("\n✓ Complete")
+ break
+
+if __name__ == "__main__":
+ asyncio.run(test_text_conversation())
+```
+
+## Audio Streaming with Voice Conversation
+
+```python
+import asyncio
+import websockets
+import json
+import base64
+import pyaudio
+
+LITELLM_API_KEY = "sk-1234"
+LITELLM_URL = 'ws://localhost:4000/v1/realtime?model=bedrock-sonic'
+
+# Audio configuration
+INPUT_RATE = 16000 # Nova Sonic expects 16kHz input
+OUTPUT_RATE = 24000 # Nova Sonic outputs 24kHz
+CHUNK = 1024
+
+async def audio_conversation():
+ # Initialize PyAudio
+ p = pyaudio.PyAudio()
+
+ # Input stream (microphone)
+ input_stream = p.open(
+ format=pyaudio.paInt16,
+ channels=1,
+ rate=INPUT_RATE,
+ input=True,
+ frames_per_buffer=CHUNK
+ )
+
+ # Output stream (speakers)
+ output_stream = p.open(
+ format=pyaudio.paInt16,
+ channels=1,
+ rate=OUTPUT_RATE,
+ output=True,
+ frames_per_buffer=CHUNK
+ )
+
+ async with websockets.connect(
+ LITELLM_URL,
+ additional_headers={"Authorization": f"Bearer {LITELLM_API_KEY}"}
+ ) as ws:
+ # Wait for session.created
+ await ws.recv()
+ print("✓ Connected")
+
+ # Configure session with audio
+ session_update = {
+ "type": "session.update",
+ "session": {
+ "instructions": "You are a friendly voice assistant.",
+ "modalities": ["text", "audio"],
+ "voice": "matthew",
+ "input_audio_format": "pcm16",
+ "output_audio_format": "pcm16"
+ }
+ }
+ await ws.send(json.dumps(session_update))
+ print("🎤 Speak into your microphone...")
+
+ async def send_audio():
+ """Capture and send audio from microphone"""
+ while True:
+ audio_data = input_stream.read(CHUNK, exception_on_overflow=False)
+ audio_b64 = base64.b64encode(audio_data).decode('utf-8')
+ await ws.send(json.dumps({
+ "type": "input_audio_buffer.append",
+ "audio": audio_b64
+ }))
+ await asyncio.sleep(0.01)
+
+ async def receive_audio():
+ """Receive and play audio responses"""
+ while True:
+ response = await ws.recv()
+ event = json.loads(response)
+
+ if event['type'] == 'response.audio.delta':
+ audio_b64 = event.get('delta', '')
+ if audio_b64:
+ audio_bytes = base64.b64decode(audio_b64)
+ output_stream.write(audio_bytes)
+
+ elif event['type'] == 'response.text.delta':
+ print(event['delta'], end='', flush=True)
+
+ elif event['type'] == 'response.done':
+ print("\n✓ Response complete")
+
+ # Run both tasks concurrently
+ await asyncio.gather(send_audio(), receive_audio())
+
+if __name__ == "__main__":
+ try:
+ asyncio.run(audio_conversation())
+ except KeyboardInterrupt:
+ print("\n\nGoodbye!")
+```
+
+## Using Tools/Function Calling
+
+```python
+import asyncio
+import websockets
+import json
+from datetime import datetime
+
+LITELLM_API_KEY = "sk-1234"
+LITELLM_URL = 'ws://localhost:4000/v1/realtime?model=bedrock-sonic'
+
+# Define tools
+TOOLS = [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Get current weather for a location",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "location": {
+ "type": "string",
+ "description": "City name"
+ }
+ },
+ "required": ["location"]
+ }
+ }
+ }
+]
+
+def get_weather(location: str) -> dict:
+ """Simulated weather function"""
+ return {
+ "location": location,
+ "temperature": 72,
+ "conditions": "sunny"
+ }
+
+async def conversation_with_tools():
+ async with websockets.connect(
+ LITELLM_URL,
+ additional_headers={"Authorization": f"Bearer {LITELLM_API_KEY}"}
+ ) as ws:
+ # Wait for session.created
+ await ws.recv()
+
+ # Configure session with tools
+ session_update = {
+ "type": "session.update",
+ "session": {
+ "instructions": "You are a helpful assistant with access to tools.",
+ "modalities": ["text"],
+ "tools": TOOLS
+ }
+ }
+ await ws.send(json.dumps(session_update))
+
+ # Send a message that requires a tool
+ message = {
+ "type": "conversation.item.create",
+ "item": {
+ "type": "message",
+ "role": "user",
+ "content": [{"type": "input_text", "text": "What's the weather in San Francisco?"}]
+ }
+ }
+ await ws.send(json.dumps(message))
+ await ws.send(json.dumps({"type": "response.create"}))
+
+ # Handle responses and tool calls
+ while True:
+ response = await ws.recv()
+ event = json.loads(response)
+
+ if event['type'] == 'response.text.delta':
+ print(event['delta'], end='', flush=True)
+
+ elif event['type'] == 'response.function_call_arguments.done':
+ # Execute the tool
+ function_name = event['name']
+ arguments = json.loads(event['arguments'])
+
+ print(f"\n🔧 Calling {function_name}({arguments})")
+ result = get_weather(**arguments)
+
+ # Send tool result back
+ tool_result = {
+ "type": "conversation.item.create",
+ "item": {
+ "type": "function_call_output",
+ "call_id": event['call_id'],
+ "output": json.dumps(result)
+ }
+ }
+ await ws.send(json.dumps(tool_result))
+ await ws.send(json.dumps({"type": "response.create"}))
+
+ elif event['type'] == 'response.done':
+ print("\n✓ Complete")
+ break
+
+if __name__ == "__main__":
+ asyncio.run(conversation_with_tools())
+```
+
+## Configuration Options
+
+### Voice Options
+Available voices: `matthew`, `joanna`, `ruth`, `stephen`, `gregory`, `amy`
+
+### Audio Formats
+- **Input**: 16kHz PCM16 (mono)
+- **Output**: 24kHz PCM16 (mono)
+
+### Modalities
+- `["text"]` - Text only
+- `["audio"]` - Audio only
+- `["text", "audio"]` - Both text and audio
+
+## Example Test Scripts
+
+Complete working examples are available in the LiteLLM repository:
+
+- **Basic audio streaming**: `test_bedrock_realtime_client.py`
+- **Simple text test**: `test_bedrock_realtime_simple.py`
+- **Tool calling**: `test_bedrock_realtime_tools.py`
+
+## Requirements
+
+```bash
+pip install litellm websockets pyaudio
+```
+
+## AWS Configuration
+
+Ensure your AWS credentials are configured:
+
+```bash
+export AWS_ACCESS_KEY_ID=your_access_key
+export AWS_SECRET_ACCESS_KEY=your_secret_key
+export AWS_REGION_NAME=us-east-1
+```
+
+Or use AWS CLI configuration:
+
+```bash
+aws configure
+```
+
+## Troubleshooting
+
+### Connection Issues
+- Ensure LiteLLM proxy is running on the correct port
+- Verify AWS credentials are properly configured
+- Check that the Bedrock model is available in your region
+
+### Audio Issues
+- Verify PyAudio is properly installed
+- Check microphone/speaker permissions
+- Ensure correct sample rates (16kHz input, 24kHz output)
+
+### Tool Calling Issues
+- Ensure tools are properly defined in session.update
+- Verify tool results are sent back with correct call_id
+- Check that response.create is sent after tool result
+
+## Related Resources
+
+- [OpenAI Realtime API Documentation](https://platform.openai.com/docs/guides/realtime)
+- [Amazon Bedrock Nova Sonic Documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/nova-sonic.html)
+- [LiteLLM Realtime API Documentation](/docs/realtime)
diff --git a/docs/my-website/docs/providers/dashscope.md b/docs/my-website/docs/providers/dashscope.md
index 565776d6c4c..3df0fbab1ba 100644
--- a/docs/my-website/docs/providers/dashscope.md
+++ b/docs/my-website/docs/providers/dashscope.md
@@ -1,7 +1,7 @@
-# Dashscope (Qwen API)
+# Dashscope API (Qwen models)
https://dashscope.console.aliyun.com/
-**We support ALL Qwen models, just set `dashscope/` as a prefix when sending completion requests**
+**We support ALL Qwen models (from Alibaba Cloud), just set `dashscope/` as a prefix when sending completion requests**
## API Key
```python
@@ -9,6 +9,26 @@ https://dashscope.console.aliyun.com/
os.environ['DASHSCOPE_API_KEY']
```
+## API Base
+You can optionally specify the API base URL depending on your region:
+
+| Region | API Base |
+|--------|----------|
+| **International** | `https://dashscope-intl.aliyuncs.com/compatible-mode/v1` |
+| **China/Beijing** | `https://dashscope.aliyuncs.com/compatible-mode/v1` |
+
+```python
+# Set via environment variable
+os.environ['DASHSCOPE_API_BASE'] = "https://dashscope-intl.aliyuncs.com/compatible-mode/v1"
+
+# Or pass directly in the completion call
+response = completion(
+ model="dashscope/qwen-turbo",
+ messages=[{"role": "user", "content": "hello"}],
+ api_base="https://dashscope-intl.aliyuncs.com/compatible-mode/v1"
+)
+```
+
## Sample Usage
```python
from litellm import completion
@@ -43,9 +63,7 @@ for chunk in response:
```
-## Supported Models - ALL Qwen Models Supported!
-We support ALL Qwen models, just set `dashscope/` as a prefix when sending completion requests
-
+## All supported Models
[DashScope Model List](https://help.aliyun.com/zh/model-studio/compatibility-of-openai-with-dashscope?spm=a2c4g.11186623.help-menu-2400256.d_2_8_0.1efd516e2tTXBn&scm=20140722.H_2833609._.OR_help-T_cn~zh-V_1#7f9c78ae99pwz)
diff --git a/docs/my-website/docs/providers/elevenlabs.md b/docs/my-website/docs/providers/elevenlabs.md
index 5cf62f51203..b4ed3d3346b 100644
--- a/docs/my-website/docs/providers/elevenlabs.md
+++ b/docs/my-website/docs/providers/elevenlabs.md
@@ -243,6 +243,13 @@ ElevenLabs provides high-quality text-to-speech capabilities through their TTS A
| Supported Operations | `/audio/speech` |
| Link to Provider Doc | [ElevenLabs TTS API ↗](https://elevenlabs.io/docs/api-reference/text-to-speech) |
+### Supported Models
+
+| Model | Route | Description |
+|-------|-------|-------------|
+| Eleven v3 | `elevenlabs/eleven_v3` | Most expressive model. 70+ languages, audio tags support for sound effects and pauses. |
+| Eleven Multilingual v2 | `elevenlabs/eleven_multilingual_v2` | Default TTS model. 29 languages, stable and production-ready. |
+
### Quick Start
#### LiteLLM Python SDK
@@ -265,6 +272,26 @@ with open("test_output.mp3", "wb") as f:
f.write(audio.read())
```
+#### Using Eleven v3 with Audio Tags
+
+Eleven v3 supports [audio tags](https://elevenlabs.io/docs/overview/capabilities/text-to-speech#audio-tags) for adding sound effects and pauses directly in the text:
+
+```python showLineNumbers title="Eleven v3 with audio tags"
+import litellm
+import os
+
+os.environ["ELEVENLABS_API_KEY"] = "your-elevenlabs-api-key"
+
+audio = litellm.speech(
+ model="elevenlabs/eleven_v3",
+ input='Welcome back. applause Today we have a special guest. Let me introduce them.',
+ voice="alloy",
+)
+
+with open("eleven_v3_output.mp3", "wb") as f:
+ f.write(audio.read())
+```
+
#### Advanced Usage: Overriding Parameters and ElevenLabs-Specific Features
```python showLineNumbers title="Advanced TTS with custom parameters"
diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md
index 23a02f7365c..6de2263916c 100644
--- a/docs/my-website/docs/providers/gemini.md
+++ b/docs/my-website/docs/providers/gemini.md
@@ -1196,6 +1196,8 @@ When responding to Computer Use tool calls, include the URL and screenshot:
+
+
## Thought Signatures
Thought signatures are encrypted representations of the model's internal reasoning process for a given turn in a conversation. By passing thought signatures back to the model in subsequent requests, you provide it with the context of its previous thoughts, allowing it to build upon its reasoning and maintain a coherent line of inquiry.
@@ -1840,6 +1842,57 @@ content = response.get('choices', [{}])[0].get('message', {}).get('content')
print(content)
```
+## gemini-robotics-er-1.5-preview Usage
+
+```python
+from litellm import api_base
+from openai import OpenAI
+import os
+import base64
+
+client = OpenAI(base_url="http://0.0.0.0:4000", api_key="sk-12345")
+base64_image = base64.b64encode(open("closeup-object-on-table-many-260nw-1216144471.webp", "rb").read()).decode()
+
+import json
+import re
+tools = [{"codeExecution": {}}]
+response = client.chat.completions.create(
+ model="gemini/gemini-robotics-er-1.5-preview",
+ messages=[
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Point to no more than 10 items in the image. The label returned should be an identifying name for the object detected. The answer should follow the json format: [{\"point\": [y, x], \"label\": }, ...]. The points are in [y, x] format normalized to 0-1000."
+ },
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}
+ }
+ ]
+ }
+ ],
+ tools=tools
+)
+
+# Extract JSON from markdown code block if present
+content = response.choices[0].message.content
+# Look for triple-backtick JSON block
+match = re.search(r'```json\s*(.*?)\s*```', content, re.DOTALL)
+if match:
+ json_str = match.group(1)
+else:
+ json_str = content
+
+try:
+ data = json.loads(json_str)
+ print(json.dumps(data, indent=2))
+except Exception as e:
+ print("Error parsing response as JSON:", e)
+ print("Response content:", content)
+```
+
## Usage - PDF / Videos / etc. Files
### Inline Data (e.g. audio stream)
diff --git a/docs/my-website/docs/providers/github_copilot.md b/docs/my-website/docs/providers/github_copilot.md
index 306c9f949ec..e9fd3444f5f 100644
--- a/docs/my-website/docs/providers/github_copilot.md
+++ b/docs/my-website/docs/providers/github_copilot.md
@@ -35,11 +35,10 @@ from litellm import completion
response = completion(
model="github_copilot/gpt-4",
- messages=[{"role": "user", "content": "Write a Python function to calculate fibonacci numbers"}],
- extra_headers={
- "editor-version": "vscode/1.85.1",
- "Copilot-Integration-Id": "vscode-chat"
- }
+ messages=[
+ {"role": "system", "content": "You are a helpful coding assistant"},
+ {"role": "user", "content": "Write a Python function to calculate fibonacci numbers"}
+ ]
)
print(response)
```
@@ -50,11 +49,7 @@ from litellm import completion
stream = completion(
model="github_copilot/gpt-4",
messages=[{"role": "user", "content": "Explain async/await in Python"}],
- stream=True,
- extra_headers={
- "editor-version": "vscode/1.85.1",
- "Copilot-Integration-Id": "vscode-chat"
- }
+ stream=True
)
for chunk in stream:
@@ -134,11 +129,7 @@ client = OpenAI(
# Non-streaming response
response = client.chat.completions.create(
model="github_copilot/gpt-4",
- messages=[{"role": "user", "content": "How do I optimize this SQL query?"}],
- extra_headers={
- "editor-version": "vscode/1.85.1",
- "Copilot-Integration-Id": "vscode-chat"
- }
+ messages=[{"role": "user", "content": "How do I optimize this SQL query?"}]
)
print(response.choices[0].message.content)
@@ -156,11 +147,7 @@ response = litellm.completion(
model="litellm_proxy/github_copilot/gpt-4",
messages=[{"role": "user", "content": "Review this code for bugs"}],
api_base="http://localhost:4000",
- api_key="your-proxy-api-key",
- extra_headers={
- "editor-version": "vscode/1.85.1",
- "Copilot-Integration-Id": "vscode-chat"
- }
+ api_key="your-proxy-api-key"
)
print(response.choices[0].message.content)
@@ -174,8 +161,6 @@ print(response.choices[0].message.content)
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-proxy-api-key" \
- -H "editor-version: vscode/1.85.1" \
- -H "Copilot-Integration-Id: vscode-chat" \
-d '{
"model": "github_copilot/gpt-4",
"messages": [{"role": "user", "content": "Explain this error message"}]
@@ -211,9 +196,11 @@ export GITHUB_COPILOT_API_KEY_FILE="api-key.json"
### Headers
-GitHub Copilot supports various editor-specific headers:
+LiteLLM automatically injects the required GitHub Copilot headers (simulating VSCode). You don't need to specify them manually.
-```python showLineNumbers title="Common Headers"
+If you want to override the defaults (e.g., to simulate a different editor), you can use `extra_headers`:
+
+```python showLineNumbers title="Custom Headers (Optional)"
extra_headers = {
"editor-version": "vscode/1.85.1", # Editor version
"editor-plugin-version": "copilot/1.155.0", # Plugin version
diff --git a/docs/my-website/docs/providers/litellm_proxy.md b/docs/my-website/docs/providers/litellm_proxy.md
index bfefc8a787c..918ac6755a5 100644
--- a/docs/my-website/docs/providers/litellm_proxy.md
+++ b/docs/my-website/docs/providers/litellm_proxy.md
@@ -227,6 +227,28 @@ response = litellm.completion(
)
```
+## OAuth2/JWT Authentication
+
+If your LiteLLM Proxy requires OAuth2/JWT authentication (e.g., Azure AD, Keycloak, Okta), the SDK can automatically obtain and refresh tokens for you.
+
+```python
+import litellm
+from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler
+
+litellm.proxy_auth = ProxyAuthHandler(
+ credential=AzureADCredential(),
+ scope="api://my-litellm-proxy/.default"
+)
+litellm.api_base = "https://my-proxy.example.com"
+
+response = litellm.completion(
+ model="gpt-4",
+ messages=[{"role": "user", "content": "Hello!"}]
+)
+```
+
+[Learn more about SDK Proxy Authentication (OAuth2/JWT Auto-Refresh) →](../proxy_auth)
+
## Sending `tags` to LiteLLM Proxy
Tags allow you to categorize and track your API requests for monitoring, debugging, and analytics purposes. You can send tags as a list of strings to the LiteLLM Proxy using the `extra_body` parameter.
diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md
index 80645a51ac5..23940e1c54e 100644
--- a/docs/my-website/docs/providers/openai.md
+++ b/docs/my-website/docs/providers/openai.md
@@ -230,7 +230,70 @@ os.environ["OPENAI_BASE_URL"] = "https://your_host/v1" # OPTIONAL
These also support the `OPENAI_BASE_URL` environment variable, which can be used to specify a custom API endpoint.
-## OpenAI Vision Models
+### OpenAI Web Search Models
+
+OpenAI has two ways to use web search, depending on the endpoint:
+
+| Approach | Endpoint | Models | How to enable |
+|----------|----------|--------|---------------|
+| **Search Models** | `/chat/completions` | `gpt-5-search-api`, `gpt-4o-search-preview`, `gpt-4o-mini-search-preview` | Pass `web_search_options` parameter |
+| **Web Search Tool** | `/responses` | `gpt-5`, `gpt-4.1`, `gpt-4o`, and other regular models | Pass `web_search_preview` tool |
+
+
+
+
+```python showLineNumbers
+from litellm import completion
+
+response = completion(
+ model="openai/gpt-5-search-api",
+ messages=[{"role": "user", "content": "What is the capital of France?"}],
+ web_search_options={
+ "search_context_size": "medium" # Options: "low", "medium", "high"
+ }
+)
+```
+
+
+
+
+```python showLineNumbers
+from litellm import responses
+
+response = responses(
+ model="openai/gpt-5",
+ input="What is the capital of France?",
+ tools=[{
+ "type": "web_search_preview",
+ "search_context_size": "low"
+ }]
+)
+```
+
+
+
+
+```yaml
+model_list:
+ # Search model for /chat/completions
+ - model_name: gpt-5-search-api
+ litellm_params:
+ model: openai/gpt-5-search-api
+ api_key: os.environ/OPENAI_API_KEY
+
+ # Regular model for /responses with web_search_preview tool
+ - model_name: gpt-5
+ litellm_params:
+ model: openai/gpt-5
+ api_key: os.environ/OPENAI_API_KEY
+```
+
+
+
+
+For full details, see the [Web Search guide](../completion/web_search.md).
+
+## OpenAI Vision Models
| Model Name | Function Call |
|-----------------------|-----------------------------------------------------------------|
| gpt-4o | `response = completion(model="gpt-4o", messages=messages)` |
diff --git a/docs/my-website/docs/providers/openai/responses_api.md b/docs/my-website/docs/providers/openai/responses_api.md
index 75eab1afac5..7799c93ccf2 100644
--- a/docs/my-website/docs/providers/openai/responses_api.md
+++ b/docs/my-website/docs/providers/openai/responses_api.md
@@ -37,6 +37,24 @@ for event in response:
print(event)
```
+#### Web Search
+```python showLineNumbers title="OpenAI Responses with Web Search"
+import litellm
+
+response = litellm.responses(
+ model="openai/gpt-5",
+ input="What is the capital of France?",
+ tools=[{
+ "type": "web_search_preview",
+ "search_context_size": "medium" # Options: "low", "medium", "high"
+ }]
+)
+
+print(response)
+```
+
+For full details, see the [Web Search guide](../../completion/web_search.md).
+
#### Image Generation with Streaming
```python showLineNumbers title="OpenAI Streaming Image Generation"
import litellm
diff --git a/docs/my-website/docs/providers/perplexity.md b/docs/my-website/docs/providers/perplexity.md
index 2fcb49c60fa..e3991c63bff 100644
--- a/docs/my-website/docs/providers/perplexity.md
+++ b/docs/my-website/docs/providers/perplexity.md
@@ -120,6 +120,370 @@ All models listed here https://docs.perplexity.ai/docs/model-cards are supported
+## Agent API (Responses API)
+
+Requires v1.72.6+
+
+
+### Using Presets
+
+Presets provide optimized defaults for specific use cases. Start with a preset for quick setup:
+
+
+
+
+```python
+from litellm import responses
+import os
+
+os.environ['PERPLEXITY_API_KEY'] = ""
+
+# Using the pro-search preset
+response = responses(
+ model="perplexity/preset/pro-search",
+ input="What are the latest developments in AI?",
+ custom_llm_provider="perplexity",
+)
+
+print(response.output)
+```
+
+
+
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+ - model_name: perplexity-pro-search
+ litellm_params:
+ model: perplexity/preset/pro-search
+ api_key: os.environ/PERPLEXITY_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+```bash
+curl http://0.0.0.0:4000/v1/responses \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer anything" \
+ -d '{
+ "model": "perplexity-pro-search",
+ "input": "What are the latest developments in AI?"
+ }'
+```
+
+
+
+
+### Using Third-Party Models
+
+Access models from OpenAI, Anthropic, Google, xAI, and other providers through Perplexity's unified API:
+
+
+
+
+```python
+from litellm import responses
+import os
+
+os.environ['PERPLEXITY_API_KEY'] = ""
+
+response = responses(
+ model="perplexity/openai/gpt-5.2",
+ input="Explain quantum computing in simple terms",
+ custom_llm_provider="perplexity",
+ max_output_tokens=500,
+)
+
+print(response.output)
+```
+
+
+
+
+```python
+from litellm import responses
+import os
+
+os.environ['PERPLEXITY_API_KEY'] = ""
+
+response = responses(
+ model="perplexity/anthropic/claude-sonnet-4-5",
+ input="Write a short story about a robot learning to paint",
+ custom_llm_provider="perplexity",
+ max_output_tokens=500,
+)
+
+print(response.output)
+```
+
+
+
+
+```python
+from litellm import responses
+import os
+
+os.environ['PERPLEXITY_API_KEY'] = ""
+
+response = responses(
+ model="perplexity/google/gemini-2.5-flash",
+ input="Explain the concept of neural networks",
+ custom_llm_provider="perplexity",
+ max_output_tokens=500,
+)
+
+print(response.output)
+```
+
+
+
+
+```python
+from litellm import responses
+import os
+
+os.environ['PERPLEXITY_API_KEY'] = ""
+
+response = responses(
+ model="perplexity/xai/grok-4-1-fast-non-reasoning",
+ input="What makes a good AI assistant?",
+ custom_llm_provider="perplexity",
+ max_output_tokens=500,
+)
+
+print(response.output)
+```
+
+
+
+
+### Web Search Tool
+
+Enable web search capabilities to access real-time information:
+
+```python
+from litellm import responses
+import os
+
+os.environ['PERPLEXITY_API_KEY'] = ""
+
+response = responses(
+ model="perplexity/openai/gpt-5.2",
+ input="What's the weather in San Francisco today?",
+ custom_llm_provider="perplexity",
+ tools=[{"type": "web_search"}],
+ instructions="You have access to a web_search tool. Use it for questions about current events.",
+)
+
+print(response.output)
+```
+
+### Function Calling
+
+The Agent API supports custom function tools. Pass function tools through unchanged:
+
+```python
+from litellm import responses
+import os
+
+os.environ['PERPLEXITY_API_KEY'] = ""
+
+response = responses(
+ model="perplexity/openai/gpt-5.2",
+ input="What's the weather in San Francisco?",
+ custom_llm_provider="perplexity",
+ tools=[
+ {"type": "web_search"},
+ {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Get the current weather for a location",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "location": {"type": "string"},
+ "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
+ },
+ },
+ },
+ },
+ ],
+ instructions="Use tools when appropriate.",
+)
+
+print(response.output)
+```
+
+### Structured Outputs
+
+Request JSON schema structured outputs via the `text` parameter:
+
+```python
+from litellm import responses
+import os
+
+os.environ['PERPLEXITY_API_KEY'] = ""
+
+response = responses(
+ model="perplexity/preset/pro-search",
+ input="Extract key facts about the Eiffel Tower",
+ custom_llm_provider="perplexity",
+ text={
+ "format": {
+ "type": "json_schema",
+ "name": "facts",
+ "schema": {
+ "type": "object",
+ "properties": {
+ "name": {"type": "string"},
+ "height_meters": {"type": "number"},
+ "year_built": {"type": "integer"},
+ },
+ "required": ["name", "height_meters", "year_built"],
+ },
+ "strict": True,
+ }
+ },
+)
+
+print(response.output)
+```
+
+
+### Reasoning Effort (Responses API)
+
+Control the reasoning effort level for reasoning-capable models:
+
+```python
+from litellm import responses
+import os
+
+os.environ['PERPLEXITY_API_KEY'] = ""
+
+response = responses(
+ model="perplexity/openai/gpt-5.2",
+ input="Solve this complex problem step by step",
+ custom_llm_provider="perplexity",
+ reasoning={"effort": "high"}, # Options: low, medium, high
+ max_output_tokens=1000,
+)
+
+print(response.output)
+```
+
+### Multi-Turn Conversations
+
+Use message arrays for multi-turn conversations with context:
+
+```python
+from litellm import responses
+import os
+
+os.environ['PERPLEXITY_API_KEY'] = ""
+
+response = responses(
+ model="perplexity/anthropic/claude-sonnet-4-5",
+ input=[
+ {"type": "message", "role": "system", "content": "You are a helpful assistant."},
+ {"type": "message", "role": "user", "content": "What are the latest AI developments?"},
+ ],
+ custom_llm_provider="perplexity",
+ instructions="Provide detailed, well-researched answers.",
+ max_output_tokens=800,
+)
+
+print(response.output)
+```
+
+### Streaming Responses
+
+Stream responses for real-time output:
+
+```python
+from litellm import responses
+import os
+
+os.environ['PERPLEXITY_API_KEY'] = ""
+
+response = responses(
+ model="perplexity/openai/gpt-5.2",
+ input="Tell me a story about space exploration",
+ custom_llm_provider="perplexity",
+ stream=True,
+ max_output_tokens=500,
+)
+
+for chunk in response:
+ if hasattr(chunk, 'type'):
+ if chunk.type == "response.output_text.delta":
+ print(chunk.delta, end="", flush=True)
+```
+
+### Supported Third-Party Models
+
+| Provider | Model Name | Function Call |
+|----------|------------|---------------|
+| OpenAI | gpt-5.2 | `responses(model="perplexity/openai/gpt-5.2", ...)` |
+| OpenAI | gpt-5.1 | `responses(model="perplexity/openai/gpt-5.1", ...)` |
+| OpenAI | gpt-5-mini | `responses(model="perplexity/openai/gpt-5-mini", ...)` |
+| Anthropic | claude-opus-4-6 | `responses(model="perplexity/anthropic/claude-opus-4-6", ...)` |
+| Anthropic | claude-opus-4-5 | `responses(model="perplexity/anthropic/claude-opus-4-5", ...)` |
+| Anthropic | claude-sonnet-4-5 | `responses(model="perplexity/anthropic/claude-sonnet-4-5", ...)` |
+| Anthropic | claude-haiku-4-5 | `responses(model="perplexity/anthropic/claude-haiku-4-5", ...)` |
+| Google | gemini-3-pro-preview | `responses(model="perplexity/google/gemini-3-pro-preview", ...)` |
+| Google | gemini-3-flash-preview | `responses(model="perplexity/google/gemini-3-flash-preview", ...)` |
+| Google | gemini-2.5-pro | `responses(model="perplexity/google/gemini-2.5-pro", ...)` |
+| Google | gemini-2.5-flash | `responses(model="perplexity/google/gemini-2.5-flash", ...)` |
+| xAI | grok-4-1-fast-non-reasoning | `responses(model="perplexity/xai/grok-4-1-fast-non-reasoning", ...)` |
+| Perplexity | sonar | `responses(model="perplexity/perplexity/sonar", ...)` |
+
+### Available Presets
+
+| Preset Name | Function Call |
+|-------------|---------------|
+| fast-search | `responses(model="perplexity/preset/fast-search", ...)` |
+| pro-search | `responses(model="perplexity/preset/pro-search", ...)` |
+| deep-research | `responses(model="perplexity/preset/deep-research", ...)` |
+| advanced-deep-research | `responses(model="perplexity/preset/advanced-deep-research", ...)` |
+
+### Complete Example
+
+```python
+from litellm import responses
+import os
+
+os.environ['PERPLEXITY_API_KEY'] = ""
+
+# Comprehensive example with multiple features
+response = responses(
+ model="perplexity/openai/gpt-5.2",
+ input="Research the latest developments in quantum computing and provide sources",
+ custom_llm_provider="perplexity",
+ tools=[
+ {"type": "web_search"},
+ {"type": "fetch_url"}
+ ],
+ instructions="Use web_search to find relevant information and fetch_url to retrieve detailed content from sources. Provide citations for all claims.",
+ max_output_tokens=1000,
+ temperature=0.7,
+)
+
+print(f"Response ID: {response.id}")
+print(f"Model: {response.model}")
+print(f"Status: {response.status}")
+print(f"Output: {response.output}")
+print(f"Usage: {response.usage}")
+```
+
:::info
For more information about passing provider-specific parameters, [go here](../completion/provider_specific_params.md)
diff --git a/docs/my-website/docs/providers/sarvam.md b/docs/my-website/docs/providers/sarvam.md
new file mode 100644
index 00000000000..6a292456781
--- /dev/null
+++ b/docs/my-website/docs/providers/sarvam.md
@@ -0,0 +1,92 @@
+# Sarvam.ai
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+LiteLLM supports all the text models from [Sarvam ai](https://docs.sarvam.ai/api-reference-docs/chat/chat-completions)
+
+## Usage
+
+```python
+import os
+from litellm import completion
+
+# Set your Sarvam API key
+os.environ["SARVAM_API_KEY"] = ""
+
+messages = [{"role": "user", "content": "Hello"}]
+
+response = completion(
+ model="sarvam/sarvam-m",
+ messages=messages,
+)
+print(response)
+```
+
+## Usage with LiteLLM Proxy Server
+
+Here's how to call a Sarvam.ai model with the LiteLLM Proxy Server
+
+1. **Modify the `config.yaml`:**
+
+ ```yaml
+ model_list:
+ - model_name: my-model
+ litellm_params:
+ model: sarvam/ # add sarvam/ prefix to route as Sarvam provider
+ api_key: api-key # api key to send your model
+ ```
+
+2. **Start the proxy:**
+
+ ```bash
+ $ litellm --config /path/to/config.yaml
+ ```
+
+3. **Send a request to LiteLLM Proxy Server:**
+
+
+
+
+
+ ```python
+ import openai
+
+ client = openai.OpenAI(
+ api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys
+ base_url="http://0.0.0.0:4000" # litellm-proxy-base url
+ )
+
+ response = client.chat.completions.create(
+ model="my-model",
+ messages=[
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ )
+
+ print(response)
+ ```
+
+
+
+
+ ```shell
+ curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Authorization: Bearer sk-1234' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "my-model",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ]
+ }'
+ ```
+
+
+
diff --git a/docs/my-website/docs/providers/scaleway.md b/docs/my-website/docs/providers/scaleway.md
new file mode 100644
index 00000000000..ea57c24db30
--- /dev/null
+++ b/docs/my-website/docs/providers/scaleway.md
@@ -0,0 +1,62 @@
+
+# Scaleway
+LiteLLM supports all [models available on Scaleway Generative APIs ↗](https://www.scaleway.com/en/docs/generative-apis/reference-content/supported-models/).
+
+## Usage with LiteLLM Python SDK
+
+```python
+import os
+from litellm import completion
+
+os.environ["SCW_SECRET_KEY"] = "your-scaleway-secret-key"
+
+messages = [{"role": "user", "content": "Write a short poem"}]
+response = completion(model="scaleway/qwen3-235b-a22b-instruct-2507", messages=messages)
+print(response)
+```
+
+## Usage with LiteLLM Proxy
+
+### 1. Set Scaleway models in config.yaml
+
+```yaml
+model_list:
+ - model_name: scaleway-model
+ litellm_params:
+ model: scaleway/qwen3-235b-a22b-instruct-2507
+ api_key: "os.environ/SCW_SECRET_KEY" # ensure you have `SCW_SECRET_KEY` in your .env
+```
+
+### 2. Start proxy
+
+```bash
+litellm --config config.yaml
+```
+
+### 3. Query proxy
+
+Assuming the proxy is running on [http://localhost:4000](http://localhost:4000):
+```bash
+curl http://localhost:4000/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer YOUR_LITELLM_MASTER_KEY" \
+ -d '{
+ "model": "scaleway-model",
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant."
+ },
+ {
+ "role": "user",
+ "content": "Write a short poem"
+ }
+ ]
+ }'
+```
+`-H "Authorization: Bearer YOUR_LITELLM_MASTER_KEY" ` is only required if you have set a LiteLLM master key
+
+
+## Supported features
+
+Scaleway provider supports all features in [Generative APIs reference documentation ↗](https://www.scaleway.com/en/developers/api/generative-apis/), such as streaming, structured outputs and tool calling.
diff --git a/docs/my-website/docs/providers/vertex_speech.md b/docs/my-website/docs/providers/vertex_speech.md
index d0acacb5aec..751782a323c 100644
--- a/docs/my-website/docs/providers/vertex_speech.md
+++ b/docs/my-website/docs/providers/vertex_speech.md
@@ -312,6 +312,7 @@ Gemini models with audio output capabilities using the chat completions API.
- Only supports `pcm16` audio format
- Streaming not yet supported
- Must set `modalities: ["audio"]`
+- When using via LiteLLM Proxy, must include `"allowed_openai_params": ["audio", "modalities"]` in the request body to enable audio parameters
:::
### Quick Start
@@ -372,7 +373,8 @@ curl http://0.0.0.0:4000/v1/chat/completions \
"model": "gemini-tts",
"messages": [{"role": "user", "content": "Say hello in a friendly voice"}],
"modalities": ["audio"],
- "audio": {"voice": "Kore", "format": "pcm16"}
+ "audio": {"voice": "Kore", "format": "pcm16"},
+ "allowed_openai_params": ["audio", "modalities"]
}'
```
@@ -389,6 +391,7 @@ response = client.chat.completions.create(
messages=[{"role": "user", "content": "Say hello in a friendly voice"}],
modalities=["audio"],
audio={"voice": "Kore", "format": "pcm16"},
+ extra_body={"allowed_openai_params": ["audio", "modalities"]}
)
print(response)
```
diff --git a/docs/my-website/docs/providers/watsonx/rerank.md b/docs/my-website/docs/providers/watsonx/rerank.md
new file mode 100644
index 00000000000..0900ce96781
--- /dev/null
+++ b/docs/my-website/docs/providers/watsonx/rerank.md
@@ -0,0 +1,52 @@
+# watsonx.ai Rerank
+
+## Overview
+
+| Property | Details |
+|----------|--------------------------------------------------------------------------|
+| Description | watsonx.ai rerank integration |
+| Provider Route on LiteLLM | `watsonx/` |
+| Supported Operations | `/ml/v1/text/rerank` |
+| Link to Provider Doc | [IBM WatsonX.ai ↗](https://cloud.ibm.com/apidocs/watsonx-ai#text-rerank) |
+
+## Quick Start
+
+### **LiteLLM SDK**
+
+```python
+import os
+from litellm import rerank
+
+os.environ["WATSONX_APIKEY"] = "YOUR_WATSONX_APIKEY"
+os.environ["WATSONX_API_BASE"] = "YOUR_WATSONX_API_BASE"
+os.environ["WATSONX_PROJECT_ID"] = "YOUR_WATSONX_PROJECT_ID"
+
+query="Best programming language for beginners?"
+documents=[
+ "Python is great for beginners due to simple syntax.",
+ "JavaScript runs in browsers and is versatile.",
+ "Rust has a steep learning curve but is very safe.",
+]
+
+response = rerank(
+ model="watsonx/cross-encoder/ms-marco-minilm-l-12-v2",
+ query=query,
+ documents=documents,
+ top_n=2,
+ return_documents=True,
+)
+
+print(response)
+```
+
+### **LiteLLM Proxy**
+
+```yaml
+model_list:
+ - model_name: cross-encoder/ms-marco-minilm-l-12-v2
+ litellm_params:
+ model: watsonx/cross-encoder/ms-marco-minilm-l-12-v2
+ api_key: os.environ/WATSONX_APIKEY
+ api_base: os.environ/WATSONX_API_BASE
+ project_id: os.environ/WATSONX_PROJECT_ID
+```
diff --git a/docs/my-website/docs/providers/xai_realtime.md b/docs/my-website/docs/providers/xai_realtime.md
new file mode 100644
index 00000000000..b36908c4686
--- /dev/null
+++ b/docs/my-website/docs/providers/xai_realtime.md
@@ -0,0 +1,308 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# xAI Voice Agent (Realtime API)
+
+xAI's Grok Voice Agent provides real-time voice conversation capabilities through WebSocket connections, enabling natural bidirectional audio interactions.
+
+| Feature | Description | Comments |
+| --- | --- | --- |
+| LiteLLM AI Gateway | ✅ | |
+| LiteLLM Python SDK | ✅ | Full support via `litellm.realtime()` |
+
+## Quick Start
+
+### Supported Model
+
+| Model | Context | Features |
+|-------|---------|----------|
+| `xai/grok-4-1-fast-non-reasoning` | 2M tokens | Voice conversation, Function calling, Vision, Audio, Web search, Caching |
+
+**Note:** xAI Realtime API uses the non-reasoning variant for optimal real-time performance.
+
+## Python SDK Usage
+
+### Basic Realtime Connection
+
+```python
+import asyncio
+from litellm import realtime
+
+async def test_xai_realtime():
+ """
+ Test xAI Grok Voice Agent via LiteLLM SDK
+ """
+ # Initialize realtime connection
+ ws = await realtime(
+ model="xai/grok-4-1-fast-non-reasoning",
+ api_key="your-xai-api-key", # or set XAI_API_KEY env var
+ )
+
+ # Connection established, xAI sends "conversation.created" event
+ print("Connected to xAI Grok Voice Agent")
+
+ # Send a message
+ await ws.send_text(json.dumps({
+ "type": "conversation.item.create",
+ "item": {
+ "type": "message",
+ "role": "user",
+ "content": [{
+ "type": "input_text",
+ "text": "Hello! How are you?"
+ }]
+ }
+ }))
+
+ # Request a response
+ await ws.send_text(json.dumps({
+ "type": "response.create"
+ }))
+
+ # Listen for responses
+ async for message in ws:
+ data = json.loads(message)
+ print(f"Received: {data['type']}")
+
+ if data['type'] == 'response.done':
+ break
+
+ await ws.close()
+
+# Run the async function
+asyncio.run(test_xai_realtime())
+```
+
+### With Audio Input/Output
+
+```python
+import asyncio
+import json
+from litellm import realtime
+
+async def xai_voice_conversation():
+ """
+ Voice conversation with xAI Grok Voice Agent
+ """
+ ws = await realtime(
+ model="xai/grok-4-1-fast-non-reasoning",
+ api_key="your-xai-api-key",
+ )
+
+ # Send audio data (base64 encoded PCM16 24kHz)
+ await ws.send_text(json.dumps({
+ "type": "conversation.item.create",
+ "item": {
+ "type": "message",
+ "role": "user",
+ "content": [{
+ "type": "input_audio",
+ "audio": "base64_encoded_audio_data_here"
+ }]
+ }
+ }))
+
+ # Request response with audio
+ await ws.send_text(json.dumps({
+ "type": "response.create",
+ "response": {
+ "modalities": ["text", "audio"],
+ "instructions": "Please respond in a friendly tone."
+ }
+ }))
+
+ # Process streaming audio response
+ async for message in ws:
+ data = json.loads(message)
+
+ if data['type'] == 'response.audio.delta':
+ # Handle audio chunks
+ audio_chunk = data['delta']
+ # Process audio_chunk (play it, save it, etc.)
+
+ elif data['type'] == 'response.done':
+ break
+
+ await ws.close()
+
+asyncio.run(xai_voice_conversation())
+```
+
+## LiteLLM Proxy (AI Gateway) Usage
+
+Load balance across multiple xAI deployments or combine with other providers.
+
+### 1. Add Model to Config
+
+```yaml
+model_list:
+ - model_name: grok-voice-agent
+ litellm_params:
+ model: xai/grok-4-1-fast-non-reasoning
+ api_key: os.environ/XAI_API_KEY
+ model_info:
+ mode: realtime
+
+ # Optional: Add fallback to OpenAI
+ - model_name: grok-voice-agent
+ litellm_params:
+ model: openai/gpt-4o-realtime-preview-2024-10-01
+ api_key: os.environ/OPENAI_API_KEY
+ model_info:
+ mode: realtime
+```
+
+### 2. Start Proxy
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+### 3. Test Connection
+
+#### Python Client
+
+```python
+import asyncio
+import websockets
+import json
+
+async def test_proxy():
+ url = "ws://0.0.0.0:4000/v1/realtime?model=grok-voice-agent"
+
+ async with websockets.connect(
+ url,
+ extra_headers={
+ "Authorization": "Bearer sk-1234", # Your LiteLLM proxy key
+ "OpenAI-Beta": "realtime=v1"
+ }
+ ) as ws:
+ # Wait for conversation.created event from xAI
+ message = await ws.recv()
+ print(f"Connected: {message}")
+
+ # Send a message
+ await ws.send(json.dumps({
+ "type": "conversation.item.create",
+ "item": {
+ "type": "message",
+ "role": "user",
+ "content": [{
+ "type": "input_text",
+ "text": "Hello from LiteLLM proxy!"
+ }]
+ }
+ }))
+
+ # Request response
+ await ws.send(json.dumps({
+ "type": "response.create"
+ }))
+
+ # Listen for response
+ async for message in ws:
+ data = json.loads(message)
+ print(f"Event: {data['type']}")
+
+ if data['type'] == 'response.done':
+ break
+
+asyncio.run(test_proxy())
+```
+
+#### Node.js Client
+
+```javascript
+// test.js - Run with: node test.js
+const WebSocket = require("ws");
+
+const url = "ws://0.0.0.0:4000/v1/realtime?model=grok-voice-agent";
+
+const ws = new WebSocket(url, {
+ headers: {
+ "Authorization": "Bearer sk-1234",
+ "OpenAI-Beta": "realtime=v1",
+ },
+});
+
+ws.on("open", function open() {
+ console.log("Connected to xAI via LiteLLM proxy");
+
+ // Send a message
+ ws.send(JSON.stringify({
+ type: "conversation.item.create",
+ item: {
+ type: "message",
+ role: "user",
+ content: [{
+ type: "input_text",
+ text: "What's the weather like?"
+ }]
+ }
+ }));
+
+ // Request response
+ ws.send(JSON.stringify({
+ type: "response.create",
+ response: {
+ modalities: ["text"],
+ instructions: "Please assist the user."
+ }
+ }));
+});
+
+ws.on("message", function incoming(message) {
+ const data = JSON.parse(message.toString());
+ console.log(`Event: ${data.type}`);
+
+ if (data.type === 'response.done') {
+ ws.close();
+ }
+});
+
+ws.on("error", function handleError(error) {
+ console.error("Error: ", error);
+});
+```
+
+## Key Differences from OpenAI
+
+xAI's Grok Voice Agent has some differences from OpenAI's Realtime API:
+
+| Feature | xAI | OpenAI | LiteLLM Handling |
+|---------|-----|--------|------------------|
+| Initial Event | `conversation.created` | `session.created` | ⚠️ Passed through as-is |
+| WebSocket URL | `wss://api.x.ai/v1/realtime` | `wss://api.openai.com/v1/realtime` | ✅ Auto-configured |
+| Model | `grok-4-1-fast-non-reasoning` | `gpt-4o-realtime-preview` | ✅ Via model prefix |
+| Audio Format | PCM16 24kHz mono | PCM16 24kHz mono | ✅ Compatible |
+| Context Window | 2M tokens | 128K tokens | N/A |
+
+**What LiteLLM Handles:**
+- ✅ Automatic URL routing to correct provider
+- ✅ Authentication headers (no `OpenAI-Beta` header for xAI)
+- ✅ WebSocket connection management
+- ✅ All other event types are compatible
+
+**What You Need to Handle:**
+- ⚠️ Initial event type difference (`conversation.created` vs `session.created`)
+
+**Tip:** Make your client compatible with both event types:
+```python
+# Handle both providers
+if event['type'] in ['session.created', 'conversation.created']:
+ print("Connection established")
+```
+
+## Related Documentation
+
+- [xAI Chat/Text Models](/docs/providers/xai)
+- [LiteLLM Realtime API Overview](/docs/realtime)
+- [xAI Official Documentation](https://docs.x.ai/docs)
+
+## Support
+
+For issues or questions:
+- [LiteLLM GitHub Issues](https://github.com/BerriAI/litellm/issues)
+- [xAI Documentation](https://docs.x.ai/docs)
diff --git a/docs/my-website/docs/proxy/access_groups.md b/docs/my-website/docs/proxy/access_groups.md
new file mode 100644
index 00000000000..59904575da8
--- /dev/null
+++ b/docs/my-website/docs/proxy/access_groups.md
@@ -0,0 +1,122 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Access Groups
+
+Access Groups simplify how you define and manage resource access across your organization. Instead of configuring models, MCP servers, and agents separately on each key or team, you create one group that bundles the resources you want to grant, then attach that group to your keys or teams.
+
+## Overview
+
+**Access Groups** let you define a reusable set of allowed resources—models, MCP servers, and agents—in a single place. One group can grant access to all three resource types. Simply attach the group to a key or team, and they get access to everything defined in that group.
+
+- **Unified resource control** – One group controls access to models, MCP servers, and agents together
+- **Reusable** – Define once, attach to many keys or teams
+- **Easy to maintain** – Update the group (add or remove resources) and all attached keys and teams automatically reflect the change
+- **Clear visibility** – See exactly which resources each group grants and which keys/teams use it
+
+
+
+### How It Works
+
+**Key concept:** Define resources in a group → Attach group to key or team → Key/team gets access to all resources in the group
+
+| Resource Type | What the group controls |
+| --------------- | -------------------------------------------------------------------- |
+| **Models** | Which LLM models keys/teams can use (e.g., `gpt-4`, `claude-3-opus`) |
+| **MCP Servers** | Which MCP servers are available for tool calling |
+| **Agents** | Which agents can be invoked |
+
+## How to Create and Use Access Groups in the UI
+
+### 1. Navigate to Access Groups
+
+Go to the Admin UI (e.g. `http://localhost:4000/ui` or your `PROXY_BASE_URL/ui`) and click **Access Groups** in the sidebar.
+
+
+
+### 2. Create an Access Group
+
+Click **Create Access Group** and give your group a name.
+
+
+
+
+
+### 3. Define Resources in the Group
+
+Use the tabs to select which models, MCP servers, and agents this group grants access to:
+
+- **Models tab** – Select the LLM models
+- **MCP Servers tab** – Select MCP servers (for tool calling)
+- **Agents tab** – Select agents
+
+
+
+
+
+
+
+### 4. Attach the Access Group to a Key
+
+When creating or editing a virtual key, expand **Optional Settings** and select your Access Group. The key will inherit access to all models, MCP servers, and agents defined in that group.
+
+1. Go to **Virtual Keys** and click **+ Create New Key**
+2. Expand **Optional Settings**
+3. In the Access Group field, select the group you created
+4. Save the key
+
+
+
+
+
+
+
+
+
+
+
+### 5. Attach the Access Group to a Team
+
+You can also attach an Access Group to a team when creating or editing the team. All keys associated with that team will then have access to the resources defined in the group.
+
+## Use Cases
+
+### Team-based Access
+
+Create groups like "Engineering", "Data Science", or "Product" with the models, MCP servers, and agents each team needs. Attach the group to the team—no need to configure each resource on every key.
+
+### Environment Separation
+
+- **Production group** – Production models, approved MCP servers, and production agents
+- **Development group** – Cost-efficient models, experimental MCP tools, and dev agents
+
+Attach the appropriate group to keys or teams based on environment.
+
+### Simplified Onboarding
+
+New developers get a key with an Access Group instead of manually configuring models, MCP servers, and agents. Add them to the right team or give them a key with the correct group.
+
+### Centralized Updates
+
+When you add a new model or MCP server to a group, every key and team attached to that group automatically gains access. Remove a resource from the group and it’s revoked everywhere at once.
+
+## Access Group vs. Model Access Groups
+
+LiteLLM has two related concepts:
+
+| Feature | **Access Groups** (this page) | **Model Access Groups** |
+| ---------- | ----------------------------------------------------------------------- | ------------------------------------------------------- |
+| Definition | Define in the UI; one group can include models, MCP servers, and agents | Defined in config or via API; groups are model-centric |
+| Scope | Models + MCP servers + agents | Models only |
+| Attach to | Keys, teams | Keys, teams |
+| Use when | You want unified control over models, MCP, and agents from the UI | You need config-based or API-based model access control |
+
+For config-based model access with `access_groups` in `model_info`, see [Model Access Groups](./model_access_groups.md).
+
+## Related Documentation
+
+- [Virtual Keys](./virtual_keys.md) – Creating and managing API keys
+- [Role-based Access Controls](./access_control.md) – Organizations, teams, and user roles
+- [Model Access Groups](./model_access_groups.md) – Config-based model access groups
+- [MCP Control](../mcp_control.md) – MCP server setup and access control
diff --git a/docs/my-website/docs/proxy/admin_ui_sso.md b/docs/my-website/docs/proxy/admin_ui_sso.md
index 7b299429db7..f88d3480446 100644
--- a/docs/my-website/docs/proxy/admin_ui_sso.md
+++ b/docs/my-website/docs/proxy/admin_ui_sso.md
@@ -23,26 +23,75 @@ From v1.76.0, SSO is now Free for up to 5 users.
-1. Add Okta credentials to your .env
+#### Step 1: Create an OIDC Application in Okta
+
+In your Okta Admin Console, create a new **OIDC Web Application**. See [Okta's guide on creating OIDC app integrations](https://help.okta.com/en-us/content/topics/apps/apps_app_integration_wizard_oidc.htm) for detailed instructions.
+
+When configuring the application:
+- **Sign-in redirect URI**: `https:///sso/callback`
+- **Sign-out redirect URI** (optional): `https://`
+
+
+
+After creating the app, copy your **Client ID** and **Client Secret** from the application's General tab:
+
+
+
+#### Step 2: Assign Users to the Application
+
+Ensure users are assigned to the app in the **Assignments** tab. If Federation Broker Mode is enabled, you may need to disable it to assign users manually.
+
+#### Step 3: Configure Authorization Server Access Policy
+
+:::warning Important
+This step is required. Without an Access Policy for your app, users will get a `no_matching_policy` error when attempting to log in.
+:::
+
+1. Go to **Security** → **API**
+
+
+
+2. Select the **default** authorization server (or your custom one)
+
+
+
+3. Click on **Access Policies** tab, create a new policy assigned to your LiteLLM app
+4. Add a rule that allows the **Authorization Code** grant type
+
+
+
+See [Okta's Access Policy documentation](https://help.okta.com/en-us/content/topics/security/api-access-management/access-policies.htm) for more details.
+
+#### Step 4: Configure LiteLLM Environment Variables
```bash
-GENERIC_CLIENT_ID = ""
-GENERIC_CLIENT_SECRET = ""
-GENERIC_AUTHORIZATION_ENDPOINT = "/authorize" # https://dev-2kqkcd6lx6kdkuzt.us.auth0.com/authorize
-GENERIC_TOKEN_ENDPOINT = "/token" # https://dev-2kqkcd6lx6kdkuzt.us.auth0.com/oauth/token
-GENERIC_USERINFO_ENDPOINT = "/userinfo" # https://dev-2kqkcd6lx6kdkuzt.us.auth0.com/userinfo
-GENERIC_CLIENT_STATE = "random-string" # [OPTIONAL] REQUIRED BY OKTA, if not set random state value is generated
-GENERIC_SSO_HEADERS = "Content-Type=application/json, X-Custom-Header=custom-value" # [OPTIONAL] Comma-separated list of additional headers to add to the request - e.g. Content-Type=application/json, etc.
+GENERIC_CLIENT_ID=""
+GENERIC_CLIENT_SECRET=""
+GENERIC_AUTHORIZATION_ENDPOINT="https:///oauth2/default/v1/authorize"
+GENERIC_TOKEN_ENDPOINT="https:///oauth2/default/v1/token"
+GENERIC_USERINFO_ENDPOINT="https:///oauth2/default/v1/userinfo"
+GENERIC_CLIENT_STATE="random-string"
+PROXY_BASE_URL="https://"
```
-You can get your domain specific auth/token/userinfo endpoints at `/.well-known/openid-configuration`
+:::tip
+You can find all OAuth endpoints at `https:///.well-known/openid-configuration`
+:::
-2. Add proxy url as callback_url on Okta
+#### Step 5: Test the SSO Flow
-On Okta, add the 'callback_url' as `/sso/callback`
+1. Start your LiteLLM proxy
+2. Navigate to `https:///ui`
+3. Click the SSO login button
+4. Authenticate with Okta and verify you're redirected back to LiteLLM
+#### Troubleshooting
-
+| Error | Cause | Solution |
+|-------|-------|----------|
+| `redirect_uri` error | Redirect URI not configured | Add `/sso/callback` to Sign-in redirect URIs in Okta |
+| `access_denied` | User not assigned to app | Assign the user in the Assignments tab |
+| `no_matching_policy` | Missing Access Policy | Create an Access Policy in the Authorization Server (see Step 3) |
@@ -174,6 +223,7 @@ GENERIC_USER_FIRST_NAME_ATTRIBUTE = "first_name"
GENERIC_USER_LAST_NAME_ATTRIBUTE = "last_name"
GENERIC_USER_ROLE_ATTRIBUTE = "given_role"
GENERIC_USER_PROVIDER_ATTRIBUTE = "provider"
+GENERIC_USER_EXTRA_ATTRIBUTES = "department,employee_id,manager" # comma-separated list of additional fields to extract from SSO response
GENERIC_CLIENT_STATE = "some-state" # if the provider needs a state parameter
GENERIC_INCLUDE_CLIENT_ID = "false" # some providers enforce that the client_id is not in the body
GENERIC_SCOPE = "openid profile email" # default scope openid is sometimes not enough to retrieve basic user info like first_name and last_name located in profile scope
@@ -190,6 +240,40 @@ Use `GENERIC_USER_ROLE_ATTRIBUTE` to specify which attribute in the SSO token co
Nested attribute paths are supported (e.g., `claims.role` or `attributes.litellm_role`).
+**Capturing Additional SSO Fields**
+
+Use `GENERIC_USER_EXTRA_ATTRIBUTES` to extract additional fields from the SSO provider response beyond the standard user attributes (id, email, name, etc.). This is useful when you need to access custom organization-specific data (e.g., department, employee ID, groups) in your [custom SSO handler](./custom_sso.md).
+
+```shell
+# Comma-separated list of field names to extract
+GENERIC_USER_EXTRA_ATTRIBUTES="department,employee_id,manager,groups"
+```
+
+**Accessing Extra Fields in Custom SSO Handler:**
+
+```python
+from litellm.proxy.management_endpoints.types import CustomOpenID
+
+async def custom_sso_handler(userIDPInfo: CustomOpenID):
+ # Access the extra fields
+ extra_fields = getattr(userIDPInfo, 'extra_fields', None) or {}
+
+ user_department = extra_fields.get("department")
+ employee_id = extra_fields.get("employee_id")
+ user_groups = extra_fields.get("groups", [])
+
+ # Use these fields for custom logic (e.g., team assignment, access control)
+ # ...
+```
+
+**Nested Field Paths:**
+
+Dot notation is supported for nested fields:
+
+```shell
+GENERIC_USER_EXTRA_ATTRIBUTES="org_info.department,org_info.cost_center,metadata.employee_type"
+```
+
- Set Redirect URI, if your provider requires it
- Set a redirect url = `/sso/callback`
```shell
diff --git a/docs/my-website/docs/proxy/call_hooks.md b/docs/my-website/docs/proxy/call_hooks.md
index fe865f67e09..17354725fd5 100644
--- a/docs/my-website/docs/proxy/call_hooks.md
+++ b/docs/my-website/docs/proxy/call_hooks.md
@@ -19,6 +19,7 @@ import Image from '@theme/IdealImage';
| `async_post_call_success_hook` | Modify outgoing response (non-streaming) | After successful LLM API call, for non-streaming responses |
| `async_post_call_failure_hook` | Transform error responses sent to clients | After failed LLM API call |
| `async_post_call_streaming_hook` | Modify outgoing response (streaming) | After successful LLM API call, for streaming responses |
+| `async_post_call_response_headers_hook` | Inject custom HTTP response headers | After LLM API call (both success and failure) |
See a complete example with our [parallel request rate limiter](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/hooks/parallel_request_limiter.py)
@@ -115,6 +116,18 @@ class MyCustomHandler(CustomLogger): # https://docs.litellm.ai/docs/observabilit
async for item in response:
yield item
+ async def async_post_call_response_headers_hook(
+ self,
+ data: dict,
+ user_api_key_dict: UserAPIKeyAuth,
+ response: Any,
+ request_headers: Optional[Dict[str, str]] = None,
+ ) -> Optional[Dict[str, str]]:
+ """
+ Inject custom headers into HTTP response (runs for both success and failure).
+ """
+ return {"x-custom-header": "custom-value"}
+
proxy_handler_instance = MyCustomHandler()
```
@@ -389,3 +402,31 @@ proxy_handler_instance = MyErrorTransformer()
```
**Result:** Clients receive `"Your prompt is too long..."` instead of `"ContextWindowExceededError: Prompt exceeds context window"`.
+
+## Advanced - Inject Custom HTTP Response Headers
+
+Use `async_post_call_response_headers_hook` to inject custom HTTP headers into responses. This hook runs for **both successful and failed** LLM API calls.
+
+```python
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.proxy.proxy_server import UserAPIKeyAuth
+from typing import Any, Dict, Optional
+
+class CustomHeaderLogger(CustomLogger):
+ def __init__(self):
+ super().__init__()
+
+ async def async_post_call_response_headers_hook(
+ self,
+ data: dict,
+ user_api_key_dict: UserAPIKeyAuth,
+ response: Any,
+ request_headers: Optional[Dict[str, str]] = None,
+ ) -> Optional[Dict[str, str]]:
+ """
+ Inject custom headers into all responses (success and failure).
+ """
+ return {"x-custom-header": "custom-value"}
+
+proxy_handler_instance = CustomHeaderLogger()
+```
diff --git a/docs/my-website/docs/proxy/cli.md b/docs/my-website/docs/proxy/cli.md
index 9244f75b756..d3624000a32 100644
--- a/docs/my-website/docs/proxy/cli.md
+++ b/docs/my-website/docs/proxy/cli.md
@@ -1,7 +1,10 @@
# CLI Arguments
-Cli arguments, --host, --port, --num_workers
-## --host
+This page documents all command-line interface (CLI) arguments available for the LiteLLM proxy server.
+
+## Server Configuration
+
+### --host
- **Default:** `'0.0.0.0'`
- The host for the server to listen on.
- **Usage:**
@@ -14,7 +17,7 @@ Cli arguments, --host, --port, --num_workers
litellm
```
-## --port
+### --port
- **Default:** `4000`
- The port to bind the server to.
- **Usage:**
@@ -27,9 +30,9 @@ Cli arguments, --host, --port, --num_workers
litellm
```
-## --num_workers
- - **Default:** `1`
- - The number of uvicorn workers to spin up.
+### --num_workers
+ - **Default:** Number of logical CPUs in the system, or `4` if that cannot be determined
+ - The number of uvicorn / gunicorn workers to spin up.
- **Usage:**
```shell
litellm --num_workers 4
@@ -40,55 +43,273 @@ Cli arguments, --host, --port, --num_workers
litellm
```
-## --api_base
+### --config
+ - **Short form:** `-c`
- **Default:** `None`
- - The API base for the model litellm should call.
+ - Path to the proxy configuration file (e.g., config.yaml).
+ - **Usage:**
+ ```shell
+ litellm --config path/to/config.yaml
+ ```
+
+### --log_config
+ - **Default:** `None`
+ - **Type:** `str`
+ - Path to the logging configuration file for uvicorn.
+ - **Usage:**
+ ```shell
+ litellm --log_config path/to/log_config.conf
+ ```
+
+### --keepalive_timeout
+ - **Default:** `None`
+ - **Type:** `int`
+ - Set the uvicorn keepalive timeout in seconds (uvicorn timeout_keep_alive parameter).
+ - **Usage:**
+ ```shell
+ litellm --keepalive_timeout 30
+ ```
+ - **Usage - set Environment Variable:** `KEEPALIVE_TIMEOUT`
+ ```shell
+ export KEEPALIVE_TIMEOUT=30
+ litellm
+ ```
+
+### --max_requests_before_restart
+ - **Default:** `None`
+ - **Type:** `int`
+ - Restart worker after this many requests. This is useful for mitigating memory growth over time.
+ - For uvicorn: maps to `limit_max_requests`
+ - For gunicorn: maps to `max_requests`
+ - **Usage:**
+ ```shell
+ litellm --max_requests_before_restart 10000
+ ```
+ - **Usage - set Environment Variable:** `MAX_REQUESTS_BEFORE_RESTART`
+ ```shell
+ export MAX_REQUESTS_BEFORE_RESTART=10000
+ litellm
+ ```
+
+## Server Backend Options
+
+### --run_gunicorn
+ - **Default:** `False`
+ - **Type:** `bool` (Flag)
+ - Starts proxy via gunicorn instead of uvicorn. Better for managing multiple workers in production.
+ - **Usage:**
+ ```shell
+ litellm --run_gunicorn
+ ```
+
+### --run_hypercorn
+ - **Default:** `False`
+ - **Type:** `bool` (Flag)
+ - Starts proxy via hypercorn instead of uvicorn. Supports HTTP/2.
+ - **Usage:**
+ ```shell
+ litellm --run_hypercorn
+ ```
+
+### --skip_server_startup
+ - **Default:** `False`
+ - **Type:** `bool` (Flag)
+ - Skip starting the server after setup (useful for database migrations only).
+ - **Usage:**
+ ```shell
+ litellm --skip_server_startup
+ ```
+
+## SSL/TLS Configuration
+
+### --ssl_keyfile_path
+ - **Default:** `None`
+ - **Type:** `str`
+ - Path to the SSL keyfile. Use this when you want to provide SSL certificate when starting proxy.
+ - **Usage:**
+ ```shell
+ litellm --ssl_keyfile_path /path/to/key.pem --ssl_certfile_path /path/to/cert.pem
+ ```
+ - **Usage - set Environment Variable:** `SSL_KEYFILE_PATH`
+ ```shell
+ export SSL_KEYFILE_PATH=/path/to/key.pem
+ litellm
+ ```
+
+### --ssl_certfile_path
+ - **Default:** `None`
+ - **Type:** `str`
+ - Path to the SSL certfile. Use this when you want to provide SSL certificate when starting proxy.
+ - **Usage:**
+ ```shell
+ litellm --ssl_certfile_path /path/to/cert.pem --ssl_keyfile_path /path/to/key.pem
+ ```
+ - **Usage - set Environment Variable:** `SSL_CERTFILE_PATH`
+ ```shell
+ export SSL_CERTFILE_PATH=/path/to/cert.pem
+ litellm
+ ```
+
+### --ciphers
+ - **Default:** `None`
+ - **Type:** `str`
+ - Ciphers to use for the SSL setup. Only used with `--run_hypercorn`.
+ - **Usage:**
+ ```shell
+ litellm --run_hypercorn --ssl_keyfile_path /path/to/key.pem --ssl_certfile_path /path/to/cert.pem --ciphers "ECDHE+AESGCM"
+ ```
+
+## Model Configuration
+
+### --model or -m
+ - **Default:** `None`
+ - The model name to pass to LiteLLM.
+ - **Usage:**
+ ```shell
+ litellm --model gpt-3.5-turbo
+ ```
+
+### --alias
+ - **Default:** `None`
+ - An alias for the model, for user-friendly reference. Use this to give a litellm model name (e.g., "huggingface/codellama/CodeLlama-7b-Instruct-hf") a more user-friendly name ("codellama").
+ - **Usage:**
+ ```shell
+ litellm --alias my-gpt-model
+ ```
+
+### --api_base
+ - **Default:** `None`
+ - The API base for the model LiteLLM should call.
- **Usage:**
```shell
litellm --model huggingface/tinyllama --api_base https://k58ory32yinf1ly0.us-east-1.aws.endpoints.huggingface.cloud
```
-## --api_version
- - **Default:** `None`
+### --api_version
+ - **Default:** `2024-07-01-preview`
- For Azure services, specify the API version.
- **Usage:**
```shell
litellm --model azure/gpt-deployment --api_version 2023-08-01 --api_base https://"
```
-## --model or -m
+### --headers
- **Default:** `None`
- - The model name to pass to Litellm.
+ - Headers for the API call (as JSON string).
- **Usage:**
```shell
- litellm --model gpt-3.5-turbo
+ litellm --model my-model --headers '{"Authorization": "Bearer token"}'
```
-## --test
- - **Type:** `bool` (Flag)
- - Proxy chat completions URL to make a test request.
- - **Usage:**
- ```shell
- litellm --test
- ```
-
-## --health
- - **Type:** `bool` (Flag)
- - Runs a health check on all models in config.yaml
- - **Usage:**
- ```shell
- litellm --health
- ```
-
-## --alias
+### --add_key
- **Default:** `None`
- - An alias for the model, for user-friendly reference.
+ - Add a key to the model configuration.
- **Usage:**
```shell
- litellm --alias my-gpt-model
+ litellm --add_key my-api-key
```
-## --debug
+### --save
+ - **Type:** `bool` (Flag)
+ - Save the model-specific config.
+ - **Usage:**
+ ```shell
+ litellm --model gpt-3.5-turbo --save
+ ```
+
+## Model Parameters
+
+### --temperature
+ - **Default:** `None`
+ - **Type:** `float`
+ - Set the temperature for the model.
+ - **Usage:**
+ ```shell
+ litellm --temperature 0.7
+ ```
+
+### --max_tokens
+ - **Default:** `None`
+ - **Type:** `int`
+ - Set the maximum number of tokens for the model output.
+ - **Usage:**
+ ```shell
+ litellm --max_tokens 50
+ ```
+
+### --request_timeout
+ - **Default:** `None`
+ - **Type:** `int`
+ - Set the timeout in seconds for completion calls.
+ - **Usage:**
+ ```shell
+ litellm --request_timeout 300
+ ```
+
+### --max_budget
+ - **Default:** `None`
+ - **Type:** `float`
+ - Set max budget for API calls. Works for hosted models like OpenAI, TogetherAI, Anthropic, etc.
+ - **Usage:**
+ ```shell
+ litellm --max_budget 100.0
+ ```
+
+### --drop_params
+ - **Type:** `bool` (Flag)
+ - Drop any unmapped params.
+ - **Usage:**
+ ```shell
+ litellm --drop_params
+ ```
+
+### --add_function_to_prompt
+ - **Type:** `bool` (Flag)
+ - If a function passed but unsupported, pass it as a part of the prompt.
+ - **Usage:**
+ ```shell
+ litellm --add_function_to_prompt
+ ```
+
+## Database Configuration
+
+### --iam_token_db_auth
+ - **Default:** `False`
+ - **Type:** `bool` (Flag)
+ - Connects to an RDS database using IAM token authentication instead of a password. This is useful for AWS RDS instances that are configured to use IAM database authentication.
+ - When enabled, LiteLLM will generate an IAM authentication token to connect to the database.
+ - **Required Environment Variables:**
+ - `DATABASE_HOST` - The RDS database host
+ - `DATABASE_PORT` - The database port
+ - `DATABASE_USER` - The database user
+ - `DATABASE_NAME` - The database name
+ - `DATABASE_SCHEMA` (optional) - The database schema
+ - **Usage:**
+ ```shell
+ litellm --iam_token_db_auth
+ ```
+ - **Usage - set Environment Variable:** `IAM_TOKEN_DB_AUTH`
+ ```shell
+ export IAM_TOKEN_DB_AUTH=True
+ export DATABASE_HOST=mydb.us-east-1.rds.amazonaws.com
+ export DATABASE_PORT=5432
+ export DATABASE_USER=mydbuser
+ export DATABASE_NAME=mydb
+ litellm
+ ```
+
+### --use_prisma_db_push
+ - **Default:** `False`
+ - **Type:** `bool` (Flag)
+ - Use `prisma db push` instead of `prisma migrate` for database schema updates. This is useful when you want to quickly sync your database schema without creating migration files.
+ - **Usage:**
+ ```shell
+ litellm --use_prisma_db_push
+ ```
+
+## Debugging
+
+### --debug
- **Default:** `False`
- **Type:** `bool` (Flag)
- Enable debugging mode for the input.
@@ -102,10 +323,10 @@ Cli arguments, --host, --port, --num_workers
litellm
```
-## --detailed_debug
+### --detailed_debug
- **Default:** `False`
- **Type:** `bool` (Flag)
- - Enable debugging mode for the input.
+ - Enable detailed debugging mode to view verbose debug logs.
- **Usage:**
```shell
litellm --detailed_debug
@@ -116,80 +337,76 @@ Cli arguments, --host, --port, --num_workers
litellm
```
-#### --temperature
- - **Default:** `None`
- - **Type:** `float`
- - Set the temperature for the model.
- - **Usage:**
- ```shell
- litellm --temperature 0.7
- ```
-
-## --max_tokens
- - **Default:** `None`
- - **Type:** `int`
- - Set the maximum number of tokens for the model output.
- - **Usage:**
- ```shell
- litellm --max_tokens 50
- ```
-
-## --request_timeout
- - **Default:** `6000`
- - **Type:** `int`
- - Set the timeout in seconds for completion calls.
- - **Usage:**
- ```shell
- litellm --request_timeout 300
- ```
-
-## --drop_params
+### --local
+ - **Default:** `False`
- **Type:** `bool` (Flag)
- - Drop any unmapped params.
+ - For local debugging purposes.
- **Usage:**
```shell
- litellm --drop_params
+ litellm --local
```
-## --add_function_to_prompt
+## Testing & Health Checks
+
+### --test
- **Type:** `bool` (Flag)
- - If a function passed but unsupported, pass it as a part of the prompt.
+ - Proxy chat completions URL to make a test request to.
- **Usage:**
```shell
- litellm --add_function_to_prompt
+ litellm --test
```
-## --config
- - Configure Litellm by providing a configuration file path.
+### --test_async
+ - **Default:** `False`
+ - **Type:** `bool` (Flag)
+ - Calls async endpoints `/queue/requests` and `/queue/response`.
- **Usage:**
```shell
- litellm --config path/to/config.yaml
+ litellm --test_async
```
-## --telemetry
+### --num_requests
+ - **Default:** `10`
+ - **Type:** `int`
+ - Number of requests to hit async endpoint with (used with `--test_async`).
+ - **Usage:**
+ ```shell
+ litellm --test_async --num_requests 100
+ ```
+
+### --health
+ - **Type:** `bool` (Flag)
+ - Runs a health check on all models in config.yaml.
+ - **Usage:**
+ ```shell
+ litellm --health
+ ```
+
+## Other Options
+
+### --version
+ - **Short form:** `-v`
+ - **Type:** `bool` (Flag)
+ - Print LiteLLM version and exit.
+ - **Usage:**
+ ```shell
+ litellm --version
+ ```
+
+### --telemetry
- **Default:** `True`
- **Type:** `bool`
- - Help track usage of this feature.
+ - Help track usage of this feature. Turn off for privacy.
- **Usage:**
```shell
litellm --telemetry False
```
-
-## --log_config
- - **Default:** `None`
- - **Type:** `str`
- - Specify a log configuration file for uvicorn.
- - **Usage:**
- ```shell
- litellm --log_config path/to/log_config.conf
- ```
-
-## --skip_server_startup
+### --use_queue
- **Default:** `False`
- **Type:** `bool` (Flag)
- - Skip starting the server after setup (useful for DB migrations only).
+ - To use celery workers for async endpoints.
- **Usage:**
```shell
- litellm --skip_server_startup
- ```
\ No newline at end of file
+ litellm --use_queue
+ ```
diff --git a/docs/my-website/docs/proxy/cli_sso.md b/docs/my-website/docs/proxy/cli_sso.md
index cde6bf266d4..ad0f033f802 100644
--- a/docs/my-website/docs/proxy/cli_sso.md
+++ b/docs/my-website/docs/proxy/cli_sso.md
@@ -28,6 +28,37 @@ EXPERIMENTAL_UI_LOGIN="True" litellm --config config.yaml
:::
+### Configuration
+
+#### JWT Token Expiration
+
+By default, CLI authentication tokens expire after **24 hours**. You can customize this expiration time by setting the `LITELLM_CLI_JWT_EXPIRATION_HOURS` environment variable when starting your LiteLLM Proxy:
+
+```bash
+# Set CLI JWT tokens to expire after 48 hours
+export LITELLM_CLI_JWT_EXPIRATION_HOURS=48
+export EXPERIMENTAL_UI_LOGIN="True"
+litellm --config config.yaml
+```
+
+Or in a single command:
+
+```bash
+LITELLM_CLI_JWT_EXPIRATION_HOURS=48 EXPERIMENTAL_UI_LOGIN="True" litellm --config config.yaml
+```
+
+**Examples:**
+- `LITELLM_CLI_JWT_EXPIRATION_HOURS=12` - Tokens expire after 12 hours
+- `LITELLM_CLI_JWT_EXPIRATION_HOURS=168` - Tokens expire after 7 days (168 hours)
+- `LITELLM_CLI_JWT_EXPIRATION_HOURS=720` - Tokens expire after 30 days (720 hours)
+
+:::tip
+You can check your current token's age and expiration status using:
+```bash
+litellm-proxy whoami
+```
+:::
+
### Steps
1. **Install the CLI**
diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md
index 89e1e2910e4..9e3b5e90978 100644
--- a/docs/my-website/docs/proxy/config_settings.md
+++ b/docs/my-website/docs/proxy/config_settings.md
@@ -94,7 +94,7 @@ litellm_settings:
# /chat/completions, /completions, /embeddings, /audio/transcriptions
mode: default_off # if default_off, you need to opt in to caching on a per call basis
ttl: 600 # ttl for caching
- disable_copilot_system_to_assistant: False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
+ disable_copilot_system_to_assistant: False # DEPRECATED - GitHub Copilot API supports system prompts.
callback_settings:
otel:
@@ -197,7 +197,7 @@ router_settings:
| disable_add_transform_inline_image_block | boolean | For Fireworks AI models - if true, turns off the auto-add of `#transform=inline` to the url of the image_url, if the model is not a vision model. |
| disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). |
| enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. |
-| disable_copilot_system_to_assistant | boolean | If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. Useful for tools (like Claude Code) that send system messages, which Copilot does not support. |
+| disable_copilot_system_to_assistant | boolean | **DEPRECATED** - GitHub Copilot API supports system prompts. |
### general_settings - Reference
@@ -321,6 +321,7 @@ router_settings:
| redis_host | string | The host address for the Redis server. **Only set this if you have multiple instances of LiteLLM Proxy and want current tpm/rpm tracking to be shared across them** |
| redis_password | string | The password for the Redis server. **Only set this if you have multiple instances of LiteLLM Proxy and want current tpm/rpm tracking to be shared across them** |
| redis_port | string | The port number for the Redis server. **Only set this if you have multiple instances of LiteLLM Proxy and want current tpm/rpm tracking to be shared across them**|
+| redis_db | int | The database number for the Redis server. **Only set this if you have multiple instances of LiteLLM Proxy and want current tpm/rpm tracking to be shared across them**|
| enable_pre_call_check | boolean | If true, checks if a call is within the model's context window before making the call. [More information here](reliability) |
| content_policy_fallbacks | array of objects | Specifies fallback models for content policy violations. [More information here](reliability) |
| fallbacks | array of objects | Specifies fallback models for all types of errors. [More information here](reliability) |
@@ -357,7 +358,8 @@ router_settings:
| redis_url | str | URL for Redis server. **Known performance issue with Redis URL.** |
| cache_responses | boolean | Flag to enable caching LLM Responses, if cache set under `router_settings`. If true, caches responses. Defaults to False. |
| router_general_settings | RouterGeneralSettings | [SDK-Only] Router general settings - contains optimizations like 'async_only_mode'. [Docs](../routing.md#router-general-settings) |
-| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Currently supported: 'router_budget_limiting', 'prompt_caching' |
+| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `deployment_affinity`, `forward_client_headers_by_model_group` |
+| deployment_affinity_ttl_seconds | int | TTL (seconds) for user-key → deployment affinity mapping when `deployment_affinity` is enabled (configured at Router init / proxy startup). Defaults to `3600` (1 hour). |
| ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. |
| search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search.md) |
| guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) |
@@ -394,7 +396,7 @@ router_settings:
| ATHINA_API_KEY | API key for Athina service
| ATHINA_BASE_URL | Base URL for Athina service (defaults to `https://log.athina.ai`)
| AUTH_STRATEGY | Strategy used for authentication (e.g., OAuth, API key)
-| AUTO_REDIRECT_UI_LOGIN_TO_SSO | Flag to enable automatic redirect of UI login page to SSO when SSO is configured. Default is **true**
+| AUTO_REDIRECT_UI_LOGIN_TO_SSO | Flag to enable automatic redirect of UI login page to SSO when SSO is configured. Default is **false**
| AUDIO_SPEECH_CHUNK_SIZE | Chunk size for audio speech processing. Default is 1024
| ANTHROPIC_API_KEY | API key for Anthropic service
| ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com
@@ -449,9 +451,12 @@ router_settings:
| BATCH_STATUS_POLL_INTERVAL_SECONDS | Interval in seconds for polling batch status. Default is 3600 (1 hour)
| BATCH_STATUS_POLL_MAX_ATTEMPTS | Maximum number of attempts for polling batch status. Default is 24 (for 24 hours)
| BEDROCK_MAX_POLICY_SIZE | Maximum size for Bedrock policy. Default is 75
+| BEDROCK_MIN_THINKING_BUDGET_TOKENS | Minimum thinking budget in tokens for Bedrock reasoning models. Bedrock returns a 400 error if budget_tokens is below this value. Requests with lower values are clamped to this minimum. Default is 1024
| BERRISPEND_ACCOUNT_ID | Account ID for BerriSpend service
| BRAINTRUST_API_KEY | API key for Braintrust integration
| BRAINTRUST_API_BASE | Base URL for Braintrust API. Default is https://api.braintrustdata.com/v1
+| BRAINTRUST_MOCK | Enable mock mode for Braintrust integration testing. When set to true, intercepts Braintrust API calls and returns mock responses without making actual network calls. Default is false
+| BRAINTRUST_MOCK_LATENCY_MS | Mock latency in milliseconds for Braintrust API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| CACHED_STREAMING_CHUNK_DELAY | Delay in seconds for cached streaming chunks. Default is 0.02
| CHATGPT_API_BASE | Base URL for ChatGPT API. Default is https://chatgpt.com/backend-api/codex
| CHATGPT_AUTH_FILE | Filename for ChatGPT authentication data. Default is "auth.json"
@@ -462,6 +467,7 @@ router_settings:
| CHATGPT_USER_AGENT_SUFFIX | Suffix to append to the ChatGPT user agent string
| CIRCLE_OIDC_TOKEN | OpenID Connect token for CircleCI
| CIRCLE_OIDC_TOKEN_V2 | Version 2 of the OpenID Connect token for CircleCI
+| CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours. Can also be set via LITELLM_CLI_JWT_EXPIRATION_HOURS
| CLOUDZERO_API_KEY | CloudZero API key for authentication
| CLOUDZERO_CONNECTION_ID | CloudZero connection ID for data submission
| CLOUDZERO_EXPORT_INTERVAL_MINUTES | Interval in minutes for CloudZero data export operations
@@ -488,6 +494,7 @@ router_settings:
| DATABASE_USER | Username for database connection
| DATABASE_USERNAME | Alias for database user
| DATABRICKS_API_BASE | Base URL for Databricks API
+| DATABRICKS_API_KEY | API key (Personal Access Token) for Databricks API authentication
| DATABRICKS_CLIENT_ID | Client ID for Databricks OAuth M2M authentication (Service Principal application ID)
| DATABRICKS_CLIENT_SECRET | Client secret for Databricks OAuth M2M authentication
| DATABRICKS_USER_AGENT | Custom user agent string for Databricks API requests. Used for partner telemetry attribution
@@ -504,15 +511,19 @@ router_settings:
| DD_AGENT_HOST | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API
| DD_AGENT_PORT | Port of DataDog agent for log intake. Default is 10518
| DD_API_KEY | API key for Datadog integration
+| DD_APP_KEY | Application key for Datadog Cost Management integration. Required along with DD_API_KEY for cost metrics
| DD_SITE | Site URL for Datadog (e.g., datadoghq.com)
| DD_SOURCE | Source identifier for Datadog logs
| DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE | Resource name for Datadog tracing of streaming chunk yields. Default is "streaming.chunk.yield"
| DD_ENV | Environment identifier for Datadog logs. Only supported for `datadog_llm_observability` callback
| DD_SERVICE | Service identifier for Datadog logs. Defaults to "litellm-server"
| DD_VERSION | Version identifier for Datadog logs. Defaults to "unknown"
+| DATADOG_MOCK | Enable mock mode for Datadog integration testing. When set to true, intercepts Datadog API calls and returns mock responses without making actual network calls. Default is false
+| DATADOG_MOCK_LATENCY_MS | Mock latency in milliseconds for Datadog API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| DEBUG_OTEL | Enable debug mode for OpenTelemetry
| DEFAULT_ALLOWED_FAILS | Maximum failures allowed before cooling down a model. Default is 3
| DEFAULT_A2A_AGENT_TIMEOUT | Default timeout in seconds for A2A (Agent-to-Agent) protocol requests. Default is 6000
+| DEFAULT_ACCESS_GROUP_CACHE_TTL | Time-to-live in seconds for cached access group information. Default is 600 (10 minutes)
| DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS | Default maximum tokens for Anthropic chat completions. Default is 4096
| DEFAULT_BATCH_SIZE | Default batch size for operations. Default is 512
| DEFAULT_CHUNK_OVERLAP | Default chunk overlap for RAG text splitters. Default is 200
@@ -531,17 +542,25 @@ router_settings:
| DEFAULT_IMAGE_WIDTH | Default width for images. Default is 300
| DEFAULT_IN_MEMORY_TTL | Default time-to-live for in-memory cache in seconds. Default is 5
| DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL | Default time-to-live in seconds for management objects (User, Team, Key, Organization) in memory cache. Default is 60 seconds.
-| DEFAULT_MAX_LRU_CACHE_SIZE | Default maximum size for LRU cache. Default is 16
+| DEFAULT_MAX_LRU_CACHE_SIZE | Default maximum size for LRU cache. Default is 64
| DEFAULT_MAX_RECURSE_DEPTH | Default maximum recursion depth. Default is 100
| DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER | Default maximum recursion depth for sensitive data masker. Default is 10
| DEFAULT_MAX_RETRIES | Default maximum retry attempts. Default is 2
| DEFAULT_MAX_TOKENS | Default maximum tokens for LLM calls. Default is 4096
| DEFAULT_MAX_TOKENS_FOR_TRITON | Default maximum tokens for Triton models. Default is 2000
| DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE | Default maximum size for redis batch cache. Default is 1000
+| DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL | Default embedding model for MCP semantic tool filtering. Default is "text-embedding-3-small"
+| DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD | Default similarity threshold for MCP semantic tool filtering. Default is 0.3
+| DEFAULT_MCP_SEMANTIC_FILTER_TOP_K | Default number of top results to return for MCP semantic tool filtering. Default is 10
+| MCP_NPM_CACHE_DIR | Directory for npm cache used by STDIO MCP servers. In containers the default (~/.npm) may not exist or be read-only. Default is `/tmp/.npm_mcp_cache`
+| MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL | Default TTL in seconds for MCP OAuth2 token cache. Default is 3600
+| MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE | Maximum number of entries in MCP OAuth2 token cache. Default is 200
+| MCP_OAUTH2_TOKEN_CACHE_MIN_TTL | Minimum TTL in seconds for MCP OAuth2 token cache. Default is 10
+| MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS | Seconds to subtract from token expiry when computing cache TTL. Default is 60
| DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20
| DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10
| DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602
-| DEFAULT_NUM_WORKERS_LITELLM_PROXY | Default number of workers for LiteLLM proxy. Default is 4. **We strongly recommend setting NUM Workers to Number of vCPUs available**
+| DEFAULT_NUM_WORKERS_LITELLM_PROXY | Default number of workers for LiteLLM proxy when `NUM_WORKERS` is not set. Default is 1. **We strongly recommend setting NUM_WORKERS to the number of vCPUs available** (e.g. `NUM_WORKERS=8` or `--num_workers 8`).
| DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD | Default threshold for prompt injection similarity. Default is 0.7
| DEFAULT_POLLING_INTERVAL | Default polling interval for schedulers in seconds. Default is 0.03
| DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET | Default reasoning effort disable thinking budget. Default is 0
@@ -586,7 +605,6 @@ router_settings:
| EMAIL_BUDGET_ALERT_TTL | Time-to-live for budget alert deduplication in seconds. Default is 86400 (24 hours)
| ENKRYPTAI_API_BASE | Base URL for EnkryptAI Guardrails API. **Default is https://api.enkryptai.com**
| ENKRYPTAI_API_KEY | API key for EnkryptAI Guardrails service
-| EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING | Flag to enable new multi-instance rate limiting. **Default is False**
| FIREWORKS_AI_4_B | Size parameter for Fireworks AI 4B model. Default is 4
| FIREWORKS_AI_16_B | Size parameter for Fireworks AI 16B model. Default is 16
| FIREWORKS_AI_56_B_MOE | Size parameter for Fireworks AI 56B MOE model. Default is 56
@@ -611,6 +629,8 @@ router_settings:
| GALILEO_USERNAME | Username for Galileo authentication
| GOOGLE_SECRET_MANAGER_PROJECT_ID | Project ID for Google Secret Manager
| GCS_BUCKET_NAME | Name of the Google Cloud Storage bucket
+| GCS_MOCK | Enable mock mode for GCS integration testing. When set to true, intercepts GCS API calls and returns mock responses without making actual network calls. Default is false
+| GCS_MOCK_LATENCY_MS | Mock latency in milliseconds for GCS API calls when mock mode is enabled. Simulates network round-trip time. Default is 150ms
| GCS_PATH_SERVICE_ACCOUNT | Path to the Google Cloud service account JSON file
| GCS_FLUSH_INTERVAL | Flush interval for GCS logging (in seconds). Specify how often you want a log to be sent to GCS. **Default is 20 seconds**
| GCS_BATCH_SIZE | Batch size for GCS logging. Specify after how many logs you want to flush to GCS. If `BATCH_SIZE` is set to 10, logs are flushed every 10 logs. **Default is 2048**
@@ -628,6 +648,7 @@ router_settings:
| GENERIC_TOKEN_ENDPOINT | Token endpoint for generic OAuth providers
| GENERIC_USER_DISPLAY_NAME_ATTRIBUTE | Attribute for user's display name in generic auth
| GENERIC_USER_EMAIL_ATTRIBUTE | Attribute for user's email in generic auth
+| GENERIC_USER_EXTRA_ATTRIBUTES | Comma-separated list of additional fields to extract from generic SSO provider response (e.g., "department,employee_id,groups"). Accessible via `CustomOpenID.extra_fields` in custom SSO handlers. Supports dot notation for nested fields
| GENERIC_USER_FIRST_NAME_ATTRIBUTE | Attribute for user's first name in generic auth
| GENERIC_USER_ID_ATTRIBUTE | Attribute for user ID in generic auth
| GENERIC_USER_LAST_NAME_ATTRIBUTE | Attribute for user's last name in generic auth
@@ -636,6 +657,10 @@ router_settings:
| GENERIC_USERINFO_ENDPOINT | Endpoint to fetch user information in generic OAuth
| GENERIC_LOGGER_ENDPOINT | Endpoint URL for the Generic Logger callback to send logs to
| GENERIC_LOGGER_HEADERS | JSON string of headers to include in Generic Logger callback requests
+| GENERIC_ROLE_MAPPINGS_DEFAULT_ROLE | Default LiteLLM role to assign when no role mapping matches in generic SSO. Used with GENERIC_ROLE_MAPPINGS_ROLES
+| GENERIC_ROLE_MAPPINGS_GROUP_CLAIM | The claim/attribute name in the SSO token that contains the user's groups. Used for role mapping
+| GENERIC_ROLE_MAPPINGS_ROLES | Python dict string mapping LiteLLM roles to SSO group names. Example: `{"proxy_admin": ["admin-group"], "internal_user": ["users"]}`
+| GENERIC_USER_ROLE_MAPPINGS | Alternative to GENERIC_ROLE_MAPPINGS_ROLES for configuring user role mappings from SSO
| GEMINI_API_BASE | Base URL for Gemini API. Default is https://generativelanguage.googleapis.com
| GALILEO_BASE_URL | Base URL for Galileo platform
| GALILEO_PASSWORD | Password for Galileo authentication
@@ -672,6 +697,8 @@ router_settings:
| HCP_VAULT_CERT_ROLE | Role for [Hashicorp Vault Secret Manager Auth](../secret.md#hashicorp-vault)
| HELICONE_API_KEY | API key for Helicone service
| HELICONE_API_BASE | Base URL for Helicone service, defaults to `https://api.helicone.ai`
+| HELICONE_MOCK | Enable mock mode for Helicone integration testing. When set to true, intercepts Helicone API calls and returns mock responses without making actual network calls. Default is false
+| HELICONE_MOCK_LATENCY_MS | Mock latency in milliseconds for Helicone API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| HOSTNAME | Hostname for the server, this will be [emitted to `datadog` logs](https://docs.litellm.ai/docs/proxy/logging#datadog)
| HOURS_IN_A_DAY | Hours in a day for calculation purposes. Default is 24
| HIDDENLAYER_API_BASE | Base URL for HiddenLayer API. Defaults to `https://api.hiddenlayer.ai`
@@ -697,6 +724,8 @@ router_settings:
| LANGFUSE_FLUSH_INTERVAL | Interval for flushing Langfuse logs
| LANGFUSE_TRACING_ENVIRONMENT | Environment for Langfuse tracing
| LANGFUSE_HOST | Host URL for Langfuse service
+| LANGFUSE_MOCK | Enable mock mode for Langfuse integration testing. When set to true, intercepts Langfuse API calls and returns mock responses without making actual network calls. Default is false
+| LANGFUSE_MOCK_LATENCY_MS | Mock latency in milliseconds for Langfuse API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| LANGFUSE_PUBLIC_KEY | Public key for Langfuse authentication
| LANGFUSE_RELEASE | Release version of Langfuse integration
| LANGFUSE_SECRET_KEY | Secret key for Langfuse authentication
@@ -708,6 +737,8 @@ router_settings:
| LANGSMITH_PROJECT | Project name for Langsmith integration
| LANGSMITH_SAMPLING_RATE | Sampling rate for Langsmith logging
| LANGSMITH_TENANT_ID | Tenant ID for Langsmith multi-tenant deployments
+| LANGSMITH_MOCK | Enable mock mode for Langsmith integration testing. When set to true, intercepts Langsmith API calls and returns mock responses without making actual network calls. Default is false
+| LANGSMITH_MOCK_LATENCY_MS | Mock latency in milliseconds for Langsmith API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| LANGTRACE_API_KEY | API key for Langtrace service
| LASSO_API_BASE | Base URL for Lasso API
| LASSO_API_KEY | API key for Lasso service
@@ -718,9 +749,14 @@ router_settings:
| LITERAL_API_KEY | API key for Literal integration
| LITERAL_API_URL | API URL for Literal service
| LITERAL_BATCH_SIZE | Batch size for Literal operations
+| LITELLM_ANTHROPIC_BETA_HEADERS_URL | Custom URL for fetching Anthropic beta headers configuration. Default is the GitHub main branch URL
| LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints
+| LITELLM_ASSETS_PATH | Path to directory for UI assets and logos. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/assets` in Docker.
+| LITELLM_CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours
| LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API
+| LITELLM_DEPLOYMENT_ENVIRONMENT | Environment name for the deployment (e.g., "production", "staging"). Used as a fallback when OTEL_ENVIRONMENT_NAME is not set. Sets the `environment` tag in telemetry data
| LITELLM_DD_AGENT_PORT | Port of DataDog agent for LiteLLM-specific log intake. Default is 10518
+| LITELLM_DD_LLM_OBS_PORT | Port for Datadog LLM Observability agent. Default is 8126
| LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
| LITELLM_DROP_PARAMS | Parameters to drop in LiteLLM requests
| LITELLM_MODIFY_PARAMS | Parameters to modify in LiteLLM requests
@@ -731,11 +767,15 @@ router_settings:
| LITELLM_MIGRATION_DIR | Custom migrations directory for prisma migrations, used for baselining db in read-only file systems.
| LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM
| LITELLM_UI_API_DOC_BASE_URL | Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. Defaults to `PROXY_BASE_URL` when unset.
+| LITELLM_UI_PATH | Path to directory for Admin UI files. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/ui` in Docker.
| LITELM_ENVIRONMENT | Environment of LiteLLM Instance, used by logging services. Currently only used by DeepEval.
| LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false.
| LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours).
+| LITELLM_KEY_ROTATION_GRACE_PERIOD | Duration to keep old key valid after rotation (e.g. "24h", "2d"). Default is empty (immediate revoke). Used for scheduled rotations and as fallback when not specified in regenerate request.
| LITELLM_LICENSE | License key for LiteLLM usage
+| LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS | Set to `True` to use the local bundled Anthropic beta headers config only, disabling remote fetching. Default is `False`
| LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM
+| LITELLM_LOCAL_POLICY_TEMPLATES | When set to "true", uses local backup policy templates instead of fetching from GitHub. Policy templates are fetched from https://raw.githubusercontent.com/BerriAI/litellm/main/policy_templates.json by default, with automatic fallback to local backup on failure
| LITELLM_LOG | Enable detailed logging for LiteLLM
| LITELLM_MODEL_COST_MAP_URL | URL for fetching model cost map data. Default is https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json
| LITELLM_LOG_FILE | File path to write LiteLLM logs to. When set, logs will be written to both console and the specified file
@@ -743,6 +783,10 @@ router_settings:
| LITELLM_METER_NAME | Name for OTEL Meter
| LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS | Optionally enable semantic logs for OTEL
| LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL
+| LITELLM_ENABLE_PYROSCOPE | If true, enables Pyroscope CPU profiling. Profiles are sent to PYROSCOPE_SERVER_ADDRESS. Off by default. See [Pyroscope profiling](/proxy/pyroscope_profiling).
+| PYROSCOPE_APP_NAME | Application name reported to Pyroscope. Required when LITELLM_ENABLE_PYROSCOPE is true. No default.
+| PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default.
+| PYROSCOPE_SAMPLE_RATE | Optional. Sample rate for Pyroscope profiling (integer). No default; when unset, the pyroscope-io library default is used.
| LITELLM_MASTER_KEY | Master key for proxy authentication
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
| LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers
@@ -755,6 +799,7 @@ router_settings:
| LITELLM_USER_AGENT | Custom user agent string for LiteLLM API requests. Used for partner telemetry attribution
| LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging
| LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
+| LITELLM_ASYNCIO_QUEUE_MAXSIZE | Maximum size for asyncio queues (e.g. log queues, spend update queues, and cookbook examples such as realtime audio in `nova_sonic_realtime.py`). Bounds in-memory growth to prevent OOM. Default is 1000.
| LOGFIRE_TOKEN | Token for Logfire logging service
| LOGFIRE_BASE_URL | Base URL for Logfire logging service (useful for self hosted deployments)
| LOGGING_WORKER_CONCURRENCY | Maximum number of concurrent coroutine slots for the logging worker on the asyncio event loop. Default is 100. Setting too high will flood the event loop with logging tasks which will lower the overall latency of the requests.
@@ -781,6 +826,8 @@ router_settings:
| MAXIMUM_TRACEBACK_LINES_TO_LOG | Maximum number of lines to log in traceback in LiteLLM Logs UI. Default is 100
| MAX_RETRY_DELAY | Maximum delay in seconds for retrying requests. Default is 8.0
| MAX_LANGFUSE_INITIALIZED_CLIENTS | Maximum number of Langfuse clients to initialize on proxy. Default is 50. This is set since langfuse initializes 1 thread everytime a client is initialized. We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times.
+| MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH | Maximum header length for MCP semantic filter tools. Default is 150
+| MAX_POLICY_ESTIMATE_IMPACT_ROWS | Maximum number of rows returned when estimating the impact of a policy. Default is 1000
| MIN_NON_ZERO_TEMPERATURE | Minimum non-zero temperature value. Default is 0.0001
| MINIMUM_PROMPT_CACHE_TOKEN_COUNT | Minimum token count for caching a prompt. Default is 1024
| MISTRAL_API_BASE | Base URL for Mistral API. Default is https://api.mistral.ai
@@ -797,6 +844,8 @@ router_settings:
| MICROSOFT_USER_ID_ATTRIBUTE | Field name for user ID in Microsoft SSO response. Default is `id`
| MICROSOFT_USER_LAST_NAME_ATTRIBUTE | Field name for user last name in Microsoft SSO response. Default is `surname`
| MICROSOFT_USERINFO_ENDPOINT | Custom userinfo endpoint URL for Microsoft SSO (overrides default Microsoft Graph userinfo endpoint)
+| MODEL_COST_MAP_MAX_SHRINK_RATIO | Maximum allowed shrinkage ratio when validating a fetched model cost map against the local backup. Rejects the fetched map if it is smaller than this fraction of the backup. Default is 0.5
+| MODEL_COST_MAP_MIN_MODEL_COUNT | Minimum number of models a fetched cost map must contain to be considered valid. Default is 50
| NO_DOCS | Flag to disable Swagger UI documentation
| NO_REDOC | Flag to disable Redoc documentation
| NO_PROXY | List of addresses to bypass proxy
@@ -816,6 +865,7 @@ router_settings:
| OPENMETER_EVENT_TYPE | Type of events sent to OpenMeter
| ONYX_API_BASE | Base URL for Onyx Security AI Guard service (defaults to https://ai-guard.onyx.security)
| ONYX_API_KEY | API key for Onyx Security AI Guard service
+| ONYX_TIMEOUT | Timeout in seconds for Onyx Guard server requests. Default is 10
| OTEL_ENDPOINT | OpenTelemetry endpoint for traces
| OTEL_EXPORTER_OTLP_ENDPOINT | OpenTelemetry endpoint for traces
| OTEL_ENVIRONMENT_NAME | Environment name for OpenTelemetry
@@ -839,6 +889,8 @@ router_settings:
| POD_NAME | Pod name for the server, this will be [emitted to `datadog` logs](https://docs.litellm.ai/docs/proxy/logging#datadog) as `POD_NAME`
| POSTHOG_API_KEY | API key for PostHog analytics integration
| POSTHOG_API_URL | Base URL for PostHog API (defaults to https://us.i.posthog.com)
+| POSTHOG_MOCK | Enable mock mode for PostHog integration testing. When set to true, intercepts PostHog API calls and returns mock responses without making actual network calls. Default is false
+| POSTHOG_MOCK_LATENCY_MS | Mock latency in milliseconds for PostHog API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| PREDIBASE_API_BASE | Base URL for Predibase API
| PRESIDIO_ANALYZER_API_BASE | Base URL for Presidio Analyzer service
| PRESIDIO_ANONYMIZER_API_BASE | Base URL for Presidio Anonymizer service
@@ -876,6 +928,8 @@ router_settings:
| ROUTER_MAX_FALLBACKS | Maximum number of fallbacks for router. Default is 5
| RUNWAYML_DEFAULT_API_VERSION | Default API version for RunwayML service. Default is "2024-11-06"
| RUNWAYML_POLLING_TIMEOUT | Timeout in seconds for RunwayML image generation polling. Default is 600 (10 minutes)
+| S3_VECTORS_DEFAULT_DIMENSION | Default vector dimension for S3 Vectors RAG ingestion. Default is 1024
+| S3_VECTORS_DEFAULT_DISTANCE_METRIC | Default distance metric for S3 Vectors RAG ingestion. Options: "cosine", "euclidean". Default is "cosine"
| SECRET_MANAGER_REFRESH_INTERVAL | Refresh interval in seconds for secret manager. Default is 86400 (24 hours)
| SEPARATE_HEALTH_APP | If set to '1', runs health endpoints on a separate ASGI app and port. Default: '0'.
| SEPARATE_HEALTH_PORT | Port for the separate health endpoints app. Only used if SEPARATE_HEALTH_APP=1. Default: 4001.
diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md
index a5674bf2bc5..56a8b9566db 100644
--- a/docs/my-website/docs/proxy/configs.md
+++ b/docs/my-website/docs/proxy/configs.md
@@ -469,6 +469,7 @@ credential_list:
api_version: "2023-05-15"
credential_info:
description: "Production credentials for EU region"
+ custom_llm_provider: "azure"
```
#### Key Parameters
diff --git a/docs/my-website/docs/proxy/custom_pricing.md b/docs/my-website/docs/proxy/custom_pricing.md
index 8f4a4c450f5..b61da85bb1d 100644
--- a/docs/my-website/docs/proxy/custom_pricing.md
+++ b/docs/my-website/docs/proxy/custom_pricing.md
@@ -9,6 +9,7 @@ LiteLLM provides flexible cost tracking and pricing customization for all LLM pr
- **Custom Pricing** - Override default model costs or set pricing for custom models
- **Cost Per Token** - Track costs based on input/output tokens (most common)
- **Cost Per Second** - Track costs based on runtime (e.g., Sagemaker)
+- **Zero-Cost Models** - Bypass budget checks for free/on-premises models by setting costs to 0
- **[Provider Discounts](./provider_discounts.md)** - Apply percentage-based discounts to specific providers
- **[Provider Margins](./provider_margins.md)** - Add fees/margins to LLM costs for internal billing
- **Base Model Mapping** - Ensure accurate cost tracking for Azure deployments
@@ -106,6 +107,51 @@ There are other keys you can use to specify costs for different scenarios and mo
These keys evolve based on how new models handle multimodality. The latest version can be found at [https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json).
+## Zero-Cost Models (Bypass Budget Checks)
+
+**Use Case**: You have on-premises or free models that should be accessible even when users exceed their budget limits.
+
+**Solution** ✅: Set both `input_cost_per_token` and `output_cost_per_token` to `0` (explicitly) to bypass all budget checks for that model.
+
+:::info
+
+When a model is configured with zero cost, LiteLLM will automatically skip ALL budget checks (user, team, team member, end-user, organization, and global proxy budget) for requests to that model.
+
+**Important**: Both costs must be **explicitly set to 0**. If costs are `null` or undefined, the model will be treated as having cost and budget checks will apply.
+
+:::
+
+### Configuration Example
+
+```yaml
+model_list:
+ # On-premises model - free to use
+ - model_name: on-prem-llama
+ litellm_params:
+ model: ollama/llama3
+ api_base: http://localhost:11434
+ model_info:
+ input_cost_per_token: 0 # 👈 Explicitly set to 0
+ output_cost_per_token: 0 # 👈 Explicitly set to 0
+
+ # Paid cloud model - budget checks apply
+ - model_name: gpt-4
+ litellm_params:
+ model: gpt-4
+ api_key: os.environ/OPENAI_API_KEY
+ # No model_info - uses default pricing from cost map
+```
+
+### Behavior
+
+With the above configuration:
+
+- **User over budget** → Can still use `on-prem-llama` ✅, but blocked from `gpt-4` ❌
+- **Team over budget** → Can still use `on-prem-llama` ✅, but blocked from `gpt-4` ❌
+- **End-user over budget** → Can still use `on-prem-llama` ✅, but blocked from `gpt-4` ❌
+
+This ensures your free/on-premises models remain accessible regardless of budget constraints, while paid models are still properly governed.
+
## Set 'base_model' for Cost Tracking (e.g. Azure deployments)
**Problem**: Azure returns `gpt-4` in the response when `azure/gpt-4-1106-preview` is used. This leads to inaccurate cost tracking
diff --git a/docs/my-website/docs/proxy/custom_sso.md b/docs/my-website/docs/proxy/custom_sso.md
index bbd7f41bee1..8b7adeb0c5a 100644
--- a/docs/my-website/docs/proxy/custom_sso.md
+++ b/docs/my-website/docs/proxy/custom_sso.md
@@ -142,6 +142,18 @@ async def custom_sso_handler(userIDPInfo: OpenID) -> SSOUserDefinedValues:
f"No ID found for user. userIDPInfo.id is None {userIDPInfo}"
)
+ #################################################
+ # Access extra fields from SSO provider (requires GENERIC_USER_EXTRA_ATTRIBUTES env var)
+ # Example: Set GENERIC_USER_EXTRA_ATTRIBUTES="department,employee_id,groups"
+ extra_fields = getattr(userIDPInfo, 'extra_fields', None) or {}
+ user_department = extra_fields.get("department")
+ employee_id = extra_fields.get("employee_id")
+ user_groups = extra_fields.get("groups", [])
+
+ print(f"User department: {user_department}") # noqa
+ print(f"Employee ID: {employee_id}") # noqa
+ print(f"User groups: {user_groups}") # noqa
+ #################################################
#################################################
# Run your custom code / logic here
diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md
index 7393e73ba87..0761e0e9fa8 100644
--- a/docs/my-website/docs/proxy/deploy.md
+++ b/docs/my-website/docs/proxy/deploy.md
@@ -200,6 +200,7 @@ Example `requirements.txt`
```shell
litellm[proxy]==1.57.3 # Specify the litellm version you want to use
+litellm-enterprise
prometheus_client
langfuse
prisma
diff --git a/docs/my-website/docs/proxy/embedding.md b/docs/my-website/docs/proxy/embedding.md
index 2adaaa24735..0e7c2d55c44 100644
--- a/docs/my-website/docs/proxy/embedding.md
+++ b/docs/my-website/docs/proxy/embedding.md
@@ -6,6 +6,16 @@ import TabItem from '@theme/TabItem';
See supported Embedding Providers & Models [here](https://docs.litellm.ai/docs/embedding/supported_embedding)
+## Supported Input Formats
+
+The `/v1/embeddings` endpoint follows the [OpenAI embeddings API specification](https://platform.openai.com/docs/api-reference/embeddings/create). The following input formats are supported:
+
+| Format | Example |
+|--------|---------|
+| String | `"input": "Hello"` |
+| Array of strings | `"input": ["Hello", "World"]` |
+| Array of tokens (integers) | `"input": [1234, 5678, 9012]` |
+| Array of token arrays | `"input": [[1234, 5678], [9012, 3456]]` |
## Quick start
Here's how to route between GPT-J embedding (sagemaker endpoint), Amazon Titan embedding (Bedrock) and Azure OpenAI embedding on the proxy server:
diff --git a/docs/my-website/docs/proxy/forward_client_headers.md b/docs/my-website/docs/proxy/forward_client_headers.md
index 5477ffe87aa..2155a7517be 100644
--- a/docs/my-website/docs/proxy/forward_client_headers.md
+++ b/docs/my-website/docs/proxy/forward_client_headers.md
@@ -6,6 +6,52 @@ Control which model groups can forward client headers to the underlying LLM prov
By default, LiteLLM does not forward client headers to LLM provider APIs for security reasons. However, you can selectively enable header forwarding for specific model groups using the `forward_client_headers_to_llm_api` setting.
+## How it Works
+
+LiteLLM does **not** forward all client headers to the LLM provider. Instead, it uses an **allowlist** approach — only headers matching specific rules are forwarded. This ensures sensitive headers (like your LiteLLM API key) are never accidentally sent to upstream providers.
+
+```mermaid
+sequenceDiagram
+ participant Client as Client (SDK / curl)
+ participant Proxy as LiteLLM Proxy
+ participant Filter as Header Filter (Allowlist)
+ participant LLM as LLM Provider (OpenAI, Anthropic, etc.)
+
+ Client->>Proxy: Request with all headers
(Authorization, x-trace-id,
x-custom-header, anthropic-beta, etc.)
+
+ Proxy->>Filter: Check forward_client_headers_to_llm_api
setting for this model group
+
+ Note over Filter: Allowlist rules:
1. Headers starting with "x-" ✅
2. "anthropic-beta" ✅
3. "x-stainless-*" ❌ (blocked)
4. All other headers ❌ (blocked)
+
+ Filter-->>Proxy: Return only allowed headers
+
+ Proxy->>LLM: Request with filtered headers
(x-trace-id, x-custom-header,
anthropic-beta)
+
+ LLM-->>Proxy: Response
+ Proxy-->>Client: Response
+```
+
+### Header Allowlist Rules
+
+The following rules determine which headers are forwarded (see [`_get_forwardable_headers`](https://github.com/litellm/litellm/blob/main/litellm/proxy/litellm_pre_call_utils.py) in `litellm/proxy/litellm_pre_call_utils.py`):
+
+| Rule | Example | Forwarded? |
+|---|---|---|
+| Headers starting with `x-` | `x-trace-id`, `x-custom-header`, `x-request-source` | ✅ Yes |
+| `anthropic-beta` header | `anthropic-beta: prompt-caching-2024-07-31` | ✅ Yes |
+| Headers starting with `x-stainless-*` | `x-stainless-lang`, `x-stainless-arch` | ❌ No (causes OpenAI SDK issues) |
+| Standard HTTP headers | `Authorization`, `Content-Type`, `Host` | ❌ No |
+| Other provider headers | `Accept`, `User-Agent` | ❌ No |
+
+### Additional Header Mechanisms
+
+| Mechanism | Description | Reference |
+|---|---|---|
+| **`x-pass-` prefix** | Headers prefixed with `x-pass-` are always forwarded with the prefix stripped, regardless of settings. E.g., `x-pass-anthropic-beta: value` → `anthropic-beta: value`. Works for all pass-through endpoints. | [Source code](https://github.com/litellm/litellm/blob/main/litellm/passthrough/utils.py) |
+| **`openai-organization`** | Forwarded only when `forward_openai_org_id: true` is set in `general_settings`. | [Forward OpenAI Org ID](#enable-globally) |
+| **User information headers** | When `add_user_information_to_llm_headers: true`, LiteLLM adds `x-litellm-user-id`, `x-litellm-org-id`, etc. | [User Information Headers](#user-information-headers-optional) |
+| **Vertex AI pass-through** | Uses a separate, stricter allowlist: only `anthropic-beta` and `content-type`. | [Source code](https://github.com/litellm/litellm/blob/main/litellm/constants.py) |
+
## Configuration
## Enable Globally
diff --git a/docs/my-website/docs/proxy/guardrails/custom_code_guardrail.md b/docs/my-website/docs/proxy/guardrails/custom_code_guardrail.md
new file mode 100644
index 00000000000..8cbc247ae5e
--- /dev/null
+++ b/docs/my-website/docs/proxy/guardrails/custom_code_guardrail.md
@@ -0,0 +1,332 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Custom Code Guardrail
+
+Write custom guardrail logic using Python-like code that runs in a sandboxed environment.
+
+## Quick Start
+
+### 1. Define the guardrail in config
+
+```yaml
+model_list:
+ - model_name: gpt-4
+ litellm_params:
+ model: gpt-4
+ api_key: os.environ/OPENAI_API_KEY
+
+guardrails:
+ - guardrail_name: block-ssn
+ litellm_params:
+ guardrail: custom_code
+ mode: pre_call
+ custom_code: |
+ def apply_guardrail(inputs, request_data, input_type):
+ for text in inputs["texts"]:
+ if regex_match(text, r"\d{3}-\d{2}-\d{4}"):
+ return block("SSN detected")
+ return allow()
+```
+
+### 2. Start proxy
+
+```bash
+litellm --config config.yaml
+```
+
+### 3. Test
+
+```bash
+curl -X POST http://localhost:4000/chat/completions \
+ -H "Authorization: Bearer sk-1234" \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "gpt-4",
+ "messages": [{"role": "user", "content": "My SSN is 123-45-6789"}],
+ "guardrails": ["block-ssn"]
+ }'
+```
+
+## Configuration
+
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `guardrail` | string | ✅ | Must be `custom_code` |
+| `mode` | string | ✅ | When to run: `pre_call`, `post_call`, `during_call` |
+| `custom_code` | string | ✅ | Python-like code with `apply_guardrail` function |
+| `default_on` | bool | ❌ | Run on all requests (default: `false`) |
+
+## Writing Custom Code
+
+### Function Signature
+
+Your code must define an `apply_guardrail` function. It can be either sync or async:
+
+```python
+# Sync version
+def apply_guardrail(inputs, request_data, input_type):
+ # inputs: see table below
+ # request_data: {"model": "...", "user_id": "...", "team_id": "...", "metadata": {...}}
+ # input_type: "request" or "response"
+
+ return allow() # or block() or modify()
+
+# Async version (recommended when using HTTP primitives)
+async def apply_guardrail(inputs, request_data, input_type):
+ response = await http_post("https://api.example.com/check", body={"text": inputs["texts"][0]})
+ if response["success"] and response["body"].get("flagged"):
+ return block("Content flagged")
+ return allow()
+```
+
+### `inputs` Parameter
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `texts` | `List[str]` | Extracted text from the request/response |
+| `images` | `List[str]` | Extracted images (for image guardrails) |
+| `tools` | `List[dict]` | Tools sent to the LLM |
+| `tool_calls` | `List[dict]` | Tool calls returned from the LLM |
+| `structured_messages` | `List[dict]` | Full messages with role info (system/user/assistant) |
+| `model` | `str` | The model being used |
+
+### `request_data` Parameter
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `model` | `str` | Model name |
+| `user_id` | `str` | User ID from API key |
+| `team_id` | `str` | Team ID from API key |
+| `end_user_id` | `str` | End user ID |
+| `metadata` | `dict` | Request metadata |
+
+### Return Values
+
+| Function | Description |
+|----------|-------------|
+| `allow()` | Let request/response through |
+| `block(reason)` | Reject with message |
+| `modify(texts=[], images=[], tool_calls=[])` | Transform content |
+
+## Built-in Primitives
+
+### Regex
+
+| Function | Description |
+|----------|-------------|
+| `regex_match(text, pattern)` | Returns `True` if pattern found |
+| `regex_replace(text, pattern, replacement)` | Replace all matches |
+| `regex_find_all(text, pattern)` | Return list of matches |
+
+### JSON
+
+| Function | Description |
+|----------|-------------|
+| `json_parse(text)` | Parse JSON string, returns `None` on error |
+| `json_stringify(obj)` | Convert to JSON string |
+| `json_schema_valid(obj, schema)` | Validate against JSON schema |
+
+### URL
+
+| Function | Description |
+|----------|-------------|
+| `extract_urls(text)` | Extract all URLs from text |
+| `is_valid_url(url)` | Check if URL is valid |
+| `all_urls_valid(text)` | Check all URLs in text are valid |
+
+### Code Detection
+
+| Function | Description |
+|----------|-------------|
+| `detect_code(text)` | Returns `True` if code detected |
+| `detect_code_languages(text)` | Returns list of detected languages |
+| `contains_code_language(text, ["sql", "python"])` | Check for specific languages |
+
+### Text Utilities
+
+| Function | Description |
+|----------|-------------|
+| `contains(text, substring)` | Check if substring exists |
+| `contains_any(text, [substr1, substr2])` | Check if any substring exists |
+| `word_count(text)` | Count words |
+| `char_count(text)` | Count characters |
+| `lower(text)` / `upper(text)` / `trim(text)` | String transforms |
+
+### HTTP Requests (Async)
+
+Make async HTTP requests to external APIs for additional validation or content moderation.
+
+| Function | Description |
+|----------|-------------|
+| `await http_request(url, method, headers, body, timeout)` | General async HTTP request |
+| `await http_get(url, headers, timeout)` | Async GET request |
+| `await http_post(url, body, headers, timeout)` | Async POST request |
+
+**Response format:**
+```python
+{
+ "status_code": 200, # HTTP status code
+ "body": {...}, # Response body (parsed JSON or string)
+ "headers": {...}, # Response headers
+ "success": True, # True if status code is 2xx
+ "error": None # Error message if request failed
+}
+```
+
+**Note:** When using HTTP primitives, define your function as `async def apply_guardrail(...)` for non-blocking execution.
+
+## Examples
+
+### Block PII (SSN)
+
+```python
+def apply_guardrail(inputs, request_data, input_type):
+ for text in inputs["texts"]:
+ if regex_match(text, r"\d{3}-\d{2}-\d{4}"):
+ return block("SSN detected")
+ return allow()
+```
+
+### Redact Email Addresses
+
+```python
+def apply_guardrail(inputs, request_data, input_type):
+ pattern = r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}"
+ modified = []
+ for text in inputs["texts"]:
+ modified.append(regex_replace(text, pattern, "[EMAIL REDACTED]"))
+ return modify(texts=modified)
+```
+
+### Block SQL Injection
+
+```python
+def apply_guardrail(inputs, request_data, input_type):
+ if input_type != "request":
+ return allow()
+ for text in inputs["texts"]:
+ if contains_code_language(text, ["sql"]):
+ return block("SQL code not allowed")
+ return allow()
+```
+
+### Validate JSON Response
+
+```python
+def apply_guardrail(inputs, request_data, input_type):
+ if input_type != "response":
+ return allow()
+
+ schema = {
+ "type": "object",
+ "required": ["name", "value"]
+ }
+
+ for text in inputs["texts"]:
+ obj = json_parse(text)
+ if obj is None:
+ return block("Invalid JSON response")
+ if not json_schema_valid(obj, schema):
+ return block("Response missing required fields")
+ return allow()
+```
+
+### Check URLs in Response
+
+```python
+def apply_guardrail(inputs, request_data, input_type):
+ if input_type != "response":
+ return allow()
+ for text in inputs["texts"]:
+ if not all_urls_valid(text):
+ return block("Response contains invalid URLs")
+ return allow()
+```
+
+### Call External Moderation API (Async)
+
+```python
+async def apply_guardrail(inputs, request_data, input_type):
+ # Call an external moderation API
+ for text in inputs["texts"]:
+ response = await http_post(
+ "https://api.example.com/moderate",
+ body={"text": text, "user_id": request_data["user_id"]},
+ headers={"Authorization": "Bearer YOUR_API_KEY"},
+ timeout=10
+ )
+
+ if not response["success"]:
+ # API call failed - decide whether to allow or block
+ return allow()
+
+ if response["body"].get("flagged"):
+ return block(response["body"].get("reason", "Content flagged"))
+
+ return allow()
+```
+
+### Combine Multiple Checks
+
+```python
+def apply_guardrail(inputs, request_data, input_type):
+ modified = []
+
+ for text in inputs["texts"]:
+ # Redact SSN
+ text = regex_replace(text, r"\d{3}-\d{2}-\d{4}", "[SSN]")
+ # Redact credit cards
+ text = regex_replace(text, r"\d{16}", "[CARD]")
+ modified.append(text)
+
+ # Block SQL in requests
+ if input_type == "request":
+ for text in inputs["texts"]:
+ if contains_code_language(text, ["sql"]):
+ return block("SQL injection blocked")
+
+ return modify(texts=modified)
+```
+
+## Sandbox Restrictions
+
+Custom code runs in a restricted environment:
+
+- ❌ No `import` statements
+- ❌ No file I/O
+- ❌ No `exec()` or `eval()`
+- ✅ HTTP requests via built-in `http_request`, `http_get`, `http_post` primitives
+- ✅ Only LiteLLM-provided primitives available
+
+## Per-Request Usage
+
+Enable guardrail per request:
+
+```bash
+curl -X POST http://localhost:4000/chat/completions \
+ -H "Authorization: Bearer sk-1234" \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "gpt-4",
+ "messages": [{"role": "user", "content": "Hello"}],
+ "guardrails": ["block-ssn"]
+ }'
+```
+
+## Default On
+
+Run guardrail on all requests:
+
+```yaml
+litellm_settings:
+ guardrails:
+ - guardrail_name: block-ssn
+ litellm_params:
+ guardrail: custom_code
+ mode: pre_call
+ default_on: true
+ custom_code: |
+ def apply_guardrail(inputs, request_data, input_type):
+ ...
+```
diff --git a/docs/my-website/docs/proxy/guardrails/grayswan.md b/docs/my-website/docs/proxy/guardrails/grayswan.md
index d6efaf15504..6c0ccbc293d 100644
--- a/docs/my-website/docs/proxy/guardrails/grayswan.md
+++ b/docs/my-website/docs/proxy/guardrails/grayswan.md
@@ -13,20 +13,26 @@ Cygnal returns a `violation` score between `0` and `1` (higher means more likely
### 1. Obtain Credentials
-1. Create a Gray Swan account and generate a Cygnal API key.
+1. Log in to our Gray Swan platform and generate a Cygnal API key.
+
+ For existing customers, you should already have access to our [platform](https://platform.grayswan.ai).
+
+ For new users, please register at this [page](https://hubs.ly/Q03-sX1J0) and we are more than happy to give you an onboarding!
+
+
2. Configure environment variables for the LiteLLM proxy host:
-```bash
-export GRAYSWAN_API_KEY="your-grayswan-key"
-export GRAYSWAN_API_BASE="https://api.grayswan.ai"
-```
+ ```bash
+ export GRAYSWAN_API_KEY="your-grayswan-key"
+ export GRAYSWAN_API_BASE="https://api.grayswan.ai"
+ ```
### 2. Configure `config.yaml`
-Add a guardrail entry that references the Gray Swan integration. Below is a balanced example that monitors both input and output but only blocks once the violation score reaches the configured threshold.
+Add a guardrail entry that references the Gray Swan integration. Below is our recommmended settings.
```yaml
-model_list:
+model_list: # this part is a standard litellm configuration for reference
- model_name: openai/gpt-4.1-mini
litellm_params:
model: openai/gpt-4.1-mini
@@ -40,13 +46,14 @@ guardrails:
api_key: os.environ/GRAYSWAN_API_KEY
api_base: os.environ/GRAYSWAN_API_BASE # optional
optional_params:
- on_flagged_action: monitor # or "block"
+ on_flagged_action: passthrough # or "block" or "monitor"
violation_threshold: 0.5 # score >= threshold is flagged
reasoning_mode: hybrid # off | hybrid | thinking
- categories:
- safety: "Detect jailbreaks and policy violations"
- policy_id: "your-cygnal-policy-id"
+ policy_id: "your-cygnal-policy-id" # Optional: Your Cygnal policy ID. Defaults to a content safety policy if empty.
+ streaming_end_of_stream_only: true # For streaming API, only send the assembled message to Cygnal (post_call only). Defaults to false.
default_on: true
+ guardrail_timeout: 30 # Defaults to 30 seconds. Change accordingly.
+ fail_open: true # Defaults to true; set to false to propagate guardrail errors.
general_settings:
master_key: "your-litellm-master-key"
@@ -65,13 +72,13 @@ litellm --config config.yaml --port 4000
## Choosing Guardrail Modes
-Gray Swan can run during `pre_call`, `during_call`, and `post_call` stages. Combine modes based on your latency and coverage requirements.
+Gray Swan can run during `pre_call`, `during_call`, and `post_call` stages. Combine modes based on your latency and coverage requirements.
| Mode | When it Runs | Protects | Typical Use Case |
|--------------|-------------------|-----------------------|------------------|
| `pre_call` | Before LLM call | User input only | Block prompt injection before it reaches the model |
| `during_call`| Parallel to call | User input only | Low-latency monitoring without blocking |
-| `post_call` | After response | Full conversation | Scan output for policy violations, leaked secrets, or IPI |
+| `post_call` | After response | Model Outputs | Scan output for policy violations, leaked secrets, or IPI |
When using `during_call` with `on_flagged_action: block` or `on_flagged_action: passthrough`:
@@ -81,87 +88,110 @@ When using `during_call` with `on_flagged_action: block` or `on_flagged_action:
- The guardrail exception prevents the response from reaching the user, but **does not cancel the running LLM task**
- This means you pay full LLM costs while returning an error/passthrough message to the user
-**Recommendation:** For cost-sensitive applications, use `pre_call` and `post_call` instead of `during_call` for blocking or passthrough modes. Reserve `during_call` for `monitor` mode where you want low-latency logging without impacting the user experience.
+**Recommendation:** Use `pre_call` and `post_call` instead of `during_call` for `passthrough` (or `block`) `on_flagged_action` (see our recommended configuration above). Reserve `during_call` for `monitor` mode ONLY when you want low-latency logging without impacting the user experience.
-
-
+---
-```yaml
-guardrails:
- - guardrail_name: "cygnal-monitor-only"
- litellm_params:
- guardrail: grayswan
- mode: "during_call"
- api_key: os.environ/GRAYSWAN_API_KEY
- optional_params:
- on_flagged_action: monitor
- violation_threshold: 0.6
- default_on: true
+## Work with Claude Code
+
+Follow the official litellm [guide](https://docs.litellm.ai/docs/tutorials/claude_responses_api) on setting up Claude Code with litellm, with the guardrail part mentioned above added to your litellm configuration. Cygnal natively supports coding agent policies defense. Define your own policy or use the provided coding policies on the platform. The example config we show above is also the recommended setup for Claude Code (with the `policy_id` replaced with an appropriate one).
+
+---
+
+## Per-request overrides via `extra_body`
+
+You can override parts of the Gray Swan guardrail configuration on a per-request basis by passing `litellm_metadata.guardrails[*].grayswan.extra_body`.
+
+`extra_body` is merged into the Cygnal request body and takes precedence over specific fields from `config.yaml`, which are `policy_id`, `violation_threshold`, and `reasoning_mode`.
+
+If you include a `metadata` field inside `extra_body`, it is forwarded to the Cygnal API as-is under the request body's `metadata` field.
+
+Example:
+
+```bash
+curl -X POST "http://0.0.0.0:4000/v1/messages?beta=true" \
+ -H "Authorization: Bearer token" \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "openrouter/anthropic/claude-sonnet-4.5",
+ "messages": [{"role": "user", "content": "hello"}],
+ "litellm_metadata": {
+ "guardrails": [
+ {
+ "cygnal-monitor": {
+ "extra_body": {
+ "policy_id": "specific policy id you want to use",
+ "metadata": {
+ "user": "health-check"
+ }
+ }
+ }
+ }
+ ]
+ }
+ }'
```
-Best for visibility without blocking. Alerts are logged via LiteLLM’s standard logging callbacks.
+OpenAI client:
-
-
+```python
+from openai import OpenAI
-```yaml
-guardrails:
- - guardrail_name: "cygnal-block-input"
- litellm_params:
- guardrail: grayswan
- mode: "pre_call"
- api_key: os.environ/GRAYSWAN_API_KEY
- optional_params:
- on_flagged_action: block
- violation_threshold: 0.4
- categories:
- pii: "Detect sensitive data"
- default_on: true
+client = OpenAI(api_key="anything", base_url="http://0.0.0.0:4000")
+
+resp = client.responses.create(
+ model="openrouter/anthropic/claude-sonnet-4.5",
+ input="hello",
+ extra_body={
+ "litellm_metadata": {
+ "guardrails": [
+ {
+ "cygnal-monitor": {
+ "extra_body": {
+ "policy_id": "69038214e5cdb6befc5e991e",
+ "metadata": {"trace_id": "trace-123"},
+ }
+ }
+ }
+ ]
+ }
+ },
+)
```
-Stops malicious or sensitive prompts before any tokens are generated.
+Anthropic client:
-
-
+```python
+from anthropic import Anthropic
-```yaml
-guardrails:
- - guardrail_name: "cygnal-full-coverage"
- litellm_params:
- guardrail: grayswan
- mode: [pre_call, post_call]
- api_key: os.environ/GRAYSWAN_API_KEY
- optional_params:
- on_flagged_action: block
- violation_threshold: 0.5
- reasoning_mode: thinking
- policy_id: "policy-id-from-grayswan"
- default_on: true
+client = Anthropic(api_key="anything", base_url="http://0.0.0.0:4000")
+
+resp = client.messages.create(
+ model="openrouter/anthropic/claude-sonnet-4.5",
+ max_tokens=256,
+ messages=[{"role": "user", "content": "hello"}],
+ extra_body={
+ "litellm_metadata": {
+ "guardrails": [
+ {
+ "cygnal-monitor": {
+ "extra_body": {
+ "policy_id": "69038214e5cdb6befc5e991e",
+ "metadata": {"trace_id": "trace-123"},
+ }
+ }
+ }
+ ]
+ }
+ },
+)
```
-Provides the strongest enforcement by inspecting both prompts and responses.
+Notes:
-
-
-
-```yaml
-guardrails:
- - guardrail_name: "cygnal-passthrough"
- litellm_params:
- guardrail: grayswan
- mode: [pre_call, post_call]
- api_key: os.environ/GRAYSWAN_API_KEY
- optional_params:
- on_flagged_action: passthrough
- violation_threshold: 0.5
- default_on: true
-```
-
-Allows requests to proceed without raising a 400 error when content is flagged. Instead of blocking, the model response content is replaced with a detailed violation message including violation score, violated rules, and detection flags (mutation, IPI). **Supported Response Formats:** OpenAI chat/text completions, Anthropic Messages API. Other response types (embeddings, images, etc.) will log a warning and return unchanged.
-
-
-
+- The guardrail name (for example, `cygnal-monitor`) must match the `guardrail_name` in `config.yaml`.
+- Per-request guardrail overrides may require a premium license, depending on your proxy settings.
---
@@ -170,9 +200,14 @@ Allows requests to proceed without raising a 400 error when content is flagged.
| Parameter | Type | Description |
|---------------------------------------|-----------------|-------------|
| `api_key` | string | Gray Swan Cygnal API key. Reads from `GRAYSWAN_API_KEY` if omitted. |
+| `api_base` | string | Override for the Gray Swan API base URL. Defaults to `https://api.grayswan.ai` or `GRAYSWAN_API_BASE`. |
| `mode` | string or list | Guardrail stages (`pre_call`, `during_call`, `post_call`). |
| `optional_params.on_flagged_action` | string | `monitor` (log only), `block` (raise `HTTPException`), or `passthrough` (replace response content with violation message, no 400 error). |
-| `.optional_params.violation_threshold`| number (0-1) | Scores at or above this value are considered violations. |
+| `optional_params.violation_threshold` | number (0-1) | Scores at or above this value are considered violations. |
| `optional_params.reasoning_mode` | string | `off`, `hybrid`, or `thinking`. Enables Cygnal's reasoning capabilities. |
| `optional_params.categories` | object | Map of custom category names to descriptions. |
| `optional_params.policy_id` | string | Gray Swan policy identifier. |
+| `guardrail_timeout` | number | Timeout in seconds for the Cygnal request. Defaults to 30. |
+| `fail_open` | boolean | If true, errors contacting Cygnal are logged and the request proceeds; if false, errors propagate. Defaults to treu. |
+| `streaming_end_of_stream_only` | boolean | For streaming `post_call`, only send the final assembled response to Cygnal. Defaults to false. |
+| `default_on` | boolean | Run the guardrail on every request by default. |
diff --git a/docs/my-website/docs/proxy/guardrails/guardrail_policies.md b/docs/my-website/docs/proxy/guardrails/guardrail_policies.md
new file mode 100644
index 00000000000..e2cb839203e
--- /dev/null
+++ b/docs/my-website/docs/proxy/guardrails/guardrail_policies.md
@@ -0,0 +1,396 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# [Beta] Guardrail Policies
+
+Use policies to group guardrails and control which ones run for specific teams, keys, or models.
+
+## Why use policies?
+
+- Enable/disable specific guardrails for teams, keys, or models
+- Group guardrails into a single policy
+- Inherit from existing policies and override what you need
+
+## Quick Start
+
+
+
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+ - model_name: gpt-4
+ litellm_params:
+ model: openai/gpt-4
+
+# 1. Define your guardrails
+guardrails:
+ - guardrail_name: pii_masking
+ litellm_params:
+ guardrail: presidio
+ mode: pre_call
+
+ - guardrail_name: prompt_injection
+ litellm_params:
+ guardrail: lakera
+ mode: pre_call
+ api_key: os.environ/LAKERA_API_KEY
+
+# 2. Create a policy
+policies:
+ my-policy:
+ guardrails:
+ add:
+ - pii_masking
+ - prompt_injection
+
+# 3. Attach the policy
+policy_attachments:
+ - policy: my-policy
+ scope: "*" # apply to all requests
+```
+
+
+
+
+**Step 1: Create a Policy**
+
+Go to **Policies** tab and click **+ Create New Policy**. Fill in the policy name, description, and select guardrails to add.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Response headers show what ran:
+
+```
+x-litellm-applied-policies: my-policy
+x-litellm-applied-guardrails: pii_masking,prompt_injection
+```
+
+## Add guardrails for a specific team
+
+:::info
+✨ Enterprise only feature for team/key-based policy attachments. [Get a free trial](https://www.litellm.ai/enterprise#trial)
+:::
+
+You have a global baseline, but want to add extra guardrails for a specific team.
+
+
+
+
+```yaml showLineNumbers title="config.yaml"
+policies:
+ global-baseline:
+ guardrails:
+ add:
+ - pii_masking
+
+ finance-team-policy:
+ inherit: global-baseline
+ guardrails:
+ add:
+ - strict_compliance_check
+ - audit_logger
+
+policy_attachments:
+ - policy: global-baseline
+ scope: "*"
+
+ - policy: finance-team-policy
+ teams:
+ - finance # team alias from /team/new
+```
+
+
+
+
+**Option 1: Create a team-scoped attachment**
+
+Go to **Policies** > **Attachments** tab and click **+ Create New Attachment**. Select the policy and the teams to scope it to.
+
+
+
+
+
+**Option 2: Attach from team settings**
+
+Go to **Teams** > click on a team > **Settings** tab > under **Policies**, select the policies to attach.
+
+
+
+
+
+
+
+
+
+
+Now the `finance` team gets `pii_masking` + `strict_compliance_check` + `audit_logger`, while everyone else just gets `pii_masking`.
+
+## Remove guardrails for a specific team
+
+:::info
+✨ Enterprise only feature for team/key-based policy attachments. [Get a free trial](https://www.litellm.ai/enterprise#trial)
+:::
+
+You have guardrails running globally, but want to disable some for a specific team (e.g., internal testing).
+
+```yaml showLineNumbers title="config.yaml"
+policies:
+ global-baseline:
+ guardrails:
+ add:
+ - pii_masking
+ - prompt_injection
+
+ internal-team-policy:
+ inherit: global-baseline
+ guardrails:
+ remove:
+ - pii_masking # don't need PII masking for internal testing
+
+policy_attachments:
+ - policy: global-baseline
+ scope: "*"
+
+ - policy: internal-team-policy
+ teams:
+ - internal-testing # team alias from /team/new
+```
+
+Now the `internal-testing` team only gets `prompt_injection`, while everyone else gets both guardrails.
+
+## Inheritance
+
+Start with a base policy and build on it:
+
+```yaml showLineNumbers title="config.yaml"
+policies:
+ base:
+ guardrails:
+ add:
+ - pii_masking
+ - toxicity_filter
+
+ strict:
+ inherit: base
+ guardrails:
+ add:
+ - prompt_injection
+
+ relaxed:
+ inherit: base
+ guardrails:
+ remove:
+ - toxicity_filter
+```
+
+What you get:
+- `base` → `[pii_masking, toxicity_filter]`
+- `strict` → `[pii_masking, toxicity_filter, prompt_injection]`
+- `relaxed` → `[pii_masking]`
+
+## Model Conditions
+
+Run guardrails only for specific models:
+
+```yaml showLineNumbers title="config.yaml"
+policies:
+ gpt4-safety:
+ guardrails:
+ add:
+ - strict_content_filter
+ condition:
+ model: "gpt-4.*" # regex - matches gpt-4, gpt-4-turbo, gpt-4o
+
+ bedrock-compliance:
+ guardrails:
+ add:
+ - audit_logger
+ condition:
+ model: # exact match list
+ - bedrock/claude-3
+ - bedrock/claude-2
+```
+
+## Attachments
+
+Policies don't do anything until you attach them. Attachments tell LiteLLM *where* to apply each policy.
+
+**Global** - runs on every request:
+
+```yaml showLineNumbers title="config.yaml"
+policy_attachments:
+ - policy: default
+ scope: "*"
+```
+
+**Team-specific** (uses team alias from `/team/new`):
+
+```yaml showLineNumbers title="config.yaml"
+policy_attachments:
+ - policy: hipaa-compliance
+ teams:
+ - healthcare-team # team alias
+ - medical-research # team alias
+```
+
+**Key-specific** (uses key alias from `/key/generate`, wildcards supported):
+
+```yaml showLineNumbers title="config.yaml"
+policy_attachments:
+ - policy: internal-testing
+ keys:
+ - "dev-*" # key alias pattern
+ - "test-*" # key alias pattern
+```
+
+**Tag-based** (matches keys/teams by metadata tags, wildcards supported):
+
+```yaml showLineNumbers title="config.yaml"
+policy_attachments:
+ - policy: hipaa-compliance
+ tags:
+ - "healthcare"
+ - "health-*" # wildcard - matches health-team, health-dev, etc.
+```
+
+Tags are read from key and team `metadata.tags`. For example, a key created with `metadata: {"tags": ["healthcare"]}` would match the attachment above.
+
+## Test Policy Matching
+
+Debug which policies and guardrails apply for a given context. Use this to verify your policy configuration before deploying.
+
+
+
+
+Go to **Policies** > **Test** tab. Enter a team alias, key alias, model, or tags and click **Test** to see which policies match and what guardrails would be applied.
+
+
+
+
+
+
+```bash
+curl -X POST "http://localhost:4000/policies/resolve" \
+ -H "Authorization: Bearer " \
+ -H "Content-Type: application/json" \
+ -d '{
+ "tags": ["healthcare"],
+ "model": "gpt-4"
+ }'
+```
+
+Response:
+
+```json
+{
+ "effective_guardrails": ["pii_masking"],
+ "matched_policies": [
+ {
+ "policy_name": "hipaa-compliance",
+ "matched_via": "tag:healthcare",
+ "guardrails_added": ["pii_masking"]
+ }
+ ]
+}
+```
+
+
+
+
+## Config Reference
+
+### `policies`
+
+```yaml
+policies:
+ :
+ description: ...
+ inherit: ...
+ guardrails:
+ add: [...]
+ remove: [...]
+ condition:
+ model: ...
+```
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `description` | `string` | Optional. What this policy does. |
+| `inherit` | `string` | Optional. Parent policy to inherit guardrails from. |
+| `guardrails.add` | `list[string]` | Guardrails to enable. |
+| `guardrails.remove` | `list[string]` | Guardrails to disable (useful with inheritance). |
+| `condition.model` | `string` or `list[string]` | Optional. Only apply when model matches. Supports regex. |
+
+### `policy_attachments`
+
+```yaml
+policy_attachments:
+ - policy: ...
+ scope: ...
+ teams: [...]
+ keys: [...]
+ models: [...]
+ tags: [...]
+```
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `policy` | `string` | **Required.** Name of the policy to attach. |
+| `scope` | `string` | Use `"*"` to apply globally. |
+| `teams` | `list[string]` | Team aliases (from `/team/new`). Supports `*` wildcard. |
+| `keys` | `list[string]` | Key aliases (from `/key/generate`). Supports `*` wildcard. |
+| `models` | `list[string]` | Model names. Supports `*` wildcard. |
+| `tags` | `list[string]` | Tag patterns (from key/team `metadata.tags`). Supports `*` wildcard. |
+
+### Response Headers
+
+| Header | Description |
+|--------|-------------|
+| `x-litellm-applied-policies` | Policies that matched this request |
+| `x-litellm-applied-guardrails` | Guardrails that actually ran |
+| `x-litellm-policy-sources` | Why each policy matched (e.g., `hipaa=tag:healthcare; baseline=scope:*`) |
+
+## How it works
+
+Example config:
+
+```yaml showLineNumbers title="config.yaml"
+policies:
+ base:
+ guardrails:
+ add: [pii_masking]
+
+ finance-policy:
+ inherit: base
+ guardrails:
+ add: [audit_logger]
+
+policy_attachments:
+ - policy: base
+ scope: "*"
+ - policy: finance-policy
+ teams: [finance]
+```
+
+```mermaid
+flowchart TD
+ A["Request with team_alias='finance'"] --> B["Matches policies: base, finance-policy"]
+ B --> C["Resolves guardrails: pii_masking, audit_logger"]
+```
+
+1. Request comes in with `team_alias='finance'`
+2. Matches `base` (via `scope: "*"`) and `finance-policy` (via `teams: [finance]`)
+3. Resolves guardrails: `base` adds `pii_masking`, `finance-policy` inherits and adds `audit_logger`
+4. Final guardrails: `pii_masking`, `audit_logger`
diff --git a/docs/my-website/docs/proxy/guardrails/onyx_security.md b/docs/my-website/docs/proxy/guardrails/onyx_security.md
index 85b0ba9f830..d240902eb52 100644
--- a/docs/my-website/docs/proxy/guardrails/onyx_security.md
+++ b/docs/my-website/docs/proxy/guardrails/onyx_security.md
@@ -128,6 +128,7 @@ guardrails:
mode: ["pre_call", "post_call", "during_call"] # Run at multiple stages
api_key: os.environ/ONYX_API_KEY
api_base: os.environ/ONYX_API_BASE
+ timeout: 10.0 # Optional, defaults to 10 seconds
```
### Required Parameters
@@ -137,6 +138,7 @@ guardrails:
### Optional Parameters
- **`api_base`**: Onyx API base URL (defaults to `https://ai-guard.onyx.security`)
+- **`timeout`**: Request timeout in seconds (defaults to `10.0`)
## Environment Variables
@@ -145,4 +147,5 @@ You can set these environment variables instead of hardcoding values in your con
```shell
export ONYX_API_KEY="your-api-key-here"
export ONYX_API_BASE="https://ai-guard.onyx.security" # Optional
+export ONYX_TIMEOUT=10 # Optional, timeout in seconds
```
diff --git a/docs/my-website/docs/proxy/guardrails/policy_tags.md b/docs/my-website/docs/proxy/guardrails/policy_tags.md
new file mode 100644
index 00000000000..11840116c31
--- /dev/null
+++ b/docs/my-website/docs/proxy/guardrails/policy_tags.md
@@ -0,0 +1,139 @@
+# Tag-Based Policy Attachments
+
+Apply guardrail policies automatically to any key or team that has a specific tag. Instead of attaching policies one-by-one, tag your keys and let the policy engine handle the rest.
+
+**Example:** Your security team requires all healthcare-related keys to run PII masking and PHI detection. Tag those keys with `health`, create a single tag-based attachment, and every matching key gets the guardrails automatically.
+
+## 1. Create a Policy with Guardrails
+
+Navigate to **Policies** in the left sidebar. You'll see a list of existing policies along with their guardrails.
+
+
+
+Click **+ Add New Policy**. In the modal, enter a name for your policy (e.g., `high-risk-policy2`). You can also type to search existing policy names if you want to reference them.
+
+
+
+Scroll down to **Guardrails to Add**. Click the dropdown to see all available guardrails configured on your proxy — select the ones this policy should enforce.
+
+
+
+After selecting your guardrails, they appear as chips in the input field. The **Resolved Guardrails** section below shows the final set that will be applied (including any inherited from a parent policy).
+
+
+
+Click **Create Policy** to save.
+
+
+
+## 2. Add a Tag Attachment for the Policy
+
+After creating the policy, switch to the **Attachments** tab. This is where you define *where* the policy applies.
+
+
+
+Click **+ Add New Attachment**. The Attachments page explains the available scopes: Global, Teams, Keys, Models, and **Tags**.
+
+
+
+In the **Create Policy Attachment** modal, first select the policy you just created from the dropdown.
+
+
+
+Choose **Specific (teams, keys, models, or tags)** as the scope type. This expands the form to show fields for Teams, Keys, Models, and Tags.
+
+
+
+Scroll down to the **Tags** field and type the tag to match — here we enter `health`. You can enter any string, or use a wildcard pattern like `health-*` to match all tags starting with `health-` (e.g., `health-team`, `health-dev`).
+
+
+
+## 3. Check the Impact of the Attachment
+
+Before creating the attachment, click **Estimate Impact** to preview how many keys and teams would be affected. This is your blast-radius check — make sure the scope is what you expect before applying.
+
+
+
+The **Impact Preview** appears inline, showing exactly how many keys and teams would be affected. In this example: "This attachment would affect **1 key** and **0 teams**", with the key alias `hi` listed.
+
+
+
+Once you're satisfied with the impact, click **Create Attachment** to save.
+
+
+
+The attachment now appears in the table with the policy name `high-risk-policy2` and tag `health` visible.
+
+
+
+## 4. Create a Key with the Tag
+
+Navigate to **Virtual Keys** in the left sidebar. Click **+ Create New Key**.
+
+
+
+Enter a key name and select a model. Then expand **Optional Settings** and scroll down to the **Tags** field.
+
+
+
+In the **Tags** field, type `health` and press Enter. This is the tag the policy engine will match against.
+
+
+
+The tag `health` now appears as a chip in the Tags field. Confirm your settings look correct.
+
+
+
+Click **Create Key** at the bottom of the form.
+
+
+
+A dialog appears with your new virtual key. Click **Copy Virtual Key** — you'll need this to test in the next step.
+
+
+
+## 5. Test the Key and Validate the Policy is Applied
+
+Navigate to **Playground** in the left sidebar to test the key interactively.
+
+
+
+Under **Virtual Key Source**, select "Virtual Key" and paste the key you just copied into the input field.
+
+
+
+Select a model from the **Select Model** dropdown.
+
+
+
+Type a message and press Enter. If a guardrail blocks the request, you'll see it in the response. In this example, the `testing-pl` guardrail detected an email pattern and returned a 403 error — confirming the policy is working.
+
+
+
+**Using curl:**
+
+You can also verify via the command line. The response headers confirm which policies and guardrails were applied:
+
+```bash
+curl -v http://localhost:4000/chat/completions \
+ -H "Authorization: Bearer " \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "gpt-4o",
+ "messages": [{"role": "user", "content": "say hi"}]
+ }'
+```
+
+Check the response headers:
+
+```
+x-litellm-applied-policies: high-risk-policy2
+x-litellm-applied-guardrails: pii-pre-guard,phi-pre-guard,testing-pl
+x-litellm-policy-sources: high-risk-policy2=tag:health
+```
+
+| Header | What it tells you |
+|--------|-------------------|
+| `x-litellm-applied-policies` | Which policies matched this request |
+| `x-litellm-applied-guardrails` | Which guardrails actually ran |
+| `x-litellm-policy-sources` | **Why** each policy matched — `tag:health` confirms it was the tag |
diff --git a/docs/my-website/docs/proxy/guardrails/policy_templates.md b/docs/my-website/docs/proxy/guardrails/policy_templates.md
new file mode 100644
index 00000000000..f0c93ca44c7
--- /dev/null
+++ b/docs/my-website/docs/proxy/guardrails/policy_templates.md
@@ -0,0 +1,296 @@
+# Policy Templates
+
+Policy templates provide pre-configured guardrail policies that you can use as a starting point for your organization. Instead of manually creating policies and guardrails, you can select a template that matches your use case and deploy it with one click.
+
+## Using Policy Templates
+
+### In the UI
+
+1. Navigate to **Policies → Templates** tab in the LiteLLM Admin UI
+2. Browse available templates (e.g., "PII Protection", "Cost Control", "HR Compliance")
+3. Click **"Use Template"** on any template
+4. Review the guardrails that will be created:
+ - Existing guardrails are marked with a green checkmark
+ - New guardrails can be selected/deselected
+5. Click **"Create X Guardrails & Use Template"**
+6. Review and customize the pre-filled policy form
+7. Click **"Create Policy"** to save
+
+### Workflow
+
+```
+Select Template → Review Guardrails → Create Selected → Edit Policy → Save
+```
+
+The system automatically:
+- ✅ Detects which guardrails already exist
+- ✅ Creates only the missing guardrails you select
+- ✅ Pre-fills the policy form with template data
+- ✅ Lets you customize before saving
+
+## Available Templates
+
+Templates are fetched from [GitHub](https://raw.githubusercontent.com/BerriAI/litellm/main/policy_templates.json) with automatic fallback to local backup.
+
+### Current Templates
+
+#### 1. Advanced PII Protection (Australia)
+- **Complexity:** High
+- **Use Case:** Comprehensive PII detection for Australian organizations
+- **Guardrails:**
+ - Australian tax identifiers (TFN, ABN, Medicare)
+ - Australian passports
+ - International PII (SSN, passports, national IDs)
+ - Contact information (email, phone, address)
+ - Financial data (credit cards, IBAN)
+ - API credentials (AWS, GitHub, Slack) - **BLOCKS** requests
+ - Network infrastructure (IP addresses)
+ - Protected class information (gender, race, religion, disability, etc.)
+
+#### 2. Baseline PII Protection
+- **Complexity:** Low
+- **Use Case:** Basic protection for internal tools and testing
+- **Guardrails:**
+ - Australian tax identifiers
+ - API credentials
+ - Financial data
+
+## Creating Your Own Policy Templates
+
+You can contribute policy templates for the entire LiteLLM community to use.
+
+### Template Structure
+
+Templates are defined in JSON format with the following structure:
+
+```json
+{
+ "id": "unique-template-id",
+ "title": "Display Title",
+ "description": "Detailed description of what this template protects",
+ "icon": "ShieldCheckIcon",
+ "iconColor": "text-purple-500",
+ "iconBg": "bg-purple-50",
+ "guardrails": [
+ "guardrail-name-1",
+ "guardrail-name-2"
+ ],
+ "complexity": "Low|Medium|High",
+ "guardrailDefinitions": [
+ {
+ "guardrail_name": "example-guardrail",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "patterns": [
+ {
+ "pattern_type": "prebuilt",
+ "pattern_name": "email",
+ "action": "MASK"
+ }
+ ],
+ "pattern_redaction_format": "[{pattern_name}_REDACTED]"
+ },
+ "guardrail_info": {
+ "description": "What this guardrail does"
+ }
+ }
+ ],
+ "templateData": {
+ "policy_name": "policy-name",
+ "description": "Policy description",
+ "guardrails_add": ["guardrail-name-1", "guardrail-name-2"],
+ "guardrails_remove": []
+ }
+}
+```
+
+### Field Descriptions
+
+#### Display Fields
+- **id**: Unique identifier (lowercase with hyphens)
+- **title**: User-facing name shown in UI
+- **description**: Detailed explanation of what the template protects
+- **icon**: Icon name (must be available in UI icon map)
+- **iconColor**: Tailwind CSS text color class
+- **iconBg**: Tailwind CSS background color class
+- **guardrails**: Array of guardrail names (for display only)
+- **complexity**: Badge showing difficulty ("Low", "Medium", or "High")
+
+#### Guardrail Definitions
+- **guardrailDefinitions**: Array of complete guardrail configurations
+ - Each must be a valid guardrail object that can be sent to `/guardrails` POST endpoint
+ - If a guardrail already exists, it will be skipped
+ - Can be empty `[]` if template uses only existing guardrails
+
+#### Policy Configuration
+- **templateData**: Object that pre-fills the policy form
+ - **policy_name**: Suggested name (user can edit)
+ - **description**: Policy description
+ - **guardrails_add**: Array of guardrail names to include
+ - **guardrails_remove**: Array to remove (usually `[]` for templates)
+ - **inherit**: (Optional) Parent policy name for inheritance
+
+### Example Template
+
+Here's a complete example for a HIPAA compliance template:
+
+```json
+{
+ "id": "hipaa-compliance",
+ "title": "HIPAA Compliance Policy",
+ "description": "Healthcare compliance policy that masks PHI and enforces HIPAA regulations for healthcare applications.",
+ "icon": "ShieldCheckIcon",
+ "iconColor": "text-red-500",
+ "iconBg": "bg-red-50",
+ "guardrails": [
+ "phi-detector",
+ "medical-record-blocker",
+ "patient-id-masker"
+ ],
+ "complexity": "High",
+ "guardrailDefinitions": [
+ {
+ "guardrail_name": "phi-detector",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "patterns": [
+ {
+ "pattern_type": "prebuilt",
+ "pattern_name": "us_ssn",
+ "action": "MASK"
+ },
+ {
+ "pattern_type": "prebuilt",
+ "pattern_name": "email",
+ "action": "MASK"
+ },
+ {
+ "pattern_type": "prebuilt",
+ "pattern_name": "us_phone",
+ "action": "MASK"
+ }
+ ],
+ "pattern_redaction_format": "[PHI_REDACTED]"
+ },
+ "guardrail_info": {
+ "description": "Detects and masks Protected Health Information (PHI)"
+ }
+ }
+ ],
+ "templateData": {
+ "policy_name": "hipaa-compliance-policy",
+ "description": "HIPAA compliance policy for healthcare applications",
+ "guardrails_add": [
+ "phi-detector",
+ "medical-record-blocker",
+ "patient-id-masker"
+ ],
+ "guardrails_remove": []
+ }
+}
+```
+
+## Contributing Templates
+
+To contribute a policy template for everyone to use:
+
+### Step 1: Create Your Template JSON
+
+1. Create a JSON file following the structure above
+2. Test it locally by adding it to your local `policy_templates.json`
+3. Verify all guardrails work correctly
+4. Ensure descriptions are clear and helpful
+
+### Step 2: Submit a Pull Request
+
+1. Fork the [LiteLLM repository](https://github.com/BerriAI/litellm)
+2. Add your template to `policy_templates.json` at the root
+3. Add your template to `litellm/policy_templates_backup.json` (keep both in sync)
+4. Create a pull request with:
+ - Clear description of what the template protects
+ - Use case examples
+ - Any relevant compliance frameworks (HIPAA, GDPR, SOC 2, etc.)
+
+### Guidelines
+
+**DO:**
+- ✅ Use clear, descriptive names
+- ✅ Include comprehensive descriptions
+- ✅ Test all guardrails thoroughly
+- ✅ Document pattern sources (e.g., "Based on NIST guidelines")
+- ✅ Group related guardrails logically
+- ✅ Consider different complexity levels
+
+**DON'T:**
+- ❌ Include credentials or secrets
+- ❌ Use overly broad patterns that may have false positives
+- ❌ Duplicate existing templates
+- ❌ Use custom code without thorough testing
+
+## Using Templates Offline
+
+For air-gapped or offline deployments, set the environment variable:
+
+```bash
+export LITELLM_LOCAL_POLICY_TEMPLATES=true
+```
+
+This forces the system to use the local backup (`litellm/policy_templates_backup.json`) instead of fetching from GitHub.
+
+## Template Sources
+
+- **GitHub (default):** https://raw.githubusercontent.com/BerriAI/litellm/main/policy_templates.json
+- **Local backup:** `litellm/policy_templates_backup.json`
+
+Templates are automatically fetched from GitHub on each request, with fallback to local backup on any failure.
+
+## Available Pattern Types
+
+When creating guardrails for templates, you can use these prebuilt patterns:
+
+### Identity Documents
+- `passport_australia`, `passport_us`, `passport_uk`, `passport_germany`, etc.
+- `us_ssn`, `us_ssn_no_dash`
+- `au_tfn`, `au_abn`, `au_medicare`
+- `nl_bsn_contextual`
+- `br_cpf`, `br_rg`, `br_cnpj`
+
+### Financial
+- `visa`, `mastercard`, `amex`, `discover`, `credit_card`
+- `iban`
+
+### Contact Information
+- `email`
+- `us_phone`, `br_phone_landline`, `br_phone_mobile`
+- `street_address`
+- `br_cep` (Brazilian postal code)
+
+### Credentials
+- `aws_access_key`, `aws_secret_key`
+- `github_token`
+- `slack_token`
+- `generic_api_key`
+
+### Network
+- `ipv4`, `ipv6`
+
+### Protected Class
+- `gender_sexual_orientation`
+- `race_ethnicity_national_origin`
+- `religion`
+- `age_discrimination`
+- `disability`
+- `marital_family_status`
+- `military_status`
+- `public_assistance`
+
+See the [full patterns list](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json) for all available patterns.
+
+## Related Docs
+
+- [Guardrail Policies](./guardrail_policies)
+- [Policy Tags](./policy_tags)
+- [Content Filter Patterns](../hooks/content_filter)
+- [Custom Code Guardrails](../hooks/custom_code)
diff --git a/docs/my-website/docs/proxy/guardrails/quick_start.md b/docs/my-website/docs/proxy/guardrails/quick_start.md
index 4a8dc4e6fe4..ddb215fcb66 100644
--- a/docs/my-website/docs/proxy/guardrails/quick_start.md
+++ b/docs/my-website/docs/proxy/guardrails/quick_start.md
@@ -203,8 +203,12 @@ Your response headers will include `x-litellm-applied-guardrails` with the guard
x-litellm-applied-guardrails: aporia-pre-guard
```
+### Guardrail Policies
-
+Need more control? Use [Guardrail Policies](./guardrail_policies.md) to:
+- Group guardrails into reusable policies
+- Enable/disable guardrails for specific teams, keys, or models
+- Inherit from existing policies and override specific guardrails
## **Using Guardrails Client Side**
@@ -401,14 +405,10 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
## **Proxy Admin Controls**
-### ✨ Monitoring Guardrails
+### Monitoring Guardrails
Monitor which guardrails were executed and whether they passed or failed. e.g. guardrail going rogue and failing requests we don't intend to fail
-:::info
-
-✨ This is an Enterprise only feature [Get a free trial](https://www.litellm.ai/enterprise#trial)
-
:::
#### Setup
diff --git a/docs/my-website/docs/proxy/guardrails/zscaler_ai_guard.md b/docs/my-website/docs/proxy/guardrails/zscaler_ai_guard.md
index 94f31c3bfdf..2e626004238 100644
--- a/docs/my-website/docs/proxy/guardrails/zscaler_ai_guard.md
+++ b/docs/my-website/docs/proxy/guardrails/zscaler_ai_guard.md
@@ -100,7 +100,7 @@ In cases where encounter other errors when apply Zscaler AI Guard, return exampl
}
}
```
-## 6. Sending User Information to Zscaler AI Guard for Analysis (Optional)
+## 6. Sending User Information to Zscaler AI Guard (Optional)
If you need to send end-user information to Zscaler AI Guard for analysis, you can set the configuration in the environment variables to True and include the relevant information in custom_headers on Zscaler AI Guard.
- To send user_api_key_alias:
@@ -133,4 +133,30 @@ curl -i http://localhost:8165/v1/chat/completions \
"zguard_policy_id":
}
}'
+```
+
+## 8. Set Custom Zscaler AI Guard Policy on Litellm Team OR Key Metadata (Optional)
+In addition to setting `zguard_policy_id` in a request or the configuration file, you can also set it in the metadata for LiteLLM Team or Key. The `zguard_policy_id` is determined using the following order of precedence: request, Key, Team, config file. This logic is illustrated below:
+```
+user_api_key_metadata = metadata.get("user_api_key_metadata", {}) or {}
+team_metadata = metadata.get("team_metadata", {}) or {}
+policy_id = (
+ metadata.get("zguard_policy_id")
+ if "zguard_policy_id" in metadata
+ else (
+ user_api_key_metadata.get("zguard_policy_id")
+ if "zguard_policy_id" in user_api_key_metadata
+ else (
+ team_metadata.get("zguard_policy_id")
+ if "zguard_policy_id" in team_metadata
+ else self.policy_id
+ )
+ )
+ )
+```
+You can leverage this feature to apply multiple policies configured on the Zscaler AI Guard (ZGuard) to traffic from different applications. (Note: It is recommended to map policies using either Team or Key metadata, but not a mix of both.)
+
+Example set in Team/Key Metadata, you can set From UI:
+```
+{"zguard_policy_id": 100}
```
\ No newline at end of file
diff --git a/docs/my-website/docs/proxy/keys_teams_router_settings.md b/docs/my-website/docs/proxy/keys_teams_router_settings.md
new file mode 100644
index 00000000000..ec59e8f271b
--- /dev/null
+++ b/docs/my-website/docs/proxy/keys_teams_router_settings.md
@@ -0,0 +1,150 @@
+import Image from '@theme/IdealImage';
+
+# UI - Router Settings for Keys and Teams
+
+Configure router settings at the key and team level to achieve granular control over routing behavior, fallbacks, retries, and other router configurations. This enables you to customize routing behavior for specific keys or teams without affecting global settings.
+
+## Overview
+
+Router Settings for Keys and Teams allows you to configure router behavior at different levels of granularity. Previously, router settings could only be configured globally, applying the same routing strategy, fallbacks, timeouts, and retry policies to all requests across your entire proxy instance.
+
+With key-level and team-level router settings, you can now:
+
+- **Customize routing strategies** per key or team (e.g., use `least-busy` for high-priority keys, `latency-based-routing` for others)
+- **Configure different fallback chains** for different keys or teams
+- **Set key-specific or team-specific timeouts** and retry policies
+- **Apply different reliability settings** (cooldowns, allowed failures) per key or team
+- **Override global settings** when needed for specific use cases
+
+
+
+## Summary
+
+Router settings follow a **hierarchical resolution order**: **Keys > Teams > Global**. When a request is made:
+
+1. **Key-level settings** are checked first. If router settings are configured for the API key being used, those settings are applied.
+2. **Team-level settings** are checked next. If the key belongs to a team and that team has router settings configured, those settings are used (unless key-level settings exist).
+3. **Global settings** are used as the final fallback. If neither key nor team settings are found, the global router settings from your proxy configuration are applied.
+
+This hierarchical approach ensures that the most specific settings take precedence, allowing you to fine-tune routing behavior for individual keys or teams while maintaining sensible defaults at the global level.
+
+## How Router Settings Resolution Works
+
+Router settings are resolved in the following priority order:
+
+### Resolution Order: Key > Team > Global
+
+1. **Key-level router settings** (highest priority)
+ - Applied when router settings are configured directly on an API key
+ - Takes precedence over all other settings
+ - Useful for individual key customization
+
+2. **Team-level router settings** (medium priority)
+ - Applied when the API key belongs to a team with router settings configured
+ - Only used if no key-level settings exist
+ - Useful for applying consistent settings across multiple keys in a team
+
+3. **Global router settings** (lowest priority)
+ - Applied from your proxy configuration file or database
+ - Used as the default when no key or team settings are found
+ - Previously, this was the only option available
+
+## How to Configure Router Settings
+
+### Configuring Router Settings for Keys
+
+Follow these steps to configure router settings for an API key:
+
+1. Navigate to [http://localhost:4000/ui/?login=success](http://localhost:4000/ui/?login=success)
+
+
+
+2. Click "+ Create New Key" (or edit an existing key)
+
+
+
+3. Click "Optional Settings"
+
+
+
+4. Click "Router Settings"
+
+
+
+5. Configure your desired router settings. For example, click "Fallbacks" to configure fallback models:
+
+
+
+6. Click "Select a model to begin configuring fallbacks" and configure your fallback chain:
+
+
+
+### Configuring Router Settings for Teams
+
+Follow these steps to configure router settings for a team:
+
+1. Navigate to [http://localhost:4000/ui/?login=success](http://localhost:4000/ui/?login=success)
+
+
+
+2. Click "Teams"
+
+
+
+3. Click "+ Create New Team" (or edit an existing team)
+
+
+
+4. Click "Router Settings"
+
+
+
+5. Configure your desired router settings. For example, click "Fallbacks" to configure fallback models:
+
+
+
+6. Click "Select a model to begin configuring fallbacks" and configure your fallback chain:
+
+
+
+## Use Cases
+
+### Different Routing Strategies per Key
+
+Configure different routing strategies for different use cases:
+
+- **High-priority production keys**: Use `latency-based-routing` for optimal performance
+- **Development keys**: Use `simple-shuffle` for simplicity
+- **Cost-sensitive keys**: Use `cost-based-routing` to minimize expenses
+
+### Team-Level Consistency
+
+Apply consistent router settings across all keys in a team:
+
+- Set team-wide fallback chains for reliability
+- Configure team-specific timeout policies
+- Apply uniform retry policies across team members
+
+### Override Global Settings
+
+Override global settings for specific scenarios:
+
+- Production keys may need stricter timeout policies than development
+- Certain teams may require different fallback models
+- Individual keys may need custom retry policies for specific use cases
+
+### Gradual Rollout
+
+Test new router settings on specific keys or teams before applying globally:
+
+- Configure new routing strategies on a test key first
+- Validate fallback chains on a small team before global rollout
+- A/B test different timeout values across different keys
+
+## Related Features
+
+- [Router Settings Reference](./config_settings.md#router_settings---reference) - Complete reference of all router settings
+- [Load Balancing](./load_balancing.md) - Learn about routing strategies and load balancing
+- [Reliability](./reliability.md) - Configure fallbacks, retries, and error handling
+- [Keys](./keys.md) - Manage API keys and their settings
+- [Teams](./teams.md) - Organize keys into teams
diff --git a/docs/my-website/docs/proxy/litellm_managed_files.md b/docs/my-website/docs/proxy/litellm_managed_files.md
index 7aba173f35b..6272180bd40 100644
--- a/docs/my-website/docs/proxy/litellm_managed_files.md
+++ b/docs/my-website/docs/proxy/litellm_managed_files.md
@@ -11,7 +11,7 @@ import Image from '@theme/IdealImage';
This is a free LiteLLM Enterprise feature.
-Available via the `litellm[proxy]` package or any `litellm` docker image.
+Available via the `litellm` docker image. If you are using the pip package, you must install [`litellm-enterprise`](https://pypi.org/project/litellm-enterprise/).
:::
diff --git a/docs/my-website/docs/proxy/load_balancing.md b/docs/my-website/docs/proxy/load_balancing.md
index 42f6ef1aa51..186307d6498 100644
--- a/docs/my-website/docs/proxy/load_balancing.md
+++ b/docs/my-website/docs/proxy/load_balancing.md
@@ -69,6 +69,67 @@ router_settings:
redis_port: 1992
```
+## Enforce Model Rate Limits
+
+Strictly enforce RPM/TPM limits set on deployments. When limits are exceeded, requests are blocked **before** reaching the LLM provider with a `429 Too Many Requests` error.
+
+:::info
+By default, `rpm` and `tpm` values are only used for **routing decisions** (picking deployments with capacity). With `enforce_model_rate_limits`, they become **hard limits**.
+:::
+
+### Quick Start
+
+```yaml
+model_list:
+ - model_name: gpt-4
+ litellm_params:
+ model: openai/gpt-4
+ api_key: os.environ/OPENAI_API_KEY
+ rpm: 60 # 60 requests per minute
+ tpm: 90000 # 90k tokens per minute
+
+router_settings:
+ optional_pre_call_checks:
+ - enforce_model_rate_limits # 👈 Enables strict enforcement
+```
+
+### How It Works
+
+| Limit Type | Enforcement | Accuracy |
+|------------|-------------|----------|
+| **RPM** | Hard limit - blocked at exact threshold | 100% accurate |
+| **TPM** | Best-effort - may slightly exceed | Blocked when already over limit |
+
+**Why TPM is best-effort:** Token count is unknown until the LLM responds. TPM is checked before each request (blocks if already over), and tracked after (adds actual tokens used).
+
+### Error Response
+
+```json
+{
+ "error": {
+ "message": "Model rate limit exceeded. RPM limit=60, current usage=60",
+ "type": "rate_limit_error",
+ "code": 429
+ }
+}
+```
+
+Response includes `retry-after: 60` header.
+
+### Multi-Instance Deployment
+
+For multiple LiteLLM proxy instances, add Redis to share rate limit state:
+
+```yaml
+router_settings:
+ optional_pre_call_checks:
+ - enforce_model_rate_limits
+ redis_host: redis.example.com
+ redis_port: 6379
+ redis_password: your-password
+```
+
+
:::info
Detailed information about [routing strategies can be found here](../routing)
:::
diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md
index 56fb420e6cf..1abb127dfda 100644
--- a/docs/my-website/docs/proxy/logging.md
+++ b/docs/my-website/docs/proxy/logging.md
@@ -1338,6 +1338,7 @@ litellm_settings:
s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3
s3_path: my-test-path # [OPTIONAL] set path in bucket you want to write logs to
s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 buckets
+ s3_use_virtual_hosted_style: false # [OPTIONAL] use virtual-hosted-style URLs (bucket.endpoint/key) instead of path-style (endpoint/bucket/key). Useful for S3-compatible services like MinIO
s3_strip_base64_files: false # [OPTIONAL] remove base64 files before storing in s3
```
diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md
index a42d91a7d5f..994788a3ad9 100644
--- a/docs/my-website/docs/proxy/prod.md
+++ b/docs/my-website/docs/proxy/prod.md
@@ -250,11 +250,133 @@ The migrate deploy command:
### Read-only File System
-If you see a `Permission denied` error, it means the LiteLLM pod is running with a read-only file system.
+Running LiteLLM with `readOnlyRootFilesystem: true` is a Kubernetes security best practice that prevents container processes from writing to the root filesystem. LiteLLM fully supports this configuration.
-To fix this, just set `LITELLM_MIGRATION_DIR="/path/to/writeable/directory"` in your environment.
+#### Quick Fix for Permission Errors
-LiteLLM will use this directory to write migration files.
+If you see a `Permission denied` error, it means the LiteLLM pod is running with a read-only file system. LiteLLM needs writable directories for:
+- **Database migrations**: Set `LITELLM_MIGRATION_DIR="/path/to/writable/directory"`
+- **Admin UI**: Set `LITELLM_UI_PATH="/path/to/writable/directory"`
+- **UI assets/logos**: Set `LITELLM_ASSETS_PATH="/path/to/writable/directory"`
+
+#### Complete Read-Only Filesystem Setup (Kubernetes)
+
+For production deployments with enhanced security, use this configuration:
+
+**Option 1: Using EmptyDir Volumes with InitContainer (Recommended)**
+
+This approach copies the pre-built UI from the Docker image to writable emptyDir volumes at pod startup.
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: litellm-proxy
+spec:
+ template:
+ spec:
+ initContainers:
+ - name: setup-ui
+ image: ghcr.io/berriai/litellm:main-stable
+ command:
+ - sh
+ - -c
+ - |
+ cp -r /var/lib/litellm/ui/* /app/var/litellm/ui/ && \
+ cp -r /var/lib/litellm/assets/* /app/var/litellm/assets/
+ volumeMounts:
+ - name: ui-volume
+ mountPath: /app/var/litellm/ui
+ - name: assets-volume
+ mountPath: /app/var/litellm/assets
+
+ containers:
+ - name: litellm
+ image: ghcr.io/berriai/litellm:main-stable
+ env:
+ - name: LITELLM_NON_ROOT
+ value: "true"
+ - name: LITELLM_UI_PATH
+ value: "/app/var/litellm/ui"
+ - name: LITELLM_ASSETS_PATH
+ value: "/app/var/litellm/assets"
+ - name: LITELLM_MIGRATION_DIR
+ value: "/app/migrations"
+ - name: PRISMA_BINARY_CACHE_DIR
+ value: "/app/cache/prisma-python/binaries"
+ - name: XDG_CACHE_HOME
+ value: "/app/cache"
+ securityContext:
+ readOnlyRootFilesystem: true
+ runAsNonRoot: true
+ runAsUser: 101
+ capabilities:
+ drop:
+ - ALL
+ volumeMounts:
+ - name: config
+ mountPath: /app/config.yaml
+ subPath: config.yaml
+ readOnly: true
+ - name: ui-volume
+ mountPath: /app/var/litellm/ui
+ - name: assets-volume
+ mountPath: /app/var/litellm/assets
+ - name: cache
+ mountPath: /app/cache
+ - name: migrations
+ mountPath: /app/migrations
+
+ volumes:
+ - name: config
+ configMap:
+ name: litellm-config
+ - name: ui-volume
+ emptyDir:
+ sizeLimit: 100Mi
+ - name: assets-volume
+ emptyDir:
+ sizeLimit: 10Mi
+ - name: cache
+ emptyDir:
+ sizeLimit: 500Mi
+ - name: migrations
+ emptyDir:
+ sizeLimit: 64Mi
+```
+
+**Option 2: Without UI (API-only deployment)**
+
+If you don't need the admin UI, you can run with minimal configuration:
+
+```yaml
+env:
+ - name: LITELLM_NON_ROOT
+ value: "true"
+ - name: LITELLM_MIGRATION_DIR
+ value: "/app/migrations"
+securityContext:
+ readOnlyRootFilesystem: true
+```
+
+The proxy will log a warning about the UI but API endpoints will work normally.
+
+#### Environment Variables for Read-Only Filesystems
+
+| Variable | Purpose | Default |
+|----------|---------|---------|
+| `LITELLM_UI_PATH` | Admin UI directory | `/var/lib/litellm/ui` (Docker) |
+| `LITELLM_ASSETS_PATH` | UI assets/logos | `/var/lib/litellm/assets` (Docker) |
+| `LITELLM_MIGRATION_DIR` | Database migrations | Package directory |
+| `PRISMA_BINARY_CACHE_DIR` | Prisma binary cache | System default |
+| `XDG_CACHE_HOME` | General cache directory | System default |
+
+#### Important Notes
+
+1. **Migrations**: Always set `LITELLM_MIGRATION_DIR` to a writable emptyDir path
+2. **Prisma Cache**: Set `PRISMA_BINARY_CACHE_DIR` and `XDG_CACHE_HOME` to writable paths
+3. **Server Root Path**: If using a custom `server_root_path`, you must pre-process UI files in your Dockerfile as the proxy cannot modify files at runtime with read-only filesystem
+4. **Automatic Detection**: The UI is automatically detected as pre-restructured if it contains a `.litellm_ui_ready` marker file (created by the official Docker images)
## 10. Use a Separate Health Check App
:::info
diff --git a/docs/my-website/docs/proxy/prometheus.md b/docs/my-website/docs/proxy/prometheus.md
index cd2b3b68f37..93a0675f097 100644
--- a/docs/my-website/docs/proxy/prometheus.md
+++ b/docs/my-website/docs/proxy/prometheus.md
@@ -121,8 +121,8 @@ Use this to track overall LiteLLM Proxy usage.
| Metric Name | Description |
|----------------------|--------------------------------------|
-| `litellm_proxy_failed_requests_metric` | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "exception_status", "exception_class", "route"` |
-| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route"` |
+| `litellm_proxy_failed_requests_metric` | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "user_email", "exception_status", "exception_class", "route", "model_id"` |
+| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route", "model_id"` |
### Callback Logging Metrics
@@ -130,7 +130,12 @@ Monitor failures while shipping logs to downstream callbacks like `s3_v3` cold s
| Metric Name | Description |
|----------------------|--------------------------------------|
-| `litellm_callback_logging_failures_metric` | Total number of failed attempts to emit logs to a configured callback. Labels: `"callback_name"`. Use this to alert on callback delivery issues such as repeated failures when writing to `s3_v3`. |
+| `litellm_callback_logging_failures_metric` | Total number of failed attempts to emit logs to a configured callback. Labels: `"callback_name"`. Use this to alert on callback delivery issues such as repeated failures when writing to `s3_v3`, `langfuse`, or `langfuse_otel` and other otel providers |
+
+**Supported Callbacks:**
+- `S3Logger` - S3 v2 cold storage failures
+- `langfuse` - Langfuse logging failures
+- `otel` - OpenTelemetry logging failures
## LLM Provider Metrics
@@ -191,10 +196,10 @@ Use this for LLM API Error monitoring and tracking remaining rate limits and tok
| Metric Name | Description |
|----------------------|--------------------------------------|
-| `litellm_request_total_latency_metric` | Total latency (seconds) for a request to LiteLLM Proxy Server - tracked for labels "end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model" |
+| `litellm_request_total_latency_metric` | Total latency (seconds) for a request to LiteLLM Proxy Server - tracked for labels "end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model", "model_id" |
| `litellm_overhead_latency_metric` | Latency overhead (seconds) added by LiteLLM processing - tracked for labels "model_group", "api_provider", "api_base", "litellm_model_name", "hashed_api_key", "api_key_alias" |
| `litellm_llm_api_latency_metric` | Latency (seconds) for just the LLM API call - tracked for labels "model", "hashed_api_key", "api_key_alias", "team", "team_alias", "requested_model", "end_user", "user" |
-| `litellm_llm_api_time_to_first_token_metric` | Time to first token for LLM API call - tracked for labels `model`, `hashed_api_key`, `api_key_alias`, `team`, `team_alias` [Note: only emitted for streaming requests] |
+| `litellm_llm_api_time_to_first_token_metric` | Time to first token for LLM API call - tracked for labels `model`, `hashed_api_key`, `api_key_alias`, `team`, `team_alias`, `requested_model`, `end_user`, `user`, `model_id` [Note: only emitted for streaming requests] |
## Tracking `end_user` on Prometheus
diff --git a/docs/my-website/docs/proxy/prompt_management.md b/docs/my-website/docs/proxy/prompt_management.md
index 0c7ff96f538..08307ba99ec 100644
--- a/docs/my-website/docs/proxy/prompt_management.md
+++ b/docs/my-website/docs/proxy/prompt_management.md
@@ -11,6 +11,7 @@ Run experiments or change the specific model (e.g. from gpt-4o to gpt4o-mini fin
| Native LiteLLM GitOps (.prompt files) | [Get Started](native_litellm_prompt) |
| Langfuse | [Get Started](https://langfuse.com/docs/prompts/get-started) |
| Humanloop | [Get Started](../observability/humanloop) |
+| Generic Prompt Management API | [Get Started](../adding_provider/generic_prompt_management_api) |
## Onboarding Prompts via config.yaml
@@ -34,7 +35,7 @@ prompts:
- prompt_id: "my_prompt_id"
litellm_params:
prompt_id: "my_prompt_id"
- prompt_integration: "dotprompt" # or langfuse, bitbucket, gitlab, custom
+ prompt_integration: "dotprompt" # or langfuse, bitbucket, gitlab, generic_prompt_management, custom
# integration-specific parameters below
```
@@ -46,6 +47,7 @@ The `prompt_integration` field determines where and how prompts are loaded:
- **`langfuse`**: Fetch prompts from Langfuse prompt management
- **`bitbucket`**: Load from BitBucket repository `.prompt` files (team-based access control)
- **`gitlab`**: Load from GitLab repository `.prompt` files (team-based access control)
+- **`generic_prompt_management`**: Integrate any prompt management system via a simple API endpoint (no PR required)
- **`custom`**: Use your own custom prompt management implementation
Each integration has its own configuration parameters and access control mechanisms.
@@ -207,6 +209,57 @@ System: You are a helpful assistant.
User: {{user_message}}
```
+
+
+
+
+```yaml
+prompts:
+ - prompt_id: "simple_prompt"
+ litellm_params:
+ prompt_integration: "generic_prompt_management"
+ provider_specific_query_params:
+ project_name: litellm
+ slug: hello-world-prompt-2bac
+ api_base: http://localhost:8080
+ api_key: os.environ/GENERIC_PROMPT_API_KEY
+ ignore_prompt_manager_model: true # optional
+ ignore_prompt_manager_optional_params: true # optional
+```
+
+**What you need to implement:**
+
+A GET endpoint at `/beta/litellm_prompt_management` that returns:
+
+```json
+{
+ "prompt_id": "simple_prompt",
+ "prompt_template": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant."
+ },
+ {
+ "role": "user",
+ "content": "Help me with {task}"
+ }
+ ],
+ "prompt_template_model": "gpt-4",
+ "prompt_template_optional_params": {
+ "temperature": 0.7,
+ "max_tokens": 500
+ }
+}
+```
+
+**Benefits:**
+- No PR required - integrate any prompt management system
+- Full control over your prompt storage and versioning
+- Support for variable substitution with `{variable}` syntax
+- Custom query parameters for filtering and access control
+
+**Learn more:** [Generic Prompt Management API Documentation](../adding_provider/generic_prompt_management_api)
+
diff --git a/docs/my-website/docs/proxy/pyroscope_profiling.md b/docs/my-website/docs/proxy/pyroscope_profiling.md
new file mode 100644
index 00000000000..fa3db3a8782
--- /dev/null
+++ b/docs/my-website/docs/proxy/pyroscope_profiling.md
@@ -0,0 +1,43 @@
+# Grafana Pyroscope CPU profiling
+
+LiteLLM proxy can send continuous CPU profiles to [Grafana Pyroscope](https://grafana.com/docs/pyroscope/latest/) when enabled via environment variables. This is optional and off by default.
+
+## Quick start
+
+1. **Install the optional dependency** (required only when enabling Pyroscope):
+
+ ```bash
+ pip install pyroscope-io
+ ```
+
+ Or install the proxy extra:
+
+ ```bash
+ pip install "litellm[proxy]"
+ ```
+
+2. **Set environment variables** before starting the proxy:
+
+ | Variable | Required | Description |
+ |----------|----------|-------------|
+ | `LITELLM_ENABLE_PYROSCOPE` | Yes (to enable) | Set to `true` to enable Pyroscope profiling. |
+ | `PYROSCOPE_APP_NAME` | Yes (when enabled) | Application name shown in the Pyroscope UI. |
+ | `PYROSCOPE_SERVER_ADDRESS` | Yes (when enabled) | Pyroscope server URL (e.g. `http://localhost:4040`). |
+ | `PYROSCOPE_SAMPLE_RATE` | No | Sample rate (integer). If unset, the pyroscope-io library default is used. |
+
+3. **Start the proxy**; profiling will begin automatically when the proxy starts.
+
+ ```bash
+ export LITELLM_ENABLE_PYROSCOPE=true
+ export PYROSCOPE_APP_NAME=litellm-proxy
+ export PYROSCOPE_SERVER_ADDRESS=http://localhost:4040
+ litellm --config config.yaml
+ ```
+
+4. **View profiles** in the Pyroscope (or Grafana) UI and select your `PYROSCOPE_APP_NAME`.
+
+## Notes
+
+- **Optional dependency**: `pyroscope-io` is an optional dependency. If it is not installed and `LITELLM_ENABLE_PYROSCOPE=true`, the proxy will log a warning and continue without profiling.
+- **Platform support**: The `pyroscope-io` package uses a native extension and is not available on all platforms (e.g. Windows is excluded by the package).
+- **Other settings**: See [Configuration settings](/proxy/config_settings) for all proxy environment variables.
diff --git a/docs/my-website/docs/proxy/release_cycle.md b/docs/my-website/docs/proxy/release_cycle.md
index 10dd6d8b3c5..b3e056b0243 100644
--- a/docs/my-website/docs/proxy/release_cycle.md
+++ b/docs/my-website/docs/proxy/release_cycle.md
@@ -22,4 +22,10 @@ Stable releases come out every week (typically Sunday)
- 'patch' bumps: extremely minor addition that doesn't affect any existing functionality or add any user-facing features. (e.g. a 'created_at' column in a database table)
- 'minor' bumps: add a new feature or a new database table that is backward compatible.
-- 'major' bumps: break backward compatibility.
\ No newline at end of file
+- 'major' bumps: break backward compatibility.
+
+### Enterprise Support
+
+
+- Stable releases come out every week. Once a new one is available, we no longer provide support for an older one.
+- If there is a MAJOR change (according to semvar conventions - e.g. 1.x.x -> 2.x.x), we can provide support for upto 90 days on the prior stable image.
diff --git a/docs/my-website/docs/proxy/request_tags.md b/docs/my-website/docs/proxy/request_tags.md
new file mode 100644
index 00000000000..c78c48229b4
--- /dev/null
+++ b/docs/my-website/docs/proxy/request_tags.md
@@ -0,0 +1,58 @@
+# Request Tags for Spend Tracking
+
+Add tags to model deployments to track spend by environment, AWS account, or any custom label.
+
+Tags appear in the `request_tags` field of LiteLLM spend logs.
+
+## Config Setup
+
+Set tags on model deployments in `config.yaml`:
+
+```yaml title="config.yaml"
+model_list:
+ - model_name: gpt-4
+ litellm_params:
+ model: azure/gpt-4-prod
+ api_key: os.environ/AZURE_PROD_API_KEY
+ api_base: https://prod.openai.azure.com/
+ tags: ["AWS_IAM_PROD"] # 👈 Tag for production
+
+ - model_name: gpt-4-dev
+ litellm_params:
+ model: azure/gpt-4-dev
+ api_key: os.environ/AZURE_DEV_API_KEY
+ api_base: https://dev.openai.azure.com/
+ tags: ["AWS_IAM_DEV"] # 👈 Tag for development
+```
+
+## Make Request
+
+Requests just specify the model - tags are automatically applied:
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+ -H 'Authorization: Bearer sk-1234' \
+ -H 'Content-Type: application/json' \
+ -d '{
+ "model": "gpt-4",
+ "messages": [{"role": "user", "content": "Hello"}]
+ }'
+```
+
+## Spend Logs
+
+The tag from the model config appears in `LiteLLM_SpendLogs`:
+
+```json
+{
+ "request_id": "chatcmpl-abc123",
+ "request_tags": ["AWS_IAM_PROD"],
+ "spend": 0.002,
+ "model": "gpt-4"
+}
+```
+
+## Related
+
+- [Spend Tracking Overview](cost_tracking.md)
+- [Tag Budgets](tag_budgets.md) - Set budget limits per tag
diff --git a/docs/my-website/docs/proxy/sync_anthropic_beta_headers.md b/docs/my-website/docs/proxy/sync_anthropic_beta_headers.md
new file mode 100644
index 00000000000..e1645082d97
--- /dev/null
+++ b/docs/my-website/docs/proxy/sync_anthropic_beta_headers.md
@@ -0,0 +1,128 @@
+# Auto Sync Anthropic Beta Headers
+
+Automatically keep your Anthropic beta headers configuration up to date without restarting your service. **This allows you to support new Anthropic beta features across all providers without restarting your service.**
+
+## Overview
+
+When Anthropic releases new beta features (e.g., new tool capabilities, extended context windows), you typically need to restart your LiteLLM service to get the latest beta header mappings for different providers (Anthropic, Bedrock, Vertex AI, Azure AI).
+
+With auto-sync, LiteLLM automatically pulls the latest configuration from GitHub's [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) without requiring a restart. This means:
+
+- **Zero downtime** when new beta features are released
+- **Always up-to-date** provider support mappings
+- **Automatic updates** - set it once and forget it
+
+## Quick Start
+
+**Manual sync:**
+```bash
+curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
+ -H "Authorization: Bearer YOUR_ADMIN_TOKEN" \
+ -H "Content-Type: application/json"
+```
+
+**Automatic sync every 24 hours:**
+```bash
+curl -X POST "https://your-proxy-url/schedule/anthropic_beta_headers_reload?hours=24" \
+ -H "Authorization: Bearer YOUR_ADMIN_TOKEN" \
+ -H "Content-Type: application/json"
+```
+
+## API Endpoints
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `/reload/anthropic_beta_headers` | POST | Manual sync |
+| `/schedule/anthropic_beta_headers_reload?hours={hours}` | POST | Schedule periodic sync |
+| `/schedule/anthropic_beta_headers_reload` | DELETE | Cancel scheduled sync |
+| `/schedule/anthropic_beta_headers_reload/status` | GET | Check sync status |
+
+**Authentication:** Requires admin role or master key
+
+## Python Example
+
+```python
+import requests
+
+def sync_anthropic_beta_headers(proxy_url, admin_token):
+ response = requests.post(
+ f"{proxy_url}/reload/anthropic_beta_headers",
+ headers={"Authorization": f"Bearer {admin_token}"}
+ )
+ return response.json()
+
+# Usage
+result = sync_anthropic_beta_headers("https://your-proxy-url", "your-admin-token")
+print(result['message'])
+```
+
+## Configuration
+
+**Custom beta headers config URL:**
+```bash
+export LITELLM_ANTHROPIC_BETA_HEADERS_URL="https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/anthropic_beta_headers_config.json"
+```
+
+**Use local beta headers config:**
+```bash
+export LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True
+```
+
+## Scheduling Automatic Reloads
+
+Schedule automatic reloads to ensure your proxy always has the latest beta header mappings:
+
+```bash
+# Reload every 24 hours
+curl -X POST "https://your-proxy-url/schedule/anthropic_beta_headers_reload?hours=24" \
+ -H "Authorization: Bearer YOUR_ADMIN_TOKEN"
+```
+
+**Check reload status:**
+```bash
+curl -X GET "https://your-proxy-url/schedule/anthropic_beta_headers_reload/status" \
+ -H "Authorization: Bearer YOUR_ADMIN_TOKEN"
+```
+
+**Response:**
+```json
+{
+ "scheduled": true,
+ "interval_hours": 24,
+ "last_run": "2026-02-13T10:00:00",
+ "next_run": "2026-02-14T10:00:00"
+}
+```
+
+**Cancel scheduled reload:**
+```bash
+curl -X DELETE "https://your-proxy-url/schedule/anthropic_beta_headers_reload" \
+ -H "Authorization: Bearer YOUR_ADMIN_TOKEN"
+```
+
+## Environment Variables
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `LITELLM_ANTHROPIC_BETA_HEADERS_URL` | URL to fetch beta headers config from | GitHub main branch |
+| `LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS` | Set to `True` to use local config only | `False` |
+
+## How It Works
+
+1. **Initial Load:** On startup, LiteLLM loads the beta headers configuration from the remote URL (or local file if configured)
+2. **Caching:** The configuration is cached in memory to avoid repeated fetches on every request
+3. **Scheduled Reload:** If configured, the proxy checks every 10 seconds whether it's time to reload based on your schedule
+4. **Manual Reload:** You can trigger an immediate reload via the API endpoint
+5. **Multi-Pod Support:** In multi-pod deployments, the reload configuration is stored in the database so all pods stay in sync
+
+## Benefits
+
+- **No Restarts Required:** Add support for new Anthropic beta features without downtime
+- **Provider Compatibility:** Automatically get updated mappings for Bedrock, Vertex AI, Azure AI, etc.
+- **Performance:** Configuration is cached and only reloaded when needed
+- **Reliability:** Falls back to local configuration if remote fetch fails
+
+## Related
+
+- [Model Cost Map Sync](./sync_models_github.md) - Auto-sync model pricing data
+- [Anthropic Beta Headers](../completion/anthropic.md#beta-features) - Using Anthropic beta features
diff --git a/docs/my-website/docs/proxy/team_budgets.md b/docs/my-website/docs/proxy/team_budgets.md
index 03d18797133..01b07f23a33 100644
--- a/docs/my-website/docs/proxy/team_budgets.md
+++ b/docs/my-website/docs/proxy/team_budgets.md
@@ -8,7 +8,6 @@ import TabItem from '@theme/TabItem';
# Pre-Requisites
- You must set up a Postgres database (e.g. Supabase, Neon, etc.)
-- To enable team member rate limits, set the environment variable `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING=true` **before starting the proxy server**. Without this, team member rate limits will not be enforced.
## Default Budget for Auto-Generated JWT Teams
diff --git a/docs/my-website/docs/proxy/ui/page_visibility.md b/docs/my-website/docs/proxy/ui/page_visibility.md
new file mode 100644
index 00000000000..06b06f33219
--- /dev/null
+++ b/docs/my-website/docs/proxy/ui/page_visibility.md
@@ -0,0 +1,121 @@
+import Image from '@theme/IdealImage';
+
+# Control Page Visibility for Internal Users
+
+Configure which navigation tabs and pages are visible to internal users (non-admin developers) in the LiteLLM UI.
+
+Use this feature to simplify the UI and control which pages your internal users/developers can see when signing in.
+
+## Overview
+
+By default, all pages accessible to internal users are visible in the navigation sidebar. The page visibility control allows admins to restrict which pages internal users can see, creating a more focused and streamlined experience.
+
+
+## Configure Page Visibility
+
+### 1. Navigate to Settings
+
+Click the **Settings** icon in the sidebar.
+
+
+
+### 2. Go to Admin Settings
+
+Click **Admin Settings** from the settings menu.
+
+
+
+### 3. Select UI Settings
+
+Click **UI Settings** to access the page visibility controls.
+
+
+
+### 4. Open Page Visibility Configuration
+
+Click **Configure Page Visibility** to expand the configuration panel.
+
+
+
+### 5. Select Pages to Make Visible
+
+Check the boxes for the pages you want internal users to see. Pages are organized by category for easy navigation.
+
+
+
+**Available pages include:**
+- Virtual Keys
+- Playground
+- Models + Endpoints
+- Agents
+- MCP Servers
+- Search Tools
+- Vector Stores
+- Logs
+- Teams
+- Organizations
+- Usage
+- Budgets
+- And more...
+
+### 6. Save Your Configuration
+
+Click **Save Page Visibility Settings** to apply the changes.
+
+
+
+### 7. Verify Changes
+
+Internal users will now only see the selected pages in their navigation sidebar.
+
+
+
+## Reset to Default
+
+To restore all pages to internal users:
+
+1. Open the Page Visibility configuration
+2. Click **Reset to Default (All Pages)**
+3. Click **Save Page Visibility Settings**
+
+This will clear the restriction and show all accessible pages to internal users.
+
+## API Configuration
+
+You can also configure page visibility programmatically using the API:
+
+### Get Current Settings
+
+```bash
+curl -X GET 'http://localhost:4000/ui_settings/get' \
+ -H 'Authorization: Bearer '
+```
+
+### Update Page Visibility
+
+```bash
+curl -X PATCH 'http://localhost:4000/ui_settings/update' \
+ -H 'Authorization: Bearer ' \
+ -H 'Content-Type: application/json' \
+ -d '{
+ "enabled_ui_pages_internal_users": [
+ "api-keys",
+ "agents",
+ "mcp-servers",
+ "logs",
+ "teams"
+ ]
+ }'
+```
+
+### Clear Page Visibility Restrictions
+
+```bash
+curl -X PATCH 'http://localhost:4000/ui_settings/update' \
+ -H 'Authorization: Bearer ' \
+ -H 'Content-Type: application/json' \
+ -d '{
+ "enabled_ui_pages_internal_users": null
+ }'
+```
+
diff --git a/docs/my-website/docs/proxy/ui_logs.md b/docs/my-website/docs/proxy/ui_logs.md
index 61f328011c3..8cfe818ebfd 100644
--- a/docs/my-website/docs/proxy/ui_logs.md
+++ b/docs/my-website/docs/proxy/ui_logs.md
@@ -25,7 +25,10 @@ View Spend, Token Usage, Key, Team Name for Each Request to LiteLLM
## Tracking - Request / Response Content in Logs Page
-If you want to view request and response content on LiteLLM Logs, you need to opt in with this setting
+If you want to view request and response content on LiteLLM Logs, you can enable it in either place:
+
+- **From the UI (no restart):** Use [UI Spend Log Settings](./ui_spend_log_settings.md) — open Logs → Settings → enable "Store Prompts in Spend Logs" → Save. Takes effect immediately and overrides config.
+- **From config:** Add this to your `proxy_config.yaml` (requires restart):
```yaml
general_settings:
@@ -34,6 +37,40 @@ general_settings:
+## Tracing Tools
+
+View which tools were provided and called in your completion requests.
+
+
+
+**Example:** Make a completion request with tools:
+
+```bash
+curl -X POST 'http://localhost:4000/chat/completions' \
+ -H 'Authorization: Bearer sk-1234' \
+ -H 'Content-Type: application/json' \
+ -d '{
+ "model": "gpt-4",
+ "messages": [{"role": "user", "content": "What is the weather?"}],
+ "tools": [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Get the current weather",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "location": {"type": "string"}
+ }
+ }
+ }
+ }
+ ]
+ }'
+```
+
+Check the Logs page to see all tools provided and which ones were called.
## Stop storing Error Logs in DB
@@ -57,7 +94,10 @@ general_settings:
If you're storing spend logs, it might be a good idea to delete them regularly to keep the database fast.
-LiteLLM lets you configure this in your `proxy_config.yaml`:
+You can set the retention period in either place:
+
+- **From the UI (no restart):** [UI Spend Log Settings](./ui_spend_log_settings.md) — Logs → Settings → set Retention Period → Save.
+- **From config:** Add the following to your `proxy_config.yaml` (requires restart):
```yaml
general_settings:
diff --git a/docs/my-website/docs/proxy/ui_spend_log_settings.md b/docs/my-website/docs/proxy/ui_spend_log_settings.md
new file mode 100644
index 00000000000..5e04974e3a7
--- /dev/null
+++ b/docs/my-website/docs/proxy/ui_spend_log_settings.md
@@ -0,0 +1,92 @@
+import Image from '@theme/IdealImage';
+
+# UI Spend Log Settings
+
+Configure spend log behavior directly from the Admin UI—no config file edits or proxy restart required. This is especially useful for cloud deployments where updating the config is difficult or requires a long release process.
+
+## Overview
+
+Previously, spend log options (such as storing request/response content and retention period) had to be set in `proxy_config.yaml` under `general_settings`. Changing them required editing the config and restarting the proxy, which was a pain point for users-especially in cloud environments—who don't have easy access to the config or whose deployment process makes config updates slow.
+
+
+
+**UI Spend Log Settings** lets you:
+
+- **Store prompts in spend logs** – Enable or disable storing request and response content in the spend logs table (only affects logs created after you change the setting)
+- **Set retention period** – Configure how long spend logs are kept before automatic cleanup (e.g. `7d`, `30d`)
+- **Apply changes immediately** – No proxy restart needed; settings take effect for new requests as soon as you save
+
+:::warning UI overrides config
+Settings changed in the UI **override** the values in your config file. For example, if `store_prompts_in_spend_logs` is explicitly set to `false` in `general_settings`, turning it on in the UI will still enable storing prompts. Use the UI when you want runtime control without redeploying.
+:::
+
+## Settings You Can Configure
+
+| Setting | Description |
+| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **Store Prompts in Spend Logs** | When enabled, request messages and response content are stored for **new** spend logs so you can view them in the Logs UI. Logs created before you enabled this will not have request/response content. When disabled, only metadata (e.g. tokens, cost, model) is stored for new logs. |
+| **Retention Period** | Maximum time to keep spend logs before they are automatically deleted (e.g. `7d`, `30d`). Optional; if not set, logs are retained according to your config or default behavior. |
+
+The same options can be set in config via [general_settings](./config_settings.md#general_settings---reference) (`store_prompts_in_spend_logs`, `maximum_spend_logs_retention_period`). Values set in the UI take precedence.
+
+## How to Configure Spend Log Settings in the UI
+
+### 1. Open the Logs page
+
+Navigate to the Admin UI (e.g. `http://localhost:4000/ui` or your `PROXY_BASE_URL/ui`) and click **Logs**.
+
+
+
+
+
+### 2. Open Logs settings
+
+Click the **Settings** (gear) icon on the Logs page to open the spend log settings panel.
+
+
+
+### 3. Enable Store Prompts in Spend Logs (optional)
+
+Turn on **Store Prompts in Spend Logs** if you want request and response content to be stored for new requests and visible when you open those log entries. This only affects logs created after you enable it; existing logs will not gain request/response content. Leave it off if you only need metadata (tokens, cost, model, etc.).
+
+
+
+### 4. Set the retention period (optional)
+
+Optionally set the **Retention Period** (e.g. `7d`, `30d`) to control how long spend logs are kept before automatic cleanup. Uses the same format as the config option `maximum_spend_logs_retention_period`.
+
+
+
+### 5. Save settings
+
+Click **Save Settings**. Changes take effect immediately for new requests; no proxy restart is required. Existing logs are not updated.
+
+
+
+### 6. Verify: view request and response in a log
+
+After enabling **Store Prompts in Spend Logs**, make a new request through the proxy, then open that log entry (or any other log created after you enabled the setting). The log details view will include the request and response content. Logs that existed before you turned the setting on will not have this content.
+
+
+
+
+
+## Use Cases
+
+### Cloud and managed deployments
+
+When the proxy runs in a managed or cloud environment, config may be in a separate repo, require a long release, or be controlled by another team. Using the UI lets you change spend log behavior (e.g. enable prompt storage for debugging or set retention) without going through that process.
+
+### Quick toggles for debugging
+
+Temporarily enable **Store Prompts in Spend Logs** to inspect request/response content on new requests when debugging, then turn it off again from the UI without editing config or restarting. Only logs created while the setting was on will contain the content.
+
+### Retention without redeploying
+
+Adjust how long spend logs are retained (e.g. shorten to reduce storage or extend for compliance) and have the new retention period and cleanup job take effect immediately.
+
+## Related Documentation
+
+- [Getting Started with UI Logs](./ui_logs.md) – Overview of what gets logged and config-based options
+- [Config Settings](./config_settings.md) – `store_prompts_in_spend_logs`, `disable_spend_logs`, `maximum_spend_logs_retention_period` in `general_settings`
+- [Spend Logs Deletion](./spend_logs_deletion.md) – How retention and cleanup work
diff --git a/docs/my-website/docs/proxy/ui_team_soft_budget_alerts.md b/docs/my-website/docs/proxy/ui_team_soft_budget_alerts.md
new file mode 100644
index 00000000000..17c42e57c9a
--- /dev/null
+++ b/docs/my-website/docs/proxy/ui_team_soft_budget_alerts.md
@@ -0,0 +1,130 @@
+import Image from '@theme/IdealImage';
+
+# Team Soft Budget Alerts
+
+Set a soft budget on a team and get email alerts when spending crosses the threshold — without blocking any requests.
+
+## Overview
+
+A **soft budget** is a spending threshold that triggers email notifications when exceeded, but **does not block requests**. This is different from a hard budget (`max_budget`), which rejects requests once the limit is reached.
+
+
+
+Team soft budget alerts let you:
+
+- **Get notified early** — receive email alerts when a team's spend crosses the soft budget threshold
+- **Keep requests flowing** — unlike hard budgets, soft budgets never block API calls
+- **Target specific recipients** — send alerts to specific email addresses (e.g. team leads, finance), not just the team members
+- **Work without global alerting** — team soft budget alerts are sent via email independently of Slack or other global alerting configuration
+
+:::warning Email integration required
+Team soft budget alerts are sent via email. You must have an active email integration (SendGrid, Resend, or SMTP) configured on your proxy for alerts to be delivered. See [Email Notifications](./email.md) for setup instructions.
+:::
+
+:::info Automatically active
+Team soft budget alerts are **automatically active** once you configure a soft budget and at least one alerting email on a team. No additional proxy configuration or restart is needed — alerts are checked on every request.
+:::
+
+## How It Works
+
+On every API request made with a key belonging to a team, the proxy checks:
+
+1. Does the team have a `soft_budget` set?
+2. Is the team's current `spend` >= the `soft_budget`?
+3. Are there any emails configured in `soft_budget_alerting_emails`?
+
+If all three conditions are met, an email alert is sent to the configured recipients. Alerts are **deduplicated** so the same alert is only sent once within a 24-hour window.
+
+## How to Set Up Team Soft Budget Alerts
+
+### 1. Navigate to the Admin UI
+
+Go to the Admin UI (e.g. `http://localhost:4000/ui` or your `PROXY_BASE_URL/ui`).
+
+
+
+### 2. Go to Teams
+
+Click **Teams** in the sidebar.
+
+
+
+### 3. Select a team
+
+Click on the team you want to configure soft budget alerts for.
+
+
+
+### 4. Open team Settings
+
+Click the **Settings** tab to view the team's configuration.
+
+
+
+### 5. Edit Settings
+
+Click **Edit Settings** to modify the team's budget configuration.
+
+
+
+### 6. Set the Soft Budget
+
+Click the **Soft Budget (USD)** field and enter your desired threshold. For example, enter `0.01` for testing or a higher value like `500` for production.
+
+
+
+### 7. Add alerting emails
+
+Click the **Soft Budget Alerting Emails** field and enter one or more comma-separated email addresses that should receive the alert.
+
+
+
+### 8. Save Changes
+
+Click **Save Changes**. The soft budget alert is now active — no proxy restart required.
+
+
+
+### 9. Verify: email alert received
+
+Once the team's spend crosses the soft budget, an email alert is sent to the configured recipients. Below is an example of the alert email:
+
+
+
+## Settings Reference
+
+| Setting | Description |
+| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
+| **Soft Budget (USD)** | The spending threshold that triggers an email alert. Requests are **not** blocked when this limit is exceeded. |
+| **Soft Budget Alerting Emails** | Comma-separated email addresses that receive the alert when the soft budget is crossed. At least one email is required for alerts to be sent. |
+
+:::tip Soft Budget vs. Max Budget
+
+- **Soft Budget**: Advisory threshold — sends email alerts but does **not** block requests.
+- **Max Budget**: Hard limit — blocks requests once the budget is exceeded.
+
+You can set both on the same team to get early warnings (soft) and a hard stop (max).
+:::
+
+## API Configuration
+
+You can also configure team soft budgets via the API when creating or updating a team:
+
+```bash
+curl -X POST 'http://localhost:4000/team/update' \
+ --header 'Authorization: Bearer sk-1234' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "team_id": "your-team-id",
+ "soft_budget": 500.00,
+ "metadata": {
+ "soft_budget_alerting_emails": ["lead@example.com", "finance@example.com"]
+ }
+ }'
+```
+
+## Related Documentation
+
+- [Email Notifications](./email.md) – Configure email integrations (Resend, SMTP) for LiteLLM Proxy
+- [Alerting](./alerting.md) – Set up Slack and other alerting channels
+- [Cost Tracking](./cost_tracking.md) – Track and manage spend across teams, keys, and users
diff --git a/docs/my-website/docs/proxy/users.md b/docs/my-website/docs/proxy/users.md
index a389f0bd443..8517db51a8f 100644
--- a/docs/my-website/docs/proxy/users.md
+++ b/docs/my-website/docs/proxy/users.md
@@ -68,13 +68,6 @@ You can:
**Step-by step tutorial on setting, resetting budgets on Teams here (API or using Admin UI)**
-> **Prerequisite:**
-> To enable team member rate limits, you must set the environment variable `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING=true` before starting the proxy server. Without this, team member rate limits will not be enforced.
-
-👉 [https://docs.litellm.ai/docs/proxy/team_budgets](https://docs.litellm.ai/docs/proxy/team_budgets)
-
-:::
-
#### **Add budgets to teams**
```shell
@@ -822,12 +815,10 @@ Expected Response:
}
```
-### [BETA] Multi-instance rate limiting
+### Multi-instance rate limiting
-Enable multi-instance rate limiting with the env var `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING="True"`
**Important Notes:**
-- Setting `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING="True"` is required for team member rate limits to function, not just for multi-instance scenarios.
- **Rate limits do not apply to proxy admin users.**
- When testing rate limits, use internal user roles (non-admin) to ensure limits are enforced as expected.
diff --git a/docs/my-website/docs/proxy/virtual_keys.md b/docs/my-website/docs/proxy/virtual_keys.md
index 38ff4ede280..c74aa75ff4a 100644
--- a/docs/my-website/docs/proxy/virtual_keys.md
+++ b/docs/my-website/docs/proxy/virtual_keys.md
@@ -549,11 +549,14 @@ curl 'http://localhost:4000/key/sk-1234/regenerate' \
"models": [
"gpt-4",
"gpt-3.5-turbo"
- ]
+ ],
+ "grace_period": "48h"
}'
```
+**Grace period (optional)**: Set `grace_period` (e.g. `"24h"`, `"2d"`, `"1w"`) to keep the old key valid for a transitional period. Both old and new keys work until the grace period elapses, enabling seamless cutover without production downtime. Omitted or empty = immediate revoke. Can also be set via `LITELLM_KEY_ROTATION_GRACE_PERIOD` env var for scheduled rotations.
+
**Read More**
- [Write rotated keys to secrets manager](https://docs.litellm.ai/docs/secret#aws-secret-manager)
@@ -640,11 +643,13 @@ Set these environment variables when starting the proxy:
|----------|-------------|---------|
| `LITELLM_KEY_ROTATION_ENABLED` | Enable the rotation worker | `false` |
| `LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS` | How often to scan for keys to rotate (in seconds) | `86400` (24 hours) |
+| `LITELLM_KEY_ROTATION_GRACE_PERIOD` | Duration to keep old key valid after rotation (e.g. `24h`, `2d`) | `""` (immediate revoke) |
**Example:**
```bash
export LITELLM_KEY_ROTATION_ENABLED=true
export LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS=3600 # Check every hour
+export LITELLM_KEY_ROTATION_GRACE_PERIOD=48h # Keep old key valid for 48h during cutover
litellm --config config.yaml
```
diff --git a/docs/my-website/docs/proxy_auth.md b/docs/my-website/docs/proxy_auth.md
new file mode 100644
index 00000000000..91084b34a37
--- /dev/null
+++ b/docs/my-website/docs/proxy_auth.md
@@ -0,0 +1,333 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# SDK Proxy Authentication (OAuth2/JWT Auto-Refresh)
+
+Automatically obtain and refresh OAuth2/JWT tokens when using the LiteLLM Python SDK with a LiteLLM Proxy that requires JWT authentication.
+
+## Overview
+
+When your LiteLLM Proxy is protected by an OAuth2/OIDC provider (Azure AD, Keycloak, Okta, Auth0, etc.), your SDK clients need valid JWT tokens for every request. Instead of manually managing token lifecycle, `litellm.proxy_auth` handles this automatically:
+
+- Obtains tokens from your identity provider
+- Caches tokens to avoid unnecessary requests
+- Refreshes tokens before they expire (60-second buffer)
+- Injects `Authorization: Bearer ` headers into every request
+
+## Quick Start
+
+### Azure AD
+
+
+
+
+Uses the [DefaultAzureCredential](https://learn.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential) chain (environment variables, managed identity, Azure CLI, etc.):
+
+```python
+import litellm
+from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler
+
+# One-time setup
+litellm.proxy_auth = ProxyAuthHandler(
+ credential=AzureADCredential(), # uses DefaultAzureCredential
+ scope="api://my-litellm-proxy/.default"
+)
+litellm.api_base = "https://my-proxy.example.com"
+
+# All requests now include Authorization headers automatically
+response = litellm.completion(
+ model="gpt-4",
+ messages=[{"role": "user", "content": "Hello!"}]
+)
+```
+
+
+
+
+Use a specific Azure AD app registration:
+
+```python
+import litellm
+from azure.identity import ClientSecretCredential
+from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler
+
+azure_cred = ClientSecretCredential(
+ tenant_id="your-tenant-id",
+ client_id="your-client-id",
+ client_secret="your-client-secret"
+)
+
+litellm.proxy_auth = ProxyAuthHandler(
+ credential=AzureADCredential(credential=azure_cred),
+ scope="api://my-litellm-proxy/.default"
+)
+litellm.api_base = "https://my-proxy.example.com"
+
+response = litellm.completion(
+ model="gpt-4",
+ messages=[{"role": "user", "content": "Hello!"}]
+)
+```
+
+
+
+
+**Required package:** `pip install azure-identity`
+
+### Generic OAuth2 (Okta, Auth0, Keycloak, etc.)
+
+Works with any OAuth2 provider that supports the `client_credentials` grant type:
+
+```python
+import litellm
+from litellm.proxy_auth import GenericOAuth2Credential, ProxyAuthHandler
+
+litellm.proxy_auth = ProxyAuthHandler(
+ credential=GenericOAuth2Credential(
+ client_id="your-client-id",
+ client_secret="your-client-secret",
+ token_url="https://your-idp.example.com/oauth2/token"
+ ),
+ scope="litellm_proxy_api"
+)
+litellm.api_base = "https://my-proxy.example.com"
+
+response = litellm.completion(
+ model="gpt-4",
+ messages=[{"role": "user", "content": "Hello!"}]
+)
+```
+
+### Custom Credential Provider
+
+Implement the `TokenCredential` protocol to use any authentication mechanism:
+
+```python
+import time
+import litellm
+from litellm.proxy_auth import AccessToken, ProxyAuthHandler
+
+class MyCustomCredential:
+ """Any class with a get_token(scope) -> AccessToken method works."""
+
+ def get_token(self, scope: str) -> AccessToken:
+ # Your custom logic to obtain a token
+ token = my_auth_system.get_jwt(scope=scope)
+ return AccessToken(
+ token=token,
+ expires_on=int(time.time()) + 3600
+ )
+
+litellm.proxy_auth = ProxyAuthHandler(
+ credential=MyCustomCredential(),
+ scope="my-scope"
+)
+```
+
+## Supported Endpoints
+
+Auth headers are automatically injected for:
+
+| Endpoint | Function |
+|----------|----------|
+| Chat Completions | `litellm.completion()` / `litellm.acompletion()` |
+| Embeddings | `litellm.embedding()` / `litellm.aembedding()` |
+
+## How It Works
+
+```
+┌──────────┐ ┌──────────────────┐ ┌──────────────┐ ┌──────────────┐
+│ Your │ │ ProxyAuthHandler │ │ Identity │ │ LiteLLM │
+│ Code │────▶│ (token cache) │────▶│ Provider │ │ Proxy │
+│ │ │ │◀────│ (Azure AD, │ │ │
+│ │ │ │ │ Okta, etc) │ │ │
+│ │ └────────┬─────────┘ └──────────────┘ │ │
+│ │ │ Authorization: Bearer │ │
+│ │──────────────┼───────────────────────────────────▶│ │
+│ │◀─────────────┼────────────────────────────────────│ │
+└──────────┘ │ └──────────────┘
+```
+
+1. You set `litellm.proxy_auth` once at startup
+2. On each SDK call (`completion()`, `embedding()`), the handler checks its cached token
+3. If the token is missing or expires within 60 seconds, it requests a new one from your identity provider
+4. The `Authorization: Bearer ` header is injected into the request
+5. If token retrieval fails, a warning is logged and the request proceeds without auth headers
+
+## API Reference
+
+### ProxyAuthHandler
+
+The main handler that manages the token lifecycle.
+
+```python
+from litellm.proxy_auth import ProxyAuthHandler
+
+handler = ProxyAuthHandler(
+ credential=, # required - credential provider
+ scope="" # required - OAuth2 scope to request
+)
+```
+
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `credential` | `TokenCredential` | Yes | A credential provider (AzureADCredential, GenericOAuth2Credential, or custom) |
+| `scope` | `str` | Yes | The OAuth2 scope to request tokens for |
+
+**Methods:**
+
+| Method | Returns | Description |
+|--------|---------|-------------|
+| `get_token()` | `AccessToken` | Get a valid token, refreshing if needed |
+| `get_auth_headers()` | `dict` | Get `{"Authorization": "Bearer "}` headers |
+
+### AzureADCredential
+
+Wraps any `azure-identity` credential with lazy initialization.
+
+```python
+from litellm.proxy_auth import AzureADCredential
+
+# Uses DefaultAzureCredential (recommended)
+cred = AzureADCredential()
+
+# Or wrap a specific azure-identity credential
+from azure.identity import ManagedIdentityCredential
+cred = AzureADCredential(credential=ManagedIdentityCredential())
+```
+
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `credential` | Azure `TokenCredential` | No | An azure-identity credential. If `None`, uses `DefaultAzureCredential` |
+
+### GenericOAuth2Credential
+
+Standard OAuth2 client credentials flow for any provider.
+
+```python
+from litellm.proxy_auth import GenericOAuth2Credential
+
+cred = GenericOAuth2Credential(
+ client_id="your-client-id",
+ client_secret="your-client-secret",
+ token_url="https://your-idp.com/oauth2/token"
+)
+```
+
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `client_id` | `str` | Yes | OAuth2 client ID |
+| `client_secret` | `str` | Yes | OAuth2 client secret |
+| `token_url` | `str` | Yes | Token endpoint URL |
+
+### AccessToken
+
+Dataclass representing an OAuth2 access token.
+
+```python
+from litellm.proxy_auth import AccessToken
+
+token = AccessToken(
+ token="eyJhbG...", # JWT string
+ expires_on=1234567890 # Unix timestamp
+)
+```
+
+### TokenCredential Protocol
+
+Any class implementing this protocol can be used as a credential provider:
+
+```python
+from litellm.proxy_auth import AccessToken
+
+class MyCredential:
+ def get_token(self, scope: str) -> AccessToken:
+ ...
+```
+
+## Provider-Specific Examples
+
+### Keycloak
+
+```python
+from litellm.proxy_auth import GenericOAuth2Credential, ProxyAuthHandler
+
+litellm.proxy_auth = ProxyAuthHandler(
+ credential=GenericOAuth2Credential(
+ client_id="litellm-client",
+ client_secret="your-keycloak-client-secret",
+ token_url="https://keycloak.example.com/realms/your-realm/protocol/openid-connect/token"
+ ),
+ scope="openid"
+)
+```
+
+### Okta
+
+```python
+from litellm.proxy_auth import GenericOAuth2Credential, ProxyAuthHandler
+
+litellm.proxy_auth = ProxyAuthHandler(
+ credential=GenericOAuth2Credential(
+ client_id="your-okta-client-id",
+ client_secret="your-okta-client-secret",
+ token_url="https://your-org.okta.com/oauth2/default/v1/token"
+ ),
+ scope="litellm_api"
+)
+```
+
+### Auth0
+
+```python
+from litellm.proxy_auth import GenericOAuth2Credential, ProxyAuthHandler
+
+litellm.proxy_auth = ProxyAuthHandler(
+ credential=GenericOAuth2Credential(
+ client_id="your-auth0-client-id",
+ client_secret="your-auth0-client-secret",
+ token_url="https://your-tenant.auth0.com/oauth/token"
+ ),
+ scope="https://my-proxy.example.com/api"
+)
+```
+
+### Azure AD with Managed Identity
+
+```python
+from azure.identity import ManagedIdentityCredential
+from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler
+
+litellm.proxy_auth = ProxyAuthHandler(
+ credential=AzureADCredential(
+ credential=ManagedIdentityCredential()
+ ),
+ scope="api://my-litellm-proxy/.default"
+)
+```
+
+## Combining with `use_litellm_proxy`
+
+You can use `proxy_auth` together with [`use_litellm_proxy`](./providers/litellm_proxy#send-all-sdk-requests-to-litellm-proxy) to route all SDK requests through an authenticated proxy:
+
+```python
+import os
+import litellm
+from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler
+
+# Route all requests through the proxy
+os.environ["LITELLM_PROXY_API_BASE"] = "https://my-proxy.example.com"
+litellm.use_litellm_proxy = True
+
+# Authenticate with OAuth2/JWT
+litellm.proxy_auth = ProxyAuthHandler(
+ credential=AzureADCredential(),
+ scope="api://my-litellm-proxy/.default"
+)
+
+# This request goes through the proxy with automatic JWT auth
+response = litellm.completion(
+ model="vertex_ai/gemini-2.0-flash-001",
+ messages=[{"role": "user", "content": "Hello!"}]
+)
+```
diff --git a/docs/my-website/docs/rag_ingest.md b/docs/my-website/docs/rag_ingest.md
index 1133b85f206..7adc2d70b5b 100644
--- a/docs/my-website/docs/rag_ingest.md
+++ b/docs/my-website/docs/rag_ingest.md
@@ -5,7 +5,7 @@ All-in-one document ingestion pipeline: **Upload → Chunk → Embed → Vector
| Feature | Supported |
|---------|-----------|
| Logging | Yes |
-| Supported Providers | `openai`, `bedrock`, `vertex_ai`, `gemini` |
+| Supported Providers | `openai`, `bedrock`, `vertex_ai`, `gemini`, `s3_vectors` |
:::tip
After ingesting documents, use [/rag/query](./rag_query.md) to search and generate responses with your ingested content.
@@ -75,6 +75,31 @@ curl -X POST "http://localhost:4000/v1/rag/ingest" \
}"
```
+### AWS S3 Vectors
+
+```bash showLineNumbers title="Ingest to S3 Vectors"
+curl -X POST "http://localhost:4000/v1/rag/ingest" \
+ -H "Authorization: Bearer sk-1234" \
+ -H "Content-Type: application/json" \
+ -d "{
+ \"file\": {
+ \"filename\": \"document.txt\",
+ \"content\": \"$(base64 -i document.txt)\",
+ \"content_type\": \"text/plain\"
+ },
+ \"ingest_options\": {
+ \"embedding\": {
+ \"model\": \"text-embedding-3-small\"
+ },
+ \"vector_store\": {
+ \"custom_llm_provider\": \"s3_vectors\",
+ \"vector_bucket_name\": \"my-embeddings\",
+ \"aws_region_name\": \"us-west-2\"
+ }
+ }
+ }"
+```
+
## Response
```json
@@ -265,6 +290,57 @@ When `vector_store_id` is omitted, LiteLLM automatically creates:
4. Install: `pip install 'google-cloud-aiplatform>=1.60.0'`
:::
+### vector_store (AWS S3 Vectors)
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `custom_llm_provider` | string | - | `"s3_vectors"` |
+| `vector_bucket_name` | string | **required** | S3 vector bucket name |
+| `index_name` | string | auto-create | Vector index name |
+| `dimension` | integer | auto-detect | Vector dimension (auto-detected from embedding model) |
+| `distance_metric` | string | `cosine` | Distance metric: `cosine` or `euclidean` |
+| `non_filterable_metadata_keys` | array | `["source_text"]` | Metadata keys excluded from filtering |
+| `aws_region_name` | string | `us-west-2` | AWS region |
+| `aws_access_key_id` | string | env | AWS access key |
+| `aws_secret_access_key` | string | env | AWS secret key |
+
+:::info S3 Vectors Auto-Creation
+When `index_name` is omitted, LiteLLM automatically creates:
+- S3 vector bucket (if it doesn't exist)
+- Vector index with auto-detected dimensions from your embedding model
+
+**Dimension Auto-Detection**: The vector dimension is automatically detected by making a test embedding request to your specified model. No need to manually specify dimensions!
+
+**Supported Embedding Models**: Works with any LiteLLM-supported embedding model (OpenAI, Cohere, Bedrock, Azure, etc.)
+:::
+
+**Example with auto-detection:**
+```json
+{
+ "embedding": {
+ "model": "text-embedding-3-small" // Dimension auto-detected as 1536
+ },
+ "vector_store": {
+ "custom_llm_provider": "s3_vectors",
+ "vector_bucket_name": "my-embeddings"
+ }
+}
+```
+
+**Example with custom embedding provider:**
+```json
+{
+ "embedding": {
+ "model": "cohere/embed-english-v3.0" // Dimension auto-detected as 1024
+ },
+ "vector_store": {
+ "custom_llm_provider": "s3_vectors",
+ "vector_bucket_name": "my-embeddings",
+ "distance_metric": "cosine"
+ }
+}
+```
+
## Input Examples
### File (Base64)
diff --git a/docs/my-website/docs/realtime.md b/docs/my-website/docs/realtime.md
index 0b3c823f5db..b191c82c670 100644
--- a/docs/my-website/docs/realtime.md
+++ b/docs/my-website/docs/realtime.md
@@ -3,13 +3,15 @@ import TabItem from '@theme/TabItem';
# /realtime
-Use this to loadbalance across Azure + OpenAI.
+Use this to loadbalance across Azure + OpenAI + xAI and more.
Supported Providers:
- OpenAI
- Azure
+- xAI ([see full docs](/docs/providers/xai_realtime))
- Google AI Studio (Gemini)
- Vertex AI
+- Bedrock
## Proxy Usage
@@ -45,6 +47,21 @@ model_list:
api_key: os.environ/OPENAI_API_KEY
```
+
+
+
+```yaml
+model_list:
+ - model_name: grok-voice-agent
+ litellm_params:
+ model: xai/grok-4-1-fast-non-reasoning
+ api_key: os.environ/XAI_API_KEY
+ model_info:
+ mode: realtime
+```
+
+**[See full xAI Realtime documentation →](/docs/providers/xai_realtime)**
+
diff --git a/docs/my-website/docs/rerank.md b/docs/my-website/docs/rerank.md
index 90f685d2bbd..9c76883d7fd 100644
--- a/docs/my-website/docs/rerank.md
+++ b/docs/my-website/docs/rerank.md
@@ -8,15 +8,15 @@ LiteLLM Follows the [cohere api request / response for the rerank api](https://c
## Overview
-| Feature | Supported | Notes |
-|---------|-----------|-------|
-| Cost Tracking | ✅ | Works with all supported models |
-| Logging | ✅ | Works across all integrations |
-| End-user Tracking | ✅ | |
-| Fallbacks | ✅ | Works between supported models |
-| Loadbalancing | ✅ | Works between supported models |
-| Guardrails | ✅ | Applies to input query only (not documents) |
-| Supported Providers | Cohere, Together AI, Azure AI, DeepInfra, Nvidia NIM, Infinity, Fireworks AI, Voyage AI | |
+| Feature | Supported | Notes |
+|---------|-----------------------------------------------------------------------------------------------------|-------|
+| Cost Tracking | ✅ | Works with all supported models |
+| Logging | ✅ | Works across all integrations |
+| End-user Tracking | ✅ | |
+| Fallbacks | ✅ | Works between supported models |
+| Loadbalancing | ✅ | Works between supported models |
+| Guardrails | ✅ | Applies to input query only (not documents) |
+| Supported Providers | Cohere, Together AI, Azure AI, DeepInfra, Nvidia NIM, Infinity, Fireworks AI, Voyage AI, watsonx.ai | |
## **LiteLLM Python SDK Usage**
### Quick Start
@@ -123,17 +123,18 @@ curl http://0.0.0.0:4000/rerank \
#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
-| Provider | Link to Usage |
-|-------------|--------------------|
-| Cohere (v1 + v2 clients) | [Usage](#quick-start) |
-| Together AI| [Usage](../docs/providers/togetherai) |
-| Azure AI| [Usage](../docs/providers/azure_ai#rerank-endpoint) |
-| Jina AI| [Usage](../docs/providers/jina_ai) |
-| AWS Bedrock| [Usage](../docs/providers/bedrock#rerank-api) |
-| HuggingFace| [Usage](../docs/providers/huggingface_rerank) |
-| Infinity| [Usage](../docs/providers/infinity) |
-| vLLM| [Usage](../docs/providers/vllm#rerank-endpoint) |
-| DeepInfra| [Usage](../docs/providers/deepinfra#rerank-endpoint) |
-| Vertex AI| [Usage](../docs/providers/vertex#rerank-api) |
-| Fireworks AI| [Usage](../docs/providers/fireworks_ai#rerank-endpoint) |
-| Voyage AI| [Usage](../docs/providers/voyage#rerank) |
\ No newline at end of file
+| Provider | Link to Usage |
+|--------------------------|------------------------------------------------------|
+| Cohere (v1 + v2 clients) | [Usage](#quick-start) |
+| Together AI | [Usage](../docs/providers/togetherai) |
+| Azure AI | [Usage](../docs/providers/azure_ai#rerank-endpoint) |
+| Jina AI | [Usage](../docs/providers/jina_ai) |
+| AWS Bedrock | [Usage](../docs/providers/bedrock#rerank-api) |
+| HuggingFace | [Usage](../docs/providers/huggingface_rerank) |
+| Infinity | [Usage](../docs/providers/infinity) |
+| vLLM | [Usage](../docs/providers/vllm#rerank-endpoint) |
+| DeepInfra | [Usage](../docs/providers/deepinfra#rerank-endpoint) |
+| Vertex AI | [Usage](../docs/providers/vertex#rerank-api) |
+| Fireworks AI | [Usage](../docs/providers/fireworks_ai#rerank-endpoint) |
+| Voyage AI | [Usage](../docs/providers/voyage#rerank) |
+| IBM watsonx.ai | [Usage](../docs/providers/watsonx/rerank) |
\ No newline at end of file
diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md
index 140dfd4faf8..65b7ad7773a 100644
--- a/docs/my-website/docs/response_api.md
+++ b/docs/my-website/docs/response_api.md
@@ -884,7 +884,12 @@ router = litellm.Router(
},
},
],
- optional_pre_call_checks=["responses_api_deployment_check"],
+ # `responses_api_deployment_check` ensures Requests with `previous_response_id`
+ # are routed to the same deployment. `deployment_affinity` adds sticky sessions
+ # for requests without `previous_response_id` (useful for implicit caching).
+ optional_pre_call_checks=["responses_api_deployment_check", "deployment_affinity"],
+ # Optional (default is 3600 seconds / 1 hour)
+ deployment_affinity_ttl_seconds=3600,
)
# Initial request
@@ -911,7 +916,16 @@ follow_up = await router.aresponses(
#### 1. Setup session continuity on proxy config.yaml
-To enable session continuity for Responses API in your LiteLLM proxy, set `optional_pre_call_checks: ["responses_api_deployment_check"]` in your proxy config.yaml.
+To enable session continuity for Responses API in your LiteLLM proxy, set `optional_pre_call_checks` in your proxy config.yaml.
+
+- `responses_api_deployment_check`: high priority routing when `previous_response_id` is provided
+- `deployment_affinity`: sticky sessions based on user key (applies even without `previous_response_id`)
+
+Notes:
+- User-key affinity is keyed on `metadata.user_api_key_hash` (the API key hash). The OpenAI `user` request parameter is an end-user identifier and is intentionally not used for deployment affinity.
+- `user_api_key_hash` is already SHA-256, and is used as-is (no double hashing).
+- Affinity is scoped by a stable model identifier (the model-map key, e.g. `model_map_information.model_map_key`) so model aliases map to the same stickiness bucket.
+- The mapping TTL is controlled by `deployment_affinity_ttl_seconds` (configured on Router init / proxy startup).
```yaml showLineNumbers title="config.yaml with Session Continuity"
model_list:
@@ -929,7 +943,11 @@ model_list:
api_base: https://endpoint2.openai.azure.com
router_settings:
- optional_pre_call_checks: ["responses_api_deployment_check"]
+ optional_pre_call_checks:
+ - responses_api_deployment_check
+ - deployment_affinity
+ # Optional (default is 3600 seconds / 1 hour)
+ deployment_affinity_ttl_seconds: 3600
```
#### 2. Use the OpenAI Python SDK to make requests to LiteLLM Proxy
@@ -1023,6 +1041,134 @@ curl http://localhost:4000/v1/responses \
+## Server-side compaction
+
+For long-running conversations, you can enable **server-side compaction** so that when the rendered context size crosses a threshold, the server automatically runs compaction in-stream and emits a compaction item—no separate `POST /v1/responses/compact` call is required.
+
+Supported on the OpenAI Responses API when using the `openai` or `azure` provider. Pass `context_management` with a compaction entry and `compact_threshold` (token count; minimum 1000). When the context crosses the threshold, the server compacts in-stream and continues. Chain turns with `previous_response_id` or by appending output items to your next input array. See [OpenAI Compaction guide](https://developers.openai.com/api/docs/guides/compaction) for details.
+
+For explicit control over when compaction runs, use the standalone compact endpoint (`POST /v1/responses/compact`) instead.
+
+### Python SDK
+
+```python showLineNumbers title="Server-side compaction with LiteLLM Python SDK"
+import litellm
+
+# Non-streaming: enable compaction when context exceeds 200k tokens
+response = litellm.responses(
+ model="openai/gpt-4o",
+ input="Your conversation input...",
+ context_management=[{"type": "compaction", "compact_threshold": 200000}],
+ max_output_tokens=1024,
+)
+print(response)
+
+# Streaming: same context_management, compaction runs in-stream if threshold is crossed
+stream = litellm.responses(
+ model="openai/gpt-4o",
+ input="Your conversation input...",
+ context_management=[{"type": "compaction", "compact_threshold": 200000}],
+ stream=True,
+)
+for event in stream:
+ print(event)
+```
+
+### LiteLLM Proxy (AI Gateway)
+
+Use the OpenAI SDK with your proxy as `base_url`, or call the proxy with curl. The proxy forwards `context_management` to the provider.
+
+**OpenAI Python SDK (proxy as base_url):**
+
+```python showLineNumbers title="Server-side compaction via LiteLLM Proxy"
+from openai import OpenAI
+
+client = OpenAI(
+ base_url="http://localhost:4000", # LiteLLM Proxy (AI Gateway)
+ api_key="your-proxy-api-key",
+)
+
+response = client.responses.create(
+ model="openai/gpt-4o",
+ input="Your conversation input...",
+ context_management=[{"type": "compaction", "compact_threshold": 200000}],
+ max_output_tokens=1024,
+)
+print(response)
+```
+
+**curl (proxy):**
+
+```bash title="Server-side compaction via curl to LiteLLM Proxy"
+curl -X POST "http://localhost:4000/v1/responses" \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer your-proxy-api-key" \
+ -d '{
+ "model": "openai/gpt-4o",
+ "input": "Your conversation input...",
+ "context_management": [{"type": "compaction", "compact_threshold": 200000}],
+ "max_output_tokens": 1024
+ }'
+```
+
+## Shell tool
+
+The **Shell tool** lets the model run commands in a hosted container or local runtime (OpenAI Responses API). You pass `tools=[{"type": "shell", "environment": {...}}]`; the `environment` object configures the runtime (e.g. `type: "container_auto"` for auto-provisioned containers). See [OpenAI Shell tool guide](https://developers.openai.com/api/docs/guides/tools-shell) for full options.
+
+Supported when using the `openai` or `azure` provider with a model that supports the Shell tool.
+
+### Python SDK
+
+```python showLineNumbers title="Shell tool with LiteLLM Python SDK"
+import litellm
+
+response = litellm.responses(
+ model="openai/gpt-5.2",
+ input="List files in /mnt/data and run python --version.",
+ tools=[{"type": "shell", "environment": {"type": "container_auto"}}],
+ tool_choice="auto",
+ max_output_tokens=1024,
+)
+```
+
+### LiteLLM Proxy (AI Gateway)
+
+Use the OpenAI SDK with your proxy as `base_url`, or call the proxy with curl. The proxy forwards `tools` (including `type: "shell"`) to the provider.
+
+**OpenAI Python SDK (proxy as base_url):**
+
+```python showLineNumbers title="Shell tool via LiteLLM Proxy"
+from openai import OpenAI
+
+client = OpenAI(
+ base_url="http://localhost:4000",
+ api_key="your-proxy-api-key",
+)
+
+response = client.responses.create(
+ model="openai/gpt-5.2",
+ input="List files in /mnt/data.",
+ tools=[{"type": "shell", "environment": {"type": "container_auto"}}],
+ tool_choice="auto",
+ max_output_tokens=1024,
+)
+```
+
+**curl:**
+
+```bash title="Shell tool via curl to LiteLLM Proxy"
+curl -X POST "http://localhost:4000/v1/responses" \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer your-proxy-api-key" \
+ -d '{
+ "model": "openai/gpt-5.2",
+ "input": "List files in /mnt/data.",
+ "tools": [{"type": "shell", "environment": {"type": "container_auto"}}],
+ "tool_choice": "auto",
+ "max_output_tokens": 1024
+ }'
+```
+
## Session Management
LiteLLM Proxy supports session management for all supported models. This allows you to store and fetch conversation history (state) in LiteLLM Proxy.
@@ -1228,8 +1374,3 @@ Response:
-
-
-
-
-
diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md
index 47967775e1e..67e7f681147 100644
--- a/docs/my-website/docs/routing.md
+++ b/docs/my-website/docs/routing.md
@@ -830,6 +830,12 @@ asyncio.run(router_acompletion())
+## Traffic Mirroring / Silent Experiments
+
+Traffic mirroring allows you to "mimic" production traffic to a secondary (silent) model for evaluation purposes. The silent model's response is gathered in the background and does not affect the latency or result of the primary request.
+
+[**See detailed guide on A/B Testing - Traffic Mirroring here**](./traffic_mirroring.md)
+
## Basic Reliability
### Deployment Ordering (Priority)
@@ -1582,11 +1588,13 @@ Get a slack webhook url from https://api.slack.com/messaging/webhooks
Initialize an `AlertingConfig` and pass it to `litellm.Router`. The following code will trigger an alert because `api_key=bad-key` which is invalid
```python
-from litellm.router import AlertingConfig
import litellm
+from litellm.router import Router
+from litellm.types.router import AlertingConfig
import os
+import asyncio
-router = litellm.Router(
+router = Router(
model_list=[
{
"model_name": "gpt-3.5-turbo",
@@ -1597,17 +1605,28 @@ router = litellm.Router(
}
],
alerting_config= AlertingConfig(
- alerting_threshold=10, # threshold for slow / hanging llm responses (in seconds). Defaults to 300 seconds
- webhook_url= os.getenv("SLACK_WEBHOOK_URL") # webhook you want to send alerts to
+ alerting_threshold=10,
+ webhook_url= "https:/..."
),
)
-try:
- await router.acompletion(
- model="gpt-3.5-turbo",
- messages=[{"role": "user", "content": "Hey, how's it going?"}],
- )
-except:
- pass
+
+async def main():
+ print(f"\n=== Configuration ===")
+ print(f"Slack logger exists: {router.slack_alerting_logger is not None}")
+
+ try:
+ await router.acompletion(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Hey, how's it going?"}],
+ )
+ except Exception as e:
+ print(f"\n=== Exception caught ===")
+ print(f"Waiting 10 seconds for alerts to be sent via periodic flush...")
+ await asyncio.sleep(10)
+ print(f"\n=== After waiting ===")
+ print(f"Alert should have been sent to Slack!")
+
+asyncio.run(main())
```
## Track cost for Azure Deployments
diff --git a/docs/my-website/docs/search/index.md b/docs/my-website/docs/search/index.md
index 551a495261a..8a71edead06 100644
--- a/docs/my-website/docs/search/index.md
+++ b/docs/my-website/docs/search/index.md
@@ -276,6 +276,7 @@ The response follows Perplexity's search format with the following structure:
| Firecrawl | `FIRECRAWL_API_KEY` | `firecrawl` |
| SearXNG | `SEARXNG_API_BASE` (required) | `searxng` |
| Linkup | `LINKUP_API_KEY` | `linkup` |
+| DuckDuckGo | `DUCKDUCKGO_API_BASE` | `duckduckgo` |
See the individual provider documentation for detailed setup instructions and provider-specific parameters.
diff --git a/docs/my-website/docs/traffic_mirroring.md b/docs/my-website/docs/traffic_mirroring.md
new file mode 100644
index 00000000000..3bdcb0f1614
--- /dev/null
+++ b/docs/my-website/docs/traffic_mirroring.md
@@ -0,0 +1,83 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# A/B Testing - Traffic Mirroring
+
+Traffic mirroring allows you to "mimic" production traffic to a secondary (silent) model for evaluation purposes. The silent model's response is gathered in the background and does not affect the latency or result of the primary request.
+
+This is useful for:
+- Testing a new model's performance on production prompts before switching.
+- Comparing costs and latency between different providers.
+- Debugging issues by mirroring traffic to a more verbose model.
+
+## Quick Start
+
+To enable traffic mirroring, add `silent_model` to the `litellm_params` of a deployment.
+
+
+
+
+```python
+from litellm import Router
+
+model_list = [
+ {
+ "model_name": "gpt-3.5-turbo",
+ "litellm_params": {
+ "model": "azure/chatgpt-v-2",
+ "api_key": "...",
+ "silent_model": "gpt-4" # 👈 Mirror traffic to gpt-4
+ },
+ },
+ {
+ "model_name": "gpt-4",
+ "litellm_params": {
+ "model": "openai/gpt-4",
+ "api_key": "..."
+ },
+ }
+]
+
+router = Router(model_list=model_list)
+
+# The request to "gpt-3.5-turbo" will trigger a background call to "gpt-4"
+response = await router.acompletion(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "How does traffic mirroring work?"}]
+)
+```
+
+
+
+
+Add `silent_model` to your `config.yaml`:
+
+```yaml
+model_list:
+ - model_name: primary-model
+ litellm_params:
+ model: azure/gpt-35-turbo
+ api_key: os.environ/AZURE_API_KEY
+ silent_model: evaluation-model # 👈 Mirror traffic here
+ - model_name: evaluation-model
+ litellm_params:
+ model: openai/gpt-4o
+ api_key: os.environ/OPENAI_API_KEY
+```
+
+
+
+
+## How it works
+1. **Request Received**: A request is made to a model group (e.g. `primary-model`).
+2. **Deployment Picked**: LiteLLM picks a deployment from the group.
+3. **Primary Call**: LiteLLM makes the call to the primary deployment.
+4. **Mirroring**: If `silent_model` is present, LiteLLM triggers a background call to that model.
+ - For **Sync** calls: Uses a shared thread pool.
+ - For **Async** calls: Uses `asyncio.create_task`.
+5. **Isolation**: The background call uses a `deepcopy` of the original request parameters and sets `metadata["is_silent_experiment"] = True`. It also strips out logging IDs to prevent collisions in usage tracking.
+
+## Key Features
+- **Latency Isolation**: The primary request returns as soon as it's ready. The background (silent) call does not block.
+- **Unified Logging**: Background calls are processed via the Router, meaning they are automatically logged to your configured observability tools (Langfuse, S3, etc.).
+- **Evaluation**: Use the `is_silent_experiment: True` flag in your logs to filter and compare results between the primary and mirrored calls.
diff --git a/docs/my-website/docs/troubleshoot.md b/docs/my-website/docs/troubleshoot.md
index f9ed47972eb..1539e1959f7 100644
--- a/docs/my-website/docs/troubleshoot.md
+++ b/docs/my-website/docs/troubleshoot.md
@@ -1,45 +1,43 @@
-# Troubleshooting & Support
-
-## Information to Provide When Seeking Help
+# Issue Reporting
When reporting issues, please include as much of the following as possible. It's okay if you can't provide everything—especially in production scenarios where the trigger might be unknown. Sharing most of this information will help us assist you more effectively.
-### 1. LiteLLM Configuration File
+## 1. LiteLLM Configuration File
Your `config.yaml` file (redact sensitive info like API keys). Include number of workers if not in config.
-### 2. Initialization Command
+## 2. Initialization Command
The command used to start LiteLLM (e.g., `litellm --config config.yaml --num_workers 8 --detailed_debug`).
-### 3. LiteLLM Version
+## 3. LiteLLM Version
-- Current version
-- Version when the issue first appeared (if different)
+- Current version
+- Version when the issue first appeared (if different)
- If upgraded, the version changed from → to
-### 4. Environment Variables
+## 4. Environment Variables
Non-sensitive environment variables not in your config (e.g., `NUM_WORKERS`, `LITELLM_LOG`, `LITELLM_MODE`). Do not include passwords or API keys.
-### 5. Server Specifications
+## 5. Server Specifications
CPU cores, RAM, OS, number of instances/replicas, etc.
-### 6. Database and Redis Usage
+## 6. Database and Redis Usage
- **Database:** Using database? (`DATABASE_URL` set), database type and version
- **Redis:** Using Redis? Redis version, configuration type (Standalone/Cluster/Sentinel).
-### 7. Endpoints
+## 7. Endpoints
The endpoint(s) you're using that are experiencing issues (e.g., `/chat/completions`, `/embeddings`).
-### 8. Request Example
+## 8. Request Example
A realistic example of the request causing issues, including expected vs. actual response and any error messages.
-### 9. Error Logs, Stack Traces, and Metrics
+## 9. Error Logs, Stack Traces, and Metrics
Full error logs, stack traces, and any images from service metrics (CPU, memory, request rates, etc.) that might help diagnose the issue.
@@ -57,4 +55,3 @@ Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238
Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
[](https://wa.link/huol9n) [](https://discord.gg/wuPM9dRgDw)
-
diff --git a/docs/my-website/docs/troubleshoot/max_callbacks.md b/docs/my-website/docs/troubleshoot/max_callbacks.md
new file mode 100644
index 00000000000..4b0f3e24b73
--- /dev/null
+++ b/docs/my-website/docs/troubleshoot/max_callbacks.md
@@ -0,0 +1,68 @@
+# MAX_CALLBACKS Limit
+
+## Error Message
+
+```
+Cannot add callback - would exceed MAX_CALLBACKS limit of 30. Current callbacks: 30
+```
+
+## What This Means
+
+LiteLLM limits the number of callbacks that can be registered to prevent performance degradation. Each callback runs on every LLM request, so having too many callbacks can cause exponential CPU usage and slow down your proxy.
+
+The default limit is **30 callbacks**.
+
+## When You Might Hit This Limit
+
+- **Large enterprise deployments** with many teams, each having their own guardrails
+- **Multiple logging integrations** combined with custom callbacks
+- **Per-team callback configurations** that add up across your organization
+
+## How to Override
+
+Set the `LITELLM_MAX_CALLBACKS` environment variable to increase the limit:
+
+```bash
+# Docker
+docker run -e LITELLM_MAX_CALLBACKS=100 ...
+
+# Docker Compose
+environment:
+ - LITELLM_MAX_CALLBACKS=100
+
+# Kubernetes
+env:
+ - name: LITELLM_MAX_CALLBACKS
+ value: "100"
+
+# Direct
+export LITELLM_MAX_CALLBACKS=100
+litellm --config config.yaml
+```
+
+## Recommendations
+
+1. **Start conservative** - Only increase as much as you need. If you have 60 teams with guardrails, try `LITELLM_MAX_CALLBACKS=75` to leave headroom.
+
+2. **Monitor performance** - More callbacks means more processing per request. Watch your CPU usage and response latency after increasing the limit.
+
+3. **Consolidate where possible** - If multiple teams use identical guardrails, consider using shared callback configurations rather than per-team duplicates.
+
+## Example: Large Enterprise Setup
+
+For an organization with 60+ teams, each with a guardrail callback:
+
+```yaml
+# config.yaml
+litellm_settings:
+ callbacks: ["prometheus", "langfuse"] # 2 global callbacks
+
+# Each team adds 1 guardrail callback = 60+ callbacks
+# Total: 62+ callbacks needed
+```
+
+Set the environment variable:
+
+```bash
+export LITELLM_MAX_CALLBACKS=100
+```
diff --git a/docs/my-website/docs/troubleshoot/prisma_migrations.md b/docs/my-website/docs/troubleshoot/prisma_migrations.md
new file mode 100644
index 00000000000..9d9cb585b2b
--- /dev/null
+++ b/docs/my-website/docs/troubleshoot/prisma_migrations.md
@@ -0,0 +1,113 @@
+# Troubleshooting Prisma Migration Errors
+
+Common Prisma migration issues encountered when upgrading or downgrading LiteLLM proxy versions, and how to fix them.
+
+## How Prisma Migrations Work in LiteLLM
+
+- LiteLLM uses [Prisma](https://www.prisma.io/) to manage its PostgreSQL database schema.
+- Migration history is tracked in the `_prisma_migrations` table in your database.
+- When LiteLLM starts, it runs `prisma migrate deploy` to apply any new migrations.
+- Upgrading LiteLLM applies all migrations added since your last applied version.
+
+## Common Errors
+
+### 1. `relation "X" does not exist`
+
+**Example error:**
+
+```
+ERROR: relation "LiteLLM_DeletedTeamTable" does not exist
+Migration: 20260116142756_update_deleted_keys_teams_table_routing_settings
+```
+
+**Cause:** This typically happens after a version rollback. The `_prisma_migrations` table still records migrations from the newer version as "applied," but the underlying database tables were modified, dropped, or never fully created.
+
+**How to fix:**
+
+#### Step 1 — Delete the failed migration entry and restart
+
+Remove the problematic migration from the history so it can be re-applied:
+
+```sql
+-- View recent migrations
+SELECT migration_name, finished_at, rolled_back_at, logs
+FROM "_prisma_migrations"
+ORDER BY started_at DESC
+LIMIT 10;
+
+-- Delete the failed migration entry
+DELETE FROM "_prisma_migrations"
+WHERE migration_name = '';
+```
+
+After deleting the entry, restart LiteLLM — it will re-apply the migration on startup.
+
+#### Step 2 — If that doesn't work, use `prisma db push`
+
+If deleting the migration entry and restarting doesn't resolve the issue, sync the schema directly:
+
+```bash
+DATABASE_URL="" prisma db push
+```
+
+This bypasses migration history and forces the database schema to match the Prisma schema.
+
+---
+
+### 2. `New migrations cannot be applied before the error is recovered from`
+
+**Cause:** A previous migration failed (recorded with an error in `_prisma_migrations`), and Prisma refuses to apply any new migrations until the failure is resolved.
+
+**How to fix:**
+
+1. Find the failed migration:
+
+```sql
+SELECT migration_name, finished_at, rolled_back_at, logs
+FROM "_prisma_migrations"
+WHERE finished_at IS NULL OR rolled_back_at IS NOT NULL
+ORDER BY started_at DESC;
+```
+
+2. Delete the failed entry and restart LiteLLM:
+
+```sql
+DELETE FROM "_prisma_migrations"
+WHERE migration_name = '';
+```
+
+3. If that doesn't work, use `prisma db push`:
+
+```bash
+DATABASE_URL="" prisma db push
+```
+
+---
+
+### 3. Migration state mismatch after version rollback
+
+**Cause:** You upgraded to version X (new migrations applied), rolled back to version Y, then upgraded again. The `_prisma_migrations` table has stale entries for migrations that were partially applied or correspond to a schema state that no longer exists.
+
+**Fix:**
+
+1. Inspect the migration table for problematic entries:
+
+```sql
+SELECT migration_name, started_at, finished_at, rolled_back_at, logs
+FROM "_prisma_migrations"
+ORDER BY started_at DESC
+LIMIT 20;
+```
+
+2. For each migration that shouldn't be there (i.e., from the version you rolled back from), delete the entry:
+ ```sql
+ DELETE FROM "_prisma_migrations" WHERE migration_name = '';
+ ```
+
+3. Restart LiteLLM to re-run migrations.
+
+4. If that doesn't work, use `prisma db push`:
+
+```bash
+DATABASE_URL="" prisma db push
+```
diff --git a/docs/my-website/docs/troubleshoot/spend_queue_warnings.md b/docs/my-website/docs/troubleshoot/spend_queue_warnings.md
new file mode 100644
index 00000000000..4be8b18f5cd
--- /dev/null
+++ b/docs/my-website/docs/troubleshoot/spend_queue_warnings.md
@@ -0,0 +1,46 @@
+# Spend Update Queue Full Warnings
+
+## Overview
+
+The "Spend update queue is full" warning occurs in high-volume LiteLLM proxy deployments when the internal spend tracking queue reaches capacity. This is a protective mechanism to prevent memory issues during traffic spikes.
+
+## Warning Message
+
+```
+WARNING:litellm.proxy.db.db_transaction_queue.spend_update_queue:Spend update queue is full. Aggregating entries to prevent memory issues.
+```
+
+## Root Cause
+
+The spend update queue has a default maximum size of 10,000 entries (`MAX_SIZE_IN_MEMORY_QUEUE=10000`). When this limit is reached:
+
+1. New spend tracking entries are aggregated instead of queued individually
+2. This prevents memory exhaustion but may slightly delay spend updates
+3. The warning indicates your deployment is processing requests faster than the database can handle spend updates
+
+## Solutions
+
+### 1. Increase Queue Size
+
+Set the `MAX_SIZE_IN_MEMORY_QUEUE` environment variable to a higher value:
+
+```bash
+MAX_SIZE_IN_MEMORY_QUEUE=50000
+```
+
+**Tradeoffs:**
+Higher queue sizes store more items in memory - provision at least 8GB RAM for large queues
+- Recommended for deployments with consistent high traffic
+
+### 2. Horizontal Scaling
+
+Deploy multiple proxy instances with load balancing. This distributes the spend tracking load across multiple queues, reducing the pressure on any single instance's spend update queue.
+
+
+
+## Related Configuration
+
+```yaml
+# Environment variables
+MAX_SIZE_IN_MEMORY_QUEUE: 10000 # Default queue size
+```
diff --git a/docs/my-website/docs/troubleshoot/ui_issues.md b/docs/my-website/docs/troubleshoot/ui_issues.md
new file mode 100644
index 00000000000..90912b1daeb
--- /dev/null
+++ b/docs/my-website/docs/troubleshoot/ui_issues.md
@@ -0,0 +1,49 @@
+# UI Troubleshooting
+
+If you're experiencing issues with the LiteLLM Admin UI, please include the following information when reporting.
+
+## 1. Steps to Reproduce
+
+A clear, step-by-step description of how to trigger the issue (e.g., "Navigate to Settings → Team, click 'Create Team', fill in fields, click submit → error appears").
+
+## 2. LiteLLM Version
+
+The current version of LiteLLM you're running. Check via `litellm --version` or the UI's settings page.
+
+## 3. Architecture & Deployment Setup
+
+Distributed environments are a known source of UI issues. Please describe:
+
+- **Number of LiteLLM instances/replicas** and how they are deployed (e.g., Kubernetes, Docker Compose, ECS)
+- **Load balancer** type and configuration (e.g., ALB, Nginx, Cloudflare Tunnel) — include whether sticky sessions are enabled
+- **How the UI is accessed** — directly via LiteLLM, through a reverse proxy, or behind an ingress controller
+- **Any CDN or caching layers** between the user and the LiteLLM server
+
+## 4. Network Tab Requests
+
+Open your browser's Developer Tools (F12 → Network tab), reproduce the issue, and share:
+
+- The **failing request(s)** — URL, method, status code, and response body
+- **Screenshots or HAR export** of the relevant network activity
+- Any **CORS or mixed-content errors** shown in the Console tab
+
+## 5. Environment Variables
+
+Non-sensitive environment variables related to the UI and proxy setup, such as:
+
+- `LITELLM_MASTER_KEY`
+- `PROXY_BASE_URL` / `LITELLM_PROXY_BASE_URL`
+- `UI_BASE_PATH`
+- Any SSO-related variables (e.g., `GOOGLE_CLIENT_ID`, `MICROSOFT_TENANT`)
+
+Do **not** include passwords, secrets, or API keys.
+
+## 6. Browser & Access Details
+
+- **Browser** and version (e.g., Chrome 120, Firefox 121)
+- **Access URL** used to reach the UI (redact sensitive parts)
+- Whether the issue occurs for **all users or specific roles** (Admin, Internal User, etc.)
+
+## 7. Screenshots or Screen Recordings
+
+A screenshot or short screen recording of the issue is extremely helpful. Include any visible error messages, toasts, or unexpected behavior.
diff --git a/docs/my-website/docs/tutorials/claude_agent_sdk.md b/docs/my-website/docs/tutorials/claude_agent_sdk.md
new file mode 100644
index 00000000000..c56784ba2df
--- /dev/null
+++ b/docs/my-website/docs/tutorials/claude_agent_sdk.md
@@ -0,0 +1,115 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Claude Agent SDK with LiteLLM
+
+Use Anthropic's Claude Agent SDK with any LLM provider through LiteLLM Proxy.
+
+The Claude Agent SDK provides a high-level interface for building AI agents. By pointing it to LiteLLM, you can use the same agent code with OpenAI, Bedrock, Azure, Vertex AI, or any other provider.
+
+## Quick Start
+
+### 1. Install Dependencies
+
+```bash
+pip install claude-agent-sdk
+```
+
+### 2. Start LiteLLM Proxy
+
+```yaml title="config.yaml" showLineNumbers
+model_list:
+ - model_name: bedrock-claude-sonnet-3.5
+ litellm_params:
+ model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
+ aws_region_name: "us-east-1"
+
+ - model_name: bedrock-claude-sonnet-4
+ litellm_params:
+ model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
+ aws_region_name: "us-east-1"
+
+ - model_name: bedrock-claude-sonnet-4.5
+ litellm_params:
+ model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
+ aws_region_name: "us-east-1"
+
+ - model_name: bedrock-claude-opus-4.5
+ litellm_params:
+ model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0"
+ aws_region_name: "us-east-1"
+
+ - model_name: bedrock-nova-premier
+ litellm_params:
+ model: "bedrock/amazon.nova-premier-v1:0"
+ aws_region_name: "us-east-1"
+```
+
+```bash
+litellm --config config.yaml
+```
+
+### 3. Point Agent SDK to LiteLLM
+
+| Environment Variable | Value | Description |
+|---------------------|-------|-------------|
+| `ANTHROPIC_BASE_URL` | `http://localhost:4000` | LiteLLM proxy URL |
+| `ANTHROPIC_API_KEY` | `sk-1234` | Your LiteLLM API key (not Anthropic key) |
+
+```python title="agent.py" showLineNumbers
+import os
+from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
+
+# Point to LiteLLM proxy (not Anthropic)
+os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
+os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM key
+
+# Configure agent with any model from your config
+options = ClaudeAgentOptions(
+ system_prompt="You are a helpful AI assistant.",
+ model="bedrock-claude-sonnet-4", # Use any model from config.yaml
+ max_turns=20,
+)
+
+async with ClaudeSDKClient(options=options) as client:
+ await client.query("What is LiteLLM?")
+
+ async for msg in client.receive_response():
+ if hasattr(msg, 'content'):
+ for content_block in msg.content:
+ if hasattr(content_block, 'text'):
+ print(content_block.text, end='', flush=True)
+```
+
+
+
+## Why Use LiteLLM with Agent SDK?
+
+| Feature | Benefit |
+|---------|---------|
+| **Multi-Provider** | Use the same agent code with OpenAI, Bedrock, Azure, Vertex AI, etc. |
+| **Cost Tracking** | Track spending across all agent conversations |
+| **Rate Limiting** | Set budgets and limits on agent usage |
+| **Load Balancing** | Distribute requests across multiple API keys or regions |
+| **Fallbacks** | Automatically retry with different models if one fails |
+
+## Complete Example
+
+See our [cookbook example](https://github.com/BerriAI/litellm/tree/main/cookbook/anthropic_agent_sdk) for a complete interactive CLI agent that:
+- Streams responses in real-time
+- Switches between models dynamically
+- Fetches available models from the proxy
+
+```bash
+# Clone and run the example
+git clone https://github.com/BerriAI/litellm.git
+cd litellm/cookbook/anthropic_agent_sdk
+pip install -r requirements.txt
+python main.py
+```
+
+## Related Resources
+
+- [Claude Agent SDK Documentation](https://github.com/anthropics/anthropic-agent-sdk)
+- [LiteLLM Proxy Quick Start](../proxy/quick_start)
+- [Complete Cookbook Example](https://github.com/BerriAI/litellm/tree/main/cookbook/anthropic_agent_sdk)
diff --git a/docs/my-website/docs/tutorials/claude_code_beta_headers.md b/docs/my-website/docs/tutorials/claude_code_beta_headers.md
new file mode 100644
index 00000000000..fab90d15e88
--- /dev/null
+++ b/docs/my-website/docs/tutorials/claude_code_beta_headers.md
@@ -0,0 +1,279 @@
+import Image from '@theme/IdealImage';
+
+# Claude Code - Managing Anthropic Beta Headers
+
+When using Claude Code with LiteLLM and non-Anthropic providers (Bedrock, Azure AI, Vertex AI), you need to ensure that only supported beta headers are sent to each provider. This guide explains how to add support for new beta headers or fix invalid beta header errors.
+
+## What Are Beta Headers?
+
+Anthropic uses beta headers to enable experimental features in Claude. When you use Claude Code, it may send beta headers like:
+
+```
+anthropic-beta: prompt-caching-scope-2026-01-05,advanced-tool-use-2025-11-20
+```
+
+However, not all providers support all Anthropic beta features. LiteLLM uses `anthropic_beta_headers_config.json` to manage which beta headers are supported by each provider.
+
+## Common Error Message
+
+```bash
+Error: The model returned the following errors: invalid beta flag
+```
+
+## How LiteLLM Handles Beta Headers
+
+LiteLLM uses a strict validation approach with a configuration file:
+
+```
+litellm/litellm/anthropic_beta_headers_config.json
+```
+
+This JSON file contains a **mapping** of beta headers for each provider:
+- **Keys**: Input beta header names (from Anthropic)
+- **Values**: Provider-specific header names (or `null` if unsupported)
+- **Validation**: Only headers present in the mapping with non-null values are forwarded
+
+This enforces stricter validation than just filtering unsupported headers - headers must be explicitly defined to be allowed.
+
+## Adding Support for a New Beta Header
+
+When Anthropic releases a new beta feature, you need to add it to the configuration file for each provider.
+
+### Step 1: Locate the Config File
+
+Find the file in your LiteLLM installation:
+
+```bash
+# If installed via pip
+cd $(python -c "import litellm; import os; print(os.path.dirname(litellm.__file__))")
+
+# The config file is at:
+# litellm/anthropic_beta_headers_config.json
+```
+
+### Step 2: Add the New Beta Header
+
+Open `anthropic_beta_headers_config.json` and add the new header to each provider's mapping:
+
+```json title="anthropic_beta_headers_config.json"
+{
+ "description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded.",
+ "anthropic": {
+ "advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
+ "new-feature-2026-03-01": "new-feature-2026-03-01",
+ ...
+ },
+ "azure_ai": {
+ "advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
+ "new-feature-2026-03-01": "new-feature-2026-03-01",
+ ...
+ },
+ "bedrock_converse": {
+ "advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
+ "new-feature-2026-03-01": null,
+ ...
+ },
+ "bedrock": {
+ "advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
+ "new-feature-2026-03-01": null,
+ ...
+ },
+ "vertex_ai": {
+ "advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
+ "new-feature-2026-03-01": null,
+ ...
+ }
+}
+```
+
+**Key Points:**
+- **Supported headers**: Set the value to the provider-specific header name (often the same as the key)
+- **Unsupported headers**: Set the value to `null`
+- **Header transformations**: Some providers use different header names (e.g., Bedrock maps `advanced-tool-use-2025-11-20` to `tool-search-tool-2025-10-19`)
+- **Alphabetical order**: Keep headers sorted alphabetically for maintainability
+
+### Step 3: Reload Configuration (No Restart Required!)
+
+**Option 1: Dynamic Reload Without Restart**
+
+Instead of restarting your application, you can dynamically reload the beta headers configuration using environment variables and API endpoints:
+
+```bash
+# Set environment variable to fetch from remote URL (Do this if you want to point it to some other URL)
+export LITELLM_ANTHROPIC_BETA_HEADERS_URL="https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/anthropic_beta_headers_config.json"
+
+# Manually trigger reload via API (no restart needed!)
+curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
+ -H "Authorization: Bearer YOUR_ADMIN_TOKEN"
+```
+
+**Option 2: Schedule Automatic Reloads**
+
+Set up automatic reloading to always stay up-to-date with the latest beta headers:
+
+```bash
+# Reload configuration every 24 hours
+curl -X POST "https://your-proxy-url/schedule/anthropic_beta_headers_reload?hours=24" \
+ -H "Authorization: Bearer YOUR_ADMIN_TOKEN"
+```
+
+**Option 3: Traditional Restart**
+
+If you prefer the traditional approach, restart your LiteLLM proxy or application:
+
+```bash
+# If using LiteLLM proxy
+litellm --config config.yaml
+
+# If using Python SDK
+# Just restart your Python application
+```
+
+:::tip Zero-Downtime Updates
+With dynamic reloading, you can fix invalid beta header errors **without restarting your service**! This is especially useful in production environments where downtime is costly.
+
+See [Auto Sync Anthropic Beta Headers](../proxy/sync_anthropic_beta_headers.md) for complete documentation.
+:::
+
+## Fixing Invalid Beta Header Errors
+
+If you encounter an "invalid beta flag" error, it means a beta header is being sent that the provider doesn't support.
+
+### Step 1: Identify the Problematic Header
+
+Check your logs to see which header is causing the issue:
+
+```bash
+Error: The model returned the following errors: invalid beta flag: new-feature-2026-03-01
+```
+
+### Step 2: Update the Config
+
+Set the header value to `null` for that provider:
+
+```json title="anthropic_beta_headers_config.json"
+{
+ "bedrock_converse": {
+ "new-feature-2026-03-01": null
+ }
+}
+```
+
+### Step 3: Restart and Test
+
+Restart your application and verify the header is now filtered out.
+
+## Contributing a Fix to LiteLLM
+
+Help the community by contributing your fix!
+
+### What to Include in Your PR
+
+1. **Update the config file**: Add the new beta header to `litellm/anthropic_beta_headers_config.json`
+2. **Test your changes**: Verify the header is correctly filtered/mapped for each provider
+3. **Documentation**: Include provider documentation links showing which headers are supported
+
+### Example PR Description
+
+```markdown
+## Add support for new-feature-2026-03-01 beta header
+
+### Changes
+- Added `new-feature-2026-03-01` to anthropic_beta_headers_config.json
+- Set to `null` for bedrock_converse (unsupported)
+- Set to header name for anthropic, azure_ai (supported)
+
+### Testing
+Tested with:
+- ✅ Anthropic: Header passed through correctly
+- ✅ Azure AI: Header passed through correctly
+- ✅ Bedrock Converse: Header filtered out (returns error without fix)
+
+### References
+- Anthropic docs: [link]
+- AWS Bedrock docs: [link]
+```
+
+
+## How Beta Header Filtering Works
+
+When you make a request through LiteLLM:
+
+```mermaid
+sequenceDiagram
+ participant CC as Claude Code
+ participant LP as LiteLLM
+ participant Config as Beta Headers Config
+ participant Provider as Provider (Bedrock/Azure/etc)
+
+ CC->>LP: Request with beta headers
+ Note over CC,LP: anthropic-beta: header1,header2,header3
+
+ LP->>Config: Load header mapping for provider
+ Config-->>LP: Returns mapping (header→value or null)
+
+ Note over LP: Validate & Transform:
1. Check if header exists in mapping
2. Filter out null values
3. Map to provider-specific names
+
+ LP->>Provider: Request with filtered & mapped headers
+ Note over LP,Provider: anthropic-beta: mapped-header2
(header1, header3 filtered out)
+
+ Provider-->>LP: Success response
+ LP-->>CC: Response
+```
+
+### Filtering Rules
+
+1. **Header must exist in mapping**: Unknown headers are filtered out
+2. **Header must have non-null value**: Headers with `null` values are filtered out
+3. **Header transformation**: Headers are mapped to provider-specific names (e.g., `advanced-tool-use-2025-11-20` → `tool-search-tool-2025-10-19` for Bedrock)
+
+### Example
+
+Request with headers:
+```
+anthropic-beta: advanced-tool-use-2025-11-20,computer-use-2025-01-24,unknown-header
+```
+
+For Bedrock Converse:
+- ✅ `computer-use-2025-01-24` → `computer-use-2025-01-24` (supported, passed through)
+- ❌ `advanced-tool-use-2025-11-20` → filtered out (null value in config)
+- ❌ `unknown-header` → filtered out (not in config)
+
+Result sent to Bedrock:
+```
+anthropic-beta: computer-use-2025-01-24
+```
+
+## Dynamic Configuration Management (No Restart Required!)
+
+### Environment Variables
+
+Control how LiteLLM loads the beta headers configuration:
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `LITELLM_ANTHROPIC_BETA_HEADERS_URL` | URL to fetch config from | GitHub main branch |
+| `LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS` | Set to `True` to use local config only | `False` |
+
+**Example: Use Custom Config URL**
+```bash
+export LITELLM_ANTHROPIC_BETA_HEADERS_URL="https://your-company.com/custom-beta-headers.json"
+```
+
+**Example: Use Local Config Only (No Remote Fetching)**
+```bash
+export LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True
+```
+## Provider-Specific Notes
+
+### Bedrock
+- Beta headers appear in both HTTP headers AND request body (`additionalModelRequestFields.anthropic_beta`)
+- Some headers are transformed (e.g., `advanced-tool-use` → `tool-search-tool`)
+
+### Azure AI
+- Uses same header names as Anthropic
+- Some features not yet supported (check config for null values)
+
+### Vertex AI
+- Some headers are transformed to match Vertex AI's implementation
+- Limited beta feature support compared to Anthropic
\ No newline at end of file
diff --git a/docs/my-website/docs/tutorials/claude_code_plugin_marketplace.md b/docs/my-website/docs/tutorials/claude_code_plugin_marketplace.md
index 946fb47d92a..9d93c717c4f 100644
--- a/docs/my-website/docs/tutorials/claude_code_plugin_marketplace.md
+++ b/docs/my-website/docs/tutorials/claude_code_plugin_marketplace.md
@@ -2,7 +2,7 @@ import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
-# Claude Code Plugin Marketplace
+# Claude Code Plugin Marketplace (Managed Skills)
LiteLLM AI Gateway acts as a central registry for Claude Code plugins. Admins can govern which plugins are available across the organization, and engineers can discover and install approved plugins from a single source.
@@ -252,7 +252,7 @@ curl -X POST http://localhost:4000/claude-code/plugins \
}'
```
-### 3. Share with Your Team
+### 3. Use in Claude Code
Send engineers the marketplace URL:
diff --git a/docs/my-website/docs/tutorials/claude_code_prompt_cache_routing.md b/docs/my-website/docs/tutorials/claude_code_prompt_cache_routing.md
new file mode 100644
index 00000000000..bbb29489856
--- /dev/null
+++ b/docs/my-website/docs/tutorials/claude_code_prompt_cache_routing.md
@@ -0,0 +1,43 @@
+# Claude Code - Prompt Cache Routing
+
+Claude's [Prompt Caching](https://platform.claude.com/docs/en/build-with-claude/prompt-caching) feature helps to optimize API usage through attempting to cache prompts and re-use cached prompts during subsequent API calls. This feature is used by Claude Code.
+
+When LiteLLM [load balancing](../proxy/load_balancing.md) is enabled, to ensure this prompt caching feature still works with Claude Code, LiteLLM needs to be configured to use the `PromptCachingDeploymentCheck` pre-call check. This pre-call check will ensure that API calls that used prompt caching are remembered and that subsequent API calls that try to use that prompt caching are routed to the same model deployment where a cache write occurred.
+
+## Set Up
+
+1. Configure the router so that it uses the `PromptCachingDeploymentCheck` (via setting the `optional_pre_call_checks` property), and configure the models so that they can access multiple deployments of Claude; below, we show an example for multiple AWS accounts (referred to as `account-1` and `account-2`, using the `aws_profile_name` property):
+```yaml
+router_settings:
+ optional_pre_call_checks: ["prompt_caching"]
+
+model_list:
+- litellm_params:
+ model: us.anthropic.claude-sonnet-4-5-20250929-v1:0
+ aws_profile_name: account-1
+ aws_region_name: us-west-2
+ model_info:
+ litellm_provider: bedrock
+ model_name: us.anthropic.claude-sonnet-4-5-20250929-v1:0
+- litellm_params:
+ model: us.anthropic.claude-sonnet-4-5-20250929-v1:0
+ aws_profile_name: account-2
+ aws_region_name: us-west-2
+ model_info:
+ litellm_provider: bedrock
+ model_name: us.anthropic.claude-sonnet-4-5-20250929-v1:0
+```
+2. Utilize Claude Code:
+ 1. Launch Claude Code, which will do a warm-up API call that tries to cache its warm-up prompt and its system prompt.
+ 2. Wait a few seconds, then quit Claude Code and re-open it.
+ 3. You'll notice that the warm-up API call successfully gets a cache hit (if using Claude Code in an IDE like VS Code, ensure that you don't do anything between step 2.1 and 2.2 here, otherwise there may not be a cache hit):
+ 1. Go to the [LiteLLM Request Logs page](../proxy/ui_logs.md) in the Admin UI
+ 2. Click on the individual requests to see (a) the cache creation and cache read tokens; and (b) the Model ID. In particular, the API call from step 2.1 should show a cache write, and the API call from step 2.2 should show a cache read; in addition, the Model ID should be equal (meaning the API call is getting forwarded to the same AWS account).
+
+## Related
+
+- [Claude Code - Quickstart](./claude_responses_api.md)
+- [Claude Code - Customer Tracking](./claude_code_customer_tracking.md)
+- [Claude Code - Plugin Marketplace](./claude_code_plugin_marketplace.md)
+- [Claude Code - WebSearch](./claude_code_websearch.md)
+- [Proxy - Load Balancing](../proxy/load_balancing.md)
diff --git a/docs/my-website/docs/tutorials/claude_mcp.md b/docs/my-website/docs/tutorials/claude_mcp.md
index 07c3cead0be..ab27908c8db 100644
--- a/docs/my-website/docs/tutorials/claude_mcp.md
+++ b/docs/my-website/docs/tutorials/claude_mcp.md
@@ -9,7 +9,7 @@ Note: LiteLLM supports OAuth for MCP servers as well. [Learn more](https://docs.
## Connecting MCP Servers
-You can also connect MCP servers to Claude Code via LiteLLM Proxy.
+You can connect MCP servers to Claude Code via LiteLLM Proxy.
1. Add the MCP server to your `config.yaml`
@@ -23,6 +23,7 @@ In this example, we'll add the Github MCP server to our `config.yaml`
mcp_servers:
github_mcp:
url: "https://api.githubcopilot.com/mcp"
+ transport: "http"
auth_type: oauth2
client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
@@ -34,31 +35,70 @@ mcp_servers:
In this example, we'll add the Atlassian MCP server to our `config.yaml`
```yaml title="config.yaml" showLineNumbers
-atlassian_mcp:
- server_id: atlassian_mcp_id
- url: "https://mcp.atlassian.com/v1/sse"
- transport: "sse"
- auth_type: oauth2
+mcp_servers:
+ atlassian_mcp:
+ url: "https://mcp.atlassian.com/v1/mcp"
+ transport: "http"
+ auth_type: oauth2
```
+:::important
+The server name under `mcp_servers:` (e.g. `atlassian_mcp`, `github_mcp`) **must match** the name used in the Claude Code URL path (`/mcp/`). A mismatch will cause a 404 error during OAuth.
+:::
+
2. Start LiteLLM Proxy
+Since Claude Code needs a publicly accessible URL for the OAuth callback, expose your proxy via ngrok or a similar tool.
+
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
-3. Use the MCP server in Claude Code
-
```bash
-claude mcp add --transport http litellm_proxy http://0.0.0.0:4000/github_mcp/mcp --header "Authorization: Bearer sk-LITELLM_VIRTUAL_KEY"
+# In a separate terminal — expose proxy for OAuth callbacks
+ngrok http 4000
```
-For MCP servers that require dynamic client registration (such as Atlassian), please set `x-litellm-api-key: Bearer sk-LITELLM_VIRTUAL_KEY` instead of using `Authorization: Bearer LITELLM_VIRTUAL_KEY`.
+3. Add the MCP server to Claude Code
+
+
+
+
+```bash
+claude mcp add --transport http litellm-github https://your-ngrok-url.ngrok-free.dev/mcp/github_mcp \
+ --header "x-litellm-api-key: Bearer sk-1234"
+```
+
+
+
+
+```bash
+claude mcp add --transport http litellm-atlassian https://your-ngrok-url.ngrok-free.dev/mcp/atlassian_mcp \
+ --header "x-litellm-api-key: Bearer sk-1234"
+```
+
+
+
+
+**Parameter breakdown:**
+
+| Parameter | Description |
+|-----------|-------------|
+| `--transport http` | Use HTTP transport for the MCP connection |
+| `litellm-atlassian` | The name for this MCP server **on Claude Code** — can be anything you choose |
+| `https://your-ngrok-url.ngrok-free.dev/mcp/atlassian_mcp` | The LiteLLM proxy URL. Format: `/mcp/`. The `atlassian_mcp` part **must match** the key under `mcp_servers:` in your LiteLLM proxy config |
+| `--header "x-litellm-api-key: Bearer sk-1234"` | Your LiteLLM virtual key for authentication to the proxy |
+
+You can also add the MCP server directly to your `~/.claude.json` file instead of using `claude mcp add`. [See Claude Code docs](https://docs.anthropic.com/en/docs/claude-code/mcp).
+
+:::note
+For MCP servers that require OAuth (such as Atlassian), use `x-litellm-api-key` instead of `Authorization` for the LiteLLM virtual key. The `Authorization` header is reserved for the OAuth flow.
+:::
4. Authenticate via Claude Code
@@ -68,24 +108,20 @@ a. Start Claude Code
claude
```
-b. Authenticate via Claude Code
+b. Open the MCP menu
```bash
/mcp
```
-c. Select the MCP server
+c. Select the MCP server (e.g. `litellm-atlassian`)
-```bash
-> litellm_proxy
-```
-
-d. Start Oauth flow via Claude Code
+d. Start the OAuth flow
```bash
> 1. Authenticate
2. Reconnect
- 3. Disable
+ 3. Disable
```
e. Once completed, you should see this success message:
diff --git a/docs/my-website/docs/tutorials/copilotkit_sdk.md b/docs/my-website/docs/tutorials/copilotkit_sdk.md
new file mode 100644
index 00000000000..fc4db8bfe3e
--- /dev/null
+++ b/docs/my-website/docs/tutorials/copilotkit_sdk.md
@@ -0,0 +1,99 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# CopilotKit SDK with LiteLLM
+
+Use CopilotKit SDK with any LLM provider through LiteLLM Proxy.
+
+> **Note:** CopilotKit SDK integration with LiteLLM Proxy works with LiteLLM v1.81.7-nightly or higher.
+
+
+## Quick Start
+
+### 1. Add Model to Config
+
+```yaml title="config.yaml"
+model_list:
+ - model_name: claude-sonnet-4-5
+ litellm_params:
+ model: "anthropic/claude-sonnet-4-5-20250514-v1:0"
+ api_key: "os.environ/ANTHROPIC_API_KEY"
+```
+
+### 2. Start LiteLLM Proxy
+
+```bash
+litellm --config config.yaml
+```
+
+### 3. Use CopilotKit SDK
+
+```typescript
+import OpenAI from "openai";
+import {
+ CopilotRuntime,
+ OpenAIAdapter,
+ copilotRuntimeNextJSAppRouterEndpoint,
+} from "@copilotkit/runtime";
+import { NextRequest } from "next/server";
+
+const model = "claude-sonnet-4-5";
+
+const openai = new OpenAI({
+ apiKey: process.env.OPENAI_API_KEY || "sk-12345",
+ baseURL: process.env.OPENAI_BASE_URL || "http://localhost:4000/v1",
+});
+
+const serviceAdapter = new OpenAIAdapter({ openai, model });
+const runtime = new CopilotRuntime();
+
+export const POST = async (req: NextRequest) => {
+ const { handleRequest } = copilotRuntimeNextJSAppRouterEndpoint({
+ runtime,
+ serviceAdapter,
+ endpoint: "/api/copilotkit",
+ });
+ return handleRequest(req);
+};
+```
+
+### 4. Test
+
+```bash
+curl -X POST http://localhost:3000/api/copilotkit \
+ -H "Content-Type: application/json" \
+ -d '{
+ "method": "agent/run",
+ "params": {
+ "agentId": "default"
+ },
+ "runId": "your_run_id",
+ "threadId": "your_thread_id",
+ "runId": ""your_run_id"",
+ "tools": [],
+ "context": [],
+ "forwardedProps": {},
+ "state": {},
+ "messages": [
+ {
+ "id": "166e573e-f7c6-4c0f-8685-04dbefec18be",
+ "content": "Hi",
+ "role": "user"
+ }
+ ]
+ }
+}'
+```
+
+## Environment Variables
+
+| Variable | Value | Description |
+|----------|-------|-------------|
+| `OPENAI_API_KEY` | `sk-12345` | Your LiteLLM API key |
+| `OPENAI_BASE_URL` | `http://localhost:4000/v1` | LiteLLM proxy URL |
+
+
+## Related Resources
+
+- [CopilotKit Documentation](https://docs.copilotkit.ai)
+- [LiteLLM Proxy Quick Start](../proxy/quick_start)
diff --git a/docs/my-website/docs/tutorials/livekit_xai_realtime.md b/docs/my-website/docs/tutorials/livekit_xai_realtime.md
new file mode 100644
index 00000000000..1d70186382f
--- /dev/null
+++ b/docs/my-website/docs/tutorials/livekit_xai_realtime.md
@@ -0,0 +1,190 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# LiveKit xAI Realtime Voice Agent
+
+Use LiveKit's xAI Grok Voice Agent plugin with LiteLLM Proxy to build low-latency voice AI agents.
+
+The LiveKit Agents framework provides tools for building real-time voice and video AI applications. By routing through LiteLLM Proxy, you get unified access to multiple realtime voice providers, cost tracking, rate limiting, and more.
+
+## Quick Start
+
+### 1. Install Dependencies
+
+```bash
+pip install livekit-agents[xai]
+```
+
+### 2. Start LiteLLM Proxy
+
+Create a config file with your xAI realtime model:
+
+```yaml title="config.yaml" showLineNumbers
+model_list:
+ - model_name: grok-voice-agent
+ litellm_params:
+ model: xai/grok-2-vision-1212
+ api_key: os.environ/XAI_API_KEY
+ model_info:
+ mode: realtime
+
+litellm_settings:
+ drop_params: True
+
+general_settings:
+ master_key: sk-1234 # Change this to a secure key
+```
+
+Start the proxy:
+
+```bash
+litellm --config config.yaml --port 4000
+```
+
+### 3. Configure LiveKit xAI Plugin
+
+Point LiveKit's xAI plugin to your LiteLLM proxy:
+
+```python
+from livekit.plugins import xai
+
+# Configure xAI to use LiteLLM proxy
+model = xai.realtime.RealtimeModel(
+ voice="ara", # Voice option
+ api_key="sk-1234", # Your LiteLLM proxy master key
+ base_url="http://localhost:4000", # LiteLLM proxy URL
+)
+```
+
+## Complete Example
+
+Here's a complete working example:
+
+
+
+
+```python
+#!/usr/bin/env python3
+"""
+Simple xAI realtime voice agent through LiteLLM proxy.
+"""
+import asyncio
+import json
+import websockets
+
+PROXY_URL = "ws://localhost:4000/v1/realtime"
+API_KEY = "sk-1234"
+MODEL = "grok-voice-agent"
+
+async def run_voice_agent():
+ """Connect to xAI realtime API through LiteLLM proxy"""
+ url = f"{PROXY_URL}?model={MODEL}"
+ headers = {"Authorization": f"Bearer {API_KEY}"}
+
+ async with websockets.connect(url, extra_headers=headers) as ws:
+ # Wait for initial connection event
+ initial = json.loads(await ws.recv())
+ print(f"✅ Connected: {initial['type']}")
+
+ # Send user message
+ await ws.send(json.dumps({
+ "type": "conversation.item.create",
+ "item": {
+ "type": "message",
+ "role": "user",
+ "content": [{
+ "type": "input_text",
+ "text": "Hello! Tell me a joke."
+ }]
+ }
+ }))
+
+ # Request response
+ await ws.send(json.dumps({
+ "type": "response.create",
+ "response": {"modalities": ["text", "audio"]}
+ }))
+
+ # Collect response
+ transcript = []
+ async for message in ws:
+ event = json.loads(message)
+
+ # Capture text response
+ if event['type'] == 'response.output_audio_transcript.delta':
+ transcript.append(event['delta'])
+ print(event['delta'], end='', flush=True)
+
+ # Done when response completes
+ elif event['type'] == 'response.done':
+ break
+
+ print(f"\n\n✅ Full response: {''.join(transcript)}")
+
+if __name__ == "__main__":
+ asyncio.run(run_voice_agent())
+```
+
+
+
+
+
+```python
+from livekit.agents import Agent, AgentSession, WorkerOptions, cli
+from livekit.plugins import xai
+
+class VoiceAgent(Agent):
+ def __init__(self):
+ super().__init__(
+ instructions="You are a helpful voice assistant.",
+ llm=xai.realtime.RealtimeModel(
+ voice="ara",
+ api_key="sk-1234",
+ base_url="http://localhost:4000",
+ ),
+ )
+
+if __name__ == "__main__":
+ cli.run_app(
+ WorkerOptions(
+ agent_factory=VoiceAgent,
+ )
+ )
+```
+
+
+
+
+## Running the Example
+
+1. **Start LiteLLM Proxy** (if not already running):
+ ```bash
+ litellm --config config.yaml --port 4000
+ ```
+
+2. **Run the example**:
+ ```bash
+ python your_script.py
+ ```
+
+## Expected Output
+
+```
+✅ Connected: conversation.created
+Hello! Here's a joke for you: Why don't scientists trust atoms?
+Because they make up everything!
+
+✅ Full response: Hello! Here's a joke for you: Why don't scientists trust atoms? Because they make up everything!
+```
+
+
+## Complete Working Example
+
+**[LiveKit Agent SDK Cookbook](https://github.com/BerriAI/litellm/tree/main/cookbook/livekit_agent_sdk)**
+
+
+## Learn More
+
+- [xAI Realtime API](/docs/providers/xai_realtime)
+- [LiveKit xAI Plugin](https://docs.livekit.io/agents/models/realtime/plugins/xai/)
+- [LiteLLM Realtime API](/docs/realtime)
diff --git a/docs/my-website/docs/tutorials/opencode_integration.md b/docs/my-website/docs/tutorials/opencode_integration.md
index 4c1e03f3dff..e55367833f2 100644
--- a/docs/my-website/docs/tutorials/opencode_integration.md
+++ b/docs/my-website/docs/tutorials/opencode_integration.md
@@ -270,9 +270,6 @@ model_list:
litellm_params:
model: deepseek/deepseek-chat
api_key: os.environ/DEEPSEEK_API_KEY
-
-litellm_settings:
- master_key: os.environ/LITELLM_MASTER_KEY
```
## Troubleshooting
diff --git a/docs/my-website/img/a2a_agent_spend.png b/docs/my-website/img/a2a_agent_spend.png
new file mode 100644
index 0000000000000000000000000000000000000000..15ec769392a8f68c346b245819cc07c584d1e106
GIT binary patch
literal 188367
zcmd?RWmuH$+BQsg2}nt)fFRvniXb8&-67pWHw+;l($bA0AksDT(A^*m-CaWuIeg=K
z*4mzXz2Cj*{r-M`%r*m@*L9t9ocq!HLHIi*IeZ){93&(peEHWhsz^wfKqMs8MJx=&
zZ}2!qWf5=4E~;`bkV=QCcMw0wm}|>hC@La7M|{RYLJqS=Li@D@@uEV!5SQj4BcUSR
zk$?Rz2jxG1#RTS{{^v95;;)6}C89b=NK#1hGScdv$omWE0qPp$m`C^S4;o!Po_-8)
zO~ztY#bM4tdH%`&6HW@n$AC{?zcYXRn(*P<_YVQ%3k#X)c0ImZ^Y24v%yB_0z!reV
zVw1;`AMDOT=lyVoOQupb*ckZ{I&zjes^IIKw;oV6=Aj3Zb+f
z64BCEYweTE|Cg7eqM4YQmYdx?p>FiVN)8$}y)dd;0bcQ#n3(V^+faZ0!^t5;5y1L@
zSt{`C;m4?muB@t?a^t(~IwDuHE#({26>#ddXCDfmb&&7SZUGcsnCj2RD-)2~a?70~
zC4&1QG;FnX<)9b36`69REfXU5xW6X|^~Qa5JkMpvm!ClH&&Vo53U0ZTN8Ajjvsa{L
zD~L14a7XH_^l{<20|!q7dU6mdcY!f!J8RP0o+2maKLd51$oL2r`icx7FK>)Zyz4Px
z*3r-d>bS6izMpIBv11_eIDURnk`WWNa#L|?Lgah0AzR_hxV*6z5hVvRZrf$V!o_`=
z!N~DwC{69lA0bA9R<{YzZ23ey5S7RUm_JV`eH9eMP4=uaZHr7uRJd)j4}}~bPgFEr
zZlNdX4$ZY>X${kTb@dqa^64cGijrZX6o9lpaVI+H>bBf`hN5_}o*?f&D!J~R_~W0Bm%ayw>oPrrGqcNtJ8(%m{MIlkva6z%D+4E7O1n5>KN+d)c1Z`Ntv(>V7$DRGHE
z7RWzO_r>eUit02a?PCWl!tj^Sv?yb2yzbgLe>v?@{-&u4h77d79y*bMU%7Pf52C;4
z4kR8XlvB4t6Z2~(Iv|U98b)9YYm-!I6_k8K%APf&wsHPP<9qlHVTE}2LK3HoJAEJb
zfv3HNN%hL!cx_P(oz!gJ*mRf0i_gI}5aGQwk=J;q7UQ}73M@Cg4U(6bk|Qx2}h=yRo^CJSA!tES2h#;4qT
zN>Up^ejvF8TO$a}+r8`?(!l*1gC|~b3$eL#=pOln0A(x3#W3i>Wy%-|mE)2(oR92q
zF0?93sg5k=(H23*EB&`f&-n_YS7C`=g{iD7YjolMUN**A*{mG0ui7-;xbv!s<{HsN
zEB^tShqS-sxl*ED=G;8~nWlv<|%(4tyrDTAU6Im5mc?4SM
zBeG#sc_fhgXqc-_Ll%2*Lb$2TGR*hV>z2Hd#!^3*h-N?f)~n%3^^s(qyBFmN=H=I&
z$p(t)TEY|WE&kv$M~dC~5LSYb%B}o}h~oGAhZq0=XIU0w3UvsAZk7Wuo1CSS%NN01f9X(t=euPC_>p0&ZYn_k7q
z83rf4g31ZE;Pj@m)}kpcEbrPpITehGTkz=Y?0oWEgzCWqm7_DqFub&q_h_}giV>lN{Emar{PoS(1nJCL1DU1Q@j@12ws3ThG8*Fi}roa}(bCT5Kv
z3H9VXBxkhMBW=N$Q4xy~+1c3@PW#Ez?yWUQ#S6_C{#jQly=C;JAM`uJZF@$m_fd}=
z-I8rLmG-Nt8#(#--uf@H^+GL&Ool52+r!mJB1BCb5{qT44cm;(?}%gw)*AO5B8adP)iF~s^4g`P{KP6M*e)!9V@M1+`1w`9
z*Tv_Fl#M*y=jY>2&e0cyNf9)-pW4OIn~V*o(a0z0x4NSq9S@5_rBHLJE;IS8RX?{G
z;rA+KODv#8@VCxYTdR@Wiu+!2$;QAb;^K%SlaNk2!%5=tT1Dm!zNS+}`kzl~IUUXm
z^LB5K<&J%Qghu5}jnH(F`%|iB1i0yE@r8
zJ{^(l7@Abf0KAn$T^UGX@9=>(TL!gIeQ;O;i_1n)x#g-6>*@B_D`tW|+aI7GWrOz4tt|W=#6=$9l3)%e}Abt`kFw&=^e_sG5$g
zxYD$e294A%XLN(K6kgiqc$EQ#er%iuyqdCh`v{mV%adLj9epSB>eXYmw^c%nB|_o(
zl;vs1MX5E{=6WzeW@aqCS_j^lhI;3X7jNzGzR3~iGPLNSmWdD!@mw4oakW9#hexk$
zjWQ!J3iuxh73Q}&G((n`jnx{wm%x50q8_H1Q_#Uw-Y>Ac5d#B*gPC^u
zDEqlG*{s7XP{GH<%EhszOE=$hy^xTQjtO3;o9f8)>3VN=0JKxV-4-}BcoEC*&HHsneD{0-
zT3$It`M!($w;<7BA!6^-ukts-6Z8-yWl?{(NflnW!RwZLvD{!j(o?Bq8RgHo9L}PLU>Te5zpF!$4jQtE5z}WL0wee6&a3f
z-iY_Kiw~n@j}lRfI+1zl=Z=1NH6-mn2olRu=}jNNT+}FPdl>{)`YcWTfYPD6VRIF=
zfQx1#?e0h5q;`Gx*fJJr@-o0AwPjp*QZYDB)Z>tE4-Fwh44?;eG3x4c3-P6^v6E~t
zj+2f2t>4Ib?2MzE_a|_#?HxD!z)bo+GuC;Z|F{ZHlmz-)W#c@2K3{$4@Z(uxXf)Nw
zDXEKW1$e^#5}4^mt1765&()c-v(~w<_q)}45@gSPj=xVG?zO8=E$UB&e^?eG*XY({
z{T)9YWV+a%_c4jhYDnyGz7Ri6pkZkdu+VLPX}e>iAAd3!g0tCU$|F0&!+6l+_OxP;
z0mQi|PoO-{_*6`$a(w3ZF;MR!$~F8xvOWTkW)^ts?|E}h{_cx@YI^j>2O4i9(;i?9
z#E?3GOz}}?`bj?f+;InI)!27`8Xx{?PnyH!XF6oJ3&5lD_mgjp7qYm0R6Y&z|Im_+
z;Sj{^P!;3G6-nVL^A)AzkRF#Q1&AKngv&lwpy=<+jH3fv{Vd#w2=WK_ZfqcH33BRW
zPigSjunPforU)cRg1`6DV`tcu3o1?BM#)6laV!%R
zJ^t+*TV%W8x79adlK2Pyd+8
zgO8CPWj$rsn1%@219Ov!9(rRhsvgcfq3ozQtoVK>qlZnkL}8>Lr{JM8poJPHhzaCJ
zN}tW&L*Aomq*|V=0rHD#(iyA=UFY>w@8~vP=jnW>(x;L17qb{jGr!)0KiBw?QsWUr
zCmC%EDV0X;@P^}ZHW?G*6}T(a&M8Cnwn|S$omM|!zlH5i;^LCAaowEN6%<(hfLu2(
zcv;1s->Iyr)Vx9O>pd_F6?VI31H&DjUrQ{aM?IR74!dAKUhL&0Zk7p1fKTP$@lDUk
z@c7^B)xMfq#^)J<0H#0bS2C4Z;O*^wF(;6qy5YZ|5emB5o42Lo{iQY|{wRXK*Ot1c
zXiZIGDn+vVk#|SQJ92u9qCZuC%B)t@pVRs^E@VrBwgyd
zJvB+P6N!I&Y@70z3Dc{k%;}8Nff~Q%ll)L@8m-
z%as@CTplY+o(BgiDMAhmzJ9012chf~u=5xz^)e84Y^hF5SAya2##+d1jm`3${6x8?
z3Hiu5lrhmh^LRm-_xS2VpC@!E`2f=UDQbPw&2S2nR>Gt>p@k!=ZRD)Y9~(^>?gnCz
zXvlrpn8ZX&#uEN{>0WGttwslIw_^s7Mg9bhe&*ALhRG{_nZA7O
z(WZvqvzz=fKId=))zZmx>im{qP8K{?4a=<
zOfK^>^Sjcw@@zT{Z<3B9V_&OfVBcOO_BY2E8im&QFWelh=Rl4}TJ}aYD&GUsL@*~A
z`woU6S%Mu_-O!PV&DWLL1z!G_m-5@HiGVRu=7aGc-r|r*>taz2bm~EtkiA+f!8?ehhLB1L@k~1N*R{wfGp1T1jA?pRF&i
z7uB^mj4G{fJkeg4Pu3~y^VmgnYE#^Nll#E}APGW!dL?sHEMUK24&uKsw`DQEoxPr+
zwU=J_fV>=uce`7I(5fQxNu<)X>*fZGqr>JYI%e@Kb%mjZR{6~Iof}4+z30-?VkhrI
ztBAkXpd0WD6(RSm);wRC9v;G~b0(7`L~HBPaQ0*TUbF!;W!!#5!kS$cdqBamq@Q0$
z8Y>Xqtcx+b5V^r-DztUV9k}eN)8dRw7k<-U*_OI^2Ci)tLHuX|n|F@}
zv`XVuC+Zn(XL%=gZm)6M6eZkL6>RoxCnV*(;SXlY+LoS#;F>_eFI8XYOGwbCc5V$D
z@kz?PiXeR(9oq8;D~XFv@cNxra3j@P_{WcYDA>Gi;tVcYyuqd>#Sxhejf|Onb1Q*K
z7KQyGl}ty@I*&ZTW|J3(a3XKXS>Xk@h0>9yLJa<@8HUV4om4-krei_tn|boyvnPG3
zX`rd&Kr)L7GRXL|7p?>B?~$}IM~%yqyg@XL8`5e*j}?MiM4t8V`@7u)*M@!Ln#^$1
zewi^$oX@_$5ritOI;`VQ&j7NuKbct3`idpebp53LjgSPb)Q6TlVeCh)adcu=SJg5DBJCA%9}`7(yu{3%>`3e;>GW+wE2_}aaJ?GdbXS~*
zU0pvU@f%%s{&4#qA0NM27>o7P?KV)!4oSS!XyyG16TCRvzYYq%DR)26oiG@92|OJV
z?9H5O_~?=F^Ccl3-iNS8OkXo@J3yazv+K@hKPwv6Cr{cFqTKPG%!zE%ne+(noL9FD
z8>PcdD*45Ypd%GKPVC3?MOvK2dbPvFcSU*M-45Otyp2v;GTJg^GpN8Qg4n$%+QeTS
zT5uX}@LOJ1VKb^BL%kLcbyfxV??pFjUyM+ZTuz$m5;wi5SKX}6KVeBLnS=GN?&Icx
z{M~MkA#X^iOEvSpvZx&Goq27a9HBC&=J2_;+nxyjzEtG_3MgG_^ty34+n*Ynx5_&)
zA6ny(pSD|Qikl%w78)E|D;T?ryU4G0ktmdE@PimuCq)|S>4BW_pC@t#kLSHvK1+4!
z<&^&UI2bb}WwrZ{
z07{~!$ANIbR+j&~ME9#c$0^k*T?~5GqwVy
z)`w^!KM|g{9oGyzU{1|+s+~gKk6V%K_s6|ciuF5p`{MC6ZA**
zE?Y;}du8bXrEbtDq@!loVub(Q^^E%*o!g;e8lQ9+0nOm8_r<<-(C7t0YW0_v%TEv4
z8B*($-`&G?!wn}dt}_$&7feqg_|=2<5&L7j3OR+H+#hX?Pj(l|cZiIpUC+KrWD7ks
zpzS2azCN69UP=f)T!N|`pSFfXBs;DC)Y;}_>sc;45Q&a|ZI`{~%K)@hSx~|F{C0yM5t}1hVLMf1Qo)6ak95|E2x|c7
z2D3GhXx{u|_NP;smtNOU&*K>|vq}@Qp`R^jwfb3X+nwUcY+#4nkV_95_)L9|X^%)b
z?c~wo-L^|LU}BH~W0wlN#i1WmI##5(l|A1y+eg)cFpphZM#{3uN`&+>p9$prF7Q8*
zKL5xm+Y-+)QG*`f#?^D+N-5SxPi&OHtjh%YS$dHEGr@TWA$eyZ8#4=*&H=em~MJfNXPgrNCt
zUC-!jB#+9`V*Qrc0*wdGiy_WgWUi&LbhK#?GL}uQJDNR@P4FHG44O4FnHK|*ST&13
zhLb#7EiA}t?~8w?7;sjQ?a!0iyfu<7w>6ykfQZPL?!}1pVv{hFFF!J`#c(URk?;GF
z0)2!Y-a)!Q56<2-H9ElLUF-*e{LShyHrQJFT|b@{@{3FZa&zr@*NM}55&8MJ@mHi9_z)Ve
zPh&G70_pQxSm-^W`J3d?0?P3uHiPh;D)J{>y}EFnva9vAh0(9_s5j@TX^=FxbnfGV
zoeS=2(u?#miJ0~f9AhZFgF?Ur&p+j;0vl@%b145&YSvXzHpXj(0i8A@on)=ge$YN>
zxLUzeq9SSC`l~%B-Hv58QP*rAL(Xdd$|AeHsz*+G5g3jJ-%_fw7-rf_>%~XQx2i0N
zhG+$fu#M>6q4sE;$aASFP&~uBO_Sb8w*b%DJbXsrfUYu}?b%({E{jUuqemLVf-
z&Bev9p0gQ@5g%_$^>fmxrVPXVT)&xcp@6-RySgme9vA1CN7T1y>xG-|+uN>$a3+Lx
zJ&(t|-97#gXta<#E!1$YnZB62$odCX#Ef3+h#)5!qAITue|B=#8CI!o>*^Yb=GYM@
z7~^KG+!KET^TShyd68_}z4biLe-*PFa*!apS@LIa=jg~T=?de*j
zgd<;h_LO-RL6ofx70q-M?{M&Nk}ks<`(i;ujoy==3NNZM;}c^L=aHT$nEntF9WP}j
z<4V0rT_1TGH(VK7#{WE>OMrFZvpqza*RQ%L3n1DR$*8jWl_PFjvyk3-rc#_lUnV#W
z8IDZ+QNoyY4AlYnD9}bexnuCNU%E4kDMvss}UIWf|qaP5rQ->PySm
zf_H}+l-KQ>MS8QjP5|=xSxIR3$0qK~%Ye3}x1Kbvev3sAljFH#qz|1U3gMeVI(LH7
zW}2H{kd4Dt@*8x5TSQc;eKSmst#;;_V|x^cLrN%1!|}iE?b_t<@++d2&O)Y)vTcbp
z(hnyp&J@yVUBpUn9P(l=lnZw*sg5;A64Hj%TI^0I;MiNqj(Bp9#3A~e?+et$2=N-2
zU>s%gOulK14=3bd9A7r$d52G|v-Ec)1Teq-p=C;1=Z9!V0+tSwh9kox+J++_<+rMl
zXq@NjI&&9EvUT-U1Z!-RExH@^Uj8}m+>#`hHRx6ViC`Zdg=HJ5*d`7o4}BzU)RwfN
ztR+r>h02M@A0<{uYJ4P^_|AGSpY=-Qu`U^S(V2tjQEB&+@pZTj#-4p>0RIVVLBks|
zr#mqvEA8hkX#NHcEG|O!u+Azq;!(WikG;&E-VGfFyR(fA4c}bffW&A?NNlILqd&&_
z#C={Jyc|m6Bzg8Hia_f!fC)}yRkLBm65n)MEID=E^Z3?#8K&9M0?K&L$pV{a2H@Ew
zK1UIhzc=p&Fo9DFn-WTqsmaKq5#?wdOW_UY$wF!YwZ{57p#j}thodC)@%nB+{`jfM
zA4IU6ghp}O%vf6Ok)+|wh-KAw73+BGAwslex*=7*
z(#kBm2g#0n?5TuZC%aTFrTJ$rO(a(z{*dkci-S~zc-Bx*vR-g~#b;z{#tm0OBZBV0
zSPK2fk1szY1cA1bUCV`ZP}`Kq@Ns(DG+wkEetacOXsJGQ5ZNDc+B!7(kb{^nDbM^5
zMe=`fw*njW5z4@Ji(LLXK~S^OGuMO-D=NA(WZ98A+4v}t8P|l3gBZy@DE$=*_)u!*
zwqAu9z;Wr>(5eQxNK|d2A_E`r2Ui_|y{vy<@cmaZMSq9r?J5E%^r%!$eH5*?gd)~q
zB5hA7+qh5i0^i%~;_FuVWHJ)%v(4pSR@u!B^JZTDDLwv|!1jK%j4dQlnAUy7k9@@0
z>@2oBXNPC@6IWe-pTzwVL2UCcaT}IlTE6_P`~A08E`!jD2f7!h3dO?z!C4|6P#J+>
zILvG$E{pwnF9?d;H)+Iq@GT#!{{wjjpv=5NtlmF)<4F4l0RKk9uEFTd!a%gKoONz`H%F#f8oUdO@G9o?z6#WyVo!D{Y#C0yQ@}v93h==
z3!C*~{{+pyAQO%>AYmC#9yIaioBu*E8Nwe-eC=`hZHw|h8#Ni{{>&&Gf$3x
z1Tec_s|CCs3*rBxy;%Q(FT1x`@;?E9l)$gm=}kl#82@N5)xY3NCZ476TUhuWh{$B{}MX)yCw~_;qNd(M?nlPs@3>^qb2jlr6TN;079i(J`5kcb`sQ>#e
z<+%HzxOiC@`^UA={?Ac7^#S3{fwLQx>nBaje}~{+u3rSX$RyPNu7C&xk|YO~E}>X5
zoC=Ea0W6nC$OC?KRdv`2Vg8+x#Y&)@Bw_v^X;fh3Lr;L=1)`mm8QO&m7#W)O+1&m+
z{im<`6$RKi8OHooARei$w?nO;P34!)nhFAO=}>EMI-xH6;>(Q=$FRJus{3JNHORNE
zkJJ%=z3M;rH!DGyGo8JWqQ8ExS-;~UD}d9Tt_ZN5UY2OQF=+ZHUDYrvb$7~YU>GV<=2gtd8xn0N3QtoBA`(p3BGlsKWoPwByCvMSxUE2Ea2V@5wJAaeO?08B
zsm&}kda0&wOpk^B=Aop$Z;-w8KK!q1{boSV!$TYx7>GrPAkwO<&tk4)A;t=#$$OJI
zv*&|l-UGhir#-Q~f0I;ZdbP#GSb`e4jT7leQFM0xCB_gfO&vwwAmE@U;D_qw)p5g#
zHr}OAatHSO-=<|AbqIiXT;dMS<^GR3HVc(}3icl9Jl5)-hq*!tMfs$T$LieceBqWczNLtquTf2W
ztwYr6$Wy$&+ezn9T{4k8O&y*O|D96=W-u<(G1JMw2w!;0m<^{1St4B4yc3FH7Hw}f
zC+Er4W*-?W$-Hgp`**7hQ6O>%?!yns$pth|-@bWe*hm)o0)2xflOgRw*y$wFjBMl8
z1L38qW#83Zt_en`41TpRSy1u@?H(~{ohEqs;QN`Kn>h!9=
zez?wc#;7vIxnrUOwW6Ye$01;lJ=-f@R7?y-8bOPrJ(=4gBBJ;}lnN{)p7N--+nVkk
zRhTZdeAKg2`JYo=XGR{noNNUk8hBdy`kK^AI^uTjeS18nOCm7
zfK~|(dwKJ4ZBXi*!rb=2xu5ZGcUq@8-N-Tp~qj|(uy^Y+s2gI~7&O98J`
zmFLjbR>T&j1i>7C1Szl9fz2OwjRfC#fl~i;;h4lA_tED6&k9t=Qt@S60z%v*h-pbnWwhc3fjF)
z?B6K&+T#$LD#`7mdLyqqG6QyzT%^1RGMgiliuBF6Z_KtZ-n$UDny9Q07H4CdNAq{v
zuX$8E6WkJf4HM{TNxQ^lK#3s7Ja61_1`ztK+MStnt0qy^I|
zQC*~MUGP}O(r45Y%0^47wVR959D0;CxaJ?@(FR#k?r_JZ#Z<^Y>Kf4E_U)~@ICW@X
z_V~A%b`Fp-HqOym6U2O*sD`jOmDe)ct9)8LEeO4mts~n8h2m
zIl}yM1Oj>bWI_Jc#yWH2lh0SSk0+Q+5fbRIn*Gk<4i&em^ij@|2&2+qos~1FeJ^S=
zS^TL1HrYCRu6J;QV21`ED%ZxdgXk%Z%K$!WZA`{bH$l?ogr
z7IND}!C4=mwY9OB5WoT%zvH2ii__#X{g}^;5l=i0=Pi~%G(HDCs}W&ukf`o6^P5#2D8X)So69Z)RL=qB2I`)hPyMFy{<#Bk1T`
zorNyLh%ZitwLROSb~6yhY96&v?4T=h4MUdF(1}R-Uyu1OFD;C+@>vDZRCXEO>q=E!
zb|6K_>$>qdt6oxqAKdaB!({~Odl};Olc%@P3$^~;O3-wjbt87EI-3!dsRYF*sDB1Q
z?H=wkOf(F6KJiiCs7tFRN}nkx@J+t`f;#0r^y(R4S4*aJ=Wz4W5;@1KRMoCH*5nwS
z7OiaCP$#(WlpDIMxUs#<=Dj(c>0S4MXPv3->HLTw;hSW(&?c|bY1HY9==mHeCfv6v
zT*yV5B`+e#*trlr#BsRIMhCaq^je6Br>Qkyms$IsA!{)j&O+}v=%Bs!O
zgNp;*)S5m*r~xK|qMPJbi|h11f%PF%pbfEIrX1*756`htpCzBa?v&&_wxAA(F-OwTPlt@Wa_+>#`it=`!>bf)C*A#
z`BrVz(jKJ@?!46Vrl@p}vnq5Ag+Jp^>Z$a7kro*!nQ>+1}A
zH=11LKWfmbK~$5hM!>^&CpuH)9EQc^^`(niB~%&}N*9cFO4uGph#s}09*XV@uko0c
zuWmP`qmgqQ;@9u5B#RDAGZ7w_v-t$$uyA3}*SdbAYHN5dW-siOtS=Xa+anOSX|Yc2
z0~%3S_nKdJ4Q!^uC5G~44|M934%;;KyME4)R!hWWPC9eBMEE*XJoCqYb=o;#*hbtzC{KnlP%$
zI`;>r+v79awz$*Kb)9m89q3{n{f|Qo+~a$>XhuWt0;L6BLERK6wLVZI49i}nz~yUwWhPy|lp6eb0g
z>NSSWZ0%=wzSX@s-(P%?Hq>Zez&`=HXr3Hxk{5T$!q9^@7kug)o}_^dc=B3sc5jw6
zj)F8uCZ5#;)><-6Byan1mB08Rr~_V6yNTs4BgpCq3oiya_es91BGV<-O-~A*atm~G
z0DUGIqFRcfX@sr)Og&|0G|xfAN{9i<6j8u~(B`WLx*)5MnyxSR#3|dhw~@u
zKeP>lYVH)79F$q=OM$XA^Cz?{Lty`NIwma9%e*g_E;}|PDeUPZgoGXkmQ3FRb?tki
zvt=MXS+Y57gAzwOhSQby&oEkUlZ0zW)=o;Df~LR*woNzif)rs}2}7T*>n}-(MUWIj
z<#VsWgoc>t@U{z@8zOc(lQA-;y*vr{`swtOmR!42>YJHbA-7#TFx;69(It;8oB;N~
z@50`-7}DIH!z4FvCE@zfJ>ITyghkZ6TLQN8t&u%|f1h3k!#gk**Xx$q&2Ks`rGIlc
z@92GS&Ag9lEn!N;3Rg!>(uMf^x3-cK>9!-`-Az}Bf4{y)_O4$?Q+-Fb+=EcvTB1xus2M)#jeEUis!g{@?-?NOs&vlCjFur?tH;kX4}Xq
zp-TS@+itP4<1H*PIqPM-?D<&pUdJ@2E!5lFyZ3rF&eL{Vq4$~ef-HUs9l==f;`s@3
zoI-LbJs;a3ntzS@=_kpp*10*{2>2!L-SyHX4Uc^{HOXbBIPp^EHw%mCwbY;F_R~Cs4_IM@Z{`vSX<<
zGz8Q1!<)q7hhF1+M9TU9uK4^{!sn6mYv^L4;~j6Xz3oAMk0&+4tRdVa{O%?_5QC65
z^5(GdI57N-QXI~#n9kR_o^6>HxiD2im1}<5y*&!X?@s_(>xVfe9@_Q@i+R=1uSjFVd;Wje5}w0nK2>wACGk!Xx^5qqjo$T&>I+>
z!#F)T<4XtCG@r|xsV5wid*R{6@Wp66*GRxYb+Bmknz;V%XEfw_N1ZJ;3d+MCLR!^~~n{50z|GXiF
zztG^ywfD)x>sxa%yV%GebU_f1@Iz@=WL`ONpE-7R7wP&vF+RM~oGx%?3Qg(E16E181uFhWV%7}+7ZDFbFL0e_<0(+)zDRYJAnpRLJ9
z?Jmk|{Te{EEFH+X0mv*`_*n%&+{cx>ZAsjRuur6!-(268NU#EozdM;o|N5~@8H7Q5
ze{C%|F1IJW^&Fg86ep-mYjtH~PelZ*R93E9crir}UphY-#ed}3Y!uuM5Y{xlxi}Ei
zz1HA15wM%^#
zCuN#CK852bpA}1h+1}n>VG1$4d3_BnQ~g}`uI`<*kf`VkY)SD^wDb}pB3sL{NgOTe
zc{yDLB|>C8A6a-eQ>BhdFam>Lz9?5)>}m4c`NR;X{%YgRLkAOaX33|GutT}Wg%?^y
zno*t)2?4tq#y!Z`nwaS5x&F-}*wJS5YKr+|!q6EHr1QP0P^M-EyTYCLqm1no3cEzc
z!&3eFPl$<2hU+a6em~;q^x3T~PW{ek1RE6glG>G}`Rgm39#MV~k=Z6CxoGN_At&aF
zs@Y)|)Ot1YS;Y@Og>b>ts^^`d(D?_^l^Lwj~$=_f)_zt17)5w$(C3b6A)+
zsKssXXM6in!SaUI5>K1KBTeP2i0J}TiE{C-xsI@u{aNTUZY0{D9av&>7wH
zobSaLnG9NzymizhM3`1eiK|QTE8NPeW~25fff9$vu`0PO&#+9Qd(No_hokvaNs~)G
zI+M<@wYNs>$l~V(*u*&?co)Ka{kY&hEc3~pDfIx_-++B=HERJb!6?FttspoQ!C%=&
zHpahohc42v>sGg}_r(KllHk{SoGyo7Uqu}46lAwg@}I){PnUrA@B0&2WJjM3}R|^gHA{EXgfFN>x&Sa?VpA5ei`LQ
zTnJh!4b}R=SZ>%|iUlqq1sBpX
z<9c%p#%F_SF$#=N_dc6N2M?yy5CQM2x}CI|4n`!rNp$gdV)BX@j+~YA(SQh!V$fqi
z^d~_=Kh*{q?#O$y4^1%jqkL13ljxaK_-mdzy&C6NR;`dJx#0E&PX2@G<9N?HJ-Q6h
zXNk8A_3Ma4!PBpo9uIiOcq;J?uID{iU}UmyxilMN%hMYq+>+y4&r?{B=ASN3cVDU#
zgP3s3`T@5-_|^uI^EvO?y%%VQ2*${(@7$Xy`|@0i-;5USm_mL1*7oa9t!95zQH}^4
zegjCx*@O|!H0I~WG@r6Nuo-}f#j`2Q4&{~c@$k9_3zxncq$gQ0lt^6tNIJ-N$OSbR=|I|NB{BmOG+T-qNx_z%YAwJ}%dVMuPxE@1w8se3$50pS~i-!C-
z?@tB20Jn=WWs<-Zx(nD38qo}q%1I%s<=eZs>=ZPPxxMd_6IBKG0Y5aJ3um<-6qCjp
zNg!BAtp-d-XA{bL{fVrWV?7D8C1;b|=X(fFtFra9It>g)Sl;{WBO9U$D1VXMBBY1N
zp-kfLx95L89az6&PZb@(e;rX2U4fxPS*L(;KNp&G-hYzf3)m560E<+r4>4+%I
zu;cwHXIyCmrKKsou%i_NUCCorn`*`K;Yt;E(bFJ%y{|1{XLYm_`E!L=1{*>Tt|Cfa
z1@(|&3+dWS{xH|6vWQxHKXAO(!(P?`ZxCiemW?p0(w(ibwRCFAmV{q+oUF$yr%3>s
zjzc#Pq{$0zhVA`KedQ`gpdVgzr_JibH=1L3|mUZAMfR;klQX9Si(
z1{Xcn6Ztydsw?iU5cEy95s8~(%F{MXy6V^Tlkgj*=NQ_EiEZ;kjS(-zL^wM#GV+DX
zgm?BuU@2gKsby*K1~DIVOpv!O?sH)mfflxwIgzJ8K7im3uO7rNm)FCLfOnH$dq@PX
zj05WZiGW&6-}l$Lc-)yPH$=8Lz_bHOrW-mppoN@1W-)2koo5E5MfKd7>7=D
zwwS%n?G{cIG?jem_0E}{kMGuEM=gJCwD`52#T{5^vHse3De$G>fzMXQ$m)T@Vc*0E
zeB|Uc47@2WdjNw~0tbB}Z!YwfcxRd|v%SD}UC((V!CHlPm}1G-*~)%|Rg(5Tq2%?TD(K`cX?vT3cQr;WKXvQb
z#>gko+B2oJEj9m4>#TxJyD_!yno83OyE1_ewp9H2;|^3-OYvjqM>8zdBeQABMZwOr)w;`1GVy!jmCFu
zbtvODYgnWL?bvf>ywjTVjMDOwRfa}JAPEph1twGF0SD4pXBJSTVNX4x@RhDg7JcJ7
zH#etH>0!X--)@$`>_gYDRT2ZfySJ1$=Pw5?eOOpY7bI+IYI2y(&Jt}h_hmmFlnrc@
zmr|g6mb6f$fuY*S2_ni(Z%_-3J0!1GEfC_;=z3N-^RW6pP+_>^D@sM*dffhzZk>$1g*T
zr_0}~3RoAhq!&XJVs73F0ceb>1aZkC?{|*Yo80h*)-}oX8XZ@H5;|tXeGcn;Km%en
z?vBvK=CORmM=tsUmZAc8fN0P{qlf6>rn>-nx%rumJD}ZS&T=pQNGSU+`P9503O9E}
zCEm1zT6C?$LE4F-yaOUM#p#Zr35;3vdl(m|;+_E1EbEG0q5CRbR7xS_DTJ6LvX)G8
zd|9>RpLr~rMRg`JD{+RlNX<=Fpp;$ZF|mdi`Q%;O_AKjlHq-6bnY#q|cdi>*rXPSZ
zZ&tSor|hfNt_9iUO9mo$lFJ5l^xA^14sQ7LJhu-0wt0^<>=qpE1c#11}pJ8CM>VW>*BuRTv_nGptZV4#JBcavHS+(J~Q@?pHKm&+E9
z4%pm`r6&47?~7l0{&f8oQZ-vO2r-hsGLyW>7cH_G({hV(KAP_`<#%5?^N~;vXuXfo
za?OFzTl45WLH4n`$y&)&|5+GxqlkMuv6fqx$H4x;ebZi4P@I-Ow-%Iuq+E>1ZG7GB
zKoYu7MEFZSC3wcOHGL-#Q--V<@eLrA%(NxQ~lEHpDgx)i#T^@r+($-Kh@wp{M3!4{zRgl^~Lf*Zq
z%sv-~B(bMS74ZzG)=o>vyzx70DnEMNsk1sLY&nwE)3332yz=pV2qBlk
zD!b9K(T|(yXO0n%0+9prRn1z!;zQ!r<~^7jBf%E;S(3oE$b1DQzuT2Z7v`7)2^=BU
zFbL1aXTbl*-dhL7^)+jw2_b^-?(Qyw
z-JSQGSMUAK`Tc$CtJ+k}P%wMXTC3N3y8G#;PaUp~KB$!l_Kq}1mWg9cSkJr?c;p?1
z(D8wbVjz!_15v?AoAZUam)nZVh87bikat`jaIlu>S|It?)zvez?R?NB5yPO-@yNT{
zv-dQWssAvyHr=CcQNKMM*z|>7%TQSTvlhjmoqcQ@-1>z!xq~{vMV71OTaJ)mbob)uCcj$UDDHSn1yt?UVLe5Go{kKXjPnVnjtXXd`ne9|k3vu=&Wq*ddM
z_`mx48kM~-(FA7j_c@N+fLz{MPBC9WiPx;6**;z0{L6mohD5#U#4dVB^ns9eJ-*(6!hV68ahM)GRZ5lp6%IhcpymZ1>8?ho$z^@RG_yMFo@`Uq^mjBMUkjzq8{-1GJgQH+z>
zN^dl<*P*xPirGcpvfkW*$WJxj3&Q!juDo^X#Ahu=kfe`ehD#j
zMD=tK3j2QQ9IP~QrI}spR*Sx*i1Lf|w-5O0x=luSNo;_q!)6>2osi{QVckgdZ3`kfCyDalZ_uBRJObBo6+o(l`anY>MuA+Rc+{
zky0aetyLAIjf~BkmElCn_A>^Iq>I-OXB~rZtXR`
zdA;O*uJZPSW*6j2@ixOZj-J88Ti2CF;6Xit;c6X+QKx5l@KL11a~e1(w4Q2GOT8l;i~9F10d|Dj#U+T6RuDHJeFCv!n9cT;aU(d~-|wAKvM>!3q{
zXw25fi5Q^Hu$qw9X`65x64oH{SyRADZ$9t2k#+5-25YE`
z_dmwJHNtzVRB*OOhFHrHB1C6{U;uzXwQouu|gm&3qUp*q^y2SGTDl@Jeg
z0iVJ$EkH=-^maQv`Ae-~E8w6dde)@(g}o!qfWBc{DbqMt%vu=Ic+pBJ6*q)_zD@Bj
zJSt|j3}|cbP;2iG={rK0O0hVplY2g41K(K&U!w!yo#m76S3yPl^t8TGhx=JtLdC)}O!D3@TEBy)fN9bNr$*#pS#PrsYi+q<(x``GJjy=vY%
z9FK+(l%epzy|bJb)c@$+Bxbr1Cp~vzl{8tL*#7eDcD1UE%CL)oakagc9`yE|!IAkf
zf4}_xoPuv@W4TtPa->?*usD*OdF)n(e!u(LzpKiQZTQl!nVzoU_6n%F)PpNTF
zKj*@xb6li?TGq-vRo|;{Y+1(DPG@U=dX68GDY7d18*r(9F=~xh
z_Gv;mVY?c(W`tqdsb{x<$qpX{Amb!`nj2(nMQPR@Mu}q!l=)^3ux8@G*XFMpA7!3=
z$~Ft%X-O`Hd$VPGz%F)w-SW&?$OC7t0K1NHs!^{}x%;^ZtJ$x%7Dft+E-&uI7C!HuwtlzzE$5g5av59PmRsSJZ+SRddSggI_h)nD_6)-9Av55tyf0vZvMFhQt}v!)w<(
z)Y(+I^Ew_0MYS)3`fVV0!uyIMbM;=cQEuy#I79A95+&2AjEOD1aw$FSMBXJj=XkTcw=Nc-it3mXaElk#
z)d3I5?bqtxG<<<$b|}iHnLbQ_o`EvNvag2-NSAt71}QOZz>Bb`~CJ
zcsa_^8YF6MJE-<&3}%vv^|}QtE8G`#&rDe%Bs}_0IcfWM?~_#Sqho&aJKZ2&^^MN8
z`-w%CE8iWqsD@9=>~5E7&yK0H+6H^AJi;##+~UDdaYMOSAWge`*blciuH8btnKBtL
zAKB(|hfDl(AK%&%tptoY#`|OXo%mV?#pX`}+F64vTmGs!@(|1g8Qf@OqQmi2LZ`>Vzi2t!+orm&dGa)%FP{vm}>(0^YdW=-U^pi{7rTZJ5cE_e5=bK7X>0SoKlhYOE
zO!CQb+qJRPQ`_FKi;q4AT7V8x7Y-v>diubk`Hj}rYqN4SUE7E6euh_GTf<+}vi_ME
z>^p*kT|-ct%)S(b8nN$98!i$@tsFB}QXRwfgLuh&$_R)jh8f!xwD!l((Qn+ergdr#
z$HnHk+wsL~E=SP>%${gt2j(Lh8;p+Y2FOdxT($G{_}z4Lb+6F-zfQ#oq?fr%AIux-
z@MJ8nFIKodkv1_HXAkys{~#hd`cV`&I^}IKt_b(@kF1X^^yjIq(Y#yub79kM0*xs9
zGY@mes3*!v8`mX?FxSRAXY1v{%D@uCs+Zn8eB5nDuja!omm$p!oGvmhT2=b#2Tx@8
zxvZroVjk3Ijz90yC*535n07FnQn~rRZ@49O-_0MmyFD%HZ}A}==0h2*4t?(%e=3{@
zJ&rDEd=ayE*1sX?)(K`eTkTz3M6Q18>WaUxh(zf|^7AeIk!)~dqZU9odh>AX%28#-
zo@}&E>b33eV!Aal>1q;RzMSGXfvb=sI=P%<5Ky_-5>8Dmo61(WV-1gNJB5gTF#4GI
z+5M*Ds1O0k!RUz=wydh3d6|fi242PIZl5e*|V9-h5E{U1CF|6j}XunV&-B77?sO$tI4CB
zClN>~NU4_2QM_(NKgB_=!2Y6O;&u<&?_iw5Xm}FoL@}rM1ovN;e(*CPTCG6MH^tq8
z86t1AKYuKLYXJmE-XoreI?`*^#&A3Dnd#%`$BJQ0+pX;yx}tihA;rZh55}xhPkNnj_y3$rDNNX6}+;0GU02SBWUH
z+)&c#gEC3ok{IqdRuP937Zwp7T(y4SjQKl+ZFWB-0pt%#m0yj{LFPmPkis+U#t**_
zib!nusNEFFLoLy<3Txa}o6e5In-%j!Df<&vO{)-2$LcX-Gv0|HuaYZ|fIaD*B@uL-
z=S#g6tIPZEu5Tnp$#S6E*2nJ-3e@s?Y`KHVNUt<&KMv4!>$|;}PbSArD-NgSJ?S6c
zHlE)yj_RY;X71s3xbUASo|kNJGNV~~9ng~k{$Othkoah?t&FQ~JZ}14OQh1~mO20e
z9EaVW7hj5CbO{h!PUkrm$85$YE7A+$4V~hq7*e`U&`Ybw4r9eEb$xFuIl`~SDwb^V
zFv-uv0xk(iN~zj)I*pJjF0>Q2I)X~mV|4R{whN8Ayzry0blV6bK2=AHVtxTUqo_f|
z
z)=u`*3VLnBnQNl$_3}YIa%}JEOFTBO@9v?vL?ft()~H3fyi?0f?$m0d#1l*RgxuG0
zK*UHw7VY%uH@Q-m{{+!;?up)VwQ-{~K8LoqKvK(#9!@#MYcmI^XLT|)&Ja%=98;iZ
z{&qRZy#ak1^zifRus>@@=($hcD0n9iWX)G$Zt1~cyBzy85uJ(XiJ=`xO?8^1+4S?r
zJ$^fzjd-Zz=^5L%?6j2*D6Sc))U$Ndx_u5xwQxF89&~8<8%EjBcwe?lnqNhzd0OpH
zl{6$pDovRAJ;i*oN@Xz($A*#~bZB9Ni$NG4^9;I!@g{R^78|(B==KD2gnrJgb_RJe
z-h1$_py*@9ap;0+w10m8bmQL^^8NReYAeM1_U6hQBYT$|Y1I$?>nksB&
z(DQVy+e3D~)*7=3#V_a_;P5$(tO0ySyzZ?MdzlqfoeBeydLUxSFBYM|l4a}C+t>Xt
z%JIu&KP!
z-0rV34jpu&QGB1CqSkS2M46zmb;(BcZJ9avZC|din3y6_nR$#}y-O!{YLQE^lCB_R
zIqT*5PE%`NW-_}BQevH5YznWVpd>b9Mx(^;sVAlb5)Ms*)}|xd>xv%S_KM+1`l_p6
zhb(T@PG*bhS-oGW`n+}D^yb_631Tb531roVxA8a#(mOI
zE&U;&GL-2029#GpTWw>>RpigJfe5ZQqLMA*9sYsv&gGaZ141xBb%#dy+n~aE##CoE
zxsJtjbg%)nbgWQ>yy}aUhO^z%k>ct#jM;0#?ecbjQ7UG}b@;G0Nf_CYZF!GE={83!
zVu#V)Ve^+Yi89*JvWUmkt&~B*s74ksG)BRYn#PAG`+kpK=jwsOQ@(T!<({+?wtf
z@#86WBkE{p%#1n};P18^vMC9bcE2T^p}ed~^EHb<%|jsa-4)I($(P=ZPb>C|^>-wP
ziFbFlo0HoY0Z4beio?`#8;ZB}`i5dj9lPsw6dRE(Pb-l~+Y3^nD2GLz{yYVDB<5qp
zvEQv5;_z60OoISUZksWR8;*;U*zVcVsI;|
zm!Z8Iuu4Y(f)*c5XRmDRHIX|dVH9ui*S#o`g*KI26L1Uwz>m|@|}I+
zd#>7K2T3k*$rmU*$uSSSFu%O5z(@WaqsY*L;DDgxF!|HcdzzWt|)kYWV_Q5z(0S0j2H8Psk
z*H+O|eewxDhwmLHy$mB<+}wKt!KBJ~9SkG7%^p|;Rg|(ZGfH?J+@pysY65tchbmS!
zIW9@8=7k*!9Fd;aN4rl4?Z`88g{mdAZmSWF8>a!^t<@8mjjO2{;P(I(hUD$AbDs)X
z!*_PnTsLb1#&>^~~f)%bbzn9_wv=PkCi@IK>h@b>WDp|bJaAfxR_7{0oFe~*7I_++lg
zC7yeh>ol?iOshoAz~<&mKMQkTzX+Ocru0XN%%(j0`4#`Xbvq)`uan#gON~X@N3Nrf
z+1y5Zzn<$DpV%ou&Gwhwv#s^;o1l%C1f`ADZ$0UIznt)xu1Yov^LOrJfE&tNUhsN4
zxtPsMzq&a5an^G*w!MlT3~GyWIlUmlLLaKieksIh3$ByD++ezD@{O1haooAvFOYK?~LuU#D!)k4hgd;SGC}n_z>nG_+NDjR~E>@iZz6S!$8DRa6prrQS|pqm;IW
zsnP`7G_fC{Pnb;m3*KLbR9Y2;#9b7Zzg_MnOzjp_+3t|gy>4wIlI;$;zNvi-EQQOH
zgrfkFz@5D5jJWO&!HG{~k&z2q?FHmmWpsamghF{!YWK);3XLIp*$iFJaGNgf$&W}{
zjW1t?kI4tTC^*yNa=$w&LBVGSXNi6zdlCEWAX-KNYW7n3UMK6ZjmP7@8ijaad`q5h
zf|%;0wpb%Ki+{Pvdh}TI{&2}QL1(RujXas8Q$5qxQgkWdx3H%aEL{mH#Rz+WMt
zPW^6Qi_{=A;y8dK+80-%1yAN`H~VmRohzRy#7)h7)-%>(3-|bNblXy3fB6sKPH}RG
z{%HWu-zdDu|K%Wpeu>(dbZ+lAJn|pG-!CrpHTJoCKEk|C&tijZu(iG!QAJL12GMuL
z#FyDEoeQ^#rUDQg`ONWAFP6WLlUQ48=mn|PYQa0)tFukZ*JWoNJ@Q#~x~T3p0E8;y
zRgB*GaZSt{GvKUQ#59kUqcp#j)_@%9p3dzmQfb1q@Nx86H~UIY-3IDr8b9y-xURZH9I!qcHPf&wB(5(+Ox!T
z8-lr{&nopgka)d0JrQM%SeT644)ylj!*v$!yt=e>z5E4nN>*~ex_^nmA*GE4SuBf#
zLEUQ*Y%VQg@8>3@yuIpHU0p0~9T~CskiJXdwhH2N^;l1OJ_?BpDJ=^_nh~rov8D##
z&UCPF>E1uSdL?ot2)B&;T+5dsP7jfJN+_hn!E^B5cMC{Ij!C~RR7wrnX`(j^fqnv6
ztU-xZ0hL&R;$|XZ0e7l^2_@6iiD;XX;wo$?;2kg!JTB6pPb!(Ts=Ys?$qBJPS`U&_
zEEWgbTP=M8SfG^SCiGEqdl3<4dl7}^>rZ3Q9fEV>RbCO7L!PWVsnze&J*dZwAn57F
z?qg@~<+ugn%ZXBB@^fON9?se=y^DQb_qZGtIh{wC6pc=`dH|ez!6T+`6FcJZKC4Mo
zijv%`o^OOobA`O@w<_<+W?BJVyp)%NSm|nK0+h5@yIc9`Exs!(`Y`R;`n&J2Ag)T~
z+nVBH*F{pO94p`K4xzweK{r8DhR%bIZ}??L7f-c8^>%%QmCb
zTQ<>$q}Ki0d4mldosi5fod?qSVrw+K%+Ia}mrgWR_Y<=iYVKNOIHO12K@*y4h^1U-
zB~Gf-@8rU4RJbfwXqJiwi5pNm?BtS8Bc$K)i4zmpf-Z3AbjY)x_Oe4>J#H2Yb5542
zh%6QG7KP0_MN4EO>_3VXP_G;^|Xi4t$*!_Ba#&$WBM)CwQK;JEz#XeNDLbH%cT
zjlonMtCCq_c*In>pr@56
zigJr1DNNcRNkTx&dBWK$x982*Td^+{UT$4W6%TixzPUcE3xM37Z2{aH7)>vr&@VNT
z@?$j4)w-qoEJO28!-bdi-~E*Q`Jj(YZ@R7hyB@L25VqgB!QY=RV!CBlQU4P({|!C+
zJ?lD4LM42;rhDG=LtKV{-8Oz)Qrg*NFAvho9Po{8521&h2_EZf82;>_k-#JpKGI{?ISzlb)B_OQRp?FUAtA
zCBDFSU_ZC~J*}Sl#|%RG4`#^_kU!TEy$JkpJcp!NEty0Sso~!YS&!o^`2H)Leh=B9
zBve^wabgy^#?5k8;IU7;8B%ni;`=_0%n_eOB&7f4`DPSQcsCHtR!U1TOf1=l%1EwE
z{A~i$0VueOA*eg*OLUCSHO*6neWM)YPiD{09a)YZ!h=rv7TVkFk9d!@it)JTcm4iI
zi{g~oRA22{F6YQJXK<(ex~UAanGL5I@}SA*15+d(O{)8h3lSaat@5ko?+O{&H~Yec8b^vxONWB`tW8G!kiZG}m9IEVQX8Y7hC
zWEu^o2!;$V#v-^s^3LWuS2a|6Pe0$~Gm9BgB{cw%b(e?kLgVsiQAQ})Y~4@{J$3os
z`J4%h$?br+^7qHtWHS9XvwKH(M$v`1fWa2!>-G=~OBLP@E>N%i`BA&v#*g`+su{T*
zb3Iowc8t+;-Yeu1ILrXPHk!*jJZ2-rtB+hHufIS0_v_0j{EGD%`ZLZ$igTSSI{m7?v_KOyp
z6xBqA(PwKVnGSJnTDZPi_LQ1ObDrvIe(LiXF2?=IACfU(m3`uy)=4T43iwI^n_tV$
zK6ZU2Qh_Kg~?+{sZAr)2a@_;9W023Tp<27Dr=X*b!>FoEj-I=DLy83
zLnmi_(*fI1$ZXPQiRlZ!rIJhfEQ$3YhtGxdGr_A0KAkz`G5JMTkRp0d-&9f=hY65N>F%8>Q0}8QEFD)DPT$Jpfzy2AI`$u37ctJt}A62l-|LXFP
z?x{4D@ve>`zJN9f{PA+0kp2QAU-MS&l4nw_H}n_J__3!+g+WhD8{+l7x6(aCO>T7g
z_x%gKxHe84cc$v$oJK$T#GjJW0y`R-E|mqs}&~ISJTooP6Z)CcKnz8!|e-g
z@r3>mq9b0FppI}Hj#}0dUEyGy{<6-;hWzrr6(6R_xT~GtN}GJCZQA}^cw3kPg9}`)
z$BWZMGopC@Cma3JWK=M2dss?nB9l^3ZXh-NP~*kv)fBxpNd!^=uiWLW_UkmU9H(!-
zbNjaq7>`QtGnbj%b~=vUkLtAIc0mw}5fimZbl-f#`Ve^KE8@1PD~U6%LmG4Dn`>E9
z_p_uWtRnIn;p*`LL2J^BuF-iroVwP{_n4ZJeOY;}{_V-aTi=@-8pj-sPR&ZgQNf4A
z6K&yd5By$Kl}501q}d&<+Fb?e?~R8YPC{t1Nn)BUbAh1VMl`JgL_3{7OtiH?)87;MvDe_3yZPJ4YC
zoj2BOXi01ehqgf58XL__iR4*;@bO|byLZdM6)SwJ87TzC9&0Q0t2wKDf!ZN+`j+0T
zpc|G4EO%hu5_iiGH!bCzfWem$>L!#WMKW&GM0Eo0CdH2_JH}bYheU53NorL>kg)`-HDNWaE
z?Ea(5YVrQ=4@CPg!;P9Bk^ga`{`*^hKDf8g&Ly_ZVDUCJIH3IDq()X;`4i%%EbVA<
z2$D0h{+mF%Vc|RH-*>ZBMlpfTg?(+UJ?8D`WUF6@4llU|>tHpsgNHoVIqr>14V=yc
z>0K2f!p*vkFk1%zm6=agl<|)w@P|zlUL`M$7JXi?mt?h>DGV7(W*?9`3OO?l?k}&1
z&K0=>GecqOi#;&9j}-u(zi+0qhfnfR?^vL&}YIv?#dw$Vc`urfn7o7~%1|6!M!g3iy9M
zTB^tmbxADfebxF_l8LQ2_N<3Uz`2y9nEmWjbL&N|$%c|nwtyjWgqFGna>SvP2J?8gK4A|&Of0-4yABC5NufH#aTlH3!k
z?=
z#VP;b8{k`E)g^O#RewxtzL!VKjQLnw}fP4k0>j->u6*bt=~)vV>S%AU4Mz@(z(>6GOhh>
z%twD`K5g`~Ua0YGuO9+h^$7t-DNSDOCfCs){P#&7;1
z%0M7)!po*Ml4-E~K;5j`xBy17#TAb9K}Y2D+OzdVJp_Zqhe_la(I_86I8%^$AQNwn
zu4#Wphf$Epe7Sq0baJz}Wc+&yYptZFYT*?U8wC)bWR
zQlS-~_G4~58SF;to9-9Y<|^{+_ba{omfKPW9vrEEi`k|j@_Iiif*V`0f{a36ZM;T7
z1D;<9)hF4NC?7E{l2ZSkla!tMpO5focxOle209<%Sdy>w%+ECKm#_Ur;Xsg6JEbTJ
z38D21&i`;)ee@_+T-HmwLeXU~{4>fiLI*eHfymV|$fs8>ZLnzCmO>>shm_kAxJhvO
z--h77Zt@ImMXd}hk-}+D;c>CUjbk^Xj;AIFtYbxZeyex9amVNo%Lc)>h`}LV-rwv>xj#Mn<=oIze1@$C
zfx17Pa(q5@wEOQ*LF9q|{sNb5SnIO)&1VAqKPCq6|0zd2lA>^6Js?OjVV#47Vm9vY
zDdfLF)()!Y{$C(x7XeM)*Zfut-;2neW89b^k_U{;ZO}
za*^o4cOVpVY5kf0zjpb*g3A=WPv!S+V6~i*29}^9Bs26s0`-4S`2RD2_>bSL@c77+
zJ_>C(J2*rE?1I5cj_U&X4#e+zFhlxy`mDw6H0%Cq-Zl+hrTDQU09i3lmeBDQc%J4@6?C{v
zdS4Fa>qaa4mhT4R8RG$(jhi_dX#=>~l6Q9p&!4{2t{EooqVCgpL*
zA^eHy0z4)S?3cHXKtW>lN;5!|kVxZo{~po%{4Xs9V4L#(GawyR(?mRw`=5N1L(YGW
zrOW;4cwqPw2SJkVKs!^#Y*85rAm=1mKzA|
z9jCk}a`sv%9*a?j|LX?Sr8ItT+RcI3as3>7l!~7~c?!4fM;MUhB?vxb|2hurv2g%V
zRve)J4IhHC8f22$8+82(0f<)3k6_;f6pU?o%sNdb+s$Usq2G4C_GFT)9W
z20DY#_$D`>|F!iCY%ZX+HCrIbObO`ow2WPhu*~EIf^^X>W)~NA)c*8a<8?dv#6*R{
z9*4EbUYHt^utF`{{C)0gNaAi5Y+^YMIkg(=w$_F(D$HhCeBTG^q&#QyAy**l>lrA<9
zmB1omgV+764|uf!q`DjUg~S-u%Rd3_f~bKhofJB|IbAmf>YshRdVyL0cs)q?Z(Gqn
z^4|aX!lzuAq^nyuqxvGJM$~ho4}G7kF@iLXN-{Xu8`gU
zd1z4R?zN6*(7AB8WHkNSkstlNgLmD{%wW#5P+rpgT2~0%YVuKXi
z3TdwIugLDL?sWiq%XYgpd)+D?Ej4HFPn7Ctsg`ONT>mZw%Cs;Tv^jO^fVQe*E#98}
z2_Lgb4yuxqlk0W=s*(E>!l4UW(12Wu)XHq$w4+2$Gx_@jaxi_?M{Gq*J<$E2+tL*J
zj!RK8o~ESP6rF0e%!Fo1G!$0>uw{dz4n&$TU%*^~tE#GsHL65cfDF?j&!LD_>A|3E
zv&Ut1o&(UiwMOp_U|vWyIPE$S15Q{x^zpXW1P!pT7+`lr?$9KsyT}mqbK?8D3`}LE
zu@qsF$56sYxZ^df52`Z`0C`}RLJ@PB!+%-{n$y$?7EdDqgda-<`?5}?L_LI!*j
z+yOYU`z$naItPG194>6s!(VFy1Q&anP%kro_-FiPEZ-(g7s^QGjjkJNRpe9_b)X
z_g#k60~WjJHIirlduoh?2#t_id8rl32sA{E1G-N2AIw%cbDCX*R1_%}ki~(A-uvNY
z{-{8USmCr^lg#uDa3Wd()@*XKuYKBI{Q@qC{z^neSCl!ZU2Np7Ws!p%8EMTvobr+Y?q3KjXI@f
zJ+wD?9xraA;VIH9Hq{sYl~wbfyUV|TUPNzrU!8k4hlf|J@wPj{cn&PSu0ZWjh_lT)
zAng#i@R^hOFWU51bVQd^jxJUFK!l}Q>tqUvsL#wseV{*R^SwRc_9YG7f-X;Du?DPkSmm|vGqtZnk1pU>25&8~9
zNI?FhnLrBUFCEMO_ZI(mp#H}Z`2Wj_`qTw_r-kNye`BA>r0=`5TwxgWmJzIdwmn)F
zp;c{$J~qC7^A`^$iW3o$HjA9VdI@TwIP@j9Kkl>^x=gw=e^j8PY{h09^%WGr(P
z9mW0zAoqhu=9S6;85(<6*La{Mp@csoI)zH1aukUmoa4Ag)gv}S&|AeqWg15f^||MD
zyW@Ewq&Y#kHTLw6#kA!I)&d}y3%OW
zT&FmKc%ha08wPcSJ3~2g^k=jKqQhgx&t1BRQ3!9|eFw(|K0^Tkg_x7oPAQ=N?cyhZ$!n^5ZHwlBcF?&1fPcguZ7rE}sJ#Lq
z+MMEa*e-Hi4Gmcm2DgC|%l-eit*wpy`Pate>5`~-_2=&BUr3RN1sr~Dam_csEM}$X
zUOsuA+_0=gkd}XP!-kTXMmn*UQjG8_c6BKlL6GWsAy_AkKKI${B!d<0CHxR3rxM!
zWx9iahD(liy~Ekrrj%ncfT_&N^xrNtI7@Lm?PPTC11$GG!0pXZ%$0070`STOe^XpV
zIN&+$o&kQ=>%Alm$=XZ7wM#zBqVJmjf)8VIc!^NWSo^=8;fV9_ptD)UXOBz%y4s6+
zI8gMX**L)!k#Nlv4k12*OTYqc4_=ww0Rck#sh$YpWhsHdc=~vN9&di?4m2}{Tg9FS
zWIhG~Kx=eu6I(se3Aob~fPM|;#dMRqqfUc<=c`+wp37+GSEj+&(&OdU>AQGF-8f*B
zEYLjxTE1q6^{)&c?BHk$2{NEwISOd9HURW!1N_Ag9+sh(8~vExZ+bx)zvTi^@Qi?o
z;D*|_LLQt{ssn&(AVKY(M}S=zX7iKMM~@PO+fnvW-wjRa8M~QR^8EGja(|r6{^sm@
z?<2S^#B_9{#iMPrl~MP8GN{ob0m0Gj@+_e2@cLke>dy7}q6Vy9V)wKJ$W1jUOqHtd
z*)HkFqE3taTSV*M78D5fS{XAh=&GN23M?}ustB-1MAXEXsW&B>4$aRT@R#caWtn|s&iLZ?2;w&
zS!Z>HV$Ba$YI2m8l=VAaJ`1I9bUP|m{-CQpfHguNJBS@y8Q{?GGCf>dxW{}93nAib
zC{{AqO4(IvvJS!g$*x1FCH{0o=v|DXDkr38D_@g!u{)t!`_w4$2#DQ$(%8Fp8AU*1
zbrJeM5B?{(M-mWVsJvd0>(g6l|bzv7A;rao8}aN
zbIPcbDgXrs83g7BSAYy|yS`yIfV+dSjDd0x-|`_K8Zo|_jY%c_Ues+%y}}^zPAY}7
z!hLz>8#Y@~#Rp(f##Nh-CqU;b2(9C5DB
zd7xz10`6yW|1_J`Lmroz^sx-{58rTi5#%uaZoJn{^mS@Yw^ik~?M_Y}z0H-gqdwo0
zR)(gr3irq)(@RA2h?mJi`);f9nkTL+TJ2VIz|q*uPsipCok7R*c6&VTJ-)fjkOl{?
zr#FW~yO;w!u-qZ-pF%y-X*8=?5pCtiyJ=`B-~+$^^UmG&1WzZ-rxp0+3L3GjI(S0*
zQAaB?eRUS7Wpft%YDz9vy18(V3F88S;@rANfOG1^xedf;B!l15&^59hlSeT4>mvSg
zb3R(o2yzj9S>1jEoGVcPeM4%pP{o<2lLxP(Lf`Yc&?t`|mp5d!z|#DLC@>$H6!&Va
zd%8R)?k>){_?py=oQ_pUnZSfETw#R%H9XdH&t?eeR+dd2)YpOve8=ay6~o)`^a+DX
zmV!X74CERz8azGcqmCh-7VR?JS6AW1V>?X(DXNAZGc1z0KKg?3j~1@V7(xn&v^DBH
z>f_IQBZ$U~J5#A8G7TsxbTi4Aoh~?aTsBr9jF;Z~jFuzX4Z|ZP$xYQYbbTrGc}9IH
zla+l>ccg6#PtjQJ_kA&r{3Vj^AlgHMbv#-;2gQHeF8;-LrT_&3TSqJ6lNBBu{&ouj
zvdbyZJhGPCp9Nl3%CC;T$s52t`WnRn>eSTKa>j!8&yHt-vv?n;$?@s7w~AJ!%0y-t
z&;)x^qMiS}U+1GnrI8{a7IRnSF(R%qk@Vz+~ze9(M&A4@(SFS2YGy7&>d#RItu*uobkm6Cd`Mr`=QKyZI092Mzw?w
zBH`~ug5zlEJzZ=#u1zgXZ!%hvg6!Y;?yT@HUS9UF^|p{k;XBjST`|=aqxI;aZbkQm
z{+j-@tkwB=(M4=MRibUtP^Nry_PEHCc>G~QsK>j`X_eP;I)NOf)9h4Hj}od7c=v10
z?sznX6^ye^M9xc4C)>2FU7Y_guUXURr6Cl`$iNH+;YBvKx?kGT5
zaaX$@WKLfK1bJW}{etYz-?FYd3wi)zaXg@vGH8mWh+Y4Uv;}5_ND62P+3xyNH|}aUw#y!x=SpvPJH<2_4ZbNCYs-w~+@)ia>u?mGD!a#@2_4So
zHaEFFO3%3*(g;vV(v=WQMk@7&53oddRU;DeIqAtIZmrR|pJ7FDhb0ZTY>Lda7z?}a
zu2w`en2lFeWgCq|$?UP)FyM@)My3(Y_2EqND@0KM)h-qu2hw}@Wh#XQ%w?H73ae`Y
zRcBqDnBxgRX28}7nl1nt6B+3YLUh1;f_Y-?^0_6PJwj<@EY%_ruBD&8$lwWxQibEu
z%sOmEx-5DWaC!Z{)kbYUTx_hP4Xw|dX%T#6mU+3cD8px9vbl1Bb#0)LHTV?X9h
zzFMsTt`_@sHT3R^9xHS82SX>Dtzu*4
zJN|O9Ip;%4nfdDI#XW#uJoRZC;cDX7dK<^qlyw_!F=<+wN{Z#AEm^_XBc0UNXq+|e
zajE_w&I7qH>?&r)-eBMI*(v5UynAG40hEBG+;*{PDW8Elf_U*=x!&Yt;bv51W`%4j>Qa=d&ZZcgvBgo*rlab}~TEP=d3E%qqjXK$30KVqDk
zb5K{3q{r)j@(PxTpMP!Igp?FFdzYx$BDVeUpZKx1!pQ@>f=MpOjF{mi-Qa{lwd`)M
zgl|MR%_goR4uMQiuL%PK!`3-SD}s>cM<-QpDBd{TPkgLUy8+QY(-n<%7^C%K8ID^mOl}|!!y#W8(X|^x(1%-Fd5VQT1z~1wS{01xihKq)@-@gSnog0li
zJ@D9FqGH}_$>PK-uNfaM4-9B-Mrz2tZVtTM;rNV%qlQ#hNk<#XbT$E&C)4Sn)5z64
z^k9Z`=|=JBKBqKcV@yt9=y9Xg%%Xt
zI2}#O)#Ynum$DXUsZjAmyxlghqSS3CgL$;wu{liSJD4345pKOpt3z}o(s@XB*t_}o
za3b76jOm{EiXZ2u?#=b*Fm6tfaM)h}A86fyw)M>_X?cGml^9*1Q;
zrthf&S|ZPDU0(vlgvc8;r7j`&i{68?86KNC%%Y`j`&m887kM1D%|>|wOP{>Jq+h1%Y1r?QP;b_WW&FX)+9WjIj&&j0ezFSZrO7KZRwdDT8-4Z-!IZdl>W|8gbrKt
zM{$mt%?tY+Q?-}8-u7)?Y^@twDt|bxi{)kdX((cn-$t3Pce9vlj}ooo5a=Js7+U2i
z7%eD#4}KTsjSP6|T${x8#BXZ`1%f}N8!Vm6=~@wi70KH_&5m3I|m7QYw
zd^X;#q1P+85;vunkOs0-uPXIH8{T<7hpQUphx%45Pjq}_R@;4&!4o@$gqnpyV2drp
z@gaWDp7Y7~%WJdLz*JR;;j)~=8+4ruR-60w%QMtSB&nfo1D_`GuWCv+)|qH8^@Go3
zSi`VqXfD#alUae9T!!i~*o>=1b#i9PrYBK%c~vFR#=&8vAns_VThx7HLSloO20tmi
z7V64Q7Z7!}tKdmFZ2Oo%&%3w5tsFT$(#^V>d&b{uXZYtq4E&c%7xRAIdBTNo4pCIz
z?kMBSR0GRTrMD%7AO)epyS57S$PnVazMorLb4GpTfzD@++4-!}j=u~jpIbx4${gKd
zNU4s&It&kX(8S%7{sw0hcE(Xf)%0g*ekGjYJJ0_t^+~JRhZlaPbAG;$v$r-k%t%v)
zam?359gx|5W}z=tIeph!Z&GH)g$~wNWBoxX23$r~TuclvCu^J~>CyGg^G?ZKD#({3
z?$^G_yLv42Irym`4Q9+{9nE4O+E8B~CLwRf4L4cm1szTi4nlch{>9=6?|m`GY({J%
z2cf8@Z2rft{uTaBG+e{k+^7WK#l$`luV9egbLV;PAbO|gJM(2;{|g&+X-0mAyZ{6|
z4k3d%5qsk@KIOMNxz=ivq<{u_0mAu#U2T9@Tw*VXD*Il&X;jXv(rM#HK^Ea~!f`LI
znp<`2c?|XcbGrd-J($kXy~{tKPY*fe=%~y)5A>qQ(#1eume%qC07=T^jTXu|A%VWR
zSFy9+`F!Vfk=rMo-%Ombd{RF%QAP5%BYt<}dno%2nwlnOobV&7oiCxG9bbGJmuXMW
zwo+@}cwgf(ZK|b2SCFov@$M>@gj@Z^m(ZPqOD22$j@KkhYx?L@#9dCW{w^^&*|NL5+hrjs^Il}S?nP%e3rObC8^0Ua`$STA6M!ziD0!)sM(7?4tA%zMcHOw1nQ#
zoFu&Var78Dw2uL7f4-#-9&fD?fZ9ojofYMpNh(J}LpML+MJ#=W$x~}kb=P>!v
zBZ&P>uaYlg*1@=VY)6^Yh$m_q3VF8sU@g^9#*9Ik3!P?Wu?TE+Wqt1NkRn~vziNXU
z-Azis$gjj0=Brfqb~uz%8{9_pBV)%s7Dl~P88`(H;70wIMT*_5ix_%Qvo@wtHm;w*
z>}@c+mYss8kQRl;n`{$mn_$-Zgvy9(-M4O=g&b23W`m2&s-6kzm)z!U*OmY?nx`nL
z$>R2x@4j2!C`qv%jy^qDZlT86H!*Z-trCRZQW{VzbY75DGW0Ttm_a}2Ig`+
z-TI~uD?9M^EcP5kzbl32yQ*epW{?tv)@N^UH@fg@DW=e
z6Y+?_LRy*w2*Ka#nch52{p;_~FE?PkVjnn;c4-(}`S#d!C2JZAn-%TG?Nf%HvX77~
zuBQx(GwQFg_48QovfM5DQQR|FrzvhXRF=}*=7#UvD0iHdUzPH#txI!}+j{nRWk#Z#
zO=GRnjylVa+27aOw7NG*s3}!Cz&M0)O-Mx}UNvLvo6TZQ0_C)XEsy83yo3`@P6dmeIlJ0vLiVCm4+{}sLqy`r8@%CP8
z4vdpNU4^Vx$UAQEqYq7$(UueQmc7Xe37Z>RD22gXC|hHOw22s$bEaxKr~j0k>h7Nn
zc!mh{9OlJft*)Svr$k-xSr6RS@HA%k;DU%zI;PViRoCwpFz_EKroPo#0A1sHV=M-*
zKE69zfHnU5z4UkZi0v7iWo_4j3C#Dz(hl8_1#```!+C?eIz0Y|c>nJb5tGfI*0OlQ
zXL_E%X#VJ5O1A`dj`syuo2od+kY-YkeP5dGYvc&Z8kJ0r2$>34R8AQ5rOJz-ODtJK
zuY3nk?AF2_043*=g{o$vV4u3H_>bbmvR6+NP-k-H@FROnf;%N_^vk+${7RnEtEGh#
z`#}mne))ZEc0f*h0lFKUk=n*F50ho@>D_};YAf~_*nw^blt#jR1joZuIiH;C!>T^XaBlRwfHzV(%%
zq^*dr@K(K=h;27)X2X%uqg(P7N>Lsw-ww^j709Lm)^bHg!e#lI
zHLYdj2cuu((z%P=rHL!QRdeei_7(k4UD|;Q}*l_4oRl*OH`1F$F{3j%$IHs
zM)umCM2Eg>oV{&r<694j#uugjtbNwnl)Q=_`|;x^D7Y~Xk;XY2BDIE}d$
zld;ZLOBr8t3f_@EI+)GYFEGWVy6i{q+;qf#jbS}ezvE)0LEhVrz3B5vzHYXaEE)uA
z)qo!yDBTotd9Hk_@_O4C{Ny;T0ZyFO)3TgTErTK&lm-$svoxT+ibg!SF>I2rzHxuL
z?GGU@d5t$(r7ZM{IDCH-HYpw+T@`>v!E(HlA*vw=c1BWWC=wpR#_&qbU-XgHprRj$XQ{
zy5%k_Kp#-Q&vbCfP+k(m%qo*2ygZLlh+~DHtufU;8Z56kea%sBX*a)2Glf}dCQH^y#YUdklL#h%D2dbNbagslWODt=$9!FhL
zUanQh*&%zc45)?vxVXaZ@9CJ``k-U@5PD|9=;1`0xEH=;L(E~|yy(m3GlQa&pZ_(^
z_ATG(k4^CpYPEp{+v~GI8b?2=ST|U%*~qaJVQ};!vuw;kYI8PmyWT?+2JZWNPoE}F
zDZTIP<&8+Lim#=+Jf^VhQZ@%PY^?Vy!`#Oh+d&cl-mO{U2^7ZOU1JzhR5~LFqKt00y0bM04w*aQwC0jkPv&o4fiZYb
z{hK>68aqT9t%~EPNz5(>5i&y!d{j09if5a;67SrcB7hlD`xRHN*kpJF=KjHx%
zZr0XUSM`4t@c*3yfHyNn!1+Q*Cji&`7eMZ1@2N;C@%lB%rX-q9$uVh|7C~
z;KJ8KGgSlzH1~In5@K&_e*ObyC@i+--EzXzR{y$pjTp_qsnFGb97K2&lJ<0D9yj&;!Gh
zv9kFcEF3j*e(VtF&vyae#0NC=(q&Dn9g}WpCJ*Z+DbUa`az*=Y&yDMipk;w>z7dYd
zdFt$;h(&!TP}OWGYC|g-2mZhNh<^uWLU~Z}+&(@><3roTbOq4AEc$6<)dPt9jq^>g
z6O{eQ2isO*>=Ff!0Z3)X_XkcB`T)*%-YRVdbWPmmS(1c9yhR)AU`q)A_}o4w7*;zl
z83K%mf0(WQhaUs)so!|q3Notz?2G=!VraVkEb)e%_xe5V?CaCb$Mq4*OG^g;BqF_!g9dAe7HoCC;d?lt^26G?t0OR_imN;?kP53`?h8u
z00rw02jITXtLzQLiF$cjtL~^6fSuj|CmWW4^@v#$A?GG5tWs(%fF+Dz=>A4b8~F^E
zH^yc;P^WettLeU5Z+2tv){YWX(a994o~{U8bAs18;|xXOVmku*Sny6Xfu1)jd)k@+
zL<=t0alCT6srO73W|g^PVPPN|nZ7^e6IlNOEGxb#3@)JVUY)UF{*3kcL-5@|UM|{p
zY1vUZ$#068UWI3GSq;>;ks5!u?fGMjZhi*(+=
z%ww$^FaL+9NGn2>_2@_?(e_=S^JI1L^q)c$e*S96^_Jr*z`DgDYX@jGEs3z+HNF8r
zQwCZ~bywEc?+O_fMQ168p8v6eQy+VSMKyvLPBXWP$m^kt&GLu|@2!n!gg$6b2G}Md!88o<0P_1<-kHiq_zCLabqPxV
zy)s1MS=a%j#P>)cjNBq^Aq=rVF*MnO96BRcaFFw!%33tQ<>s?YL+)BLFQSwWxSNr!owQL;|wM4=Yq)HdAZ>o_=We
zH_M?al{b@buKY5@(~`@6Q3pl6Dk-g;L0Vc1O?Up;+TPpe~w-9(=9+Kg(S
zG?(fGHjY&@%TY^#6F(4+RQ{qg3*O!H$R(a!
zpvUJ3Yd3ARGrnkZD~2sXw|0JtQ!ob$16GacZ-L7$v5Y^hAgd9*%yWN;v4ImSO?vQI
z!YX5peK}|zt7~=FS()w6aG|v*Ty*^Bbk!;rsKdN31Moa13a^y%d}=+za!yzJ#{Q7D
zqFvOegv02{^)@Nzsoz=edEQ2H*T1jGPQG&zK&(e{;bSR2B~K6RoBE
zu}HsL8u8CELYXe~X)DN|<^ibz6!ZQ|yWfCWf%d{0#sE~NubA110YO|
zptE#6BLF+B#N$lflbBZXczHkf#wgbxzB~aB&OU7eSY1x*U4LjFHGnQ2K>JWCZE$EK
zU!)R0yLeOO5OMBz)G2wmwAZO<$Z)0?=m#($p2&rVypKA!-Ph^-rXDGyKv^6B;srVT%RmG)b_S7*a#
zg}tA=H*v<)R)4t@t$dESDpKQYP8`M?q9X>&tgWE1v9}<1a9C%EM8P)lV*hwyY&Hm6
zuB0gg->-Bt4c$$BVDs+CupsjfRO=X1IvUV7+K1J<-5J2x4?bdQblt>0Z?dPx_;B-W
zPVB6!Jn5f^>&9({>}_P>=1*}w`-4Hqp9y^Lp9p
z-&Txk7|j`$NdH9oKJKv!dV|J9pWS8&uFQt`0WO>(Y$3XC1OD;MW19K;+3C7~*#?lJ
z$DnUgmOP-T(2#>~?8Tuw^BB3kFco=P$>drOTnH%2uhp@5Qfhjs&vE@vBgIwm<#Day
z5dWtkG^0TaTgDlUHib_P^x~UB_f*XG0(TRmUd9Kw7GIj49Nd~=F`n@cjJ>ouS{_-3
zA2N^V-`aFC>mfcEwRTnJhSlM9xQn)BBuYd?ED{rWY85B`VrtvL+w0!Z`mWoeUso93&Iz+goiJ&Xv4~yw?)qj
zq@W4QTpn##3*!s+*gn<>??&{dpf`gEb4Ysh`HVESe$Ccz#gJ=O2m6#)GK`QW{CZvA
zz^fnnjNvMym>P$HC*Dq&uT6?A-N<|7{7?NL>6g%%U9
ztel#%E@p~{>tA|agg=vATwJ_ZJBn4zhwsh6c8Q!{E#&wYS{
zgA3hltDLV`#8B)ojFHd+mwqn^QiK0$kj;C%YTFdJaS(kGSM^{DUAuWtqa6gT@gw5%
zwVi%9ZA0Hs@I@|YTBn!$0Vm|Wr2SR#kH_1@3#fiKVK>bf!&Z{`EXIdz!YoBQgD#l*
zk(bqlfZQ$0_GT{!Cc>@p%#Kuo_6gV3c{4D5#-=TfFnbmIz1@cjFl@SiVTia}sz{jF
z8gHh&O!S$bgos|Hxc-&5{2vSM(|k-f=Mb|N;ZxC$4V;z#2)2LA$fSH%~IHU6q
zx-d!3r-b#AFb+_`HVa3_XJs)U%zU%vsP7ZwNP)lZEu50Ps}X?k@xOT`H;(T-KlvCXU2&JjCkDr!Occ
z;sBN@N2ou27JBf;2W3DeCe51TWStj$Pd>Rt0Qe@^>krP_veS(zpGyC#q7fJWt1r7U
z`b-QptEkeWUomosC$+9S+ij4#EZ7J9P+jqtyLy)X7fC=r3VG9FvPvxU#ru1|^YqMq
zC5tEQaf+EcPt_Lm)!aKt^xK}g#+fc|);Bd+g}>o9e{7l?$>EefcM#L|db8*IV5Nba
z9Z=`((U}vO@U>;+no8j33c~ISmxwtZ=-3s>pGiyQ2S{Cv)(`O_mdWjr9BQ(}l|q1(
z?If^)Bn*%dBShA-Otda=&z0NM^eiJc-&)1`f0iZmi@iCx`s<5C=>FAqM%=xH%pd-6
ztqdj79YkgRz*mh}Q-g)$MkJ|W`&^dGSSHAQX_#(x)Hd_+b8N*>T%wfA(u41q%&-Se
zhoN`Pq_z9GouXap)Ivw?H}Y~i-ts)Je#EzmE)E_@OK?g$-=tpmu}P!#N7y%we%R+p
z-pgmlC?ib?=|n(SVj=W{V&|>p7%>&P-Z+Qhc))k>*J^A>jv9G%>3aX0VD6^y_`AFo
z{*2KP>YoY^&8~62f4TLc?6bhrSF2+MVfEagF$qT6t&J3^VJ|>H_YRyGwvw7A
zFqQpCHtOn>wxD52$+bDlqwJ>^lr!#OVvFBikDs5&zJ}|HS4apr#D|;L`Rb~K93;n*
zf-k>wbAcVbzDxmTDgJ3o)>pcxE|r^+<4LgAqe65EzqmoI+dFhfqwlu)=5(Fiz@vr0
z%TZjAcbO8Kg%RqtfO~*J!Ek;y-t8UY{cMNd)?wyZ^(p1L9oF9&F&2}@62veES3s@B
z3xDDBgH^GW(vFaa&L!Wx15ygNuW{Y4Y|<|@04K|N>pa?U9xv^?B}pcr+Qxv{%1q76
zVqUPnC+0G*&0RahkH^Xn1GZ*Lo_V;%ORc51`-B85s|(-oCn^N?kA<~`E_UUFFXaGJ
z*f*}nWL*qR88y}DNMXXB%!hgfK!-oAc~ZX-ejVOp6(Yv=`JF)62LWrr_vbWs3ZLaD
z4QUU&$iHQ$b#mKo^vN#bfq>m*vE0*qcA1ZJ#eTw}=&C4azrs*sh#WK??X^GNHUViO
z%8;kgeto-l*=W;ctO_hPr#te}2QcZ7o%=|9q#4B>`s)!Q-05VK7AWL5wZ|VFKuxhC
zosh%#%gB#BKLfmiOE7)qF>NKp)8n0eHQL`^iH}mJgEO{*ntUw$M{cCZIn-ld-8xw1
zB6NAsr?5@YfsW`kipaQ^VN{NliW}*LYT&qUWW(xxaf2y%x(BM0Jqr-_gSftK+
zRa3_eFc}?ir=+`tEyR3V0J$>b{^`ywOL)RW(5~T15}s4!ADQI8MC2
z;x#VM-69y|Fj8VAgh}uRkq@$8#6xAE1x-GqF3}()1m7t1tzn?b0e^d#$rtd@{yUC%XaobSt_*;p08X2NSjAq!xb635F&mY89
zpCqg+Tr!rICOMx-J;a71wun)aYEvr^+7chblO^?i@{KY0UeyPp|HTZmd&LmIi#nMEik{w
zr%s<`diZ}|UKpLfG98zeGm%g(&2oyt6ez1u0Bz3i%P)>g{Po-I0p>B~Dhi-kQ~_J8
z99;d2mY~r(4_giy_isYf&^(#-bs^UZ}
zCJ-dRIoK2syF04Of4ukM5^sY6(;h%SHnXh$o_-FucrOqaf>{fK8Id1GUMz0OiZFk>
z<#=-XP0Viy0=sx-{Q=u4JqO>;m!nk{LMaL!$yIMK?djeowU~aqRU?T&?B#O8(*I;f
z{PI!F7Sg52X5&n;=47N{?UR>mJ#S_-jMQA7PeD7muWKmOXx-y1@*xICnIyR{9_dkE
zO;6v-9VzjM7R{MzSRB~j$B9nXh|Cy}GyV5_n4=P@RN3k>(>je!8
z5B*4`B(8+G!e;zQ1ayLCnB|H86m_*)1gw+Br0UgSBXL`)LZhUi-S$whS?1+jXf=Lc
zY`<%8$T+VoIjEne)@pdHv)8m*y+*wsmSBq27^3Rv5*Q4vXu~)3PjF@!qCmAF9+|8=rtj3elAn7B>6O6Jq=xrqyre
zMfN5dO1fK~U&$G4T~6yk>VKL&9RwA?d9QqsE1p1+O-iR5#w(s#p3lsi2#Koi=&C>U
zJ!mZhDmG4IeNkrJ6+$dVV{pYq3>f52dTBP7L{fM~H%o8?LstAcD2u=wAoBgcv;Y)+RAm*dV;U*#Lf%#6@+Y2T
z|7r6Sxm^v&Vp>$u+P>f|>#q3Sq!uTmjNmsF7GHtCW_*#h=*F2+G4
zQSdi@wG2pt2$A|dFQuq!lPj-z6|o63DltENN9@wKTrS&;o6vQAM&RJOoOF>}@9U}$
z7REbEu^p6iAz?X`F*rSN*8o}DDi@~^uTFN`W`tE~S!nMPkAgYsu_>EAL8v#e>x64l
z+Ch|CW{Sq516~ZH0|{@m@UZko;(p5J{f%y^SpG&^{`A3K^>`7Mq9tyFM+z0~fGJz<
zq|zt;S!~5GnA(pq`OTu&_|?bwfw~IRUf{G8=;-ba*C?CTImWT=pM|f9C0l`UcXh>P
zAvcYQOOu<`!E1R?6N8c1a~5u47W-JvCR_zc%1Dn0(p>a0IxrQ$83~W#WTdB8IckFMQQ8{MYkyP#l^iZbn_hniWOFjibtgxhOYZz{OP4i
z!RM07Kd{gmD*9l&62O*)hQdwe^R!@(UI|$GsDY03n~5A!-FkWV(OBL2i-mV;uUWj3
z8N2Pfr?k>#5R~V}%@Q!GU8|WcFY|m;CSW@GVtU3D;kDUO+bcO-Q6=M%fu)HOu33w6
zC%^cU4Rs@dCo%}b&TJwalLqElYkJX?%n7;}ffH&1DtzR!a6s#1v%>+jQ8^UA=#G}4
z3;Lmp*|`#!NVPbXAoZXcq5iP%*gd&>+IPVAp=Jgt>CU|Pn^39oZJ2=${vlB%2G1s3
zxg=~x-boU?^L=o8cE6E+pIT{Dmf-z*0WFhX0(QzFe|-{d;_+raH%8N;?nOy^nPyIv
zEXkN#N1;MP4mY;?nr%vUf}P9XSTIF(R5x0f_Q*aLlq#xQ{!Y{PWvVclV3g6D^DKdZ
zLz0yF1vDmK0WrN{h7~zllHtak7X5lo=Ur(4qe)D$_lIs0eR4@w(9-$Nb1h)%&%wC`0OwFT^ZZdk
zoLk`2Dq%A*n`08S?n$HUu^;yfBys;z4Bz=NC(n@c$A+H7bw81nd`4*8Pf9VpeL)WU
zLWC!~#3c~^^)v(fPk5TZhv(zgk6;U8ZV9yyqsr&7UV)m{>bVnNVv3txeqWSpNMcM}
zJw%Hj$@$rnq-K}WqaLnhn%LvA7M=*+d1*FLlR@}>W$>$y&fmEliqGI4OBUm(?7OOs
zy4P@8CQ(X5Y@(*s_Euoz=WZprVQWX4K(Mv<8duB;DC_PPTf2hY@_t<>^wO(HaWFHT
zAq{j*7h%$<9n_3il8}|(s46DvmK(M=RvTG&N-&
zBUJTV5V31a?0Nw)Ba-=SSd
zoI$jcA|$5qeP05P@9JxrfpdKd}C(_Jra>sryh+gDF
za_shZs!389`fL9m$F#_zf>Y}TMiz@_hGdC=`_kihYGzq}YGfRx=<>wuuEy&;LM!>A
zb|HStZRPMio8`(GTyp$&PJ*R9n}uf#WkZa-uO9s~mtaSG>+)|JKa%b#RqpGBuW@l`
zbMEnPt^)MN;`A(nuTm@|ZJ4$U?$v%2I?Y`v-Y3()_MXc6y~V!m@AFrt?l=By%p3k#
zJEL36;?siGPX;|PIyXau*RG7Z-H3~7v|5hcP)*QHvKLzKY6?jaEAvnD`Y38uJmVb)
zB}Uwxu9pxtYtosm6zzM=`XfoFZV|!_YqjNbnNE#w^nWYVYvP;%Jz_oIx;fwFC`5}O
zYwVR>@$NIRn-z_598YB}1z509@Xv4-pZRZ_v`Zm1zq&wHPRx{K9oCGAZjn;ZS}u?G
zGx9}|*oGJh=fSB2O2mj-y`KoL)S&tJ_zEj1^!4<>WJre47^KfG?Wia+t==@kl;&V{
zkZ!X=u;!XGpT-6YjGb=5erYKp#e5V(LI>2xVPac#(SVKvVjL4@l6f
z8hm^8ZsBTEvmpbnIS9p+oh(cMbRwZQJ>Y2zg%g~tBl>;p!ZqI<(d^>qixN`_82w{Q
zu;&N#Wej-EiMdA`h&+_YEbjVVOkBqn_QCt}-!M(qvFql~{nBVUm{CUOcD?uSPg87T
z$(Pl~o|5S-mA|i76m|Q;_Ty?O_*89-1HoSELJ-x5DA4X#QAo-b;aXMjH^lM=E;LGN
zz^hoVCS5UJbP9+3TsW2~`uyDv>xR(Pi+&d29Lo}J`s)d&ng
zWCtLXo~w-k-`8Dj;efE+&Vx9J(Q9j|n019LFG|AXBXH$Nqn;)hc0Q=dgpwi@ho$*j
z7@C;;9uOvV`@IieKS&X|>Yxx{VQ?+?XnJu2>tqL4RKY8*C^keKhg>+eq>bPG@b}3U
z2*u`-D^HGot(Q#c1)(lSY!_O$Ty3GNAF~eMJN@E7xp^fr);cYhJ@c7Y;|c}ew@k_e
zjfWCiLM6*0d6O)PoL=>Y@1u89S6FakNeLBazidD-1{A2>sA*P_c3kNv=dLdWmL70L
zGiwcrC0hh6;h^V^wQ}`_Z-bX2!sm5GoY1i=_=^CGAD7El)m~HBKr0?qwIo7Ykb8gk
z*tmL1o1d)B?HTVAs27JA=}$p4EL1onuDTY-A#`|_md=~rvgg)LUl7;h@>_eqG0){v
zE*HwUo05N}I!F#&cez=Rr68BFjvsZHd?!}TI%`CP+qY4arg&$BU1-0xyQG|*jNb1y
zy^guP^Rl7*;}?qAbVQTyTY9d$hp?D_uHn!!dn{BAtGL+%SDJAT6pFz#JaWK})K@ti
zNi>w6u*eO0O*zTczWv^17XoIkH#DoEc>?A*`IXe_|5e
zemPx&%j+?du%=BAYO2Z5+6SKjK=zBl#33iejSB4my5%Lc9Ek-<6@HJwmCAB*At;)fn=lmd}`%Nq|Te2v_X;#ZuHcF$Ot
zxHf8!)^2YEf1;>t9%mU2QLiSrz%AZ9TbiNde_mIbEcXj1f1NM|e{3*FnSVCFcf7pE
z4E;W}GUFn~5m#b#$0MQ`>KGacV>>^nLIZC@EIuYC%v_yUxLuF^IeB~hNHa_H?GG~|
zS%p5p&qo+tX&OOcEcG#$Y$!i?1>`xczvVe~C#1OWI7JQd^BOo{2}pAZSC;ypcAG!5
zZW6qM@E*LAN6z@UNyvv#uCwiF?~^`lDvp!jB?k)ru|N7Pbjk{auqQ1@W~U~0ve~j;
zf9V#}#V{#J4Z<|*u|o2i%hnwHv69z;1Ml
z%5#iE1&Jk{UCuY-h&B6p+oi#4)Pl<;x!4hhDFqebLbwND@4-8Z7s2`s!^155>(y*>
zR|!SAWp9I@&vQr~%}0dSiKhOvPx2G~F_G@~56fHH*D5d6?4KD{PpeGsZA_vsQ`Sq$
z-uev8E+0B;$*z9$H0kp@ekao+&s&OzjX1TnGf1x27dnl8va3F-!LE1P>1}N4)*M`}
zn4MY8PdPU^teso?b~b`M3769
zmVHinEy>mGAUY6rtBA@($|A#%3$)tq{GJA1*QSM^>f=tJ>Z3{EwlgAv<(8nA