From 14193435b8672ed974ecc666038b336f03ab53a6 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:08:25 -0700 Subject: [PATCH 001/102] refactor(anthropic): rename llms/anthropic/pass_through to llms/anthropic/messages (#48510) * refactor(anthropic): rename llms/anthropic/pass_through to llms/anthropic/messages Move the unified /v1/messages code out of a directory called pass_through, since it is not the passthrough feature. The inner messages sub-package becomes native, and the Vertex AI partner model experimental_pass_through package becomes messages. The old packages stay as import shims that warn once and alias every submodule in sys.modules, so old deep imports still resolve Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(anthropic): keep Final variables out of loops in the rename shim Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): match the warning text in the rename shim test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(anthropic): resolve old pass_through imports lazily through a meta path finder Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(anthropic): mark the alias loader spec restore for the type discipline gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(anthropic): format the renamed package shim Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(anthropic): keep the rebind-ok marker on the spec restore line Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(anthropic): carry the new package search path on alias specs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): declare the shim test locals Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ARCHITECTURE.md | 4 +- .../callbacks-legacy-python/src/mapping.rs | 2 +- litellm/__init__.py | 2 +- litellm/_lazy_imports_registry.py | 2 +- litellm/caching/caching_handler.py | 8 +- litellm/integrations/otel/model/request_io.py | 2 +- litellm/integrations/shadow_eval_logger.py | 2 +- .../websearch_interception/ARCHITECTURE.md | 2 +- .../prompt_templates/factory.py | 2 +- .../renamed_package_shim.py | 87 ++++++++ .../litellm_core_utils/served_output_texts.py | 2 +- .../chat/guardrail_translation/handler.py | 10 +- .../anthropic/count_tokens/transformation.py | 2 +- .../adapters/__init__.py | 0 .../adapters/handler.py | 20 +- .../adapters/streaming_iterator.py | 14 +- .../adapters/transformation.py | 12 +- .../architecture.md | 0 .../caller_model_checks.py | 0 .../context_management/__init__.py | 0 .../context_management/constants.py | 0 .../context_management/dispatcher.py | 2 +- .../context_management/editors/__init__.py | 0 .../editors/clear_tool_uses.py | 0 .../context_management/editors/compact.py | 8 +- .../context_management/errors.py | 0 .../context_management/placeholders.py | 0 .../context_management/result.py | 0 .../convertible_blocks.py | 2 +- .../native}/agentic_streaming_iterator.py | 4 +- .../native}/fake_stream_iterator.py | 2 +- .../messages => messages/native}/handler.py | 4 +- .../native}/interceptors/README.md | 0 .../native}/interceptors/__init__.py | 0 .../native}/interceptors/advisor.py | 2 +- .../native}/interceptors/base.py | 0 .../native}/mcp_handler.py | 8 +- .../native}/mid_conversation_system.py | 0 .../native}/response_cache.py | 2 +- .../native}/streaming_iterator.py | 8 +- .../native}/transformation.py | 6 +- .../messages => messages/native}/utils.py | 2 +- .../responses_adapters/__init__.py | 0 .../responses_adapters/handler.py | 2 +- .../responses_adapters/streaming_iterator.py | 8 +- .../responses_adapters/transformation.py | 8 +- .../{pass_through => messages}/safeguards.py | 4 +- .../stream_assembly.py | 2 +- .../{pass_through => messages}/utils.py | 0 .../llms/anthropic/pass_through/__init__.py | 7 + .../llms/anthropic/prompt_cache_prediction.py | 4 +- .../anthropic/messages_transformation.py | 2 +- .../bedrock/chat/converse_transformation.py | 2 +- .../messages_transformation.py | 2 +- litellm/llms/bedrock/count_tokens/kimi_k3.py | 2 +- .../anthropic_claude_transformation.py | 4 +- .../bedrock/messages/mantle_transformation.py | 2 +- litellm/llms/custom_httpx/llm_http_handler.py | 8 +- .../databricks/messages/transformation.py | 2 +- .../llms/deepseek/messages/transformation.py | 2 +- .../github_copilot/messages/transformation.py | 2 +- .../llms/minimax/messages/transformation.py | 2 +- .../responses/count_tokens/token_counter.py | 4 +- .../responses/count_tokens/transformation.py | 6 +- .../openai_like/messages/transformation.py | 2 +- .../llms/opencode/messages/transformation.py | 2 +- .../llms/tencent/messages/transformation.py | 2 +- .../llms/vertex_ai/count_tokens/context.py | 14 +- .../experimental_pass_through/__init__.py | 3 + .../transformation.py | 2 +- .../anthropic/output_params_utils.py | 2 +- litellm/messages/dispatch.py | 2 +- .../proxy/anthropic_endpoints/endpoints.py | 2 +- litellm/proxy/guardrails/anthropic_sse.py | 16 +- .../guardrail_hooks/straiker/straiker.py | 2 +- .../anthropic_passthrough_logging_handler.py | 2 +- .../streaming_handler.py | 4 +- litellm/proxy/proxy_server.py | 4 +- litellm/router.py | 20 +- litellm/router_utils/baseline_request.py | 2 +- .../host/callbacks_legacy_python.py | 2 +- litellm/rust_bridge/host/routes/messages.py | 4 +- litellm/rust_bridge/messages/detectors.py | 4 +- litellm/utils.py | 2 +- ruff-strict.toml | 4 +- .../coverage_registry/llm_conversational.yaml | 4 +- .../coverage_registry/quota_management.yaml | 6 +- .../test_guardrail_long_tool_names_wire.py | 2 +- .../messages/test_body_hooks.py | 4 +- .../messages/test_dispatch.py | 2 +- .../test_websearch_agentic_loop_cap.py | 2 +- .../test_websearch_short_circuit.py | 18 +- .../test_websearch_streaming_wrap.py | 2 +- .../test_litellm_logging.py | 2 +- .../litellm_core_utils/test_token_counter.py | 2 +- .../test_anthropic_guardrail_handler.py | 2 +- .../test_anthropic_chat_transformation.py | 6 +- .../adapters/__init__.py | 0 ...ropic_messages_adapters_transformation.py} | 12 +- .../test_handler_context_management.py | 4 +- .../test_handler_output_config_passthrough.py | 2 +- .../adapters/test_handler_prompt_cache_key.py | 2 +- ..._handler_reasoning_effort_normalization.py | 2 +- .../test_handler_safeguard_results.py | 6 +- .../test_streaming_iterator_combined_chunk.py | 2 +- .../test_streaming_iterator_compaction.py | 2 +- .../test_streaming_iterator_empty_choices.py | 2 +- .../test_streaming_iterator_first_delta.py | 2 +- .../test_streaming_iterator_message_id.py | 4 +- ...est_streaming_iterator_mid_stream_error.py | 2 +- .../test_streaming_iterator_sse_stream.py | 2 +- .../test_streaming_iterator_stop_reason.py | 2 +- .../test_streaming_iterator_tool_args.py | 2 +- .../{pass_through => messages}/conftest.py | 0 .../context_management/__init__.py | 0 .../test_clear_tool_uses.py | 8 +- .../context_management/test_compact.py | 190 +++++++++--------- .../context_management/test_constants.py | 2 +- .../context_management/test_dispatcher.py | 2 +- .../messages => messages/native}/__init__.py | 0 .../native}/test_advisor_integration.py | 24 +-- .../test_agentic_streaming_iterator.py | 2 +- .../native}/test_anthropic_messages_effort.py | 2 +- ..._anthropic_messages_encrypted_reasoning.py | 2 +- .../test_anthropic_messages_handler.py} | 96 ++++----- ...est_anthropic_messages_per_turn_control.py | 2 +- .../native}/test_anthropic_messages_router.py | 0 .../test_anthropic_messages_service_tier.py | 2 +- .../native}/test_anthropic_messages_speed.py | 4 +- ...t_anthropic_messages_structured_outputs.py | 2 +- ...test_anthropic_messages_transformation.py} | 2 +- .../native}/test_content_after_stop_reason.py | 2 +- .../native}/test_fake_stream_iterator.py | 2 +- .../native}/test_mcp_handler.py | 22 +- .../native}/test_mid_conversation_system.py | 2 +- .../native}/test_parallel_tool_calls.py | 2 +- .../test_reasoning_auto_summary_messages.py | 6 +- .../test_reasoning_effort_translation.py | 2 +- .../test_request_optional_param_utils.py | 2 +- .../native}/test_response_cache.py | 6 +- .../native}/test_sse_wrapper.py | 2 +- .../native}/test_streaming_iterator.py | 12 +- .../responses_adapters/__init__.py | 0 .../test_responses_adapters_handler.py | 2 +- ...t_responses_adapters_streaming_iterator.py | 6 +- .../test_responses_adapters_transformation.py | 6 +- .../messages/test_advisor_orchestration.py | 138 ++++++------- ...hropic_native_passthrough_spend_logging.py | 0 .../test_caller_model_checks.py | 4 +- .../test_convertible_blocks.py | 6 +- .../test_reasoning_effort_fields.py | 10 +- .../test_safeguards.py | 8 +- .../test_stream_assembly.py | 2 +- .../pass_through/test_pass_through_shim.py | 60 ++++++ .../anthropic/test_anthropic_common_utils.py | 28 +-- .../test_anthropic_prompt_cache_prediction.py | 2 +- .../test_anthropic_claude_transformation.py | 2 +- .../custom_httpx/test_llm_http_handler.py | 14 +- ...pseek_anthropic_messages_transformation.py | 2 +- ..._github_copilot_messages_transformation.py | 2 +- .../chat/test_transformation.py | 2 +- ...test_openai_count_tokens_transformation.py | 2 +- ..._like_anthropic_messages_transformation.py | 2 +- ...ncent_anthropic_messages_transformation.py | 2 +- .../test_get_complete_url_trailing_slash.py | 2 +- .../vertex_ai/count_tokens/test_context.py | 4 +- .../test_vertex_ai_gemini_transformation.py | 2 +- ...test_vertex_and_google_ai_studio_gemini.py | 6 +- tests/unit/llms/vertex_ai/test_vertex.py | 2 +- .../experimental_pass_through/__init__.py | 0 .../test_experimental_pass_through_shim.py | 20 ++ ...artner_models_anthropic_messages_config.py | 2 +- .../test_bedrock_guardrails.py | 2 +- .../guardrail_hooks/test_headroom.py | 2 +- ...t_anthropic_passthrough_logging_handler.py | 2 +- .../test_streaming_handler_interrupt.py | 2 +- .../test_spend_tracking_utils.py | 2 +- tests/unit/proxy/test_budget_reservation.py | 4 +- .../proxy/test_common_request_processing.py | 8 +- .../proxy_logging/test_streaming_hooks.py | 2 +- .../unit/responses/mcp/test_tool_precheck.py | 2 +- .../test_fallback_event_handlers.py | 2 +- .../unit/rust_bridge/messages/test_secrets.py | 2 +- tests/unit/test_private_usage_aliases.py | 14 +- tests/unit/test_router/test_router.py | 4 +- 185 files changed, 729 insertions(+), 552 deletions(-) create mode 100644 litellm/litellm_core_utils/renamed_package_shim.py rename litellm/llms/anthropic/{pass_through => messages}/adapters/__init__.py (100%) rename litellm/llms/anthropic/{pass_through => messages}/adapters/handler.py (98%) rename litellm/llms/anthropic/{pass_through => messages}/adapters/streaming_iterator.py (99%) rename litellm/llms/anthropic/{pass_through => messages}/adapters/transformation.py (99%) rename litellm/llms/anthropic/{pass_through => messages}/architecture.md (100%) rename litellm/llms/anthropic/{pass_through => messages}/caller_model_checks.py (100%) rename litellm/llms/anthropic/{pass_through => messages}/context_management/__init__.py (100%) rename litellm/llms/anthropic/{pass_through => messages}/context_management/constants.py (100%) rename litellm/llms/anthropic/{pass_through => messages}/context_management/dispatcher.py (98%) rename litellm/llms/anthropic/{pass_through => messages}/context_management/editors/__init__.py (100%) rename litellm/llms/anthropic/{pass_through => messages}/context_management/editors/clear_tool_uses.py (100%) rename litellm/llms/anthropic/{pass_through => messages}/context_management/editors/compact.py (99%) rename litellm/llms/anthropic/{pass_through => messages}/context_management/errors.py (100%) rename litellm/llms/anthropic/{pass_through => messages}/context_management/placeholders.py (100%) rename litellm/llms/anthropic/{pass_through => messages}/context_management/result.py (100%) rename litellm/llms/anthropic/{pass_through => messages}/convertible_blocks.py (99%) rename litellm/llms/anthropic/{pass_through/messages => messages/native}/agentic_streaming_iterator.py (98%) rename litellm/llms/anthropic/{pass_through/messages => messages/native}/fake_stream_iterator.py (99%) rename litellm/llms/anthropic/{pass_through/messages => messages/native}/handler.py (99%) rename litellm/llms/anthropic/{pass_through/messages => messages/native}/interceptors/README.md (100%) rename litellm/llms/anthropic/{pass_through/messages => messages/native}/interceptors/__init__.py (100%) rename litellm/llms/anthropic/{pass_through/messages => messages/native}/interceptors/advisor.py (99%) rename litellm/llms/anthropic/{pass_through/messages => messages/native}/interceptors/base.py (100%) rename litellm/llms/anthropic/{pass_through/messages => messages/native}/mcp_handler.py (95%) rename litellm/llms/anthropic/{pass_through/messages => messages/native}/mid_conversation_system.py (100%) rename litellm/llms/anthropic/{pass_through/messages => messages/native}/response_cache.py (98%) rename litellm/llms/anthropic/{pass_through/messages => messages/native}/streaming_iterator.py (99%) rename litellm/llms/anthropic/{pass_through/messages => messages/native}/transformation.py (99%) rename litellm/llms/anthropic/{pass_through/messages => messages/native}/utils.py (99%) rename litellm/llms/anthropic/{pass_through => messages}/responses_adapters/__init__.py (100%) rename litellm/llms/anthropic/{pass_through => messages}/responses_adapters/handler.py (99%) rename litellm/llms/anthropic/{pass_through => messages}/responses_adapters/streaming_iterator.py (98%) rename litellm/llms/anthropic/{pass_through => messages}/responses_adapters/transformation.py (99%) rename litellm/llms/anthropic/{pass_through => messages}/safeguards.py (99%) rename litellm/llms/anthropic/{pass_through => messages}/stream_assembly.py (99%) rename litellm/llms/anthropic/{pass_through => messages}/utils.py (100%) create mode 100644 litellm/llms/anthropic/pass_through/__init__.py create mode 100644 litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/__init__.py rename litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/{experimental_pass_through => messages}/transformation.py (99%) rename tests/unit/llms/anthropic/{pass_through => messages}/adapters/__init__.py (100%) rename tests/unit/llms/anthropic/{pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py => messages/adapters/test_anthropic_messages_adapters_transformation.py} (99%) rename tests/unit/llms/anthropic/{pass_through => messages}/adapters/test_handler_context_management.py (95%) rename tests/unit/llms/anthropic/{pass_through => messages}/adapters/test_handler_output_config_passthrough.py (99%) rename tests/unit/llms/anthropic/{pass_through => messages}/adapters/test_handler_prompt_cache_key.py (97%) rename tests/unit/llms/anthropic/{pass_through => messages}/adapters/test_handler_reasoning_effort_normalization.py (99%) rename tests/unit/llms/anthropic/{pass_through => messages}/adapters/test_handler_safeguard_results.py (98%) rename tests/unit/llms/anthropic/{pass_through => messages}/adapters/test_streaming_iterator_combined_chunk.py (99%) rename tests/unit/llms/anthropic/{pass_through => messages}/adapters/test_streaming_iterator_compaction.py (98%) rename tests/unit/llms/anthropic/{pass_through => messages}/adapters/test_streaming_iterator_empty_choices.py (97%) rename tests/unit/llms/anthropic/{pass_through => messages}/adapters/test_streaming_iterator_first_delta.py (99%) rename tests/unit/llms/anthropic/{pass_through => messages}/adapters/test_streaming_iterator_message_id.py (96%) rename tests/unit/llms/anthropic/{pass_through => messages}/adapters/test_streaming_iterator_mid_stream_error.py (99%) rename tests/unit/llms/anthropic/{pass_through => messages}/adapters/test_streaming_iterator_sse_stream.py (98%) rename tests/unit/llms/anthropic/{pass_through => messages}/adapters/test_streaming_iterator_stop_reason.py (97%) rename tests/unit/llms/anthropic/{pass_through => messages}/adapters/test_streaming_iterator_tool_args.py (99%) rename tests/unit/llms/anthropic/{pass_through => messages}/conftest.py (100%) rename tests/unit/llms/anthropic/{pass_through => messages}/context_management/__init__.py (100%) rename tests/unit/llms/anthropic/{pass_through => messages}/context_management/test_clear_tool_uses.py (98%) rename tests/unit/llms/anthropic/{pass_through => messages}/context_management/test_compact.py (91%) rename tests/unit/llms/anthropic/{pass_through => messages}/context_management/test_constants.py (99%) rename tests/unit/llms/anthropic/{pass_through => messages}/context_management/test_dispatcher.py (98%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/__init__.py (100%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_advisor_integration.py (92%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_agentic_streaming_iterator.py (99%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_anthropic_messages_effort.py (99%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_anthropic_messages_encrypted_reasoning.py (96%) rename tests/unit/llms/anthropic/{pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py => messages/native/test_anthropic_messages_handler.py} (96%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_anthropic_messages_per_turn_control.py (98%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_anthropic_messages_router.py (100%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_anthropic_messages_service_tier.py (92%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_anthropic_messages_speed.py (96%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_anthropic_messages_structured_outputs.py (98%) rename tests/unit/llms/anthropic/{pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py => messages/native/test_anthropic_messages_transformation.py} (99%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_content_after_stop_reason.py (99%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_fake_stream_iterator.py (97%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_mcp_handler.py (95%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_mid_conversation_system.py (97%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_parallel_tool_calls.py (99%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_reasoning_auto_summary_messages.py (97%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_reasoning_effort_translation.py (99%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_request_optional_param_utils.py (98%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_response_cache.py (98%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_sse_wrapper.py (98%) rename tests/unit/llms/anthropic/{pass_through/messages => messages/native}/test_streaming_iterator.py (99%) rename tests/unit/llms/anthropic/{pass_through => messages}/responses_adapters/__init__.py (100%) rename tests/unit/llms/anthropic/{pass_through => messages}/responses_adapters/test_responses_adapters_handler.py (99%) rename tests/unit/llms/anthropic/{pass_through => messages}/responses_adapters/test_responses_adapters_streaming_iterator.py (99%) rename tests/unit/llms/anthropic/{pass_through => messages}/responses_adapters/test_responses_adapters_transformation.py (99%) rename tests/unit/llms/anthropic/{pass_through => messages}/test_anthropic_native_passthrough_spend_logging.py (100%) rename tests/unit/llms/anthropic/{pass_through => messages}/test_caller_model_checks.py (99%) rename tests/unit/llms/anthropic/{pass_through => messages}/test_convertible_blocks.py (97%) rename tests/unit/llms/anthropic/{pass_through => messages}/test_reasoning_effort_fields.py (97%) rename tests/unit/llms/anthropic/{pass_through => messages}/test_safeguards.py (99%) rename tests/unit/llms/anthropic/{pass_through => messages}/test_stream_assembly.py (98%) create mode 100644 tests/unit/llms/anthropic/pass_through/test_pass_through_shim.py create mode 100644 tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/__init__.py create mode 100644 tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/test_experimental_pass_through_shim.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 70c2d600ba3..2177e483a1b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -336,9 +336,9 @@ Each translation is isolated in its own file, making it easy to test and modify | `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` | | `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` | | `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` | -| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/pass_through/messages/transformation.py` | +| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/messages/native/transformation.py` | | `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py` | -| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` | +| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py` | | Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` | ### Example: Debugging prompt caching diff --git a/litellm-rust/crates/callbacks-legacy-python/src/mapping.rs b/litellm-rust/crates/callbacks-legacy-python/src/mapping.rs index a4aee4eb4da..cfa9e53aa7b 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/mapping.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/mapping.rs @@ -235,7 +235,7 @@ python_callbacks! { "async_log_stream_event", "async_post_mcp_tool_call_hook", ], - Dispatch::Python("litellm.llms.anthropic.pass_through.messages.handler") => [ + Dispatch::Python("litellm.llms.anthropic.messages.native.handler") => [ "async_pre_request_hook", ], Dispatch::Python("litellm.litellm_core_utils.streaming_handler") => [ diff --git a/litellm/__init__.py b/litellm/__init__.py index 9ec11dd1291..c74292c5888 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1788,7 +1788,7 @@ if TYPE_CHECKING: SagemakerNovaConfig as SagemakerNovaConfig, ) from .llms.cohere.chat.transformation import CohereChatConfig as CohereChatConfig - from .llms.anthropic.pass_through.messages.transformation import ( + from .llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig as AnthropicMessagesConfig, ) from .llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 0f8288941e2..f43e369e092 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -801,7 +801,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ), "CohereChatConfig": (".llms.cohere.chat.transformation", "CohereChatConfig"), "AnthropicMessagesConfig": ( - ".llms.anthropic.pass_through.messages.transformation", + ".llms.anthropic.messages.native.transformation", "AnthropicMessagesConfig", ), "BedrockClaudePlatformMessagesConfig": ( diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 7089ff78835..affabc6cc74 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -56,7 +56,7 @@ from litellm.types.utils import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.llms.anthropic.pass_through.messages.response_cache import ( + from litellm.llms.anthropic.messages.native.response_cache import ( AnthropicMessagesStreamCacheWriter, ) from litellm.types.utils import PromptTokensDetailsWrapper @@ -182,7 +182,7 @@ def _should_defer_streaming_cache_hit_callbacks(*, cached_result: object) -> boo spend and callback records. A plain (non-stream) replay logs here, since nothing else will. """ - from litellm.llms.anthropic.pass_through.messages.response_cache import ( + from litellm.llms.anthropic.messages.native.response_cache import ( CachedAnthropicMessagesStreamIterator, ) from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator @@ -1040,7 +1040,7 @@ class LLMCachingHandler: elif ( call_type == CallTypes.anthropic_messages.value or call_type == CallTypes.aanthropic_messages.value ) and isinstance(cached_result, dict): - from litellm.llms.anthropic.pass_through.messages.response_cache import ( + from litellm.llms.anthropic.messages.native.response_cache import ( convert_cached_anthropic_messages_result, ) @@ -1270,7 +1270,7 @@ class LLMCachingHandler: return result if not isinstance(result, AsyncIterator): return result - from litellm.llms.anthropic.pass_through.messages.response_cache import ( + from litellm.llms.anthropic.messages.native.response_cache import ( AnthropicMessagesStreamCacheWriter, ) diff --git a/litellm/integrations/otel/model/request_io.py b/litellm/integrations/otel/model/request_io.py index 287d28a0d17..82225531e23 100644 --- a/litellm/integrations/otel/model/request_io.py +++ b/litellm/integrations/otel/model/request_io.py @@ -6,7 +6,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm.integrations.otel.mappers.utils import json_or_none -from litellm.llms.anthropic.pass_through.stream_assembly import assemble_anthropic_sse_stream, is_raw_sse_stream +from litellm.llms.anthropic.messages.stream_assembly import assemble_anthropic_sse_stream, is_raw_sse_stream from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse from litellm.types.utils import ModelResponse, ModelResponseStream diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index d0ed30d40dd..3a2d7ee33bb 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -113,7 +113,7 @@ def _chat_request_from_anthropic_messages( because the logged optional_params switch dialect per provider path (the bridge's inner completion rewrites them to chat shape mid-flight); the adapter translates them alongside the messages, and sampling params copy through untranslated.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) diff --git a/litellm/integrations/websearch_interception/ARCHITECTURE.md b/litellm/integrations/websearch_interception/ARCHITECTURE.md index 255c1f1adb2..4ba8c2daa63 100644 --- a/litellm/integrations/websearch_interception/ARCHITECTURE.md +++ b/litellm/integrations/websearch_interception/ARCHITECTURE.md @@ -70,7 +70,7 @@ Claude Code (Anthropic's official CLI) sends web search requests using Anthropic Native tools are converted to LiteLLM standard format **before** sending to the provider: -1. **Conversion Point** (`litellm/llms/anthropic/pass_through/messages/handler.py`): +1. **Conversion Point** (`litellm/llms/anthropic/messages/native/handler.py`): - In `anthropic_messages()` function (lines 60-127) - Runs BEFORE the API request is made - Detects native web search tools using `is_web_search_tool()` diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 3bc8dc4e23e..de8c6e6c4cc 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -6354,7 +6354,7 @@ def is_flat_responses_tool(tool: Mapping[str, object]) -> bool: def chat_completion_tools_for(tool: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: - from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter + from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) diff --git a/litellm/litellm_core_utils/renamed_package_shim.py b/litellm/litellm_core_utils/renamed_package_shim.py new file mode 100644 index 00000000000..7f7f39a351a --- /dev/null +++ b/litellm/litellm_core_utils/renamed_package_shim.py @@ -0,0 +1,87 @@ +import importlib +import sys +import warnings +from collections.abc import Callable, Sequence +from importlib.abc import Loader, MetaPathFinder +from importlib.machinery import ModuleSpec +from importlib.util import spec_from_loader +from types import ModuleType +from typing import Final + + +class _AliasLoader(Loader): + """Hand the import system the module already loaded under its new name instead of executing a copy.""" + + def __init__(self, target: ModuleType) -> None: + self._target: Final = target + self._spec: Final = target.__spec__ + + def create_module(self, spec: ModuleSpec) -> ModuleType: + return self._target + + def exec_module(self, module: ModuleType) -> None: + module.__spec__ = self._spec # rebind-ok: the loader restores the spec the import system replaced + + +class _RenamedPackageFinder(MetaPathFinder): + def __init__(self, old_name: str, renames: tuple[tuple[str, str], ...]) -> None: + self._prefix: Final = old_name + "." + self._renames: Final = renames + + def new_name_for(self, old_module_name: str) -> str | None: + if not old_module_name.startswith(self._prefix): + return None + for old_prefix, new_prefix in self._renames: + if old_module_name == old_prefix or old_module_name.startswith(old_prefix + "."): + return new_prefix + old_module_name[len(old_prefix) :] + return None + + def find_spec( + self, + fullname: str, + path: Sequence[str] | None = None, + target: ModuleType | None = None, + ) -> ModuleSpec | None: + new_name: Final = self.new_name_for(fullname) + if new_name is None: + return None + module: Final = importlib.import_module(new_name) + spec: Final = spec_from_loader(fullname, _AliasLoader(module), is_package=hasattr(module, "__path__")) + if spec is not None and spec.submodule_search_locations is not None: + spec.submodule_search_locations = list(module.__path__) # pyright: ignore[reportAny] # module attributes are untyped + return spec + + +def alias_renamed_package( + old_name: str, + new_name: str, + *, + nested_renames: tuple[tuple[str, str], ...] = (), +) -> Callable[[str], ModuleType]: + """Make every module of the package now at ``new_name`` importable under its old dotted name. + + Nothing is imported up front: a meta path finder resolves ``old_name.x.y`` to the module object at + ``new_name.x.y`` the moment something imports it, so a deep import of the old path is the very same object + as the new path and importing one old module loads no more than importing its new counterpart would. + ``nested_renames`` lists (old, new) dotted prefixes for sub-packages that changed name inside the move, + longest first. The returned function is the shim package's module ``__getattr__``, so attribute access on + the old package reaches the renamed sub-packages too. + """ + warnings.warn( + f"{old_name} moved to {new_name}; update imports, the old path will be removed in a future release", + DeprecationWarning, + stacklevel=3, + ) + finder: Final = _RenamedPackageFinder(old_name, (*nested_renames, (old_name, new_name))) + if not any( + isinstance(entry, _RenamedPackageFinder) and entry.new_name_for(old_name + ".x") for entry in sys.meta_path + ): + sys.meta_path.insert(0, finder) + + def getattr_(name: str) -> ModuleType: + try: + return importlib.import_module(f"{old_name}.{name}") + except ModuleNotFoundError as error: + raise AttributeError(f"module {old_name!r} has no attribute {name!r}") from error + + return getattr_ diff --git a/litellm/litellm_core_utils/served_output_texts.py b/litellm/litellm_core_utils/served_output_texts.py index e23db5200a3..854785f304b 100644 --- a/litellm/litellm_core_utils/served_output_texts.py +++ b/litellm/litellm_core_utils/served_output_texts.py @@ -110,7 +110,7 @@ def served_output_texts(response: object) -> ServedTexts | None: def served_stream_output_texts(chunks: Sequence[object]) -> ServedTexts | None: if chunks and all(isinstance(chunk, ModelResponseStream) for chunk in chunks): return _chat_stream_texts(tuple(chunk for chunk in chunks if isinstance(chunk, ModelResponseStream))) - from litellm.llms.anthropic.pass_through.stream_assembly import ( + from litellm.llms.anthropic.messages.stream_assembly import ( assemble_anthropic_sse_stream, is_anthropic_sse_stream, ) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 3ab21099d3a..d371aab4581 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -26,18 +26,18 @@ from typing_extensions import ReadOnly, TypedDict, assert_never from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.anthropic.chat.transformation import AnthropicConfig -from litellm.llms.anthropic.pass_through.adapters.transformation import ( +from litellm.llms.anthropic.messages.adapters.transformation import ( OPENAI_MAX_TOOL_NAME_LENGTH, LiteLLMAnthropicMessagesAdapter, is_provider_native_tool_dict, truncate_tool_name, ) -from litellm.llms.anthropic.pass_through.convertible_blocks import ( +from litellm.llms.anthropic.messages.convertible_blocks import ( keep_non_string_tool_result_texts, scannable_tool_result_request, ) -from litellm.llms.anthropic.pass_through.messages.utils import filtered_safeguard_results, redacted_stop_details -from litellm.llms.anthropic.pass_through.stream_assembly import ( +from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details +from litellm.llms.anthropic.messages.stream_assembly import ( build_complete_streaming_response, build_usage_only_response_from_chunks, ) @@ -435,7 +435,7 @@ class AnthropicMessagesHandler(BaseTranslation): def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]: import uuid - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) from litellm.llms.base_llm.guardrail_translation.utils import ( diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index c2d87c641be..eb8e595c849 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -162,7 +162,7 @@ def _has_non_chat_user_block(message: Mapping[str, object]) -> bool: def _gets_placeholder_name(tool: Mapping[str, object]) -> bool: - from litellm.llms.anthropic.pass_through.adapters.transformation import is_provider_native_tool_dict + from litellm.llms.anthropic.messages.adapters.transformation import is_provider_native_tool_dict name: Final = tool.get("name") return not (_is_chat_tool(tool) or is_provider_native_tool_dict(tool) or (isinstance(name, str) and name.strip())) diff --git a/litellm/llms/anthropic/pass_through/adapters/__init__.py b/litellm/llms/anthropic/messages/adapters/__init__.py similarity index 100% rename from litellm/llms/anthropic/pass_through/adapters/__init__.py rename to litellm/llms/anthropic/messages/adapters/__init__.py diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py similarity index 98% rename from litellm/llms/anthropic/pass_through/adapters/handler.py rename to litellm/llms/anthropic/messages/adapters/handler.py index 01e781485a0..e5dd1d3c94f 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -15,23 +15,23 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject -from litellm.llms.anthropic.pass_through.adapters.transformation import ( +from litellm.llms.anthropic.messages.adapters.transformation import ( AnthropicAdapter, ) -from litellm.llms.anthropic.pass_through.context_management import ( +from litellm.llms.anthropic.messages.context_management import ( AnthropicContextManagementError, PolyfillResult, apply_context_management, ) -from litellm.llms.anthropic.pass_through.convertible_blocks import reject_malformed_tool_result_text -from litellm.llms.anthropic.pass_through.safeguards import ( +from litellm.llms.anthropic.messages.convertible_blocks import reject_malformed_tool_result_text +from litellm.llms.anthropic.messages.safeguards import ( SafeguardsEvaluator, build_safeguards_evaluator, read_classifier_model_setting, requested_dangerous_tool_use, with_safeguard_results, ) -from litellm.llms.anthropic.pass_through.utils import ( +from litellm.llms.anthropic.messages.utils import ( ProxySubCallLoggingKwargs, is_reasoning_auto_summary_enabled, litellm_logging_obj_from_kwargs, @@ -158,7 +158,7 @@ async def prepare_context_managed_request( drop_invalid_edits: bool = False, ) -> PolyfillResult | None: """Apply client compaction history, then optional context_management polyfill.""" - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( apply_client_compaction_block_history, ) @@ -239,7 +239,7 @@ def _polyfill_will_run( if edits is None: return False - from litellm.llms.anthropic.pass_through.context_management.constants import ( + from litellm.llms.anthropic.messages.context_management.constants import ( COMPACT_EDIT_TYPE, ) @@ -265,7 +265,7 @@ def _spec_has_non_compact_edits( if edits is None: return False - from litellm.llms.anthropic.pass_through.context_management.constants import ( + from litellm.llms.anthropic.messages.context_management.constants import ( COMPACT_EDIT_TYPE, ) @@ -298,7 +298,7 @@ def _normalize_spec_edits( if context_management_explicitly_dropped(additional_drop_params): return None - from litellm.llms.anthropic.pass_through.context_management.dispatcher import ( + from litellm.llms.anthropic.messages.context_management.dispatcher import ( normalize_spec, ) @@ -543,7 +543,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: Handles both string ("max") and dict ({"effort": "max", "summary": ...}) formats. Uses model registry to check supports_xhigh/supports_minimal. """ - from litellm.llms.anthropic.pass_through.utils import ( + from litellm.llms.anthropic.messages.utils import ( normalize_reasoning_effort_value, ) diff --git a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py similarity index 99% rename from litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py rename to litellm/llms/anthropic/messages/adapters/streaming_iterator.py index 4eb53f35a03..13acb19e454 100644 --- a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py @@ -27,8 +27,8 @@ from litellm._uuid import uuid from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items from litellm.litellm_core_utils.hidden_params import set_hidden_params -from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults -from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame +from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults +from litellm.llms.anthropic.messages.utils import anthropic_sse_frame from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.anthropic import ( AppliedEdit, @@ -46,7 +46,7 @@ from litellm.types.utils import AdapterCompletionStreamWrapper, ChatCompletionDe if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject - from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator + from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponseStream @@ -234,7 +234,7 @@ class _CombinedChunkSplitter: @staticmethod def _is_combined(chunk: "ModelResponseStream") -> bool: """True if ``chunk`` carries response content AND a finish_reason.""" - from litellm.llms.anthropic.pass_through.messages.utils import ( + from litellm.llms.anthropic.messages.native.utils import ( openai_chat_refusal_text, ) @@ -1308,7 +1308,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): delta: Final = processed_chunk["delta"] if delta.get("stop_reason") == "max_tokens": return processed_chunk - from litellm.llms.anthropic.pass_through.messages.utils import ( + from litellm.llms.anthropic.messages.native.utils import ( refusal_stop_details, ) @@ -1363,7 +1363,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): @staticmethod def _is_blank_delta(chunk: "ModelResponseStream") -> bool: from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block - from litellm.llms.anthropic.pass_through.messages.utils import ( + from litellm.llms.anthropic.messages.native.utils import ( openai_chat_refusal_text, ) @@ -1400,7 +1400,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): - Different content types in the response - Specific markers in the content """ - from litellm.llms.anthropic.pass_through.messages.utils import ( + from litellm.llms.anthropic.messages.native.utils import ( openai_chat_refusal_text, ) diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py similarity index 99% rename from litellm/llms/anthropic/pass_through/adapters/transformation.py rename to litellm/llms/anthropic/messages/adapters/transformation.py index 9b1297dbd75..5ab112d0bfe 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -10,8 +10,8 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.hidden_params import get_hidden_params -from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_part -from litellm.llms.anthropic.pass_through.utils import ( +from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_part +from litellm.llms.anthropic.messages.utils import ( is_reasoning_auto_summary_enabled, json_copy_or_original, prompt_cache_key_from_user_id, @@ -139,14 +139,14 @@ from litellm.llms.anthropic.common_utils import ( normalize_anthropic_tool_use_id, strip_encrypted_reasoning_blocks_from_anthropic_messages, ) -from litellm.llms.anthropic.pass_through.context_management import ( +from litellm.llms.anthropic.messages.context_management import ( PolyfillResult, ) -from litellm.llms.anthropic.pass_through.messages.mid_conversation_system import ( +from litellm.llms.anthropic.messages.native.mid_conversation_system import ( convert_mid_conversation_system_turns, is_system_role_message, ) -from litellm.llms.anthropic.pass_through.messages.utils import ( +from litellm.llms.anthropic.messages.native.utils import ( openai_chat_refusal_text, refusal_stop_details, ) @@ -210,7 +210,7 @@ from .streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject - from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator + from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator from litellm.types.llms.anthropic import ContentBlockContentBlockDict ToolResultContent: TypeAlias = str | list[ToolMessageContentPart] diff --git a/litellm/llms/anthropic/pass_through/architecture.md b/litellm/llms/anthropic/messages/architecture.md similarity index 100% rename from litellm/llms/anthropic/pass_through/architecture.md rename to litellm/llms/anthropic/messages/architecture.md diff --git a/litellm/llms/anthropic/pass_through/caller_model_checks.py b/litellm/llms/anthropic/messages/caller_model_checks.py similarity index 100% rename from litellm/llms/anthropic/pass_through/caller_model_checks.py rename to litellm/llms/anthropic/messages/caller_model_checks.py diff --git a/litellm/llms/anthropic/pass_through/context_management/__init__.py b/litellm/llms/anthropic/messages/context_management/__init__.py similarity index 100% rename from litellm/llms/anthropic/pass_through/context_management/__init__.py rename to litellm/llms/anthropic/messages/context_management/__init__.py diff --git a/litellm/llms/anthropic/pass_through/context_management/constants.py b/litellm/llms/anthropic/messages/context_management/constants.py similarity index 100% rename from litellm/llms/anthropic/pass_through/context_management/constants.py rename to litellm/llms/anthropic/messages/context_management/constants.py diff --git a/litellm/llms/anthropic/pass_through/context_management/dispatcher.py b/litellm/llms/anthropic/messages/context_management/dispatcher.py similarity index 98% rename from litellm/llms/anthropic/pass_through/context_management/dispatcher.py rename to litellm/llms/anthropic/messages/context_management/dispatcher.py index bce653df6d1..1557cf206ac 100644 --- a/litellm/llms/anthropic/pass_through/context_management/dispatcher.py +++ b/litellm/llms/anthropic/messages/context_management/dispatcher.py @@ -18,7 +18,7 @@ from .editors import ( from .result import PolyfillResult if TYPE_CHECKING: - from litellm.llms.anthropic.pass_through.utils import ProxySubCallLoggingKwargs + from litellm.llms.anthropic.messages.utils import ProxySubCallLoggingKwargs from litellm.router import Router from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth diff --git a/litellm/llms/anthropic/pass_through/context_management/editors/__init__.py b/litellm/llms/anthropic/messages/context_management/editors/__init__.py similarity index 100% rename from litellm/llms/anthropic/pass_through/context_management/editors/__init__.py rename to litellm/llms/anthropic/messages/context_management/editors/__init__.py diff --git a/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py similarity index 100% rename from litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py rename to litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py diff --git a/litellm/llms/anthropic/pass_through/context_management/editors/compact.py b/litellm/llms/anthropic/messages/context_management/editors/compact.py similarity index 99% rename from litellm/llms/anthropic/pass_through/context_management/editors/compact.py rename to litellm/llms/anthropic/messages/context_management/editors/compact.py index 5eb2863a241..254d1c4c058 100644 --- a/litellm/llms/anthropic/pass_through/context_management/editors/compact.py +++ b/litellm/llms/anthropic/messages/context_management/editors/compact.py @@ -21,12 +21,12 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify -from litellm.llms.anthropic.pass_through.caller_model_checks import ( +from litellm.llms.anthropic.messages.caller_model_checks import ( caller_can_call_model, caller_within_model_budget, caller_within_model_rate_limit, ) -from litellm.llms.anthropic.pass_through.utils import ( +from litellm.llms.anthropic.messages.utils import ( ProxySubCallLoggingKwargs, proxy_general_settings, proxy_sub_call_metadata, @@ -269,7 +269,7 @@ def _count_effective_tokens( threshold check matches the downstream ``input_tokens`` metric. """ # Local import to avoid pulling the adapter at module load time. - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -426,7 +426,7 @@ def _build_summary_messages( system prompt); the conversation history is translated to OpenAI shape; the summarization prompt is appended as a final user turn. """ - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) diff --git a/litellm/llms/anthropic/pass_through/context_management/errors.py b/litellm/llms/anthropic/messages/context_management/errors.py similarity index 100% rename from litellm/llms/anthropic/pass_through/context_management/errors.py rename to litellm/llms/anthropic/messages/context_management/errors.py diff --git a/litellm/llms/anthropic/pass_through/context_management/placeholders.py b/litellm/llms/anthropic/messages/context_management/placeholders.py similarity index 100% rename from litellm/llms/anthropic/pass_through/context_management/placeholders.py rename to litellm/llms/anthropic/messages/context_management/placeholders.py diff --git a/litellm/llms/anthropic/pass_through/context_management/result.py b/litellm/llms/anthropic/messages/context_management/result.py similarity index 100% rename from litellm/llms/anthropic/pass_through/context_management/result.py rename to litellm/llms/anthropic/messages/context_management/result.py diff --git a/litellm/llms/anthropic/pass_through/convertible_blocks.py b/litellm/llms/anthropic/messages/convertible_blocks.py similarity index 99% rename from litellm/llms/anthropic/pass_through/convertible_blocks.py rename to litellm/llms/anthropic/messages/convertible_blocks.py index 9d651f8d721..09d662da543 100644 --- a/litellm/llms/anthropic/pass_through/convertible_blocks.py +++ b/litellm/llms/anthropic/messages/convertible_blocks.py @@ -12,7 +12,7 @@ from typing_extensions import assert_never import litellm from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.factory import sanitize_anthropic_tool_use_id -from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages +from litellm.llms.anthropic.messages.native.utils import normalize_native_messages from litellm.types.utils import TokenCountMessagesFormat _CONVERTIBLE_BLOCK_TYPES: Final = MappingProxyType( diff --git a/litellm/llms/anthropic/pass_through/messages/agentic_streaming_iterator.py b/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py similarity index 98% rename from litellm/llms/anthropic/pass_through/messages/agentic_streaming_iterator.py rename to litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py index 922a6e21bae..25d4d7b3ce1 100644 --- a/litellm/llms/anthropic/pass_through/messages/agentic_streaming_iterator.py +++ b/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py @@ -336,7 +336,7 @@ class AgenticAnthropicStreamingIterator: await task async def aclose(self) -> None: - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( aclose_if_supported, ) @@ -379,7 +379,7 @@ class AgenticAnthropicStreamingIterator: if hasattr(result, "__aiter__"): self._follow_up_iterator = result.__aiter__() elif isinstance(result, dict): - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( diff --git a/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py b/litellm/llms/anthropic/messages/native/fake_stream_iterator.py similarity index 99% rename from litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py rename to litellm/llms/anthropic/messages/native/fake_stream_iterator.py index 2b6161d98f6..39bdfbbc53a 100644 --- a/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py +++ b/litellm/llms/anthropic/messages/native/fake_stream_iterator.py @@ -12,7 +12,7 @@ import json from collections.abc import Mapping, Sequence from typing import Final, cast -from litellm.llms.anthropic.pass_through.messages.utils import anthropic_message_delta +from litellm.llms.anthropic.messages.native.utils import anthropic_message_delta from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) diff --git a/litellm/llms/anthropic/pass_through/messages/handler.py b/litellm/llms/anthropic/messages/native/handler.py similarity index 99% rename from litellm/llms/anthropic/pass_through/messages/handler.py rename to litellm/llms/anthropic/messages/native/handler.py index e1757fbc2d3..6f97a019a96 100644 --- a/litellm/llms/anthropic/pass_through/messages/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -365,7 +365,7 @@ async def _try_websearch_short_circuit( ), ) if stream: - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) @@ -659,7 +659,7 @@ def anthropic_messages_handler( # reference the provider cannot resolve. Popped from kwargs so it never reaches the provider. skip_mcp_handler: Final = kwargs.pop("_skip_mcp_handler", False) if not skip_mcp_handler and tools and may_name_litellm_mcp_gateway(tools): # pyright: ignore[reportUnknownArgumentType] # untyped tools list - from litellm.llms.anthropic.pass_through.messages.mcp_handler import ( + from litellm.llms.anthropic.messages.native.mcp_handler import ( anthropic_messages_with_mcp, ) diff --git a/litellm/llms/anthropic/pass_through/messages/interceptors/README.md b/litellm/llms/anthropic/messages/native/interceptors/README.md similarity index 100% rename from litellm/llms/anthropic/pass_through/messages/interceptors/README.md rename to litellm/llms/anthropic/messages/native/interceptors/README.md diff --git a/litellm/llms/anthropic/pass_through/messages/interceptors/__init__.py b/litellm/llms/anthropic/messages/native/interceptors/__init__.py similarity index 100% rename from litellm/llms/anthropic/pass_through/messages/interceptors/__init__.py rename to litellm/llms/anthropic/messages/native/interceptors/__init__.py diff --git a/litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py b/litellm/llms/anthropic/messages/native/interceptors/advisor.py similarity index 99% rename from litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py rename to litellm/llms/anthropic/messages/native/interceptors/advisor.py index 85d76de5402..f69d9820ef9 100644 --- a/litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py +++ b/litellm/llms/anthropic/messages/native/interceptors/advisor.py @@ -68,7 +68,7 @@ class AdvisorOrchestrationHandler(MessagesInterceptor): custom_llm_provider: str | None, **kwargs, ) -> AnthropicMessagesResponse | AsyncIterator: - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) diff --git a/litellm/llms/anthropic/pass_through/messages/interceptors/base.py b/litellm/llms/anthropic/messages/native/interceptors/base.py similarity index 100% rename from litellm/llms/anthropic/pass_through/messages/interceptors/base.py rename to litellm/llms/anthropic/messages/native/interceptors/base.py diff --git a/litellm/llms/anthropic/pass_through/messages/mcp_handler.py b/litellm/llms/anthropic/messages/native/mcp_handler.py similarity index 95% rename from litellm/llms/anthropic/pass_through/messages/mcp_handler.py rename to litellm/llms/anthropic/messages/native/mcp_handler.py index f369e10f22c..9df3c2c32e5 100644 --- a/litellm/llms/anthropic/pass_through/messages/mcp_handler.py +++ b/litellm/llms/anthropic/messages/native/mcp_handler.py @@ -79,9 +79,9 @@ async def anthropic_messages_with_mcp( from litellm.experimental_mcp_client.tools import ( transform_mcp_tool_to_anthropic_tool, ) - from litellm.llms.anthropic.pass_through.adapters.handler import request_safeguards_evaluator - from litellm.llms.anthropic.pass_through.messages.handler import proxy_classifies_tool_uses - from litellm.llms.anthropic.pass_through.safeguards import with_safeguard_results + from litellm.llms.anthropic.messages.adapters.handler import request_safeguards_evaluator + from litellm.llms.anthropic.messages.native.handler import proxy_classifies_tool_uses + from litellm.llms.anthropic.messages.safeguards import with_safeguard_results from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) @@ -198,7 +198,7 @@ async def anthropic_messages_with_mcp( else response ) if stream: - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) diff --git a/litellm/llms/anthropic/pass_through/messages/mid_conversation_system.py b/litellm/llms/anthropic/messages/native/mid_conversation_system.py similarity index 100% rename from litellm/llms/anthropic/pass_through/messages/mid_conversation_system.py rename to litellm/llms/anthropic/messages/native/mid_conversation_system.py diff --git a/litellm/llms/anthropic/pass_through/messages/response_cache.py b/litellm/llms/anthropic/messages/native/response_cache.py similarity index 98% rename from litellm/llms/anthropic/pass_through/messages/response_cache.py rename to litellm/llms/anthropic/messages/native/response_cache.py index 81d01f1437e..d2425c3f484 100644 --- a/litellm/llms/anthropic/pass_through/messages/response_cache.py +++ b/litellm/llms/anthropic/messages/native/response_cache.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Final, cast import litellm from litellm._logging import verbose_logger from litellm.caching.caching_handler import create_cache_write_task, is_response_without_output -from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( +from litellm.llms.anthropic.messages.native.streaming_iterator import ( AnthropicMessagesStreamingResponse, BaseAnthropicMessagesStreamingIterator, aclose_if_supported, diff --git a/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py b/litellm/llms/anthropic/messages/native/streaming_iterator.py similarity index 99% rename from litellm/llms/anthropic/pass_through/messages/streaming_iterator.py rename to litellm/llms/anthropic/messages/native/streaming_iterator.py index bfa309fad83..52b3f1042f8 100644 --- a/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/native/streaming_iterator.py @@ -18,13 +18,13 @@ from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.anthropic.common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP -from litellm.llms.anthropic.pass_through.messages.utils import ( +from litellm.llms.anthropic.messages.native.utils import ( INCOMPLETE_STREAM_ERROR_MESSAGE, MESSAGE_DELTA_OPTIONAL_KEYS, anthropic_message_delta, ) -from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults -from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame +from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults +from litellm.llms.anthropic.messages.utils import anthropic_sse_frame from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) @@ -33,7 +33,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointT from litellm.types.utils import GenericStreamingChunk, ModelResponseStream if TYPE_CHECKING: - from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator + from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ: Final = PassThroughEndpointLogging() diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py similarity index 99% rename from litellm/llms/anthropic/pass_through/messages/transformation.py rename to litellm/llms/anthropic/messages/native/transformation.py index 35db54147c5..cc53858f842 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -10,8 +10,8 @@ from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import ensure_api_route -from litellm.llms.anthropic.pass_through.convertible_blocks import drop_non_string_tool_result_texts -from litellm.llms.anthropic.pass_through.safeguards import proxy_serves_safeguards +from litellm.llms.anthropic.messages.convertible_blocks import drop_non_string_tool_result_texts +from litellm.llms.anthropic.messages.safeguards import proxy_serves_safeguards from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) @@ -736,7 +736,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): litellm_logging_obj: LiteLLMLoggingObj, ) -> AsyncIterator[bytes]: """Helper function to handle Anthropic streaming responses using the existing logging handlers""" - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( BaseAnthropicMessagesStreamingIterator, ) diff --git a/litellm/llms/anthropic/pass_through/messages/utils.py b/litellm/llms/anthropic/messages/native/utils.py similarity index 99% rename from litellm/llms/anthropic/pass_through/messages/utils.py rename to litellm/llms/anthropic/messages/native/utils.py index d91a5e2fb5c..29bbd8178ba 100644 --- a/litellm/llms/anthropic/pass_through/messages/utils.py +++ b/litellm/llms/anthropic/messages/native/utils.py @@ -11,7 +11,7 @@ from litellm.llms.anthropic.common_utils import ( sanitize_tool_use_ids_in_anthropic_messages, strip_empty_content_blocks_from_anthropic_messages, ) -from litellm.llms.anthropic.pass_through.utils import is_reasoning_auto_summary_enabled +from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enabled from litellm.types.llms.anthropic import ( AnthropicMessagesRequestOptionalParams, AnthropicStopDetails, diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/__init__.py b/litellm/llms/anthropic/messages/responses_adapters/__init__.py similarity index 100% rename from litellm/llms/anthropic/pass_through/responses_adapters/__init__.py rename to litellm/llms/anthropic/messages/responses_adapters/__init__.py diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py similarity index 99% rename from litellm/llms/anthropic/pass_through/responses_adapters/handler.py rename to litellm/llms/anthropic/messages/responses_adapters/handler.py index b1347b0c79e..99a07a1e9bf 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -201,7 +201,7 @@ def _build_responses_kwargs( if isinstance(reasoning, dict): effort: Final[object] = reasoning.get("effort") if isinstance(effort, str): - from litellm.llms.anthropic.pass_through.utils import ( + from litellm.llms.anthropic.messages.utils import ( normalize_reasoning_effort_value, ) diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py similarity index 98% rename from litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py rename to litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py index e628a30fdd6..614a7c96848 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py @@ -14,13 +14,13 @@ from litellm.exceptions import MidStreamFallbackError from litellm.litellm_core_utils.prompt_templates.common_utils import ( encrypted_reasoning_signature, ) -from litellm.llms.anthropic.pass_through.messages.utils import ( +from litellm.llms.anthropic.messages.native.utils import ( INCOMPLETE_STREAM_ERROR_MESSAGE, refusal_stop_details, responses_output_refusal_text, ) -from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults -from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame +from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults +from litellm.llms.anthropic.messages.utils import anthropic_sse_frame from litellm.responses.streaming_iterator import ( partial_responses_incomplete_event_on_disconnect, stream_error_status_and_message, @@ -35,7 +35,7 @@ from .transformation import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject - from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator + from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator from litellm.types.llms.openai import ResponseIncompleteEvent diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py similarity index 99% rename from litellm/llms/anthropic/pass_through/responses_adapters/transformation.py rename to litellm/llms/anthropic/messages/responses_adapters/transformation.py index 48fc987a95a..1dd07c1739b 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -20,12 +20,12 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking, ) -from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_parts -from litellm.llms.anthropic.pass_through.messages.utils import ( +from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_parts +from litellm.llms.anthropic.messages.native.utils import ( refusal_stop_details, responses_output_refusal_text, ) -from litellm.llms.anthropic.pass_through.utils import ( +from litellm.llms.anthropic.messages.utils import ( is_reasoning_auto_summary_enabled, json_copy_or_original, prompt_cache_key_from_user_id, @@ -71,7 +71,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if raw_usage is None: return AnthropicUsage(input_tokens=0, output_tokens=0) - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) from litellm.responses.utils import ResponseAPILoggingUtils diff --git a/litellm/llms/anthropic/pass_through/safeguards.py b/litellm/llms/anthropic/messages/safeguards.py similarity index 99% rename from litellm/llms/anthropic/pass_through/safeguards.py rename to litellm/llms/anthropic/messages/safeguards.py index eededde7e92..22627a454b3 100644 --- a/litellm/llms/anthropic/pass_through/safeguards.py +++ b/litellm/llms/anthropic/messages/safeguards.py @@ -11,8 +11,8 @@ from typing_extensions import NotRequired, ReadOnly, Unpack, assert_never import litellm from litellm._logging import verbose_logger -from litellm.llms.anthropic.pass_through.caller_model_checks import CallerModelVerdict, caller_model_verdict -from litellm.llms.anthropic.pass_through.utils import ( +from litellm.llms.anthropic.messages.caller_model_checks import CallerModelVerdict, caller_model_verdict +from litellm.llms.anthropic.messages.utils import ( ProxySubCallLoggingKwargs, proxy_general_settings, proxy_sub_call_metadata, diff --git a/litellm/llms/anthropic/pass_through/stream_assembly.py b/litellm/llms/anthropic/messages/stream_assembly.py similarity index 99% rename from litellm/llms/anthropic/pass_through/stream_assembly.py rename to litellm/llms/anthropic/messages/stream_assembly.py index 90cb9f4c335..90bdad8f4bc 100644 --- a/litellm/llms/anthropic/pass_through/stream_assembly.py +++ b/litellm/llms/anthropic/messages/stream_assembly.py @@ -12,7 +12,7 @@ from litellm.llms.anthropic.chat.handler import ( ModelResponseIterator as AnthropicModelResponseIterator, ) from litellm.llms.anthropic.chat.transformation import AnthropicConfig -from litellm.llms.anthropic.pass_through.messages.utils import MESSAGE_DELTA_OPTIONAL_KEYS +from litellm.llms.anthropic.messages.native.utils import MESSAGE_DELTA_OPTIONAL_KEYS from litellm.types.utils import ( Choices, Message, diff --git a/litellm/llms/anthropic/pass_through/utils.py b/litellm/llms/anthropic/messages/utils.py similarity index 100% rename from litellm/llms/anthropic/pass_through/utils.py rename to litellm/llms/anthropic/messages/utils.py diff --git a/litellm/llms/anthropic/pass_through/__init__.py b/litellm/llms/anthropic/pass_through/__init__.py new file mode 100644 index 00000000000..c29255c09c2 --- /dev/null +++ b/litellm/llms/anthropic/pass_through/__init__.py @@ -0,0 +1,7 @@ +from litellm.litellm_core_utils.renamed_package_shim import alias_renamed_package + +__getattr__ = alias_renamed_package( + __name__, + "litellm.llms.anthropic.messages", + nested_renames=((f"{__name__}.messages", "litellm.llms.anthropic.messages.native"),), +) diff --git a/litellm/llms/anthropic/prompt_cache_prediction.py b/litellm/llms/anthropic/prompt_cache_prediction.py index 4c363dba860..7a4c707860d 100644 --- a/litellm/llms/anthropic/prompt_cache_prediction.py +++ b/litellm/llms/anthropic/prompt_cache_prediction.py @@ -24,11 +24,11 @@ from litellm.llms.anthropic.common_utils import ( ) from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( DEFAULT_ANTHROPIC_API_VERSION, AnthropicMessagesConfig, ) -from litellm.llms.anthropic.pass_through.messages.utils import AnthropicMessagesRequestUtils, prepare_native_messages +from litellm.llms.anthropic.messages.native.utils import AnthropicMessagesRequestUtils, prepare_native_messages from litellm.router_utils.baseline_request import ( BASELINE_PARAMETERS, capture_baseline_parameters, diff --git a/litellm/llms/azure_ai/anthropic/messages_transformation.py b/litellm/llms/azure_ai/anthropic/messages_transformation.py index 2aacc5771d0..6b2da96de23 100644 --- a/litellm/llms/azure_ai/anthropic/messages_transformation.py +++ b/litellm/llms/azure_ai/anthropic/messages_transformation.py @@ -5,7 +5,7 @@ Azure Anthropic messages transformation config - extends AnthropicMessagesConfig from collections.abc import Sequence from typing import Any, Final -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.azure.common_utils import BaseAzureLLM diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 4d7ec663d09..0bc07ac6b22 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1955,7 +1955,7 @@ class AmazonConverseConfig(BaseConfig): anthropic_beta_list: list, ) -> None: """Keep only compact_20260112 edits for Bedrock; add beta header or drop field.""" - from litellm.llms.anthropic.pass_through.context_management.constants import ( + from litellm.llms.anthropic.messages.context_management.constants import ( COMPACT_EDIT_TYPE, ) from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES diff --git a/litellm/llms/bedrock/claude_platform/messages_transformation.py b/litellm/llms/bedrock/claude_platform/messages_transformation.py index 57ebaa0ca71..06308cde124 100644 --- a/litellm/llms/bedrock/claude_platform/messages_transformation.py +++ b/litellm/llms/bedrock/claude_platform/messages_transformation.py @@ -1,7 +1,7 @@ from typing import Final import litellm -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( DEFAULT_ANTHROPIC_API_VERSION, AnthropicMessagesConfig, ) diff --git a/litellm/llms/bedrock/count_tokens/kimi_k3.py b/litellm/llms/bedrock/count_tokens/kimi_k3.py index 4f8b4624cd8..5eb46944f9a 100644 --- a/litellm/llms/bedrock/count_tokens/kimi_k3.py +++ b/litellm/llms/bedrock/count_tokens/kimi_k3.py @@ -32,7 +32,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( chat_completion_tools_for, is_flat_responses_tool, ) -from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter +from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam KIMI_K3_REPO: Final = "moonshotai/Kimi-K3" diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py index 805652630ed..008b977f485 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py @@ -24,7 +24,7 @@ from litellm.llms.anthropic.common_utils import ( carried_thinking_keys, validated_object_mapping, ) -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.base_llm.anthropic_messages.transformation import ( @@ -878,7 +878,7 @@ class AmazonAnthropicClaudeMessagesConfig( merge them from ``message_start`` so logging/cost sees a consistent usage object (fixes negative input costs: LIT-2411). """ - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( BaseAnthropicMessagesStreamingIterator, ) diff --git a/litellm/llms/bedrock/messages/mantle_transformation.py b/litellm/llms/bedrock/messages/mantle_transformation.py index 7b9dde42c4a..89f4cc4dd8b 100644 --- a/litellm/llms/bedrock/messages/mantle_transformation.py +++ b/litellm/llms/bedrock/messages/mantle_transformation.py @@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from pydantic import TypeAdapter -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( DEFAULT_ANTHROPIC_API_VERSION, AnthropicMessagesConfig, ) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 7b808cf2be2..91ac5a0ca9a 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -238,7 +238,7 @@ if TYPE_CHECKING: from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig @@ -2650,7 +2650,7 @@ class BaseLLMHTTPHandler: initial_response: AsyncIterator | AnthropicMessagesResponse if stream: - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( AnthropicMessagesStreamingResponse, anthropic_messages_stream_hidden_params, ) @@ -2674,7 +2674,7 @@ class BaseLLMHTTPHandler: hidden_params=stream_hidden_params, ) - from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( + from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( AgenticAnthropicStreamingIterator, ) @@ -6237,7 +6237,7 @@ class BaseLLMHTTPHandler: from typing import cast from litellm._logging import verbose_logger - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( diff --git a/litellm/llms/databricks/messages/transformation.py b/litellm/llms/databricks/messages/transformation.py index f5ccbd92fb4..e22ad85b5f9 100644 --- a/litellm/llms/databricks/messages/transformation.py +++ b/litellm/llms/databricks/messages/transformation.py @@ -8,7 +8,7 @@ from pydantic import TypeAdapter import litellm from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.databricks.common_utils import ( diff --git a/litellm/llms/deepseek/messages/transformation.py b/litellm/llms/deepseek/messages/transformation.py index 3b2dd5af260..3b2076b0085 100644 --- a/litellm/llms/deepseek/messages/transformation.py +++ b/litellm/llms/deepseek/messages/transformation.py @@ -9,7 +9,7 @@ from pydantic import TypeAdapter import litellm from litellm.litellm_core_utils.url_utils import join_api_base -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.secret_managers.main import get_secret_str diff --git a/litellm/llms/github_copilot/messages/transformation.py b/litellm/llms/github_copilot/messages/transformation.py index 613b7e42100..bc2b68cd02b 100644 --- a/litellm/llms/github_copilot/messages/transformation.py +++ b/litellm/llms/github_copilot/messages/transformation.py @@ -3,7 +3,7 @@ from typing import Final, cast # noqa: TID251 # narrows untyped request dicts from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.url_utils import ensure_api_route -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) diff --git a/litellm/llms/minimax/messages/transformation.py b/litellm/llms/minimax/messages/transformation.py index f7a263a467c..f546d2d3a16 100644 --- a/litellm/llms/minimax/messages/transformation.py +++ b/litellm/llms/minimax/messages/transformation.py @@ -4,7 +4,7 @@ MiniMax Anthropic transformation config - extends AnthropicConfig for MiniMax's import litellm from litellm.litellm_core_utils.url_utils import ensure_api_route -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.secret_managers.main import get_secret_str diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index deee502c9f2..6ae61e15953 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -15,8 +15,8 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.tool_fields import DropPolicy -from litellm.llms.anthropic.pass_through.convertible_blocks import deployment_drops_params, unconvertible_blocks_error -from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( +from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error +from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) from litellm.llms.base_llm.base_utils import BaseTokenCounter diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index 0f885146e36..b4e152ff406 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -15,9 +15,9 @@ from typing_extensions import ReadOnly, TypedDict from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes from litellm.litellm_core_utils.tool_fields import DropPolicy from litellm.litellm_core_utils.url_utils import join_api_base -from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_text_error -from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages -from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( +from litellm.llms.anthropic.messages.convertible_blocks import malformed_tool_result_text_error +from litellm.llms.anthropic.messages.native.utils import normalize_native_messages +from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) from litellm.responses.utils import ResponsesAPIRequestUtils diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index 0ab55d653e0..8daae2a8e6a 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -3,7 +3,7 @@ from typing import Final import litellm from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.common_utils import normalize_cache_control_in_anthropic_payload -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.openai_like.json_loader import SimpleProviderConfig diff --git a/litellm/llms/opencode/messages/transformation.py b/litellm/llms/opencode/messages/transformation.py index b08a6df47aa..16953a33329 100644 --- a/litellm/llms/opencode/messages/transformation.py +++ b/litellm/llms/opencode/messages/transformation.py @@ -3,7 +3,7 @@ from collections.abc import Mapping, Sequence from pydantic import TypeAdapter from litellm.litellm_core_utils.url_utils import ensure_api_route -from litellm.llms.anthropic.pass_through.messages.transformation import AnthropicMessagesConfig +from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig from litellm.llms.opencode.common_utils import go_api_base, go_headers diff --git a/litellm/llms/tencent/messages/transformation.py b/litellm/llms/tencent/messages/transformation.py index 30c3ecaf3a9..cfc9e9ed6aa 100644 --- a/litellm/llms/tencent/messages/transformation.py +++ b/litellm/llms/tencent/messages/transformation.py @@ -7,7 +7,7 @@ alongside its standard OpenAI-compatible chat completions endpoint. import litellm from litellm.litellm_core_utils.url_utils import join_api_base -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.secret_managers.main import get_secret_str diff --git a/litellm/llms/vertex_ai/count_tokens/context.py b/litellm/llms/vertex_ai/count_tokens/context.py index 2579e857bd9..fc7d6545624 100644 --- a/litellm/llms/vertex_ai/count_tokens/context.py +++ b/litellm/llms/vertex_ai/count_tokens/context.py @@ -134,7 +134,7 @@ def vertex_gemini_count_context( def unconvertible_messages_error( messages: Sequence[Mapping[str, object]] | None, messages_format: TokenCountMessagesFormat ) -> VertexGeminiCountContextError | None: - from litellm.llms.anthropic.pass_through.convertible_blocks import unconvertible_blocks_error + from litellm.llms.anthropic.messages.convertible_blocks import unconvertible_blocks_error error_message: Final = unconvertible_blocks_error(messages, messages_format) return None if error_message is None else VertexGeminiCountContextError(error_message=error_message) @@ -145,7 +145,7 @@ def malformed_tool_result_text_count_error( messages_format: TokenCountMessagesFormat, deployment: Mapping[str, object] | None, ) -> VertexGeminiCountContextError | None: - from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_count_error + from litellm.llms.anthropic.messages.convertible_blocks import malformed_tool_result_count_error error_message: Final = malformed_tool_result_count_error(messages, messages_format, deployment) return None if error_message is None else VertexGeminiCountContextError(error_message=error_message) @@ -235,7 +235,7 @@ def _view_without_edits( system: object | None, typed_system: str | Sequence[Mapping[str, object]] | None, ) -> InferenceRequestView: - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( apply_client_compaction_block_history, ) @@ -253,9 +253,9 @@ def _view_without_edits( async def _anthropic_inference_view( messages: Sequence[Mapping[str, object]], system: object | None, edits: CountContextEdits ) -> InferenceRequestView | VertexGeminiCountContextError: - from litellm.llms.anthropic.pass_through.adapters.handler import prepare_context_managed_request - from litellm.llms.anthropic.pass_through.context_management import AnthropicContextManagementError - from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages + from litellm.llms.anthropic.messages.adapters.handler import prepare_context_managed_request + from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError + from litellm.llms.anthropic.messages.native.utils import normalize_native_messages typed_system: Final = _typed_system(system) if isinstance(typed_system, VertexGeminiCountContextError): @@ -308,7 +308,7 @@ def _chat_messages( model: str, custom_llm_provider: Literal["vertex_ai", "gemini"], ) -> Sequence[AllMessageValues]: - from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter + from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter match messages_format: case "anthropic": diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/__init__.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/__init__.py new file mode 100644 index 00000000000..9756aa4ec5a --- /dev/null +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/__init__.py @@ -0,0 +1,3 @@ +from litellm.litellm_core_utils.renamed_package_shim import alias_renamed_package + +__getattr__ = alias_renamed_package(__name__, "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages") diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py similarity index 99% rename from litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py rename to litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py index eefeda190ca..6d8eb228786 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py @@ -2,7 +2,7 @@ from collections.abc import Mapping, Sequence from typing import Final from litellm.llms.anthropic.common_utils import AnthropicModelInfo -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, messages_carry_output_config, ) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py index db27427a5be..00c28a728e0 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py @@ -2,7 +2,7 @@ Shared sanitization for ``output_config`` / ``output_format`` on Vertex AI Claude. Lives in its own module so both the chat-completion transformation (``transformation.py``) and the Messages pass-through transformation -(``experimental_pass_through/transformation.py``) can import it without +(``messages/transformation.py``) can import it without forming a cycle through the parent module's heavier imports. CodeQL flagged the ``..transformation`` import path as a potential cyclic diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index 2e9d6a77a62..0e2fc2e3dcb 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -5,7 +5,7 @@ from typing import Final, TypeAlias, cast # noqa: TID251 # native binding sele from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider -from litellm.llms.anthropic.pass_through.messages import handler as main +from litellm.llms.anthropic.messages.native import handler as main from litellm.rust_bridge import detectors from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target from litellm.rust_bridge.dispatch import PublicDispatch diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index b12676e08f3..3d2f89be15f 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -19,7 +19,7 @@ from litellm.anthropic_interface.exceptions import ( ) from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.litellm_core_utils.tool_fields import first_validation_error -from litellm.llms.anthropic.pass_through.context_management import ( +from litellm.llms.anthropic.messages.context_management import ( AnthropicContextManagementError, ) from litellm.llms.base_llm.guardrail_translation.utils import ( diff --git a/litellm/proxy/guardrails/anthropic_sse.py b/litellm/proxy/guardrails/anthropic_sse.py index 2baeabdce7b..3f19f5cb386 100644 --- a/litellm/proxy/guardrails/anthropic_sse.py +++ b/litellm/proxy/guardrails/anthropic_sse.py @@ -13,15 +13,15 @@ from types import MappingProxyType from typing import Final from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.llms.anthropic.pass_through.messages.utils import filtered_safeguard_results, redacted_stop_details -from litellm.llms.anthropic.pass_through.stream_assembly import anthropic_message_delta_fields -from litellm.llms.anthropic.pass_through.stream_assembly import ( +from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details +from litellm.llms.anthropic.messages.stream_assembly import anthropic_message_delta_fields +from litellm.llms.anthropic.messages.stream_assembly import ( assemble_anthropic_sse_stream as assemble_anthropic_sse_stream, # noqa: PLC0414 # public re-export ) -from litellm.llms.anthropic.pass_through.stream_assembly import ( +from litellm.llms.anthropic.messages.stream_assembly import ( is_anthropic_sse_stream as is_anthropic_sse_stream, # noqa: PLC0414 # public re-export ) -from litellm.llms.anthropic.pass_through.stream_assembly import ( +from litellm.llms.anthropic.messages.stream_assembly import ( is_raw_sse_stream as is_raw_sse_stream, # noqa: PLC0414 # public re-export ) from litellm.types.utils import Choices, ModelResponse @@ -50,7 +50,7 @@ def anthropic_sse_error_frames(message: str) -> tuple[bytes, ...]: ) -from litellm.llms.anthropic.pass_through.stream_assembly import ( # noqa: E402 # re-export after the definitions above +from litellm.llms.anthropic.messages.stream_assembly import ( # noqa: E402 # re-export after the definitions above is_sse_error_stream as is_sse_error_stream, # noqa: PLC0414 # public re-export ) @@ -111,10 +111,10 @@ def anthropic_sse_chunks_from_response( the input the classifier judged. Their free-text ``explanation`` was written about the text the guardrail rewrote, so it is dropped unless the caller rewrites no text (``keep_explanations``). """ - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py b/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py index f2f3ec201f5..589b865618b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py +++ b/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py @@ -559,7 +559,7 @@ def _v3_answer(request_data: Mapping[str, object], model: str | None) -> Mapping return _v3_text_completion_as_chat(response) if not isinstance(response, ModelResponse) or not _v3_anthropic_messages_route(request_data): return _jsonable_dict(response) - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 15f70073c7a..c7aa530434a 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -21,7 +21,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, ) from litellm.llms.anthropic import get_anthropic_config -from litellm.llms.anthropic.pass_through.stream_assembly import ( +from litellm.llms.anthropic.messages.stream_assembly import ( build_complete_streaming_response, build_complete_streaming_response_legacy, build_usage_only_response_from_chunks, diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index b6a067b7666..c94dc16a8f5 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -311,7 +311,7 @@ class PassThroughStreamingHandler: @staticmethod def _error_frame_closed_at(endpoint_type: EndpointType, raw_bytes: Sequence[bytes]) -> tuple[str, str, int] | None: - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( closing_superseded_attempt, parse_anthropic_error_event, ) @@ -342,7 +342,7 @@ class PassThroughStreamingHandler: - Vertex AI - OpenAI """ - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( is_message_stop_chunk, # pyright: ignore[reportPrivateUsage] # both native stream paths share terminal-event detection is_provider_error_chunk, # pyright: ignore[reportPrivateUsage] # provider errors must not become cache evidence ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ca0f8892b9d..bcba76ca1b3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -374,7 +374,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( SensitiveDataMasker, mask_sensitive_keys, ) -from litellm.llms.anthropic.pass_through.convertible_blocks import ( +from litellm.llms.anthropic.messages.convertible_blocks import ( deployment_drops_params, drop_non_string_tool_result_texts, malformed_tool_result_count_error, @@ -15171,7 +15171,7 @@ def _router_default_litellm_params(router: Router | None) -> Mapping[str, object def _count_context_management( context_management: JsonValue, model: str, request_litellm_params: Mapping[str, JsonValue] ) -> CountContextManagement | None: - from litellm.llms.anthropic.pass_through.adapters.handler import context_management_explicitly_dropped + from litellm.llms.anthropic.messages.adapters.handler import context_management_explicitly_dropped if context_management is None or llm_router is None: return None diff --git a/litellm/router.py b/litellm/router.py index abcdf9068d9..0d4045e6ca8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -737,7 +737,7 @@ def _anthropic_stream_forwards_ping_live(chunk: object, has_generated_content: b """A `ping` keepalive reaches the client live whenever the stream has not committed: it carries no lifecycle, so it cannot create overlapping lifecycles on the wire, and it keeps the connection alive while lifecycle frames sit buffered for a possible fallback during a long thinking pass.""" - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import is_anthropic_ping_chunk + from litellm.llms.anthropic.messages.native.streaming_iterator import is_anthropic_ping_chunk return not has_generated_content and is_anthropic_ping_chunk(chunk) @@ -753,7 +753,7 @@ def _without_line_breaks(value: object) -> str: def _anthropic_stream_error_is_gateway_verdict(chunk: object) -> bool: """AgenticAnthropicStreamingIterator's own retrieval-failure frame is the gateway's verdict, not a provider failure: another deployment would rerun the same failed hook, so it reaches the client instead of falling back.""" - from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( + from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( is_server_fulfilled_tool_leak_error, ) @@ -917,7 +917,7 @@ def _stream_commits_now(chunk: object, has_generated_content: bool, buffered_chu pre-content buffer cap was hit) rather than keep buffering lifecycle frames toward a possible fallback. """ - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( is_anthropic_content_delta_chunk, ) @@ -6307,7 +6307,7 @@ class Router: response=response, kwargs=kwargs, ): - from litellm.llms.anthropic.pass_through.messages.utils import ( + from litellm.llms.anthropic.messages.native.utils import ( safeguard_refusal_error, ) @@ -6424,16 +6424,16 @@ class Router: including pings and provider error frames, is forwarded live and verbatim instead. """ - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( aclose_if_supported, aclose_superseded_attempt, parse_anthropic_error_event, parse_anthropic_refusal_stop_details, ) - from litellm.llms.anthropic.pass_through.messages.utils import ( + from litellm.llms.anthropic.messages.native.utils import ( safeguard_refusal_error, ) @@ -6797,7 +6797,7 @@ class Router: A retry stamps the retry headers async_function_with_retries would have for a pre-stream retry; a later hop the recovered stream makes replaces them with its own, the way a fallback's do. """ - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( aclose_if_supported, anthropic_messages_response_as_sse_events, ) @@ -9931,7 +9931,7 @@ class Router: when a content-policy fallback is configured; a plain refusal without stop_details, or any response with nothing configured, is returned to the client unchanged. """ - from litellm.llms.anthropic.pass_through.messages.utils import ( + from litellm.llms.anthropic.messages.native.utils import ( get_safeguard_refusal_stop_details, ) @@ -14148,7 +14148,7 @@ class Router: `tools` (Chat Completions, Responses and Anthropic Messages shapes) and the Anthropic Messages top-level `system` block. """ - from litellm.llms.anthropic.pass_through.messages.utils import ( + from litellm.llms.anthropic.messages.native.utils import ( anthropic_system_to_openai_message, ) diff --git a/litellm/router_utils/baseline_request.py b/litellm/router_utils/baseline_request.py index 2f6262197a1..59878421068 100644 --- a/litellm/router_utils/baseline_request.py +++ b/litellm/router_utils/baseline_request.py @@ -7,7 +7,7 @@ from typing import Final, cast from pydantic import JsonValue, TypeAdapter, ValidationError -from litellm.llms.anthropic.pass_through.messages.utils import anthropic_messages_optional_param_keys +from litellm.llms.anthropic.messages.native.utils import anthropic_messages_optional_param_keys CACHE_SETTINGS: Final = ( "system", diff --git a/litellm/rust_bridge/host/callbacks_legacy_python.py b/litellm/rust_bridge/host/callbacks_legacy_python.py index ae913882968..2d2f5072544 100644 --- a/litellm/rust_bridge/host/callbacks_legacy_python.py +++ b/litellm/rust_bridge/host/callbacks_legacy_python.py @@ -377,7 +377,7 @@ def stream_success( end: datetime.datetime, first_chunk: datetime.datetime | None, ) -> None: - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ, ) from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler diff --git a/litellm/rust_bridge/host/routes/messages.py b/litellm/rust_bridge/host/routes/messages.py index 1024141d5a2..017ff65e2a7 100644 --- a/litellm/rust_bridge/host/routes/messages.py +++ b/litellm/rust_bridge/host/routes/messages.py @@ -10,7 +10,7 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout -from litellm.llms.anthropic.pass_through.utils import is_reasoning_auto_summary_enabled +from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enabled from litellm.rust_bridge import failures from litellm.rust_bridge.public_call import known_params, optional_str from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse @@ -64,7 +64,7 @@ def body_params(request: Mapping[str, object]) -> Mapping[str, object]: def stream_hidden_params(headers: Sequence[tuple[str, str]]) -> Mapping[str, object]: - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( anthropic_messages_stream_hidden_params, ) diff --git a/litellm/rust_bridge/messages/detectors.py b/litellm/rust_bridge/messages/detectors.py index b53e80805a5..5f75858d350 100644 --- a/litellm/rust_bridge/messages/detectors.py +++ b/litellm/rust_bridge/messages/detectors.py @@ -73,7 +73,7 @@ def _resolved_provider(request: NativeCall) -> str | None: def intercepted(request: NativeCall) -> bool: """The async Messages handler delegates matching tools to a registered interceptor.""" - from litellm.llms.anthropic.pass_through.messages.interceptors import get_messages_interceptors + from litellm.llms.anthropic.messages.native.interceptors import get_messages_interceptors tools: Final = optional_sequence(request.resolved.get("tools")) if not tools: @@ -85,6 +85,6 @@ def intercepted(request: NativeCall) -> bool: def safeguards_classifier(request: NativeCall) -> bool: """Python's proxy classifier evaluates dangerous tool use for non-Claude models.""" - from litellm.llms.anthropic.pass_through.safeguards import native_route_classifies + from litellm.llms.anthropic.messages.safeguards import native_route_classifies return native_route_classifies(str(request.resolved["model"]), request.resolved.get("safeguards")) diff --git a/litellm/utils.py b/litellm/utils.py index 781aa5f4fce..6f0b1ca2e3e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9668,7 +9668,7 @@ class ProviderConfigManager: return BedrockMantleAnthropicMessagesConfig() elif litellm.LlmProviders.VERTEX_AI == provider: if "claude" in model_lower: - from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages.transformation import ( VertexAIPartnerModelsAnthropicMessagesConfig, ) diff --git a/ruff-strict.toml b/ruff-strict.toml index 398cca15df2..e9db4d95cc8 100644 --- a/ruff-strict.toml +++ b/ruff-strict.toml @@ -71,7 +71,7 @@ max-args = 5 # directly, each with a `# noqa: TID251 # `. "litellm.responses.main.responses".msg = "Import litellm.responses.dispatch.responses so the call routes through dispatch." "litellm.responses.main.aresponses".msg = "Import litellm.responses.dispatch.aresponses so the call routes through dispatch." -"litellm.llms.anthropic.pass_through.messages.handler.anthropic_messages".msg = "Import litellm.messages.anthropic_messages so the call routes through dispatch." -"litellm.llms.anthropic.pass_through.messages.handler.anthropic_messages_handler".msg = "Import litellm.messages.anthropic_messages_handler so the call routes through dispatch." +"litellm.llms.anthropic.messages.native.handler.anthropic_messages".msg = "Import litellm.messages.anthropic_messages so the call routes through dispatch." +"litellm.llms.anthropic.messages.native.handler.anthropic_messages_handler".msg = "Import litellm.messages.anthropic_messages_handler so the call routes through dispatch." "litellm.main.completion".msg = "Import litellm.completion so the call routes through dispatch." "litellm.main.acompletion".msg = "Import litellm.acompletion so the call routes through dispatch." diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index ea095d4510c..c3c5330c440 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -76,8 +76,8 @@ - {id: llm.messages.bedrock_invoke.web_search_server_tool.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: web_search_server_tool, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py", rationale: "Bedrock hosts no web_search server tool, so this only works because interception rewrites it before the upstream call and the agentic loop feeds the results back in native shape; a regression that short-circuits or forwards it instead yields raw text or AWS's 400", fail_before_fix: unproven} - {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven} - {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven} -- {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven} -- {id: llm.messages.vertex.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven} +- {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven} +- {id: llm.messages.vertex.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py", rationale: "Vertex Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven} - {id: llm.responses.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Core endpoint; OpenAI Responses native"} - {id: llm.responses.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.9 / LIT-4778", rationale: "Responses missing/empty input and missing model are rejected"} - {id: llm.responses.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Streaming via /v1/responses"} diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index df554b20204..cbaf46130d6 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -41,7 +41,7 @@ - {id: quota_management.budget.spend_counter.reseed_matches_db, module: quota_management, tier: P2, behavior: budget, variant: spend_counter, assertions: [reseed_matches_db], exercised_on: [chat_completions], source: "proxy/spend_tracking/budget_reservation.py", rationale: "Concurrent cold-counter reseeds keep the enforcement counter equal to DB spend (#26829)"} - {id: quota_management.spend_tracking.chat_completions.logs_cost, module: quota_management, tier: P0, behavior: spend_tracking, variant: chat_completions, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A paid chat call writes a nonzero spend row"} - {id: quota_management.spend_tracking.stream.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Streaming responses aggregate token counts into a spend row"} -- {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/pass_through/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"} +- {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/messages/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"} - {id: quota_management.spend_tracking.embeddings.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: embeddings, assertions: [logs_cost], exercised_on: [embeddings], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Embedding calls write nonzero spend rows"} - {id: quota_management.spend_tracking.cache_hit.zero_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_hit, assertions: [zero_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A response-cache hit logs at zero cost with the cache-hit marker"} - {id: quota_management.spend_tracking.discount_config.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: discount_config, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/management_endpoints/cost_tracking_settings.py + cost_calculator.py", rationale: "A persisted provider discount changes the real request's spend row"} @@ -62,11 +62,11 @@ - {id: quota_management.spend_tracking.cache_write.bills_cache_creation_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_write, assertions: [bills_cache_creation_rate], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "OpenAI cache-write tokens land on the spend row as cache-creation tokens billed at the cache-creation rate, not silently at the input rate (#34046)"} - {id: quota_management.spend_tracking.cost_breakdown.reports_component_costs, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_breakdown, assertions: [reports_component_costs], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "The spend row's metadata.cost_breakdown itemizes cache-read, cache-creation, output, and reasoning costs at the deployment's own rates and they sum to the row's spend (#31686)"} - {id: quota_management.spend_tracking.stream_cache_read.bills_cache_read_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream_cache_read, assertions: [bills_cache_read_rate], exercised_on: [chat_completions], source: "litellm_core_utils/streaming_chunk_builder_utils.py", rationale: "A streamed call's reassembled usage keeps the cached-token detail so cache reads bill at the cache-read discount, not full input price (#34812)"} -- {id: quota_management.spend_tracking.messages_bridge.keeps_cache_tokens, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [keeps_cache_tokens], exercised_on: [messages], source: "llms/anthropic/pass_through/responses_adapters/handler.py", rationale: "A /v1/messages request served by a Responses-only OpenAI model keeps its cache-read tokens and their discounted billing across the bridge (#34957)"} +- {id: quota_management.spend_tracking.messages_bridge.keeps_cache_tokens, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [keeps_cache_tokens], exercised_on: [messages], source: "llms/anthropic/messages/responses_adapters/handler.py", rationale: "A /v1/messages request served by a Responses-only OpenAI model keeps its cache-read tokens and their discounted billing across the bridge (#34957)"} - {id: quota_management.spend_tracking.service_tier.bills_tier_rates, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier, assertions: [bills_tier_rates], exercised_on: [chat_completions], source: "cost_calculator.py", rationale: "A priority service_tier call bills input, output, and reasoning at the deployment's *_priority rates and records the tier on the row (#35923, #35925)"} - {id: quota_management.spend_tracking.service_tier_stream.records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [chat_completions], source: "litellm_core_utils/streaming_chunk_builder_utils.py", fail_before_fix: proven, rationale: "A streamed call with no service_tier requested bills at the rates of the tier OpenAI stamps on its chunks and records that served tier on the row; the reassembled stream dropped the provider tier so the row recorded none and priced at the default rates"} - {id: quota_management.spend_tracking.service_tier_stream.responses_records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [responses], source: "responses/streaming_iterator.py", rationale: "A streamed /v1/responses call bills at the tier carried on the response.completed event's inner response and records that served tier on the spend row"} -- {id: quota_management.spend_tracking.service_tier_stream.messages_records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [messages], source: "llms/anthropic/pass_through/adapters/streaming_iterator.py", rationale: "A streamed /v1/messages call on an OpenAI-backed deployment bills at the tier OpenAI served; the Anthropic wire format has no tier field, so the spend row is the only record of it"} +- {id: quota_management.spend_tracking.service_tier_stream.messages_records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [messages], source: "llms/anthropic/messages/adapters/streaming_iterator.py", rationale: "A streamed /v1/messages call on an OpenAI-backed deployment bills at the tier OpenAI served; the Anthropic wire format has no tier field, so the spend row is the only record of it"} - {id: quota_management.spend_tracking.cost_headers.additive_components, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_headers, assertions: [additive_components], exercised_on: [chat_completions], source: "proxy/common_request_processing.py", rationale: "The x-litellm-response-cost-* component headers sum to the total, input covers only fresh tokens, and reasoning stays a subset of output (#36965)"} - {id: quota_management.spend_tracking.passthrough_stream.injects_usage_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: passthrough_stream, assertions: [injects_usage_cost], exercised_on: [openai_passthrough], source: "proxy/pass_through_endpoints/streaming_handler.py", rationale: "With include_cost_in_streaming_usage on, the /openai passthrough's final streaming usage frame carries the proxy-computed cost (#36503). Uncovered: the flag is only settable in litellm_settings, and the shared e2e stack does not turn it on yet"} - {id: quota_management.spend_tracking.websearch_interception.bills_under_request_session, module: quota_management, tier: P1, behavior: spend_tracking, variant: websearch_interception, assertions: [bills_under_request_session], exercised_on: [messages], source: "integrations/websearch_interception/handler.py", fail_before_fix: proven, rationale: "A web_search server tool the proxy intercepts into litellm.asearch writes its own asearch spend row, and that row carries the parent request's session_id so the session view counts the search and its cost next to the turn that triggered it (LIT-8063)"} diff --git a/tests/integration/messages_endpoint/test_guardrail_long_tool_names_wire.py b/tests/integration/messages_endpoint/test_guardrail_long_tool_names_wire.py index 8842e04db6b..89a388423e0 100644 --- a/tests/integration/messages_endpoint/test_guardrail_long_tool_names_wire.py +++ b/tests/integration/messages_endpoint/test_guardrail_long_tool_names_wire.py @@ -14,7 +14,7 @@ from integration._support.ports import allocate_port from integration._support.wire import Reply, Request, Wire, wire_server from pydantic import JsonValue, TypeAdapter -from litellm.llms.anthropic.pass_through.adapters.transformation import truncate_tool_name +from litellm.llms.anthropic.messages.adapters.transformation import truncate_tool_name _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _JSON_OBJECTS: Final = TypeAdapter(list[dict[str, JsonValue]]) diff --git a/tests/test_litellm_rust/messages/test_body_hooks.py b/tests/test_litellm_rust/messages/test_body_hooks.py index 188cd58e00e..59f27d5dc04 100644 --- a/tests/test_litellm_rust/messages/test_body_hooks.py +++ b/tests/test_litellm_rust/messages/test_body_hooks.py @@ -8,8 +8,8 @@ import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.llms.anthropic.pass_through.messages import interceptors -from litellm.llms.anthropic.pass_through.messages.interceptors.base import MessagesInterceptor +from litellm.llms.anthropic.messages.native import interceptors +from litellm.llms.anthropic.messages.native.interceptors.base import MessagesInterceptor from tests.test_litellm_rust.messages.data import BODY_HOOKS, MESSAGES_RESPONSE, REQUEST, call from tests.test_litellm_rust.support.callback_recorder import drain_logging from tests.test_litellm_rust.support.body_hooks import LegacyAgenticLoop, PlannedAgenticLoop, agentic_hooks diff --git a/tests/test_litellm_rust/messages/test_dispatch.py b/tests/test_litellm_rust/messages/test_dispatch.py index 50f71569d73..9430e01ea8a 100644 --- a/tests/test_litellm_rust/messages/test_dispatch.py +++ b/tests/test_litellm_rust/messages/test_dispatch.py @@ -8,7 +8,7 @@ import litellm from litellm.caching.caching import Cache from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger -from litellm.llms.anthropic.pass_through.messages import handler as python_messages +from litellm.llms.anthropic.messages.native import handler as python_messages from litellm.messages import dispatch from litellm.messages.dispatch import ( _ADISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch diff --git a/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py b/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py index 64c49f03732..351ca9beb4d 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py @@ -25,7 +25,7 @@ from litellm.integrations.websearch_interception.handler import ( ) from litellm.integrations.websearch_interception.tools import get_litellm_web_search_tool from litellm.litellm_core_utils.agentic_loop_settings import DEFAULT_MAX_AGENTIC_LOOPS -from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( +from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) from litellm.llms.base_llm.search.transformation import SearchResponse, SearchResult diff --git a/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py b/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py index 4fbc78c2f4d..fdc3771e903 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py @@ -238,7 +238,7 @@ class TestShortCircuitEntryPoint: @pytest.mark.asyncio async def test_returns_none_when_no_callbacks(self): """No callbacks configured → returns None""" - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( _try_websearch_short_circuit, ) @@ -255,7 +255,7 @@ class TestShortCircuitEntryPoint: @pytest.mark.asyncio async def test_returns_dict_when_not_streaming(self): """Non-streaming short-circuit → returns dict""" - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( _try_websearch_short_circuit, ) @@ -280,10 +280,10 @@ class TestShortCircuitEntryPoint: @pytest.mark.asyncio async def test_returns_stream_iterator_when_streaming(self): """Streaming short-circuit → returns FakeAnthropicMessagesStreamIterator""" - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( _try_websearch_short_circuit, ) @@ -322,7 +322,7 @@ class TestShortCircuitEntryPoint: """Non-WebSearchInterceptionLogger callbacks are ignored""" from unittest.mock import MagicMock - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( _try_websearch_short_circuit, ) @@ -345,10 +345,10 @@ class TestShortCircuitEntryPoint: loop. The short-circuit must use the ORIGINAL stream value so streaming callers get SSE events instead of a plain dict. """ - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( _try_websearch_short_circuit, ) @@ -378,7 +378,7 @@ class TestShortCircuitEntryPoint: still fire the short-circuit when the caller propagates the derived provider. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( _try_websearch_short_circuit, ) @@ -410,7 +410,7 @@ class TestShortCircuitSafeguardResults: NO_TOOL_USES_AVAILABLE = [{"type": "dangerous_tool_use", "status": {"type": "available", "tool_uses": {}}}] async def _short_circuit(self, *, stream: bool, kwargs: dict | None): - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( _try_websearch_short_circuit, ) diff --git a/tests/unit/integrations/websearch_interception/test_websearch_streaming_wrap.py b/tests/unit/integrations/websearch_interception/test_websearch_streaming_wrap.py index f4a46efaa1d..1401f260427 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_streaming_wrap.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_streaming_wrap.py @@ -13,7 +13,7 @@ import pytest from litellm.integrations.custom_logger import CustomLogger from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( +from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) from litellm.types.integrations.custom_logger import AgenticLoopPlan diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 561ac9b2b33..b46c65d4460 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -6575,7 +6575,7 @@ def test_handle_anthropic_messages_response_logging_keeps_the_served_response_id def test_handle_anthropic_messages_response_logging_bills_the_bridged_chat_completion_not_the_served_body(): """A chat deployment answers /v1/messages through the chat bridge. The Anthropic body served to the caller has no reasoning token field, so the spend row must come from the recorded chat completion.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter + from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter from litellm.types.utils import CompletionTokensDetailsWrapper, Usage chat_completion = ModelResponse( diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index c452e12f2aa..f222a257b8f 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -41,7 +41,7 @@ from litellm.litellm_core_utils.token_counter import ( offload_token_count, ) from litellm.litellm_core_utils.token_counter import token_counter as token_counter_new -from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter +from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 8218c120153..9e88b59adc9 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -18,7 +18,7 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import Streami from litellm.llms.anthropic.chat.guardrail_translation.handler import ( AnthropicMessagesHandler, ) -from litellm.llms.anthropic.pass_through.adapters.transformation import truncate_tool_name +from litellm.llms.anthropic.messages.adapters.transformation import truncate_tool_name from litellm.types.utils import ChatCompletionMessageToolCall, GenericGuardrailAPIInputs diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index b3b04440d16..ef002a73e52 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -26,7 +26,7 @@ from litellm.llms.anthropic.chat import ModelResponseIterator from litellm.llms.anthropic.common_utils import process_anthropic_headers from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig @@ -6152,7 +6152,7 @@ def test_translate_system_message_strips_billing_header_for_bedrock_invoke(): [ ("litellm.llms.anthropic.chat.transformation", "AnthropicConfig", False), ( - "litellm.llms.anthropic.pass_through.messages.transformation", + "litellm.llms.anthropic.messages.native.transformation", "AnthropicMessagesConfig", False, ), @@ -6194,7 +6194,7 @@ def test_translate_system_message_strips_billing_header_for_bedrock_invoke(): True, ), ( - "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation", + "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages.transformation", "VertexAIPartnerModelsAnthropicMessagesConfig", True, ), diff --git a/tests/unit/llms/anthropic/pass_through/adapters/__init__.py b/tests/unit/llms/anthropic/messages/adapters/__init__.py similarity index 100% rename from tests/unit/llms/anthropic/pass_through/adapters/__init__.py rename to tests/unit/llms/anthropic/messages/adapters/__init__.py diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py rename to tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index 658aa009683..4e4b89b764b 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -21,14 +21,14 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, _bedrock_converse_messages_pt, ) -from litellm.llms.anthropic.pass_through.adapters.transformation import ( +from litellm.llms.anthropic.messages.adapters.transformation import ( OPENAI_MAX_TOOL_NAME_LENGTH, AnthropicAdapter, LiteLLMAnthropicMessagesAdapter, create_tool_name_mapping, truncate_tool_name, ) -from litellm.llms.anthropic.pass_through.messages.mid_conversation_system import ( +from litellm.llms.anthropic.messages.native.mid_conversation_system import ( CONVERTED_SYSTEM_NOTE, ) from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig @@ -3957,7 +3957,7 @@ class TestAnthropicStreamWrapperToolArgs: return [text_chunk, tool_chunk, finish_chunk] def _make_stream_wrapper(self, chunks): - from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( + from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) @@ -4072,7 +4072,7 @@ def _make_simple_openai_response( def test_translate_openai_response_to_anthropic_with_polyfill_compaction_block(): """compaction_block from PolyfillResult must be prepended to content at index 0.""" - from litellm.llms.anthropic.pass_through.context_management.result import ( + from litellm.llms.anthropic.messages.context_management.result import ( PolyfillResult, ) @@ -4105,7 +4105,7 @@ def test_translate_openai_response_to_anthropic_with_polyfill_compaction_block() def test_translate_openai_response_to_anthropic_with_polyfill_iterations_usage(): """iterations_usage from PolyfillResult must produce usage['iterations'] with a message entry.""" - from litellm.llms.anthropic.pass_through.context_management.result import ( + from litellm.llms.anthropic.messages.context_management.result import ( PolyfillResult, ) @@ -4160,7 +4160,7 @@ def test_translate_openai_response_to_anthropic_no_polyfill_no_change(): def test_translate_openai_response_to_anthropic_with_polyfill_both_compaction_and_iterations(): """Full summary path: compaction_block and iterations_usage both present simultaneously.""" - from litellm.llms.anthropic.pass_through.context_management.result import ( + from litellm.llms.anthropic.messages.context_management.result import ( PolyfillResult, ) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_context_management.py similarity index 95% rename from tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py rename to tests/unit/llms/anthropic/messages/adapters/test_handler_context_management.py index 9b5477e3206..81cc1fbbe02 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_context_management.py @@ -5,8 +5,8 @@ from unittest.mock import patch import pytest import litellm -from litellm.llms.anthropic.pass_through.adapters.handler import LiteLLMMessagesToCompletionTransformationHandler -from litellm.llms.anthropic.pass_through.context_management import AnthropicContextManagementError +from litellm.llms.anthropic.messages.adapters.handler import LiteLLMMessagesToCompletionTransformationHandler +from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError from litellm.router import Router from litellm.types.utils import Choices, Message, ModelResponse, Usage diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_output_config_passthrough.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_output_config_passthrough.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/adapters/test_handler_output_config_passthrough.py rename to tests/unit/llms/anthropic/messages/adapters/test_handler_output_config_passthrough.py index 03553e65ad4..c34cb686928 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_output_config_passthrough.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_output_config_passthrough.py @@ -37,7 +37,7 @@ import pytest # where pytest is invoked from. sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) -from litellm.llms.anthropic.pass_through.adapters.handler import ( +from litellm.llms.anthropic.messages.adapters.handler import ( ANTHROPIC_ONLY_REQUEST_KEYS, LiteLLMMessagesToCompletionTransformationHandler, ) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py similarity index 97% rename from tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py rename to tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py index c31d85be0a8..ed6cd6134ff 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py @@ -6,7 +6,7 @@ import pytest sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) -from litellm.llms.anthropic.pass_through.adapters.handler import ( +from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py rename to tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py index 5a82b9af468..e504513dd36 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py @@ -16,7 +16,7 @@ import pytest import respx import litellm -from litellm.llms.anthropic.pass_through.adapters.handler import ( +from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_safeguard_results.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_safeguard_results.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/adapters/test_handler_safeguard_results.py rename to tests/unit/llms/anthropic/messages/adapters/test_handler_safeguard_results.py index 51964a77b67..cbe65b8a4e0 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_safeguard_results.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_safeguard_results.py @@ -11,11 +11,11 @@ import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.llms.anthropic.pass_through.adapters.handler import ( +from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, request_safeguards_evaluator, ) -from litellm.llms.anthropic.pass_through.safeguards import ToolUseUnderReview +from litellm.llms.anthropic.messages.safeguards import ToolUseUnderReview from litellm.proxy._types import LiteLLM_TeamTableCachedObj, UserAPIKeyAuth from litellm.router import Router from litellm.types.utils import ( @@ -30,7 +30,7 @@ from litellm.types.utils import ( StreamingChoices, Usage, ) -from tests.unit.llms.anthropic.pass_through.conftest import ProxyAuthState +from tests.unit.llms.anthropic.messages.conftest import ProxyAuthState SAFEGUARDS: Final = [{"type": "dangerous_tool_use", "classifier_context": {"permission_mode": "auto"}}] BASH_TOOL: Final = {"name": "Bash", "description": "run", "input_schema": {"type": "object", "properties": {}}} diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_combined_chunk.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_combined_chunk.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_combined_chunk.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_combined_chunk.py index 7400cd08dd3..d4911427e73 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_combined_chunk.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_combined_chunk.py @@ -22,7 +22,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo convert_to_streaming_response, convert_to_streaming_response_async, ) -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, _CombinedChunkSplitter, ) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_compaction.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_compaction.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_compaction.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_compaction.py index 2610824edf4..1d60f71e196 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_compaction.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_compaction.py @@ -6,7 +6,7 @@ from unittest.mock import MagicMock import pytest -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) from litellm.types.utils import Delta, StreamingChoices, Usage diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_empty_choices.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_empty_choices.py similarity index 97% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_empty_choices.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_empty_choices.py index 66825124f25..826c16f6b97 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_empty_choices.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_empty_choices.py @@ -12,7 +12,7 @@ import asyncio import json from typing import Any, AsyncIterator, Dict, List, Optional -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_first_delta.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_first_delta.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_first_delta.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_first_delta.py index 92234282bb2..32ec0ffae8e 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_first_delta.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_first_delta.py @@ -27,7 +27,7 @@ from unittest.mock import MagicMock import pytest -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) from litellm.types.utils import ( diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_message_id.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_message_id.py similarity index 96% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_message_id.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_message_id.py index a4f851c7753..b98877f25aa 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_message_id.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_message_id.py @@ -12,10 +12,10 @@ import pytest import respx import litellm -from litellm.llms.anthropic.pass_through.adapters.handler import ( +from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_mid_stream_error.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_mid_stream_error.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_mid_stream_error.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_mid_stream_error.py index 6951253349e..2cc0d86a3a8 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_mid_stream_error.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_mid_stream_error.py @@ -36,7 +36,7 @@ from litellm.exceptions import MidStreamFallbackError from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, _mid_stream_error_sse_event, ) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_sse_stream.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_sse_stream.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_sse_stream.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_sse_stream.py index ec57636a17b..79976564e5a 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_sse_stream.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_sse_stream.py @@ -13,7 +13,7 @@ import pytest from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicSSEStream, AnthropicStreamWrapper, ) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_stop_reason.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_stop_reason.py similarity index 97% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_stop_reason.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_stop_reason.py index 5f5007d52b5..48ffad5a050 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_stop_reason.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_stop_reason.py @@ -12,7 +12,7 @@ bridge emitted ``stop_reason: "end_turn"`` and Anthropic tool-runners import pytest -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) from litellm.llms.ollama.chat.transformation import ( diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_tool_args.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_tool_args.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_tool_args.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_tool_args.py index 224775ad5b9..d34a978f228 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_tool_args.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_tool_args.py @@ -17,7 +17,7 @@ from unittest.mock import MagicMock import pytest -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) from litellm.types.utils import ( diff --git a/tests/unit/llms/anthropic/pass_through/conftest.py b/tests/unit/llms/anthropic/messages/conftest.py similarity index 100% rename from tests/unit/llms/anthropic/pass_through/conftest.py rename to tests/unit/llms/anthropic/messages/conftest.py diff --git a/tests/unit/llms/anthropic/pass_through/context_management/__init__.py b/tests/unit/llms/anthropic/messages/context_management/__init__.py similarity index 100% rename from tests/unit/llms/anthropic/pass_through/context_management/__init__.py rename to tests/unit/llms/anthropic/messages/context_management/__init__.py diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py rename to tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py index 2ffa73dda21..2567d1b76db 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py @@ -10,17 +10,17 @@ from unittest.mock import AsyncMock, MagicMock import pytest import litellm -from litellm.llms.anthropic.pass_through.adapters.handler import _run_polyfill_if_enabled -from litellm.llms.anthropic.pass_through.context_management import ( +from litellm.llms.anthropic.messages.adapters.handler import _run_polyfill_if_enabled +from litellm.llms.anthropic.messages.context_management import ( AnthropicContextManagementError, PolyfillResult, apply_context_management, ) -from litellm.llms.anthropic.pass_through.context_management.constants import ( +from litellm.llms.anthropic.messages.context_management.constants import ( CLEARED_TOOL_RESULT_PLACEHOLDER, COMPACT_SUMMARY_MODEL_SETTING_KEY, ) -from litellm.llms.anthropic.pass_through.context_management.editors.clear_tool_uses import ( +from litellm.llms.anthropic.messages.context_management.editors.clear_tool_uses import ( apply_clear_tool_uses_20250919, ) from litellm.types.llms.anthropic import AppliedEdit diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py b/tests/unit/llms/anthropic/messages/context_management/test_compact.py similarity index 91% rename from tests/unit/llms/anthropic/pass_through/context_management/test_compact.py rename to tests/unit/llms/anthropic/messages/context_management/test_compact.py index 9b3084f0e7c..159c3586c53 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_compact.py @@ -21,11 +21,11 @@ import pytest from fastapi import HTTPException import litellm -from litellm.llms.anthropic.pass_through.context_management import ( +from litellm.llms.anthropic.messages.context_management import ( AnthropicContextManagementError, apply_context_management, ) -from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( +from litellm.llms.anthropic.messages.context_management.editors.compact import ( _augment_system_with_summary, _extract_summary_text, _select_last_user_question, @@ -34,7 +34,7 @@ from litellm.llms.anthropic.pass_through.context_management.editors.compact impo apply_client_compaction_block_history, apply_compact_20260112, ) -from litellm.llms.anthropic.pass_through.context_management.result import ( +from litellm.llms.anthropic.messages.context_management.result import ( PolyfillResult, ) from litellm.proxy._types import ( @@ -46,7 +46,7 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.hooks.parallel_request_limiter_v3 import RateLimitUnverifiableError -from tests.unit.llms.anthropic.pass_through.conftest import ProxyAuthState +from tests.unit.llms.anthropic.messages.conftest import ProxyAuthState MODEL = "openai/gpt-4o" @@ -321,7 +321,7 @@ async def test_trigger_below_minimum_raises(): async def test_trigger_at_minimum_does_not_raise(): """Exactly 50 000 is allowed — only strictly less than 50k is rejected.""" with patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value=None, ): result = await apply_compact_20260112( @@ -346,7 +346,7 @@ async def test_trigger_at_minimum_does_not_raise(): async def test_opt_in_gating_no_summary_model_configured(): messages = _simple_messages() with patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value=None, ): result = await apply_compact_20260112( @@ -373,7 +373,7 @@ async def test_opt_in_gating_no_summary_model_keeps_post_compaction_tail(): messages = _messages_with_compaction("prior summary text") with patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value=None, ): result = await apply_compact_20260112( @@ -455,7 +455,7 @@ async def test_slice_only_path_with_existing_compaction_block(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=500), # well under threshold @@ -492,7 +492,7 @@ async def test_slice_only_no_compaction_block_under_threshold(): messages = _simple_messages() with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=500), @@ -523,12 +523,12 @@ async def test_full_summary_path(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), # over 150k threshold patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", new_callable=AsyncMock, return_value=mock_response, ), @@ -577,7 +577,7 @@ async def test_full_summary_path_uses_router_when_available(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="my-summary-model", ), patch("litellm.token_counter", return_value=200_000), @@ -613,12 +613,12 @@ async def test_litellm_metadata_propagated_to_summary_call(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", new_callable=AsyncMock, return_value=mock_response, ) as mock_call, @@ -650,12 +650,12 @@ async def test_summary_call_failed(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", new_callable=AsyncMock, side_effect=RuntimeError("network error"), ), @@ -686,12 +686,12 @@ async def test_summary_extraction_failed_no_tags(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", new_callable=AsyncMock, return_value=mock_response, ), @@ -718,7 +718,7 @@ async def test_pause_after_compaction_ignored_warning(): """pause_after_compaction: true → warning recorded, request proceeds normally.""" messages = _simple_messages() with patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value=None, ): result = await apply_compact_20260112( @@ -741,7 +741,7 @@ async def test_pause_after_compaction_ignored_warning(): async def test_unsupported_trigger_type_falls_back_to_default(): messages = _simple_messages() with patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value=None, ): result = await apply_compact_20260112( @@ -779,12 +779,12 @@ async def test_custom_instructions_used_verbatim(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", side_effect=_fake_call_summary_model, ), ): @@ -824,12 +824,12 @@ async def test_default_instructions_appended_with_no_tool_suffix_when_no_tools() with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", side_effect=_fake_call_summary_model, ), ): @@ -860,12 +860,12 @@ async def test_default_instructions_with_tools_appends_no_tool_suffix(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", side_effect=_fake_call_summary_model, ), ): @@ -894,12 +894,12 @@ async def test_system_prompt_forwarded_to_summary_call_as_string(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", side_effect=_fake_call_summary_model, ), ): @@ -934,12 +934,12 @@ async def test_system_prompt_forwarded_to_summary_call_as_content_blocks(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", side_effect=_fake_call_summary_model, ), ): @@ -977,12 +977,12 @@ async def test_summary_call_carries_prior_compaction_summary_into_system(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", side_effect=_fake_call_summary_model, ), ): @@ -1014,12 +1014,12 @@ async def test_summary_call_omits_system_message_when_system_is_none(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", side_effect=_fake_call_summary_model, ), ): @@ -1055,12 +1055,12 @@ async def test_summary_call_does_not_emit_consecutive_user_turns(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", side_effect=_fake_call_summary_model, ), ): @@ -1085,10 +1085,10 @@ async def test_summary_call_sends_default_max_tokens(): (which require it) don't reject the request and silently fall back to ``summary_call_failed``. """ - from litellm.llms.anthropic.pass_through.context_management.constants import ( + from litellm.llms.anthropic.messages.context_management.constants import ( COMPACT_SUMMARY_MAX_TOKENS, ) - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _call_summary_model, ) @@ -1112,7 +1112,7 @@ async def test_summary_call_sends_default_max_tokens(): async def test_summary_call_honors_max_tokens_override(): """Operators can override the default summary ``max_tokens`` via ``general_settings.context_management_summary_max_tokens``.""" - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _read_summary_max_tokens_setting, ) @@ -1129,7 +1129,7 @@ async def test_summary_call_honors_max_tokens_override(): ): assert _read_summary_max_tokens_setting() == 8192 - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _call_summary_model, ) @@ -1148,10 +1148,10 @@ def test_summary_max_tokens_setting_falls_back_for_invalid_values(): """Invalid override values (non-int, non-positive, missing) fall back to the compiled default so a typo in ``general_settings`` doesn't break the summary call.""" - from litellm.llms.anthropic.pass_through.context_management.constants import ( + from litellm.llms.anthropic.messages.context_management.constants import ( COMPACT_SUMMARY_MAX_TOKENS, ) - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _read_summary_max_tokens_setting, ) @@ -1168,10 +1168,10 @@ def test_summary_max_tokens_setting_falls_back_for_invalid_values(): async def test_summary_call_sends_default_timeout(): """``timeout`` is set on the summary call so a slow or unresponsive summary model cannot hang the parent ``/v1/messages`` request indefinitely.""" - from litellm.llms.anthropic.pass_through.context_management.constants import ( + from litellm.llms.anthropic.messages.context_management.constants import ( COMPACT_SUMMARY_TIMEOUT_SECONDS, ) - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _call_summary_model, ) @@ -1235,12 +1235,12 @@ async def test_summary_model_denied_when_key_not_in_allowlist(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), ): @@ -1267,12 +1267,12 @@ async def test_summary_model_allowed_when_in_key_allowlist(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), ): @@ -1298,12 +1298,12 @@ async def test_summary_model_allowed_when_no_user_api_key_auth(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), ): @@ -1322,12 +1322,12 @@ async def test_summary_model_allowed_when_no_user_api_key_auth(): async def _compact_as(auth: UserAPIKeyAuth, summary_call: AsyncMock) -> PolyfillResult: with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", summary_call, ), ): @@ -1443,12 +1443,12 @@ async def test_summary_model_denied_when_key_over_model_budget(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.model_max_budget_limiter", limiter), @@ -1494,12 +1494,12 @@ async def test_summary_model_denied_when_user_over_model_budget(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.model_max_budget_limiter", limiter), @@ -1537,7 +1537,7 @@ async def test_summary_model_denied_when_user_over_model_budget(): async def test_summary_model_denied_when_team_member_over_model_budget(monkeypatch: pytest.MonkeyPatch): from litellm.caching.dual_cache import DualCache - from litellm.llms.anthropic.pass_through.context_management.constants import COMPACT_SUMMARY_MODEL_SETTING_KEY + from litellm.llms.anthropic.messages.context_management.constants import COMPACT_SUMMARY_MODEL_SETTING_KEY from litellm.models.team_membership import LiteLLM_TeamMembership from litellm.proxy._types import ( LiteLLM_BudgetTable, @@ -1671,12 +1671,12 @@ async def test_summary_model_denied_when_end_user_over_model_budget(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.model_max_budget_limiter", limiter), @@ -1712,12 +1712,12 @@ async def test_summary_model_allowed_when_within_model_budget(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.model_max_budget_limiter", limiter), @@ -1803,12 +1803,12 @@ async def test_summary_model_rate_limit_check_errors(limiter_error, summary_call with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging), @@ -1845,12 +1845,12 @@ async def test_summary_model_denied_when_over_rate_limit(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging), @@ -1881,12 +1881,12 @@ async def test_summary_model_allowed_when_within_rate_limit(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging), @@ -1929,12 +1929,12 @@ async def test_summary_model_allowed_while_the_caller_holds_the_keys_only_parall with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.proxy_logging_obj", _proxy_logging_like_the_live_proxy(limiter)), @@ -1965,12 +1965,12 @@ async def test_summary_model_rate_limit_skipped_for_legacy_limiter(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging), @@ -2017,12 +2017,12 @@ async def test_summary_model_denied_when_team_over_model_budget(): with ( patch( # test-quality-ok: apply_compact_20260112 reads the summary model setting as a module global, no seam - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), # test-quality-ok: forces the over-threshold branch patch( # test-quality-ok: the summary call is the observable that must NOT happen when the team is over budget - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch( # test-quality-ok: the limiter is a proxy_server module global the editor imports, no injection seam @@ -2073,12 +2073,12 @@ async def test_scoped_budget_metadata_propagated_to_summary_call(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", new_callable=AsyncMock, return_value=mock_response, ) as mock_call, @@ -2101,7 +2101,7 @@ async def test_scoped_budget_metadata_propagated_to_summary_call(): async def test_summary_call_passes_end_user_id_as_top_level_user(): """``_call_summary_model`` forwards the propagated end-user id as the top-level ``user`` kwarg that legacy limiter / prometheus end-user tracking reads.""" - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _call_summary_model, ) @@ -2124,7 +2124,7 @@ async def test_summary_call_passes_end_user_id_as_top_level_user(): async def test_summary_call_omits_user_when_no_end_user_id(): """No end-user id on the parent request means no ``user`` kwarg is sent.""" - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _call_summary_model, ) @@ -2158,12 +2158,12 @@ async def test_model_budget_metadata_propagated_to_summary_call(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", new_callable=AsyncMock, return_value=mock_response, ) as mock_call, @@ -2195,12 +2195,12 @@ async def test_summary_call_propagates_allowed_model_region(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), ): @@ -2221,7 +2221,7 @@ async def test_summary_call_omits_allowed_model_region_when_unset(): """Callers without a region restriction must not get an ``allowed_model_region=None`` kwarg, which would otherwise force the router to evaluate region filtering. """ - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _call_summary_model, ) @@ -2244,7 +2244,7 @@ async def test_summary_call_omits_allowed_model_region_when_unset(): async def test_summary_call_forwards_allowed_model_region_when_set(): """When the caller is region-restricted, the kwarg reaches the router.""" - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _call_summary_model, ) @@ -2275,7 +2275,7 @@ async def test_dispatcher_routes_compact_edit(): """compact_20260112 in the dispatcher resolves to opt-in gate when no model set.""" messages = _simple_messages() with patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value=None, ): result = await apply_context_management( @@ -2317,7 +2317,7 @@ async def test_dispatcher_trigger_below_minimum_raises_through(): async def test_run_polyfill_skipped_when_context_management_in_additional_drop_params(): """additional_drop_params=["context_management"] is the explicit opt-out.""" - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( _run_polyfill_if_enabled, ) @@ -2338,13 +2338,13 @@ async def test_run_polyfill_runs_when_litellm_drop_params_true(monkeypatch): """drop_params must not disable the polyfill: context_management is a LiteLLM-supported param (polyfilled where not native), and drop_params only exists to strip genuinely unsupported params.""" - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( _run_polyfill_if_enabled, ) monkeypatch.setattr(litellm, "drop_params", True) with patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value=None, ): result = await _run_polyfill_if_enabled( @@ -2363,7 +2363,7 @@ async def test_run_polyfill_runs_when_litellm_drop_params_true(monkeypatch): async def test_run_polyfill_skipped_when_spec_empty(): """Empty context_management_spec must also return None (no polyfill work).""" - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( _run_polyfill_if_enabled, ) @@ -2444,7 +2444,7 @@ def _openai_chat_response(): async def _call_async_adapter_handler(**handler_kwargs: Any): - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -2503,7 +2503,7 @@ async def test_async_handler_additional_drop_params_strips_context_management(): def _call_sync_adapter_handler(**handler_kwargs: Any): - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -2555,7 +2555,7 @@ async def test_prepare_context_managed_request_forwards_proxy_litellm_metadata() Anthropic-shape ``metadata`` arg (which only carries ``user_id``). Otherwise the summary subcall lands on the router with no parent attribution, and those tokens go unbilled to the caller's key/team.""" - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( prepare_context_managed_request, ) @@ -2568,7 +2568,7 @@ async def test_prepare_context_managed_request_forwards_proxy_litellm_metadata() with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), @@ -2759,7 +2759,7 @@ def test_endpoint_runs_failure_hook_on_500_context_management_error(): def test_count_effective_tokens_counts_midturn_system_correction(): - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _count_effective_tokens, ) @@ -2786,7 +2786,7 @@ def test_count_effective_tokens_counts_midturn_system_correction(): def test_build_summary_messages_keeps_midturn_system_correction_in_place(): - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _build_summary_messages, ) @@ -2820,7 +2820,7 @@ async def test_threshold_check_counts_tokens_off_the_event_loop(monkeypatch): warm_tokenizer, ) - from litellm.llms.anthropic.pass_through.context_management.constants import ( + from litellm.llms.anthropic.messages.context_management.constants import ( COMPACT_SUMMARY_MODEL_SETTING_KEY, ) from litellm.proxy.proxy_server import general_settings diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_constants.py b/tests/unit/llms/anthropic/messages/context_management/test_constants.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/context_management/test_constants.py rename to tests/unit/llms/anthropic/messages/context_management/test_constants.py index 680c2b6f79a..da0d9c38791 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_constants.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_constants.py @@ -8,7 +8,7 @@ import pytest import litellm from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.llms.anthropic.pass_through.context_management.constants import ( +from litellm.llms.anthropic.messages.context_management.constants import ( CLEARED_TOOL_RESULT_PLACEHOLDER, ) from litellm.types.utils import ( diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py b/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py rename to tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py index 66957456c95..b79e33b0c3b 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py @@ -2,7 +2,7 @@ Unit tests for the context_management polyfill dispatcher. """ -from litellm.llms.anthropic.pass_through.context_management import ( +from litellm.llms.anthropic.messages.context_management import ( apply_context_management, ) diff --git a/tests/unit/llms/anthropic/pass_through/messages/__init__.py b/tests/unit/llms/anthropic/messages/native/__init__.py similarity index 100% rename from tests/unit/llms/anthropic/pass_through/messages/__init__.py rename to tests/unit/llms/anthropic/messages/native/__init__.py diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_advisor_integration.py b/tests/unit/llms/anthropic/messages/native/test_advisor_integration.py similarity index 92% rename from tests/unit/llms/anthropic/pass_through/messages/test_advisor_integration.py rename to tests/unit/llms/anthropic/messages/native/test_advisor_integration.py index f5f30fb86c1..8bd36c6564a 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_advisor_integration.py +++ b/tests/unit/llms/anthropic/messages/native/test_advisor_integration.py @@ -70,7 +70,7 @@ async def test_full_dispatch_interceptor_fires_and_loop_completes(): The interceptor must fire, run the loop (1 advisor call), and return a clean final response with no advisor tool_use blocks. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages, ) @@ -86,7 +86,7 @@ async def test_full_dispatch_interceptor_fires_and_loop_completes(): return _text_resp("def is_prime(n): ...") # executor: final with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_handler, ): result = await anthropic_messages( @@ -121,10 +121,10 @@ async def test_max_uses_enforced_through_full_handler(): AdvisorMaxIterationsError propagates out of anthropic_messages() when the executor keeps calling the advisor past max_uses. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages, ) - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorMaxIterationsError, ) @@ -137,7 +137,7 @@ async def test_max_uses_enforced_through_full_handler(): return _advisor_call_resp() with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_handler, ): with pytest.raises(AdvisorMaxIterationsError): @@ -162,7 +162,7 @@ async def test_anthropic_provider_bypasses_interceptor(): With custom_llm_provider='anthropic', the interceptor must NOT fire. The advisor_20260301 tool is forwarded as-is to the underlying handler. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages, ) @@ -170,7 +170,7 @@ async def test_anthropic_provider_bypasses_interceptor(): # Patch the non-interceptor code path — anthropic_messages_handler with patch( - "litellm.llms.anthropic.pass_through.messages.handler.anthropic_messages_handler", + "litellm.llms.anthropic.messages.native.handler.anthropic_messages_handler", return_value=direct_response, ) as mock_native: result = await anthropic_messages( @@ -209,7 +209,7 @@ async def test_named_params_forwarded_into_advisor_executor_subcall(): them, e.g. Vertex AI rejecting ``clear_thinking_20251015`` context_management edits with: ``strategy requires thinking to be enabled or adaptive``. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages, ) @@ -238,7 +238,7 @@ async def test_named_params_forwarded_into_advisor_executor_subcall(): return _text_resp("Final answer.") with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_handler, ): await anthropic_messages( @@ -289,7 +289,7 @@ async def test_pre_request_hook_override_does_not_collide_with_explicit_kwargs() Regression for Greptile P2 on PR #27810. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages, ) @@ -326,11 +326,11 @@ async def test_pre_request_hook_override_does_not_collide_with_explicit_kwargs() with ( patch( - "litellm.llms.anthropic.pass_through.messages.handler._execute_pre_request_hooks", + "litellm.llms.anthropic.messages.native.handler._execute_pre_request_hooks", side_effect=fake_pre_request_hooks, ), patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_handler, ), ): diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_agentic_streaming_iterator.py b/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/messages/test_agentic_streaming_iterator.py rename to tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py index 56fa3015d44..90a6ae3a574 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_agentic_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py @@ -11,7 +11,7 @@ import pytest from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES -from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( +from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES, AgenticAnthropicStreamingIterator, _handle_content_block_delta, diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_effort.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_effort.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_effort.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_effort.py index 7885cc69b19..94476139313 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_effort.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_effort.py @@ -7,7 +7,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, ) from litellm.llms.anthropic.common_utils import AnthropicError -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.openai_like.json_loader import SimpleProviderConfig diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_encrypted_reasoning.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py similarity index 96% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_encrypted_reasoning.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py index ca81147da4c..cd6d2d9be96 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_encrypted_reasoning.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py @@ -1,7 +1,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( encrypted_reasoning_signature, ) -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py similarity index 96% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index bb4388b9d0e..c11f58faec4 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -83,12 +83,12 @@ async def test_openai_messages_forwards_stop_sequences_to_chat_completions( @pytest.mark.parametrize("stop_sequences", [None, []], ids=["no-stops", "empty-stops"]) -def test_anthropic_experimental_pass_through_messages_handler(stop_sequences): +def test_anthropic_messages_handler(stop_sequences): """ Test that api key is passed to litellm.responses for OpenAI models. OpenAI and Azure models are routed directly to the Responses API. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -169,12 +169,12 @@ async def test_openai_model_does_not_forward_stream_options_to_responses_api(): assert "stream_options" not in request_body -def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_and_api_base_and_custom_values(): +def test_anthropic_messages_handler_dynamic_api_key_and_api_base_and_custom_values(): """ Test that api key, api base, and extra kwargs are forwarded to litellm.completion for Azure models. Azure models are routed through chat/completions (not the Responses API). """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -201,7 +201,7 @@ async def test_anthropic_messages_sanitizes_empty_text_blocks_before_dispatch(): """Regression test for #22930. The unified /v1/messages path must strip empty text blocks before forwarding, otherwise Anthropic returns 400 "text content blocks must be non-empty".""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler msgs = [ { @@ -239,7 +239,7 @@ async def test_anthropic_messages_sanitizes_empty_text_blocks_before_dispatch(): @pytest.mark.asyncio async def test_anthropic_messages_sanitizes_tool_use_ids_before_dispatch(): - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler msgs = [ { @@ -283,14 +283,14 @@ async def _async_return(value): return value -def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provider(): +def test_anthropic_messages_handler_custom_llm_provider(): """ Test that litellm.completion is called when a custom LLM provider is given. Provider resolution now happens exactly once, inside litellm.completion itself (BerriAI/litellm#37716), so the handler passes the original unresolved model through. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -374,7 +374,7 @@ def test_openai_model_with_thinking_converts_to_reasoning(): OpenAI models are routed directly to the Responses API, so we verify that litellm.responses() is called with `reasoning` properly set. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -414,7 +414,7 @@ class TestThinkingParameterTransformation: def test_claude_model_preserves_thinking_with_budget_tokens(self): """Test that Claude models get thinking parameter passed through with exact budget_tokens.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -429,7 +429,7 @@ class TestThinkingParameterTransformation: def test_non_claude_model_converts_thinking_to_reasoning_effort(self): """Test that non-Claude models convert thinking to reasoning_effort.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -447,7 +447,7 @@ class TestThinkingParameterTransformation: def test_translate_thinking_for_model_summary_when_enabled(self): """When reasoning_auto_summary is True, summary='detailed' is injected.""" import litellm - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -465,7 +465,7 @@ class TestThinkingParameterTransformation: def test_translate_thinking_for_model_preserves_user_summary(self): """User-provided summary is always preserved regardless of flag.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -482,7 +482,7 @@ class TestThinkingSummaryPreservation: def test_thinking_summary_concise_preserved_for_openai(self): """User-provided summary='concise' should not be replaced with 'detailed'.""" - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -498,7 +498,7 @@ class TestThinkingSummaryPreservation: def test_thinking_summary_auto_preserved_for_openai(self): """User-provided summary='auto' should be preserved.""" - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -515,7 +515,7 @@ class TestThinkingSummaryPreservation: def test_summary_added_when_auto_summary_enabled(self): """When reasoning_auto_summary is True, summary='detailed' is added.""" import litellm - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -540,7 +540,7 @@ class TestThinkingSummaryPreservation: def test_no_summary_by_default_string_reasoning(self): """By default (reasoning_auto_summary=False), summary is not added for string reasoning_effort.""" import litellm - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -563,7 +563,7 @@ class TestThinkingSummaryPreservation: def test_no_summary_by_default_dict_reasoning(self): """By default (reasoning_auto_summary=False), summary is not injected into dict reasoning_effort.""" import litellm - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -586,7 +586,7 @@ class TestThinkingSummaryPreservation: def test_summary_added_when_env_var_set(self, monkeypatch): """When LITELLM_REASONING_AUTO_SUMMARY env var is true, summary is added.""" import litellm - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -613,7 +613,7 @@ class TestThinkingSummaryPreservation: def test_user_provided_summary_preserved_even_when_flag_off(self): """When user already set summary in dict reasoning_effort, it's preserved regardless of flag.""" import litellm - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -634,7 +634,7 @@ class TestThinkingSummaryPreservation: def test_openai_model_with_thinking_summary_end_to_end(self): """End-to-end: anthropic_messages_handler should preserve thinking.summary for OpenAI models.""" - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -663,7 +663,7 @@ class TestThinkingSummaryPreservation: def test_responses_adapter_preserves_summary(self): """translate_thinking_to_reasoning should include summary when user provides it.""" - from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( + from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) @@ -674,7 +674,7 @@ class TestThinkingSummaryPreservation: def test_responses_adapter_no_summary_by_default(self): """translate_thinking_to_reasoning should not include summary by default (opt-in).""" import litellm - from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( + from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) @@ -690,7 +690,7 @@ class TestThinkingSummaryPreservation: def test_translate_thinking_for_model_preserves_summary(self): """translate_thinking_for_model should include summary in reasoning_effort dict when user provides it.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -704,7 +704,7 @@ class TestThinkingSummaryPreservation: def test_translate_thinking_for_model_disabled_stays_plain_string_when_auto_summary_enabled(self): """Disabled thinking must stay a plain string even when reasoning_auto_summary is on.""" import litellm - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -743,7 +743,7 @@ def _empty_block_msgs(): def test_handler_strips_when_no_presanitized_flag(): """Sync entry point (no async wrapper): handler must still sanitize.""" - from litellm.llms.anthropic.pass_through.messages import handler, utils + from litellm.llms.anthropic.messages.native import handler, utils with patch.object( utils, @@ -763,7 +763,7 @@ def test_handler_strips_when_no_presanitized_flag(): def test_handler_skips_strip_when_presanitized(): """Async wrapper already sanitized -> handler must NOT rescan.""" - from litellm.llms.anthropic.pass_through.messages import handler, utils + from litellm.llms.anthropic.messages.native import handler, utils with patch.object( utils, @@ -784,7 +784,7 @@ def test_handler_skips_strip_when_presanitized(): def test_handler_flattens_replayed_unencrypted_web_search_results(): """Synthesized search blocks replayed as history must reach the provider as text.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler captured = {} @@ -839,7 +839,7 @@ def test_handler_flattens_replayed_unencrypted_web_search_results(): def test_presanitized_flag_not_leaked_to_provider_params(): """The private sentinel must be popped, never forwarded as a request param.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler captured = {} @@ -868,7 +868,7 @@ def test_presanitized_flag_not_leaked_to_provider_params(): @pytest.mark.asyncio async def test_async_wrapper_sets_presanitized_and_sanitizes_once(): """End-to-end: wrapper sanitizes (once) AND signals the handler to skip.""" - from litellm.llms.anthropic.pass_through.messages import handler, utils + from litellm.llms.anthropic.messages.native import handler, utils captured = {} @@ -912,7 +912,7 @@ def _gate_stubs(monkeypatch): provider config handed to the native passthrough path and ``translation_calls`` counts hits on the Anthropic->OpenAI translation handlers. """ - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler captured = {} translation_calls = {"count": 0} @@ -942,7 +942,7 @@ def _gate_stubs(monkeypatch): def test_gate_passthrough_when_supported_endpoints_opts_in(monkeypatch): """provider=openai + model_info.supported_endpoints containing /v1/messages must route to the native passthrough config, NOT the translation handlers.""" - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) from litellm.llms.openai_like.messages.transformation import ( @@ -968,7 +968,7 @@ def test_gate_passthrough_when_supported_endpoints_opts_in(monkeypatch): def test_gate_translates_when_supported_endpoints_absent(monkeypatch): """Default behavior is unchanged: without the /v1/messages opt-in, an openai deployment is translated (Responses API), never passed through natively.""" - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -990,7 +990,7 @@ def test_gate_translates_when_supported_endpoints_absent(monkeypatch): def test_gate_passthrough_skipped_when_only_chat_completions_supported(monkeypatch): """A deployment that lists only /v1/chat/completions is still translated; the opt-in is specifically the /v1/messages entry.""" - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -1023,7 +1023,7 @@ def test_gate_passthrough_forwards_cache_control_ttl_only_when_deployment_opts_i ): """The passthrough config strips cache_control.ttl unless the deployment sets model_info.cache_control_ttl to exactly true.""" - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -1374,7 +1374,7 @@ class TestMessagesStreamingSuccessLogging: @pytest.mark.asyncio async def test_responses_bridge_streaming_emits_success_logging(self, capture_success_payloads): """The Responses bridge, which is the default for openai/ deployments.""" - from litellm.llms.anthropic.pass_through.responses_adapters.handler import ( + from litellm.llms.anthropic.messages.responses_adapters.handler import ( LiteLLMMessagesToResponsesAPIHandler, ) @@ -1415,14 +1415,14 @@ class TestMessagesStreamingSuccessLogging: """The chat-completions bridge, reached via litellm.use_chat_completions_url_for_anthropic_messages. Its router lookup is stubbed to what an SDK caller with no proxy running already resolves to.""" - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) _bind_logging_worker_to_running_loop() with patch( - "litellm.llms.anthropic.pass_through.adapters.handler._proxy_router_fallback", + "litellm.llms.anthropic.messages.adapters.handler._proxy_router_fallback", return_value=None, ): sse_stream = await LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler( @@ -1475,7 +1475,7 @@ async def test_anthropic_messages_maps_provider_exception_before_failure_logging The 403 row pins the upstream status on the way through the mapper: Anthropic's documented permission_error must reach the caller as a 403, never as the mapper's APIConnectionError 500 fallthrough.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler capture = _FailureCapture() monkeypatch.setattr(litellm, "callbacks", [capture]) @@ -1517,7 +1517,7 @@ async def test_anthropic_messages_leaves_non_provider_failures_unmapped(): """The mapping boundary is for provider failures only. A request rejected before the provider call (here invalid metadata) must surface as the original exception, not as the mapper's APIConnectionError, whose message embeds a server traceback.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler def upstream_must_not_be_called(request: httpx.Request) -> httpx.Response: raise AssertionError("the provider must not be called for a request rejected locally") @@ -1563,7 +1563,7 @@ def _recording_client(seen_urls: list[str]) -> AsyncHTTPHandler: @pytest.mark.asyncio async def test_provider_messages_api_base_env_is_not_shadowed_by_the_chat_default(monkeypatch): - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler monkeypatch.delenv("DEEPSEEK_API_BASE", raising=False) monkeypatch.setenv("DEEPSEEK_ANTHROPIC_API_BASE", "https://deepseek.internal.example/anthropic") @@ -1583,7 +1583,7 @@ async def test_provider_messages_api_base_env_is_not_shadowed_by_the_chat_defaul @pytest.mark.asyncio async def test_anthropic_messages_forwards_safeguards_and_unknown_beta_to_anthropic(): """Shapes are what Claude Code 2.1.278 sends and api.anthropic.com returns, captured 2026-09-21.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler safeguards = [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}] client_betas = "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14" @@ -1631,7 +1631,7 @@ async def test_anthropic_messages_forwards_safeguards_and_unknown_beta_to_anthro @pytest.mark.asyncio async def test_anthropic_messages_streaming_forwards_safeguards_and_keeps_safeguard_results(): """Shapes are what Claude Code 2.1.278 sends and api.anthropic.com returns, captured 2026-09-21.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler safeguards = [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}] tool_verdicts = {"toolu_01": {"type": "evaluated", "outcome": "not_flagged"}} @@ -1736,7 +1736,7 @@ async def test_anthropic_messages_forwards_safeguards_and_dangerous_tool_use_bet local_beta_headers_config, client_headers ): """Bedrock Invoke takes betas in the body's `anthropic_beta` and 400s on `safeguards` without the beta, so the beta rides along with the field.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler safeguards, safeguard_results = _claude_code_auto_mode_request() captured: dict[str, object] = {} @@ -1765,7 +1765,7 @@ async def test_anthropic_messages_forwards_safeguards_and_dangerous_tool_use_bet local_beta_headers_config: None, client_headers: dict[str, str] ) -> None: """Vertex rawPredict takes the beta as the `anthropic-beta` header and 400s on `safeguards` without it, so the beta rides along with the field.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler safeguards, safeguard_results = _claude_code_auto_mode_request() captured: dict[str, object] = {} @@ -1884,7 +1884,7 @@ def _native_upstream(*, stream: bool, model: str, **extra: object) -> AsyncHTTPH async def _native_messages(model: str, upstream: AsyncHTTPHandler, *, stream: bool, **extra: object) -> object: - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler safeguards, _ = _claude_code_auto_mode_request() with patch("litellm.proxy.proxy_server.general_settings", {"safeguards_classifier_model": "classifier"}): @@ -2332,7 +2332,7 @@ async def test_anthropic_messages_bedrock_dynamic_region(): async def test_anthropic_messages_with_safeguards_and_no_classifier_keep_the_upstream_400_for_malformed_messages( malformed_messages: object, ) -> None: - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler def upstream_rejects_the_messages(request: httpx.Request) -> httpx.Response: return httpx.Response( @@ -2362,7 +2362,7 @@ async def test_anthropic_messages_with_safeguards_and_no_classifier_keep_the_ups def test_native_messages_tool_config_point_marks_last_tool_and_drops_the_key(monkeypatch): """LIT-9880 on /v1/messages: the hook wrote the tool_config point back into kwargs and the native path dropped it silently, so the tools went out without any cache breakpoint.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler captured = {} diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_per_turn_control.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_per_turn_control.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_per_turn_control.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_per_turn_control.py index 05c12c6a285..e29f962ec8c 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_per_turn_control.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_per_turn_control.py @@ -2,7 +2,7 @@ import pytest from litellm import anthropic_beta_headers_manager from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.openai_like.json_loader import SimpleProviderConfig diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py similarity index 100% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_service_tier.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_service_tier.py similarity index 92% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_service_tier.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_service_tier.py index 6472ccd1b2e..75d5b583cc8 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_service_tier.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_service_tier.py @@ -2,7 +2,7 @@ from typing import Final import pytest -from litellm.llms.anthropic.pass_through.messages.utils import AnthropicMessagesRequestUtils +from litellm.llms.anthropic.messages.native.utils import AnthropicMessagesRequestUtils REQUEST: Final[dict[str, object]] = { "max_tokens": 64, diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_speed.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_speed.py similarity index 96% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_speed.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_speed.py index 45c6ebaa39b..9be3cdf55b3 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_speed.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_speed.py @@ -1,9 +1,9 @@ import litellm import pytest -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) -from litellm.llms.anthropic.pass_through.messages.utils import ( +from litellm.llms.anthropic.messages.native.utils import ( AnthropicMessagesRequestUtils, ) diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_structured_outputs.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_structured_outputs.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_structured_outputs.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_structured_outputs.py index db3ff58d80a..5d9ad409f15 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_structured_outputs.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_structured_outputs.py @@ -8,7 +8,7 @@ from typing import Final import pytest import litellm -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.types.router import GenericLiteLLMParams diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py index 457d5d5e1c8..aa6625561ba 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py @@ -10,7 +10,7 @@ from pydantic import JsonValue import litellm from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.anthropic.wif import get_anthropic_wif_token diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_content_after_stop_reason.py b/tests/unit/llms/anthropic/messages/native/test_content_after_stop_reason.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/messages/test_content_after_stop_reason.py rename to tests/unit/llms/anthropic/messages/native/test_content_after_stop_reason.py index f57a872c82a..1accce8383b 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_content_after_stop_reason.py +++ b/tests/unit/llms/anthropic/messages/native/test_content_after_stop_reason.py @@ -17,7 +17,7 @@ from typing import List import pytest -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_fake_stream_iterator.py b/tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py similarity index 97% rename from tests/unit/llms/anthropic/pass_through/messages/test_fake_stream_iterator.py rename to tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py index 476cea2833c..233145f2706 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_fake_stream_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py @@ -2,7 +2,7 @@ import json from collections.abc import Iterable, Mapping from typing import Final -from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( +from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_mcp_handler.py b/tests/unit/llms/anthropic/messages/native/test_mcp_handler.py similarity index 95% rename from tests/unit/llms/anthropic/pass_through/messages/test_mcp_handler.py rename to tests/unit/llms/anthropic/messages/native/test_mcp_handler.py index 3aca6181d12..73c2edbce19 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_mcp_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_mcp_handler.py @@ -4,13 +4,13 @@ from unittest.mock import AsyncMock, patch import pytest -from litellm.llms.anthropic.pass_through.adapters.transformation import ( +from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) -from litellm.llms.anthropic.pass_through.messages.handler import ( +from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) -from litellm.llms.anthropic.pass_through.messages.mcp_handler import ( +from litellm.llms.anthropic.messages.native.mcp_handler import ( _build_tool_result_message, _extract_tool_use_blocks, ) @@ -39,7 +39,7 @@ def test_anthropic_messages_handler_routes_litellm_proxy_mcp_to_the_gateway(): dispatch makes the whole feature unreachable while every unit test still passes. """ with patch( - "litellm.llms.anthropic.pass_through.messages.mcp_handler.anthropic_messages_with_mcp", + "litellm.llms.anthropic.messages.native.mcp_handler.anthropic_messages_with_mcp", new=AsyncMock(return_value={"routed": True}), ) as routed: result = anthropic_messages_handler( @@ -59,7 +59,7 @@ def test_anthropic_messages_handler_routes_litellm_proxy_mcp_to_the_gateway(): def test_anthropic_messages_handler_skips_the_gateway_on_recursion(): """The gateway's own follow-up call must not re-enter the gateway.""" with patch( - "litellm.llms.anthropic.pass_through.messages.mcp_handler.anthropic_messages_with_mcp", + "litellm.llms.anthropic.messages.native.mcp_handler.anthropic_messages_with_mcp", new=AsyncMock(return_value={"routed": True}), ) as routed: with pytest.raises(ValueError, match="anthropic_messages_handler is not implemented for sync calls"): @@ -78,7 +78,7 @@ def test_anthropic_messages_handler_skips_the_gateway_on_recursion(): def test_anthropic_messages_handler_leaves_native_tools_alone(): """A plain Anthropic tool is not an MCP reference and must not reach the gateway.""" with patch( - "litellm.llms.anthropic.pass_through.messages.mcp_handler.anthropic_messages_with_mcp", + "litellm.llms.anthropic.messages.native.mcp_handler.anthropic_messages_with_mcp", new=AsyncMock(return_value={"routed": True}), ) as routed: with pytest.raises(ValueError, match="anthropic_messages_handler is not implemented for sync calls"): @@ -161,7 +161,7 @@ async def test_anthropic_messages_with_mcp_forwards_the_callers_mcp_credentials( token, per-user env) silently returns nothing while the model claims it has no access. Only a no-auth server would look healthy. """ - from litellm.llms.anthropic.pass_through.messages import mcp_handler + from litellm.llms.anthropic.messages.native import mcp_handler from litellm.responses.mcp.request_context import MCPRequestContext context = MCPRequestContext( @@ -246,7 +246,7 @@ async def test_anthropic_messages_with_mcp_hands_execution_the_requests_served_t """ from mcp.types import Tool - from litellm.llms.anthropic.pass_through.messages import mcp_handler + from litellm.llms.anthropic.messages.native import mcp_handler from litellm.responses.mcp.request_context import MCPRequestContext served = [ @@ -311,7 +311,7 @@ async def test_anthropic_messages_with_mcp_stops_when_every_tool_call_is_skipped Anthropic rejects that, so the caller would get an unhandled 400 from the middle of the loop rather than the model's own answer. """ - from litellm.llms.anthropic.pass_through.messages import mcp_handler + from litellm.llms.anthropic.messages.native import mcp_handler from litellm.responses.mcp.request_context import MCPRequestContext tool_use_response = { @@ -349,7 +349,7 @@ async def test_anthropic_messages_with_mcp_stops_when_every_tool_call_is_skipped @pytest.mark.asyncio async def test_anthropic_messages_with_mcp_classifies_only_the_response_the_client_gets(): - from litellm.llms.anthropic.pass_through.messages import mcp_handler + from litellm.llms.anthropic.messages.native import mcp_handler from litellm.responses.mcp.request_context import MCPRequestContext from litellm.router import Router @@ -409,7 +409,7 @@ async def _mcp_safeguards_turn_recording_beta_map_loads(custom_llm_provider: str import threading from litellm import anthropic_beta_headers_manager - from litellm.llms.anthropic.pass_through.messages import mcp_handler + from litellm.llms.anthropic.messages.native import mcp_handler from litellm.responses.mcp.request_context import MCPRequestContext loader_threads: Final[list[int]] = [] # mutable-ok: records the thread each beta map load ran on diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_mid_conversation_system.py b/tests/unit/llms/anthropic/messages/native/test_mid_conversation_system.py similarity index 97% rename from tests/unit/llms/anthropic/pass_through/messages/test_mid_conversation_system.py rename to tests/unit/llms/anthropic/messages/native/test_mid_conversation_system.py index f527b4912d5..a1615c01d08 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_mid_conversation_system.py +++ b/tests/unit/llms/anthropic/messages/native/test_mid_conversation_system.py @@ -1,6 +1,6 @@ from collections import Counter -from litellm.llms.anthropic.pass_through.messages.mid_conversation_system import ( +from litellm.llms.anthropic.messages.native.mid_conversation_system import ( CONVERTED_SYSTEM_NOTE, convert_mid_conversation_system_turns, ) diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_parallel_tool_calls.py b/tests/unit/llms/anthropic/messages/native/test_parallel_tool_calls.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/messages/test_parallel_tool_calls.py rename to tests/unit/llms/anthropic/messages/native/test_parallel_tool_calls.py index 229eff4275a..5b2e285fed7 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_parallel_tool_calls.py +++ b/tests/unit/llms/anthropic/messages/native/test_parallel_tool_calls.py @@ -2,7 +2,7 @@ from typing import List -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) from litellm.types.utils import ( diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_reasoning_auto_summary_messages.py b/tests/unit/llms/anthropic/messages/native/test_reasoning_auto_summary_messages.py similarity index 97% rename from tests/unit/llms/anthropic/pass_through/messages/test_reasoning_auto_summary_messages.py rename to tests/unit/llms/anthropic/messages/native/test_reasoning_auto_summary_messages.py index cc951421b10..7334b4eeaa1 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_reasoning_auto_summary_messages.py +++ b/tests/unit/llms/anthropic/messages/native/test_reasoning_auto_summary_messages.py @@ -14,7 +14,7 @@ from unittest.mock import MagicMock, patch import litellm -from litellm.llms.anthropic.pass_through.messages.handler import ( +from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -30,10 +30,10 @@ def _call_handler_and_capture_optional_params(thinking=None, **extra_kwargs): captured = {} with patch( - "litellm.llms.anthropic.pass_through.messages.handler." + "litellm.llms.anthropic.messages.native.handler." "base_llm_http_handler" ) as mock_handler, patch( - "litellm.llms.anthropic.pass_through.messages.handler." + "litellm.llms.anthropic.messages.native.handler." "ProviderConfigManager" ) as mock_pcm: # Make get_provider_anthropic_messages_config return a non-None config diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_reasoning_effort_translation.py b/tests/unit/llms/anthropic/messages/native/test_reasoning_effort_translation.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/messages/test_reasoning_effort_translation.py rename to tests/unit/llms/anthropic/messages/native/test_reasoning_effort_translation.py index 284c6ea532d..50dc0deabfd 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_reasoning_effort_translation.py +++ b/tests/unit/llms/anthropic/messages/native/test_reasoning_effort_translation.py @@ -8,7 +8,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, ) from litellm.llms.anthropic.common_utils import AnthropicError -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_request_optional_param_utils.py b/tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/messages/test_request_optional_param_utils.py rename to tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py index 7a4f3aa5f90..d525ccfec37 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_request_optional_param_utils.py +++ b/tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py @@ -9,7 +9,7 @@ Regression tests for the /v1/messages request-parse fast paths: import pytest import litellm -from litellm.llms.anthropic.pass_through.messages.utils import ( +from litellm.llms.anthropic.messages.native.utils import ( AnthropicMessagesRequestUtils, anthropic_messages_optional_param_keys, ) diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_response_cache.py b/tests/unit/llms/anthropic/messages/native/test_response_cache.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/messages/test_response_cache.py rename to tests/unit/llms/anthropic/messages/native/test_response_cache.py index 1df4ccec02c..133bcd96433 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_response_cache.py +++ b/tests/unit/llms/anthropic/messages/native/test_response_cache.py @@ -13,8 +13,8 @@ from litellm.caching.caching_handler import LLMCachingHandler from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.llms.anthropic.pass_through.messages import handler -from litellm.llms.anthropic.pass_through.messages.response_cache import ( +from litellm.llms.anthropic.messages.native import handler +from litellm.llms.anthropic.messages.native.response_cache import ( AnthropicMessagesStreamCacheWriter, CachedAnthropicMessagesStreamIterator, ) @@ -275,7 +275,7 @@ async def test_abandoned_stream_is_not_cached(local_cache, request_kwargs, monke async def test_cached_stream_replay_logs_once_when_polled_after_exhaustion(): from unittest.mock import AsyncMock, MagicMock, patch - from litellm.llms.anthropic.pass_through.messages.response_cache import ( + from litellm.llms.anthropic.messages.native.response_cache import ( CachedAnthropicMessagesStreamIterator, ) from litellm.proxy.pass_through_endpoints.streaming_handler import ( diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_sse_wrapper.py b/tests/unit/llms/anthropic/messages/native/test_sse_wrapper.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/messages/test_sse_wrapper.py rename to tests/unit/llms/anthropic/messages/native/test_sse_wrapper.py index d22478e16ef..cd6b3fa006d 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_sse_wrapper.py +++ b/tests/unit/llms/anthropic/messages/native/test_sse_wrapper.py @@ -2,7 +2,7 @@ import pytest from fastapi.testclient import TestClient -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py b/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py rename to tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py index 4ed9054f3d3..dd50f60250a 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py @@ -11,8 +11,8 @@ import pytest from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.anthropic.pass_through.messages import streaming_iterator as streaming_iterator_module -from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( +from litellm.llms.anthropic.messages.native import streaming_iterator as streaming_iterator_module +from litellm.llms.anthropic.messages.native.streaming_iterator import ( INCOMPLETE_STREAM_ERROR_MESSAGE, AnthropicMessagesStreamHiddenParams, AnthropicMessagesStreamingResponse, @@ -26,8 +26,8 @@ from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( is_provider_error_chunk, parse_anthropic_error_event, ) -from litellm.llms.anthropic.pass_through.messages.transformation import AnthropicMessagesConfig -from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator +from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig +from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig from litellm.types.utils import Choices, Message, ModelResponse @@ -1157,7 +1157,7 @@ async def test_abort_upstream_logs_warning_when_aclose_raises(caplog): async def test_enqueue_for_client_returns_false_when_already_detached(): """_enqueue_for_client must return False immediately (without touching the queue) when client_detached is already set before the call.""" - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( BaseAnthropicMessagesStreamingIterator, ) @@ -1174,7 +1174,7 @@ async def test_enqueue_for_client_returns_false_when_already_detached(): async def test_enqueue_for_client_returns_false_when_client_detaches_while_queue_full(): """_enqueue_for_client must return False (and cancel the put) when the queue is full and client_detached fires before space becomes available.""" - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( BaseAnthropicMessagesStreamingIterator, ) diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/__init__.py b/tests/unit/llms/anthropic/messages/responses_adapters/__init__.py similarity index 100% rename from tests/unit/llms/anthropic/pass_through/responses_adapters/__init__.py rename to tests/unit/llms/anthropic/messages/responses_adapters/__init__.py diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py rename to tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index 8fe6bd31082..7de722e1171 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -16,7 +16,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, ) -from litellm.llms.anthropic.pass_through.responses_adapters.handler import ( +from litellm.llms.anthropic.messages.responses_adapters.handler import ( LiteLLMMessagesToResponsesAPIHandler, _build_responses_kwargs, ) diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py rename to tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py index e1dded214bf..5af8e69a0ea 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -1,6 +1,6 @@ """ Tests for AnthropicResponsesStreamWrapper -(litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py) +(litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py) """ import asyncio @@ -18,8 +18,8 @@ from litellm.exceptions import MidStreamFallbackError from litellm.litellm_core_utils.prompt_templates.common_utils import ( encrypted_reasoning_signature, ) -from litellm.llms.anthropic.pass_through.messages.utils import INCOMPLETE_STREAM_ERROR_MESSAGE -from litellm.llms.anthropic.pass_through.responses_adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.native.utils import INCOMPLETE_STREAM_ERROR_MESSAGE +from litellm.llms.anthropic.messages.responses_adapters.streaming_iterator import ( AnthropicResponsesStreamWrapper, ) from litellm.types.llms.openai import ResponseFailedEvent, ResponsesAPIResponse diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py rename to tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index 829000c0c5f..f6fcb9e6f1e 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -1,6 +1,6 @@ """ Tests for LiteLLMAnthropicToResponsesAPIAdapter -(litellm/llms/anthropic/pass_through/responses_adapters/transformation.py) +(litellm/llms/anthropic/messages/responses_adapters/transformation.py) """ import json @@ -22,7 +22,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_PLACEHOLDER, encrypted_reasoning_signature, ) -from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( +from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig @@ -2413,7 +2413,7 @@ class TestPromptCacheBreakpointToResponses: assert log_output == self.LOG_OUTPUT def test_prompt_cache_options_forwarded_to_responses_kwargs(self): - from litellm.llms.anthropic.pass_through.responses_adapters.handler import ( + from litellm.llms.anthropic.messages.responses_adapters.handler import ( _build_responses_kwargs, ) diff --git a/tests/unit/llms/anthropic/messages/test_advisor_orchestration.py b/tests/unit/llms/anthropic/messages/test_advisor_orchestration.py index 5b0f743b836..810dfffaf8b 100644 --- a/tests/unit/llms/anthropic/messages/test_advisor_orchestration.py +++ b/tests/unit/llms/anthropic/messages/test_advisor_orchestration.py @@ -71,7 +71,7 @@ def _make_advisor_tool_use_response( def test_can_handle_edge_cases(): - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -99,7 +99,7 @@ async def test_anthropic_native_interceptor_skipped(): For provider=anthropic, can_handle() must return False. The interceptor must never call handle(). """ - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -115,7 +115,7 @@ async def test_anthropic_native_interceptor_skipped(): @pytest.mark.asyncio async def test_loop_no_advisor_call(): """Executor returns text on first try — no advisor call, loop exits immediately.""" - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, _call_messages_handler, ) @@ -124,7 +124,7 @@ async def test_loop_no_advisor_call(): executor_response = _make_text_response(final_text) with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", new_callable=AsyncMock, return_value=executor_response, ) as mock_call: @@ -157,7 +157,7 @@ async def test_loop_one_advisor_call(): Executor calls advisor once → advisor responds → executor produces final text. Total calls: 3 (executor, advisor, executor-final). """ - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -185,7 +185,7 @@ async def test_loop_one_advisor_call(): return final_resp # executor: final answer with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -217,7 +217,7 @@ async def test_loop_one_advisor_call(): @pytest.mark.asyncio async def test_loop_max_uses_raises(): """Loop exceeding max_uses must raise AdvisorMaxIterationsError.""" - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorMaxIterationsError, AdvisorOrchestrationHandler, ) @@ -238,7 +238,7 @@ async def test_loop_max_uses_raises(): return advisor_tool_use_resp with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -261,17 +261,17 @@ async def test_loop_max_uses_raises(): @pytest.mark.asyncio async def test_loop_streaming_wraps_response(): """stream=True: final response must be wrapped in FakeAnthropicMessagesStreamIterator.""" - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) executor_response = _make_text_response("Hello, world!") with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", new_callable=AsyncMock, return_value=executor_response, ): @@ -307,7 +307,7 @@ async def test_prior_advisor_blocks_replaced_in_history(): History containing server_tool_use + advisor_tool_result blocks gets collapsed to text before forwarding to the executor. """ - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -340,7 +340,7 @@ async def test_prior_advisor_blocks_replaced_in_history(): return _make_text_response("Here is the efficient version.") with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -380,7 +380,7 @@ async def test_advisor_tool_translated_for_executor(): """ The executor must receive a regular tool definition (not advisor_20260301 type). """ - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -392,7 +392,7 @@ async def test_advisor_tool_translated_for_executor(): return _make_text_response("Done.") with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -422,7 +422,7 @@ async def test_advisor_tool_translated_for_executor(): @pytest.mark.asyncio async def test_max_uses_zero_raises_on_first_advisor_call(): """max_uses=0 must cause AdvisorMaxIterationsError on the first advisor call.""" - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorMaxIterationsError, AdvisorOrchestrationHandler, ) @@ -434,7 +434,7 @@ async def test_max_uses_zero_raises_on_first_advisor_call(): return advisor_tool_use_resp # executor always tries to call advisor with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -457,7 +457,7 @@ async def test_max_uses_zero_raises_on_first_advisor_call(): @pytest.mark.asyncio async def test_missing_advisor_model_raises_value_error(): """handle() must raise ValueError when the advisor tool has no model field.""" - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -484,7 +484,7 @@ async def test_missing_advisor_model_raises_value_error(): async def test_max_uses_none_falls_back_to_default(): """When max_uses is absent, the handler uses ADVISOR_MAX_USES from constants.""" import litellm.constants as _c - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorMaxIterationsError, AdvisorOrchestrationHandler, ) @@ -498,7 +498,7 @@ async def test_max_uses_none_falls_back_to_default(): return advisor_tool_use_resp with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -532,7 +532,7 @@ ADVISOR_TOOL_WITH_CREDS = { async def _run_advisor_and_capture_subcall_kwargs(): """Run one advisor turn and return the kwargs of the advisor sub-call.""" - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -557,11 +557,11 @@ async def _run_advisor_and_capture_subcall_kwargs(): with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ), patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor.validate_url", + "litellm.llms.anthropic.messages.native.interceptors.advisor.validate_url", ), ): h = AdvisorOrchestrationHandler() @@ -581,7 +581,7 @@ async def test_advisor_creds_dropped_when_proxy_opt_in_disabled(): """On the proxy without opt-in, the caller's advisor api_base/api_key must NOT reach the sub-call (would redirect it / leak the server key).""" with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=False, ): captured = await _run_advisor_and_capture_subcall_kwargs() @@ -593,7 +593,7 @@ async def test_advisor_creds_dropped_when_proxy_opt_in_disabled(): async def test_advisor_creds_honored_when_proxy_opt_in_enabled(): """With the admin opt-in, the documented clientside routing still works.""" with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ): captured = await _run_advisor_and_capture_subcall_kwargs() @@ -626,7 +626,7 @@ def test_allow_client_side_advisor_credentials_reads_proxy_flag(): """The gate mirrors the proxy's allow_client_side_credentials opt-in.""" import sys - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _allow_client_side_advisor_credentials, ) @@ -650,7 +650,7 @@ def test_allow_client_side_advisor_credentials_defaults_true_outside_proxy(): import builtins import sys - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _allow_client_side_advisor_credentials, ) @@ -674,7 +674,7 @@ def test_advisor_gate_propagates_non_import_errors(): returning True.""" import sys - from litellm.llms.anthropic.pass_through.messages.interceptors import ( + from litellm.llms.anthropic.messages.native.interceptors import ( advisor, ) @@ -733,12 +733,12 @@ async def test_advisor_uses_tool_credentials_when_clientside_enabled(): def test_resolve_advisor_credentials_returns_none_when_gate_closed(): - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=False, ): result = _resolve_advisor_credentials(ADVISOR_TOOL_WITH_CREDS) @@ -746,18 +746,18 @@ def test_resolve_advisor_credentials_returns_none_when_gate_closed(): def test_resolve_advisor_credentials_allows_api_key_without_api_base(): - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) tool = {**ADVISOR_TOOL, "api_key": "sk-other"} with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ), patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor.validate_url", + "litellm.llms.anthropic.messages.native.interceptors.advisor.validate_url", side_effect=AssertionError("validate_url must not run without an api_base"), ), ): @@ -766,13 +766,13 @@ def test_resolve_advisor_credentials_allows_api_key_without_api_base(): def test_resolve_advisor_credentials_rejects_api_base_without_api_key(): - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) tool = {**ADVISOR_TOOL, "api_base": "https://other.example"} with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ): with pytest.raises(ValueError, match="api_base"): @@ -781,7 +781,7 @@ def test_resolve_advisor_credentials_rejects_api_base_without_api_key(): @pytest.mark.parametrize("api_base", [None, "https://other.example"]) def test_resolve_advisor_credentials_rejects_an_oidc_secret_reference_api_key(api_base): - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) @@ -793,17 +793,17 @@ def test_resolve_advisor_credentials_rejects_an_oidc_secret_reference_api_key(ap def test_resolve_advisor_credentials_validates_api_base_before_use(): - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ), patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor.validate_url" + "litellm.llms.anthropic.messages.native.interceptors.advisor.validate_url" ) as mock_validate, ): result = _resolve_advisor_credentials(ADVISOR_TOOL_WITH_CREDS) @@ -813,17 +813,17 @@ def test_resolve_advisor_credentials_validates_api_base_before_use(): def test_resolve_advisor_credentials_propagates_ssrf_error(): from litellm.litellm_core_utils.url_utils import SSRFError - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ), patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor.validate_url", + "litellm.llms.anthropic.messages.native.interceptors.advisor.validate_url", side_effect=SSRFError("URL targets a blocked address"), ), ): @@ -834,18 +834,18 @@ def test_resolve_advisor_credentials_propagates_ssrf_error(): def test_resolve_advisor_credentials_skips_validation_when_url_validation_disabled(): import litellm - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ), patch.object(litellm, "user_url_validation", False), patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor.validate_url", + "litellm.llms.anthropic.messages.native.interceptors.advisor.validate_url", side_effect=AssertionError("validate_url must not run when user_url_validation is disabled"), ), ): @@ -857,7 +857,7 @@ def test_resolve_advisor_credentials_blocks_real_cloud_metadata_address(): """End-to-end (no mocked validate_url): a caller can't redirect the advisor sub-call to the cloud-metadata address even with an api_key.""" from litellm.litellm_core_utils.url_utils import SSRFError - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) @@ -867,7 +867,7 @@ def test_resolve_advisor_credentials_blocks_real_cloud_metadata_address(): "api_base": "https://169.254.169.254/latest/meta-data/", } with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ): with pytest.raises(SSRFError): @@ -875,13 +875,13 @@ def test_resolve_advisor_credentials_blocks_real_cloud_metadata_address(): def test_resolve_advisor_credentials_rejects_non_https_api_base(): - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) tool = {**ADVISOR_TOOL, "api_key": "sk-other", "api_base": "http://8.8.8.8"} with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ): with pytest.raises(ValueError, match="https"): @@ -891,14 +891,14 @@ def test_resolve_advisor_credentials_rejects_non_https_api_base(): def test_resolve_advisor_credentials_rejects_api_base_when_ssl_verify_disabled(): import litellm - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) tool = {**ADVISOR_TOOL, "api_key": "sk-other", "api_base": "https://8.8.8.8"} with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ), patch.object(litellm, "ssl_verify", False), @@ -910,13 +910,13 @@ def test_resolve_advisor_credentials_rejects_api_base_when_ssl_verify_disabled() def test_resolve_advisor_credentials_allows_real_public_ip_address(): """End-to-end (no mocked validate_url): a globally-routable literal IP api_base is honored when paired with an api_key.""" - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) tool = {**ADVISOR_TOOL, "api_key": "sk-other", "api_base": "https://8.8.8.8"} with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ): result = _resolve_advisor_credentials(tool) @@ -935,7 +935,7 @@ async def test_advisor_sub_call_failure_is_tagged(): """When the advisor sub-call raises, the exception that propagates out of handle() must be tagged as an advisor orchestration failure.""" import litellm - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) from litellm.router_utils.cooldown_handlers import is_advisor_orchestration_failure @@ -954,7 +954,7 @@ async def test_advisor_sub_call_failure_is_tagged(): ) with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -977,7 +977,7 @@ async def test_advisor_max_iterations_failure_is_tagged(): """When the orchestration loop exceeds max_uses (the executor keeps calling the advisor), the AdvisorMaxIterationsError must be tagged so the healthy executor deployment is not cooled down.""" - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorMaxIterationsError, AdvisorOrchestrationHandler, ) @@ -993,7 +993,7 @@ async def test_advisor_max_iterations_failure_is_tagged(): return _make_advisor_tool_use_response() with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -1015,7 +1015,7 @@ async def test_executor_failure_is_not_tagged(): """A failure of the executor call (not advisor orchestration) must NOT be tagged — the selected deployment genuinely failed and should cool down.""" import litellm - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) from litellm.router_utils.cooldown_handlers import is_advisor_orchestration_failure @@ -1028,7 +1028,7 @@ async def test_executor_failure_is_not_tagged(): ) with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -1084,7 +1084,7 @@ def _router_with_advisor_deployment( @pytest.mark.asyncio async def test_advisor_sub_call_routes_through_proxy_router(): import litellm.proxy.proxy_server as proxy_server - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -1107,7 +1107,7 @@ async def test_advisor_sub_call_routes_through_proxy_router(): with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ), patch.object(proxy_server, "llm_router", router), @@ -1145,7 +1145,7 @@ async def test_advisor_sub_call_routes_through_proxy_router(): async def test_advisor_sub_call_routes_through_router_for_alias_and_wildcard(router_kwargs, advisor_model): """Alias and wildcard advisor models resolve through the router like exact model_list matches.""" import litellm.proxy.proxy_server as proxy_server - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -1168,7 +1168,7 @@ async def test_advisor_sub_call_routes_through_router_for_alias_and_wildcard(rou with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ), patch.object(proxy_server, "llm_router", router), @@ -1194,7 +1194,7 @@ async def test_advisor_sub_call_routes_through_router_for_alias_and_wildcard(rou async def test_advisor_sub_call_bypasses_router_for_unconfigured_model(): """An advisor model the router doesn't know about keeps the SDK-level path.""" import litellm.proxy.proxy_server as proxy_server - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -1219,7 +1219,7 @@ async def test_advisor_sub_call_bypasses_router_for_unconfigured_model(): with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ), patch.object(proxy_server, "llm_router", router), @@ -1243,7 +1243,7 @@ async def test_advisor_sub_call_client_override_bypasses_router(): """A caller-supplied api_key/api_base override must not be re-routed.""" import litellm import litellm.proxy.proxy_server as proxy_server - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -1274,7 +1274,7 @@ async def test_advisor_sub_call_client_override_bypasses_router(): with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ), patch.object(proxy_server, "llm_router", router), @@ -1309,7 +1309,7 @@ async def test_advisor_sub_call_client_override_bypasses_router(): @pytest.mark.asyncio async def test_advisor_context_excludes_in_sequence_system_rows(): - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -1329,7 +1329,7 @@ async def test_advisor_context_excludes_in_sequence_system_rows(): return _make_text_response("Final answer.") with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() diff --git a/tests/unit/llms/anthropic/pass_through/test_anthropic_native_passthrough_spend_logging.py b/tests/unit/llms/anthropic/messages/test_anthropic_native_passthrough_spend_logging.py similarity index 100% rename from tests/unit/llms/anthropic/pass_through/test_anthropic_native_passthrough_spend_logging.py rename to tests/unit/llms/anthropic/messages/test_anthropic_native_passthrough_spend_logging.py diff --git a/tests/unit/llms/anthropic/pass_through/test_caller_model_checks.py b/tests/unit/llms/anthropic/messages/test_caller_model_checks.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/test_caller_model_checks.py rename to tests/unit/llms/anthropic/messages/test_caller_model_checks.py index 5a45221a3ca..751f7cc3934 100644 --- a/tests/unit/llms/anthropic/pass_through/test_caller_model_checks.py +++ b/tests/unit/llms/anthropic/messages/test_caller_model_checks.py @@ -7,7 +7,7 @@ from starlette.requests import Request import litellm from litellm.caching.dual_cache import DualCache -from litellm.llms.anthropic.pass_through.caller_model_checks import ( +from litellm.llms.anthropic.messages.caller_model_checks import ( _caller_within_team_member_model_budget, caller_model_verdict, rate_limit_verdict, @@ -32,7 +32,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import ( from litellm.proxy.utils import InternalUsageCache, hash_token from litellm.router import Router from litellm.types.agents import AgentResponse -from tests.unit.llms.anthropic.pass_through.conftest import ProxyAuthState +from tests.unit.llms.anthropic.messages.conftest import ProxyAuthState CLASSIFIER: Final = "classifier" CLASSIFIER_BUDGET: Final = {CLASSIFIER: {"max_budget": 0.1, "budget_duration": "1d"}} diff --git a/tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py b/tests/unit/llms/anthropic/messages/test_convertible_blocks.py similarity index 97% rename from tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py rename to tests/unit/llms/anthropic/messages/test_convertible_blocks.py index eac75486da4..5fb6cc32dfb 100644 --- a/tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py +++ b/tests/unit/llms/anthropic/messages/test_convertible_blocks.py @@ -6,13 +6,13 @@ from typing import Final import pytest from pydantic import JsonValue, TypeAdapter -from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter -from litellm.llms.anthropic.pass_through.convertible_blocks import ( +from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter +from litellm.llms.anthropic.messages.convertible_blocks import ( malformed_tool_result_text_error, unconvertible_anthropic_blocks, unconvertible_blocks_error, ) -from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( +from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) from litellm.llms.vertex_ai.count_tokens.context import gemini_count_contents diff --git a/tests/unit/llms/anthropic/pass_through/test_reasoning_effort_fields.py b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py similarity index 97% rename from tests/unit/llms/anthropic/pass_through/test_reasoning_effort_fields.py rename to tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py index e9450d025a6..3432c9503de 100644 --- a/tests/unit/llms/anthropic/pass_through/test_reasoning_effort_fields.py +++ b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py @@ -14,7 +14,7 @@ from typing import Any, Dict import pytest import litellm -from litellm.llms.anthropic.pass_through.utils import ( +from litellm.llms.anthropic.messages.utils import ( normalize_reasoning_effort_value, ) from litellm.router_utils.reasoning_effort_capability import ( @@ -156,7 +156,7 @@ class TestAdapterAdaptiveThinking: def test_messages_adapter_adaptive_returns_medium_default(self): """Adaptive thinking returns 'medium' as default reasoning_effort.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -168,7 +168,7 @@ class TestAdapterAdaptiveThinking: def test_messages_adapter_adaptive_overridden_by_output_config(self): """For adaptive thinking, output_config.effort overrides reasoning_effort.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) from litellm.types.llms.anthropic import AnthropicMessagesRequest @@ -191,7 +191,7 @@ class TestAdapterAdaptiveThinking: def test_responses_adapter_adaptive_with_output_config(self): """Responses adapter: adaptive thinking + output_config.effort.""" - from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( + from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) @@ -204,7 +204,7 @@ class TestAdapterAdaptiveThinking: def test_responses_adapter_adaptive_default_medium(self): """Responses adapter: adaptive thinking without output_config defaults to medium.""" - from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( + from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) diff --git a/tests/unit/llms/anthropic/pass_through/test_safeguards.py b/tests/unit/llms/anthropic/messages/test_safeguards.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/test_safeguards.py rename to tests/unit/llms/anthropic/messages/test_safeguards.py index 8ea9b693139..0537d0fbd38 100644 --- a/tests/unit/llms/anthropic/pass_through/test_safeguards.py +++ b/tests/unit/llms/anthropic/messages/test_safeguards.py @@ -12,11 +12,11 @@ import litellm from litellm.caching.dual_cache import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.llms.anthropic.pass_through.caller_model_checks import ( +from litellm.llms.anthropic.messages.caller_model_checks import ( CallerModelVerdict, _caller_within_team_member_model_budget, ) -from litellm.llms.anthropic.pass_through.safeguards import ( +from litellm.llms.anthropic.messages.safeguards import ( CLASSIFIER_MAX_TOKENS, CLASSIFIER_SYSTEM_PROMPT, TOOL_RESULT_CHAR_LIMIT, @@ -32,7 +32,7 @@ from litellm.llms.anthropic.pass_through.safeguards import ( requested_dangerous_tool_use, with_safeguard_results, ) -from litellm.llms.anthropic.pass_through.utils import proxy_sub_call_metadata +from litellm.llms.anthropic.messages.utils import proxy_sub_call_metadata from litellm.proxy._types import LiteLLM_EndUserTable, Litellm_EntityType, UserAPIKeyAuth from litellm.proxy.hooks.model_max_budget_limiter import ( PROXY_VirtualKeyModelMaxBudgetLimiter, @@ -44,7 +44,7 @@ from litellm.router import Router from litellm.types.llms.anthropic import AnthropicResponseContentBlockToolUse from litellm.types.router import RouterErrors from litellm.types.utils import Choices, Message, ModelResponse -from tests.unit.llms.anthropic.pass_through.conftest import ProxyAuthState +from tests.unit.llms.anthropic.messages.conftest import ProxyAuthState DANGEROUS_TOOL_USE_REQUEST: Final = [{"type": "dangerous_tool_use", "classifier_context": {"permission_mode": "auto"}}] LS: Final = ToolUseUnderReview(id="call_ls", name="Bash", input={"command": "ls"}) diff --git a/tests/unit/llms/anthropic/pass_through/test_stream_assembly.py b/tests/unit/llms/anthropic/messages/test_stream_assembly.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/test_stream_assembly.py rename to tests/unit/llms/anthropic/messages/test_stream_assembly.py index dbd49064e4b..f7d6b8d4f8c 100644 --- a/tests/unit/llms/anthropic/pass_through/test_stream_assembly.py +++ b/tests/unit/llms/anthropic/messages/test_stream_assembly.py @@ -3,7 +3,7 @@ from typing import Final import pytest -from litellm.llms.anthropic.pass_through.stream_assembly import ( +from litellm.llms.anthropic.messages.stream_assembly import ( anthropic_message_delta_fields, assemble_anthropic_sse_stream, is_anthropic_sse_stream, diff --git a/tests/unit/llms/anthropic/pass_through/test_pass_through_shim.py b/tests/unit/llms/anthropic/pass_through/test_pass_through_shim.py new file mode 100644 index 00000000000..4016e5301db --- /dev/null +++ b/tests/unit/llms/anthropic/pass_through/test_pass_through_shim.py @@ -0,0 +1,60 @@ +import importlib +import sys +from typing import Final + +import pytest + +OLD_PACKAGE: Final = "litellm.llms.anthropic.pass_through" +NEW_PACKAGE: Final = "litellm.llms.anthropic.messages" + + +def _forget_old_package(monkeypatch: pytest.MonkeyPatch) -> None: + for name in tuple(sys.modules): + if name == OLD_PACKAGE or name.startswith(OLD_PACKAGE + "."): + monkeypatch.delitem(sys.modules, name) + + +def test_old_deep_import_is_the_new_module_object_and_warns(monkeypatch: pytest.MonkeyPatch) -> None: + _forget_old_package(monkeypatch) + with pytest.warns(DeprecationWarning, match=NEW_PACKAGE): + old_handler: Final = importlib.import_module(f"{OLD_PACKAGE}.messages.handler") + from litellm.llms.anthropic.messages.native import handler as new_handler + + assert old_handler is new_handler + assert sys.modules[f"{OLD_PACKAGE}.messages.handler"] is new_handler + assert importlib.import_module(f"{OLD_PACKAGE}.adapters.transformation") is importlib.import_module( + f"{NEW_PACKAGE}.adapters.transformation" + ) + assert importlib.import_module(f"{OLD_PACKAGE}.safeguards") is importlib.import_module(f"{NEW_PACKAGE}.safeguards") + + +def test_old_package_attribute_access_reaches_renamed_subpackage(monkeypatch: pytest.MonkeyPatch) -> None: + _forget_old_package(monkeypatch) + with pytest.warns(DeprecationWarning, match=NEW_PACKAGE): + old_package: Final = importlib.import_module(OLD_PACKAGE) + + assert old_package.messages is importlib.import_module(f"{NEW_PACKAGE}.native") # pyright: ignore[reportAny] # module attributes are untyped + assert old_package.messages.handler is importlib.import_module(f"{NEW_PACKAGE}.native.handler") # pyright: ignore[reportAny] # module attributes are untyped + with pytest.raises(AttributeError, match="no attribute 'nope'"): + _ = old_package.nope # pyright: ignore[reportAny] # module attributes are untyped + + +def test_old_transformation_import_loads_only_its_new_counterpart(monkeypatch: pytest.MonkeyPatch) -> None: + _forget_old_package(monkeypatch) + monkeypatch.delitem(sys.modules, f"{NEW_PACKAGE}.native.streaming_iterator", raising=False) + with pytest.warns(DeprecationWarning, match=NEW_PACKAGE): + old_transformation: Final = importlib.import_module(f"{OLD_PACKAGE}.messages.transformation") + + assert old_transformation is sys.modules[f"{NEW_PACKAGE}.native.transformation"] + assert f"{NEW_PACKAGE}.native.streaming_iterator" not in sys.modules, "shim imported the streaming iterator" + + +def test_old_subpackage_import_keeps_the_new_package_path_importable(monkeypatch: pytest.MonkeyPatch) -> None: + _forget_old_package(monkeypatch) + monkeypatch.delitem(sys.modules, f"{NEW_PACKAGE}.native.mcp_handler", raising=False) + with pytest.warns(DeprecationWarning, match=NEW_PACKAGE): + native: Final = importlib.import_module(f"{OLD_PACKAGE}.messages") + + assert [path.rsplit("/", 2)[-2:] for path in native.__path__] == [["messages", "native"]] # pyright: ignore[reportAny] # module attributes are untyped + assert native.__spec__ is not None and native.__spec__.name == f"{NEW_PACKAGE}.native" + assert importlib.import_module(f"{NEW_PACKAGE}.native.mcp_handler").__name__ == f"{NEW_PACKAGE}.native.mcp_handler" diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index 093b4d6cce8..eb5c1598c17 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -436,7 +436,7 @@ class TestPassthroughOAuth: def test_passthrough_oauth_no_x_api_key(self): """Passthrough endpoint should not add x-api-key for OAuth tokens.""" - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -459,7 +459,7 @@ class TestPassthroughOAuth: def test_passthrough_regular_key_uses_x_api_key(self): """Passthrough endpoint should still use x-api-key for regular API keys.""" - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -1265,7 +1265,7 @@ class TestPassthroughAuthToken: """Passthrough endpoint should use Bearer auth when only ANTHROPIC_AUTH_TOKEN is set.""" from unittest.mock import patch as mock_patch - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -1289,7 +1289,7 @@ class TestPassthroughAuthToken: """Passthrough endpoint should prefer ANTHROPIC_API_KEY over ANTHROPIC_AUTH_TOKEN.""" from unittest.mock import patch as mock_patch - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -1320,7 +1320,7 @@ class TestPassthroughAuthToken: from unittest.mock import patch as mock_patch import litellm - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -1342,7 +1342,7 @@ class TestPassthroughAuthToken: """A client-forwarded x-api-key header, whatever its casing, should satisfy validation without env credentials.""" from unittest.mock import patch as mock_patch - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -1365,7 +1365,7 @@ class TestPassthroughAuthToken: """get_complete_url should use ANTHROPIC_BASE_URL when api_base is None.""" from unittest.mock import patch as mock_patch - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -2043,7 +2043,7 @@ class TestAnthropicThinkingSignatureSelfHeal: def test_anthropic_messages_config_http_retry_helpers(self): import httpx - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -2113,7 +2113,7 @@ class TestAnthropicThinkingSignatureSelfHeal: def test_anthropic_messages_retry_honors_thinking_block_binding( self, thinking: Mapping[str, object], error_message: str, expected: bool ) -> None: - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -2865,7 +2865,7 @@ class TestWifServerOwnedAuthHeaderStrip: """The passthrough surface hands the caller's own credential upstream rather than minting. That check was case-sensitive, so X-Api-Key slipped past it and the caller's key would have travelled beside a minted Bearer.""" - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -3084,7 +3084,7 @@ class TestMessagesEnvAuthBetaMerge: def test_client_beta_survives_env_auth_injection(self, monkeypatch, clean_anthropic_env): """Regression: headers.update(auth_header) silently clobbered the client's anthropic-beta on the native /v1/messages route.""" - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -3166,7 +3166,7 @@ class TestWifLitellmParamsPlumbing: assert len(poster.requests) == 1 def test_messages_mints_from_litellm_params(self, wif_engine): - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -3245,7 +3245,7 @@ class TestWifAsyncSeam: @pytest.mark.asyncio async def test_avalidate_messages_environment_mints_off_loop(self, wif_async_engine, monkeypatch): - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -3268,7 +3268,7 @@ class TestWifAsyncSeam: async def test_avalidate_delegates_to_subclass_sync_override(self): """A provider subclass that only overrides the sync method must keep its behavior when the handler goes through the async variant.""" - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) diff --git a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py index df89774951f..7a6b6534131 100644 --- a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py +++ b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py @@ -13,7 +13,7 @@ import litellm from litellm.caching.dual_cache import DualCache from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.anthropic.count_tokens import handler as count_handler -from litellm.llms.anthropic.pass_through.messages.transformation import DEFAULT_ANTHROPIC_API_VERSION +from litellm.llms.anthropic.messages.native.transformation import DEFAULT_ANTHROPIC_API_VERSION from litellm.llms.anthropic.prompt_cache_prediction import ( CountedPromptCachePlan, NativePredictionTarget, diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py index 4d9feea4454..93367fdff16 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py @@ -29,7 +29,7 @@ from litellm.constants import ( # Ensure the project root is on the import path so `litellm` can be imported when # tests are executed from any working directory. from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.anthropic.pass_through.messages.mid_conversation_system import ( +from litellm.llms.anthropic.messages.native.mid_conversation_system import ( as_system_content_blocks, ) from litellm.llms.bedrock.common_utils import ( diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 55fad96ceb1..297e64629a8 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -979,7 +979,7 @@ async def test_async_anthropic_messages_handler_streaming_forwards_provider_resp """ from collections.abc import AsyncIterator as ABCAsyncIterator - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -1042,10 +1042,10 @@ async def test_async_anthropic_messages_handler_agentic_streaming_forwards_provi from collections.abc import AsyncIterator as ABCAsyncIterator from litellm.integrations.custom_logger import CustomLogger - from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( + from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( AgenticAnthropicStreamingIterator, ) - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -1113,7 +1113,7 @@ async def test_anthropic_messages_streaming_response_aclose_closes_upstream_stre the upstream stream so provider connections are released on client disconnect instead of lingering until garbage collection. """ - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( AnthropicMessagesStreamingResponse, ) @@ -1145,10 +1145,10 @@ async def test_anthropic_messages_streaming_response_aclose_closes_upstream_stre @pytest.mark.asyncio async def test_anthropic_messages_streaming_response_aclose_closes_agentic_upstream_stream(): - from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( + from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( AgenticAnthropicStreamingIterator, ) - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( AnthropicMessagesStreamingResponse, ) @@ -2096,7 +2096,7 @@ async def test_anthropic_post_skips_the_thinking_strip_retry_when_block_binding_ ) -> None: """A caller that pinned ``prefix_mismatch_behavior: error`` asked for the 400, so the handler must surface it instead of stripping the bound blocks and retrying.""" - from litellm.llms.anthropic.pass_through.messages.transformation import AnthropicMessagesConfig + from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig handler = BaseLLMHTTPHandler() thinking_block = {"type": "thinking", "thinking": "", "signature": "sig"} diff --git a/tests/unit/llms/deepseek/messages/test_deepseek_anthropic_messages_transformation.py b/tests/unit/llms/deepseek/messages/test_deepseek_anthropic_messages_transformation.py index 0afe57a001f..316b8576ac4 100644 --- a/tests/unit/llms/deepseek/messages/test_deepseek_anthropic_messages_transformation.py +++ b/tests/unit/llms/deepseek/messages/test_deepseek_anthropic_messages_transformation.py @@ -1,5 +1,5 @@ import litellm -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.deepseek.messages.transformation import ( diff --git a/tests/unit/llms/github_copilot/messages/test_github_copilot_messages_transformation.py b/tests/unit/llms/github_copilot/messages/test_github_copilot_messages_transformation.py index 46062f16c9a..20ff65f11d9 100644 --- a/tests/unit/llms/github_copilot/messages/test_github_copilot_messages_transformation.py +++ b/tests/unit/llms/github_copilot/messages/test_github_copilot_messages_transformation.py @@ -334,7 +334,7 @@ def test_github_copilot_config_does_not_handle_web_search_natively(): interception handler short-circuiting Copilot instead of routing to it, even though Copilot now has a BaseAnthropicMessagesConfig. The base Anthropic config (bedrock/vertex/anthropic path) must report True.""" - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) diff --git a/tests/unit/llms/microsoft_365_copilot/chat/test_transformation.py b/tests/unit/llms/microsoft_365_copilot/chat/test_transformation.py index 22cb8da3aed..aab3c82a389 100644 --- a/tests/unit/llms/microsoft_365_copilot/chat/test_transformation.py +++ b/tests/unit/llms/microsoft_365_copilot/chat/test_transformation.py @@ -5,7 +5,7 @@ import pytest from pydantic import TypeAdapter import litellm -from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter +from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter from litellm.llms.microsoft_365_copilot.chat.transformation import ( Microsoft365CopilotChatConfig, build_chat_request, diff --git a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py index 8995b3cb068..d5e7bb4f93b 100644 --- a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py @@ -2,7 +2,7 @@ from typing import Final import pytest -from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( +from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) from litellm.llms.openai.responses.count_tokens.transformation import ( diff --git a/tests/unit/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py b/tests/unit/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py index 1d5ef50ca3a..935e3e4a21d 100644 --- a/tests/unit/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py +++ b/tests/unit/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py @@ -396,7 +396,7 @@ def test_request_defaults_missing_cache_control_type_and_drops_non_dict(config): def test_native_anthropic_config_keeps_cache_control_ttl(): """Anthropic itself accepts ttl, so the normalization must stay scoped to the OpenAI-like passthrough and never reach the native Anthropic path.""" - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) diff --git a/tests/unit/llms/tencent/messages/test_tencent_anthropic_messages_transformation.py b/tests/unit/llms/tencent/messages/test_tencent_anthropic_messages_transformation.py index e5bdce9d3e0..b737ad3e9c1 100644 --- a/tests/unit/llms/tencent/messages/test_tencent_anthropic_messages_transformation.py +++ b/tests/unit/llms/tencent/messages/test_tencent_anthropic_messages_transformation.py @@ -1,5 +1,5 @@ import litellm -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.tencent.messages.transformation import ( diff --git a/tests/unit/llms/test_get_complete_url_trailing_slash.py b/tests/unit/llms/test_get_complete_url_trailing_slash.py index 9118498c2c2..cd778c20d9b 100644 --- a/tests/unit/llms/test_get_complete_url_trailing_slash.py +++ b/tests/unit/llms/test_get_complete_url_trailing_slash.py @@ -145,7 +145,7 @@ API_BASE_IS_THE_FULL_URL: Final[Mapping[str, str]] = MappingProxyType( "searchapi.search.transformation.SearchAPIConfig", "tinyfish.search.transformation.TinyfishSearchConfig", "vertex_ai.text_to_speech.transformation.VertexAITextToSpeechConfig", - "vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation." + "vertex_ai.vertex_ai_partner_models.anthropic.messages.transformation." "VertexAIPartnerModelsAnthropicMessagesConfig", ) } diff --git a/tests/unit/llms/vertex_ai/count_tokens/test_context.py b/tests/unit/llms/vertex_ai/count_tokens/test_context.py index a1643f0f699..b2a0b735b0f 100644 --- a/tests/unit/llms/vertex_ai/count_tokens/test_context.py +++ b/tests/unit/llms/vertex_ai/count_tokens/test_context.py @@ -14,7 +14,7 @@ import pytest from pydantic import JsonValue, TypeAdapter import litellm -from litellm.llms.anthropic.pass_through.adapters.transformation import ( +from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, truncate_tool_name, ) @@ -622,7 +622,7 @@ async def test_anthropic_view_applies_clear_edits_without_calling_the_compaction with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="gemini/gemini-2.5-flash", ), patch.object(litellm, "acompletion", side_effect=AssertionError("count_tokens must not summarize")) as summary, diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 75449eacfd3..f3741d2667a 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -2842,7 +2842,7 @@ def test_thinking_block_signature_is_not_forwarded_to_gemini() -> None: def test_anthropic_messages_history_replays_to_gemini_without_claude_signature() -> None: from litellm.litellm_core_utils.prompt_templates.factory import _get_dummy_thought_signature - from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter + from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter thinking: Final = "I should look the weather up before answering." anthropic_messages: Final = [ diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 3bc864cc047..0f5c21d08c7 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -20,7 +20,7 @@ from litellm.cost_calculator import batch_cost_calculator from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token, parse_prompt_tokens_details -from litellm.llms.anthropic.pass_through.messages import handler as anthropic_messages_handler +from litellm.llms.anthropic.messages.native import handler as anthropic_messages_handler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig from litellm.llms.vertex_ai.common_utils import GEMINI_MAX_ENUM_CHARACTERS, VertexAIError @@ -7544,7 +7544,7 @@ def test_gemini_candidate_with_finish_reason_no_content_chat_completion(): def test_gemini_candidate_with_finish_reason_no_content_anthropic_messages(): - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -7617,7 +7617,7 @@ def test_gemini_candidate_with_finish_reason_no_content_responses_api(): def test_gemini_candidate_other_finish_reasons_no_content(): - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) from litellm.responses.litellm_completion_transformation.transformation import ( diff --git a/tests/unit/llms/vertex_ai/test_vertex.py b/tests/unit/llms/vertex_ai/test_vertex.py index 04cf391c700..7e0baaf9e10 100644 --- a/tests/unit/llms/vertex_ai/test_vertex.py +++ b/tests/unit/llms/vertex_ai/test_vertex.py @@ -2461,7 +2461,7 @@ async def test_anthropic_message_via_anthropic_messages( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages.transformation import ( VertexAIPartnerModelsAnthropicMessagesConfig, ) from litellm.utils import ProviderConfigManager diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/__init__.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/test_experimental_pass_through_shim.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/test_experimental_pass_through_shim.py new file mode 100644 index 00000000000..4860b5dbe7e --- /dev/null +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/test_experimental_pass_through_shim.py @@ -0,0 +1,20 @@ +import importlib +import sys +from typing import Final + +import pytest + +OLD_PACKAGE: Final = "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through" +NEW_PACKAGE: Final = "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages" + + +def test_old_deep_import_is_the_new_module_object_and_warns(monkeypatch: pytest.MonkeyPatch) -> None: + for name in tuple(sys.modules): + if name == OLD_PACKAGE or name.startswith(OLD_PACKAGE + "."): + monkeypatch.delitem(sys.modules, name) + with pytest.warns(DeprecationWarning, match=NEW_PACKAGE): + old_transformation = importlib.import_module(f"{OLD_PACKAGE}.transformation") + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages import transformation + + assert old_transformation is transformation + assert sys.modules[f"{OLD_PACKAGE}.transformation"] is sys.modules[f"{NEW_PACKAGE}.transformation"] diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 136c444c9f1..6c10d524531 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -12,7 +12,7 @@ import pytest import requests import responses -from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( +from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages.transformation import ( VertexAIPartnerModelsAnthropicMessagesConfig, ) from litellm.llms.vertex_ai.vertex_ai_partner_models.main import VertexAIPartnerModels diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index 69b3e113b8f..8c8aa7af19f 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -3258,7 +3258,7 @@ async def test_streaming_hook_fails_closed_when_assembler_raises_api_error(): guardrail = _sse_guardrail() with patch( - "litellm.llms.anthropic.pass_through.stream_assembly.build_complete_streaming_response", + "litellm.llms.anthropic.messages.stream_assembly.build_complete_streaming_response", side_effect=litellm.APIError( status_code=500, message="Error building chunks for logging/streaming usage calculation", diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py index 5bdcdddf4b7..3336b20f5eb 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -2711,7 +2711,7 @@ async def test_retrieved_content_protected_when_mcp_tool_name_is_truncated(guard the OpenAI-translated view the guardrail scans, dropping the suffix. The call id read from the request's own Anthropic tool_use (never truncated) still pairs the retrieved row so it is held back.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( truncate_tool_name, ) diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index 7530703a05a..31a963d3e47 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -11,7 +11,7 @@ from pydantic import ValidationError import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.anthropic.pass_through.stream_assembly import extract_sse_data +from litellm.llms.anthropic.messages.stream_assembly import extract_sse_data from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( AnthropicPassthroughLoggingHandler, ) diff --git a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py index 4aa7e9cfbae..ae95c27950c 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py +++ b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py @@ -20,7 +20,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.tinyfish_passthr from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.llms.anthropic.pass_through.messages.streaming_iterator import aclose_superseded_attempt +from litellm.llms.anthropic.messages.native.streaming_iterator import aclose_superseded_attempt from litellm.proxy.pass_through_endpoints.streaming_handler import ( PassThroughStreamingHandler, ) diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 78f5e1ace0c..86d064050eb 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -5074,7 +5074,7 @@ async def test_spend_log_request_id_is_the_message_id_a_bridged_streaming_caller adapter mints itself, and it is the only request id that call ever shows the caller, so GET /spend/logs?request_id=msg_... has to land on the row.""" from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.llms.anthropic.pass_through.responses_adapters.streaming_iterator import ( + from litellm.llms.anthropic.messages.responses_adapters.streaming_iterator import ( AnthropicResponsesStreamWrapper, ) from litellm.types.llms.openai import ( diff --git a/tests/unit/proxy/test_budget_reservation.py b/tests/unit/proxy/test_budget_reservation.py index 4635e145914..42b41a2e3f2 100644 --- a/tests/unit/proxy/test_budget_reservation.py +++ b/tests/unit/proxy/test_budget_reservation.py @@ -13,10 +13,10 @@ import litellm from litellm.caching.dual_cache import DualCache from litellm.types.caching import RedisPipelineIncrementOperation from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES -from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( +from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( AgenticAnthropicStreamingIterator, ) -from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( +from litellm.llms.anthropic.messages.native.streaming_iterator import ( AnthropicMessagesStreamingResponse, ) from litellm.proxy._types import ( diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 561577094df..5c99e5a01ea 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -8640,10 +8640,10 @@ class TestStreamingClientDisconnectBilling: through the translate_completion_output_params_streaming result to the inner chat stream's collected chunks or a disconnect bills nothing. """ - from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( + from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicSSEStream, ) - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( AnthropicAdapter, ) from litellm.router import FallbackAwareAnthropicMessagesStream @@ -9180,7 +9180,7 @@ def _apply_stream_usage_tracking( @pytest.mark.asyncio async def test_disconnect_bills_partial_default_route_messages_stream_through_the_responses_adapter(self): - from litellm.llms.anthropic.pass_through.responses_adapters.streaming_iterator import ( + from litellm.llms.anthropic.messages.responses_adapters.streaming_iterator import ( AnthropicResponsesStreamWrapper, ) from litellm.router import FallbackAwareAnthropicMessagesStream @@ -11923,7 +11923,7 @@ class TestResponseCostFromLoggingObjForBridgedMessages: def test_recomputed_cost_prices_the_reasoning_tokens_of_the_bridged_chat_completion(self): from litellm.types.utils import CompletionTokensDetailsWrapper, ModelResponse, Usage - from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter + from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter model = "dashscope/qwen-turbo" pricing = litellm.get_model_info(model) diff --git a/tests/unit/proxy/utils/proxy_logging/test_streaming_hooks.py b/tests/unit/proxy/utils/proxy_logging/test_streaming_hooks.py index 50f50478ad3..8f8f10565e9 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_streaming_hooks.py +++ b/tests/unit/proxy/utils/proxy_logging/test_streaming_hooks.py @@ -23,7 +23,7 @@ from litellm.exceptions import GuardrailRaisedException from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( +from litellm.llms.anthropic.messages.native.streaming_iterator import ( BaseAnthropicMessagesStreamingIterator, ) from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/unit/responses/mcp/test_tool_precheck.py b/tests/unit/responses/mcp/test_tool_precheck.py index 832151ce2a6..a3b2c3a0efb 100644 --- a/tests/unit/responses/mcp/test_tool_precheck.py +++ b/tests/unit/responses/mcp/test_tool_precheck.py @@ -18,7 +18,7 @@ GATEWAY_REGISTRY_MODULE: Final = "litellm.proxy._experimental.mcp_server.mcp_ser GATEWAY_HANDLER_MODULES: Final = ( "litellm.responses.mcp.litellm_proxy_mcp_handler", "litellm.responses.mcp.chat_completions_handler", - "litellm.llms.anthropic.pass_through.messages.mcp_handler", + "litellm.llms.anthropic.messages.native.mcp_handler", ) FUNCTION_TOOL: Final = { diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index 75b6ab04446..53375ee0ab4 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -16,7 +16,7 @@ from pydantic import TypeAdapter import litellm from litellm import Router from litellm.integrations.custom_logger import CustomLogger -from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import FakeAnthropicMessagesStreamIterator +from litellm.llms.anthropic.messages.native.fake_stream_iterator import FakeAnthropicMessagesStreamIterator from litellm.litellm_core_utils import get_llm_provider_logic from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper diff --git a/tests/unit/rust_bridge/messages/test_secrets.py b/tests/unit/rust_bridge/messages/test_secrets.py index 64208182025..f6439753809 100644 --- a/tests/unit/rust_bridge/messages/test_secrets.py +++ b/tests/unit/rust_bridge/messages/test_secrets.py @@ -9,7 +9,7 @@ import pytest import litellm from litellm.integrations.custom_secret_manager import CustomSecretManager -from litellm.llms.anthropic.pass_through.messages.handler import anthropic_messages +from litellm.llms.anthropic.messages.native.handler import anthropic_messages from litellm.rust_bridge.host import settings from litellm.rust_bridge.messages.entrypoints import NATIVE_AMESSAGES, NATIVE_MESSAGES from litellm.rust_bridge.public_call import NativeCall diff --git a/tests/unit/test_private_usage_aliases.py b/tests/unit/test_private_usage_aliases.py index dcd18a5c75e..88ae584f34f 100644 --- a/tests/unit/test_private_usage_aliases.py +++ b/tests/unit/test_private_usage_aliases.py @@ -2108,49 +2108,49 @@ PROPERTY_CASES: Final = ( CLASS_PROPERTY_CASES: Final = () LLMS_ALIAS_CASES: Final = ( ( - "litellm.llms.anthropic.pass_through.adapters.transformation", + "litellm.llms.anthropic.messages.adapters.transformation", "LiteLLMAnthropicMessagesAdapter", "_translate_openai_usage_to_anthropic_usage", "translate_openai_usage_to_anthropic_usage", False, ), ( - "litellm.llms.anthropic.pass_through.adapters.transformation", + "litellm.llms.anthropic.messages.adapters.transformation", "LiteLLMAnthropicMessagesAdapter", "_translate_openai_usage_to_anthropic_usage_delta", "translate_openai_usage_to_anthropic_usage_delta", False, ), ( - "litellm.llms.anthropic.pass_through.adapters.transformation", + "litellm.llms.anthropic.messages.adapters.transformation", "LiteLLMAnthropicMessagesAdapter", "_translate_streaming_openai_chunk_to_anthropic_content_block", "translate_streaming_openai_chunk_to_anthropic_content_block", False, ), ( - "litellm.llms.anthropic.pass_through.context_management.dispatcher", + "litellm.llms.anthropic.messages.context_management.dispatcher", "", "_normalize_spec", "normalize_spec", False, ), ( - "litellm.llms.anthropic.pass_through.messages.streaming_iterator", + "litellm.llms.anthropic.messages.native.streaming_iterator", "", "_is_message_stop_chunk", "is_message_stop_chunk", False, ), ( - "litellm.llms.anthropic.pass_through.messages.streaming_iterator", + "litellm.llms.anthropic.messages.native.streaming_iterator", "", "_is_provider_error_chunk", "is_provider_error_chunk", False, ), ( - "litellm.llms.anthropic.pass_through.messages.transformation", + "litellm.llms.anthropic.messages.native.transformation", "", "_messages_carry_output_config", "messages_carry_output_config", diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 3004f83b1df..e2ee80b7874 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -39,8 +39,8 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.plumbing.context import set_request_root_span from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging -from litellm.llms.anthropic.pass_through.messages.streaming_iterator import closing_superseded_attempt -from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( +from litellm.llms.anthropic.messages.native.streaming_iterator import closing_superseded_attempt +from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES, ) from litellm.llms.bedrock.common_utils import BedrockError From 0657ed46cf435b85ae73ee3963e92f4efa5ca335 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:34:55 -0700 Subject: [PATCH 002/102] fix(langfuse_otel)!: derive trace-level name, input, output and tags on traces litellm starts (#42705) * fix(langfuse_otel): derive trace-level name, input, output and tags on the generation span Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(langfuse_otel): keep every reasoning summary in the derived observation output Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(langfuse_otel): leave joined traces untouched by derived trace fields Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(langfuse_otel): restore preset cache key fallback and type the derived-field helpers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(langfuse_otel): keep trace tags off joined traces entirely Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(langfuse_otel): tolerate out-of-band upstream requests and add sink outage chaos cell Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): build the langfuse rig config without mutating the loaded yaml Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(langfuse_otel): flatten output items with itertools.chain to satisfy LIT014 budget Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(langfuse_otel): type the _RecordingSpan and _emitted unit helpers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(langfuse_otel): keep joined-trace tags, mask trace io, leave v2 trace io to the root Joined traces (existing_trace_id) keep the caller tags they carried before, and only the derived trace fields are skipped for them. mask_input and mask_output now redact the derived trace input and output, matching the vanilla Langfuse callback. The v2 mapper derives the trace name only, because the root observation already carries the trace input and output there. Uses the public preset cache key helper, moves imports to module top, restores the original formatting of the unit test file, and adds integration cells for the v2 llm_only scope and SDK sync/async calls. * fix(langfuse_otel): split string caller tags the way Langfuse reads them A caller tags value given as a comma list or a JSON array string (body metadata.tags or the langfuse_tags header) used to reach Langfuse raw, where ingestion split it into separate tags. Merging it with the derived tags kept it as one tag, so parse it the same way before the merge * test(otel): expect langfuse.trace.name on the v2 langfuse_otel generation span * fix(langfuse_otel): read string mask flags the way the classic callback does The langfuse_mask_input / langfuse_mask_output headers arrive as strings, so "false" redacted the derived trace io. Parse them with str_to_bool and fall back to truthiness, as the classic langfuse callback does * fix(langfuse_otel): derive trace fields only on traces litellm starts A litellm span that joins a trace an app or caller started (inbound traceparent, the app's active span, a caller-provided parent span) no longer gets a derived trace name, input, output or tags, like a request joining an existing_trace_id. Langfuse applies the later-starting span's trace update, so the derived values overwrote the app's own trace name and io. OTel v2 derives the name on the request root only when it has no parent, and the generation span keeps main's caller-only trace name. * test(langfuse_otel): keep main's OTel v2 generation key set in the arize family cell The OTel v2 generation span no longer carries a derived langfuse.trace.name, so the pinned key set goes back to main's. * fix(langfuse_otel): name the OTel v2 request root once, from the request's own call The v2 Langfuse logger re-derived langfuse.trace.name on the request root for every LLM call in the request. A post_call llm_as_a_judge guardrail's judge call replaced the caller's trace_name on the root with litellm-acompletion, and with another v2 callback (otel, arize_phoenix) listed before langfuse_otel that vendor's root span gained langfuse.trace.name on every request. The derived name now fills only an unnamed root this logger's provider started, and never comes from a call litellm makes for its own features. A caller trace_name still names the root on every call, as on main. * fix(langfuse_otel): keep the derived trace fields from pushing other attributes off a full span The derived trace name, tags, input and output were written after the OpenInference attributes with no room check, so a span near the SDK's attribute limit (about 34 tools at the default 128) evicted keys main exported, such as openinference.span.kind, metadata and llm.model_name, and raised the dropped count. They are now written last and only into the room the limit leaves next to the error attributes a failed call adds, so a full span exports what it would without them. * test(langfuse_otel): give the full-span cells' derived request a caller trace id, as the joined one's traceparent does * chore(langfuse_otel): drop the docstrings and comments the house rule disallows and type the new tests' parameters * test(langfuse_otel): set the span attribute limit the full-span tests run at and keep Final locals out of the app-trace loop --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- .../integrations/langfuse/langfuse_otel.py | 389 ++++++++--- litellm/integrations/otel/langfuse_logger.py | 23 +- litellm/integrations/otel/mappers/langfuse.py | 4 +- litellm/types/integrations/langfuse_otel.py | 2 + .../test_langfuse_otel_trace_fields.py | 660 ++++++++++++++++++ .../test_langfuse_otel_trace_fields_sdk.py | 362 ++++++++++ .../integrations/otel/test_langfuse_logger.py | 135 +++- .../otel/test_otel_v2_vendor_mappers.py | 26 + tests/unit/integrations/test_langfuse_otel.py | 416 ++++++++++- 9 files changed, 1898 insertions(+), 119 deletions(-) create mode 100644 tests/integration/observability/test_langfuse_otel_trace_fields.py create mode 100644 tests/integration/sdk/test_langfuse_otel_trace_fields_sdk.py diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index 9364a3d4216..34d2c9fce0d 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -1,19 +1,32 @@ import base64 import json import os -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from datetime import datetime -from typing import TYPE_CHECKING, Any, Final, Optional +from itertools import chain +from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, runtime_checkable +from opentelemetry import trace + +import litellm from litellm._logging import verbose_logger +from litellm.constants import REDACTED_BY_LITELLM +from litellm.integrations._types.open_inference import ErrorAttributes from litellm.integrations.arize import _utils from litellm.integrations.langfuse.langfuse_otel_attributes import ( LangfuseLLMObsOTELAttributes, serialize_langfuse_datetime, ) -from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig +from litellm.integrations.opentelemetry import ( + HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE, + LITELLM_PROXY_REQUEST_SPAN_NAME, + OpenTelemetry, + OpenTelemetryConfig, +) from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads +from litellm.secret_managers.main import str_to_bool from litellm.types.integrations.langfuse_otel import ( LangfuseSpanAttributes, ) @@ -31,6 +44,86 @@ LANGFUSE_CLOUD_EU_ENDPOINT: Final = "https://cloud.langfuse.com/api/public/otel" LANGFUSE_CLOUD_US_ENDPOINT: Final = "https://us.cloud.langfuse.com/api/public/otel" LANGFUSE_INGESTION_VERSION_HEADER: Final = "x-langfuse-ingestion-version" LANGFUSE_INGESTION_VERSION: Final = "4" +_ERROR_ATTRIBUTES: Final = frozenset( + { + ErrorAttributes.ERROR_CODE, + HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE, + ErrorAttributes.ERROR_TYPE, + ErrorAttributes.ERROR_MESSAGE, + ErrorAttributes.ERROR_LLM_PROVIDER, + ErrorAttributes.ERROR_STACK_TRACE, + } +) + + +@runtime_checkable +class _Gettable(Protocol): + def get(self, key: str, default: object = None) -> object: ... + + +def _as_gettable(value: object) -> _Gettable | None: + return value if isinstance(value, _Gettable) else None + + +def _attr(item: object, name: str, default: object = None) -> object: + return getattr(item, name, default) + + +def _preset_cache_key(kwargs: Mapping[str, object]) -> object: + if litellm.cache is None: + return None + return litellm.cache.get_preset_cache_key_from_kwargs(**kwargs) + + +def _mask_flag(value: object) -> bool: + parsed: Final = str_to_bool(value) if isinstance(value, str) else None + return bool(value) if parsed is None else parsed + + +def _caller_tag_list(value: object) -> tuple[str, ...]: + parsed: Final = ( + safe_json_loads(value, default=value) if isinstance(value, str) and value.strip().startswith("[") else value + ) + if isinstance(parsed, list): + return tuple(tag if isinstance(tag, str) else json.dumps(tag, default=str) for tag in parsed) + if isinstance(parsed, str) and parsed: + return tuple(tag.strip() for tag in parsed.split(",")) if "," in parsed else (parsed,) + return () + + +def _parent_of(span: object) -> trace.SpanContext | None: + parent: Final = getattr(span, "parent", None) + return parent if isinstance(parent, trace.SpanContext) else None + + +def _litellm_proxy_span(kwargs: Mapping[str, object]) -> trace.Span | None: + litellm_params: Final = _as_gettable(kwargs.get("litellm_params")) or {} + carried: Final = ( + (_as_gettable(litellm_params.get(key)) or {}).get("litellm_parent_otel_span") + for key in ("metadata", "litellm_metadata") + ) + return next( + ( + span + for span in carried + if isinstance(span, trace.Span) and getattr(span, "name", None) == LITELLM_PROXY_REQUEST_SPAN_NAME + ), + None, + ) + + +def _starts_its_own_trace(span: object, kwargs: Mapping[str, object]) -> bool: + if span is trace.get_current_span(): + return False + parent: Final = _parent_of(span) + if parent is None: + return True + proxy_span: Final = _litellm_proxy_span(kwargs) + return ( + proxy_span is not None + and _parent_of(proxy_span) is None + and proxy_span.get_span_context().span_id == parent.span_id + ) class LangfuseOtelLogger(OpenTelemetry): @@ -129,98 +222,42 @@ class LangfuseOtelLogger(OpenTelemetry): safe_set_attribute(span, enum_attr.value, value) @staticmethod - def _set_observation_output(span: Span, response_obj): - """Helper to set observation output attributes.""" - from litellm.integrations.arize._utils import safe_set_attribute - from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - + def _observation_output(response_obj: _Gettable | None) -> str | None: if not response_obj or not hasattr(response_obj, "get"): - return + return None + return _extract_output_items(response_obj) or _extract_choices_output(response_obj) - choices: Final = response_obj.get("choices", []) - if choices: - first_choice: Final = choices[0] - message: Final = first_choice.get("message", {}) - tool_calls: Final = message.get("tool_calls") - if tool_calls: - transformed_tool_calls: Final = [] - for tool_call in tool_calls: - function = tool_call.get("function", {}) - arguments_str = function.get("arguments", "{}") - try: - arguments_obj = json.loads(arguments_str) if isinstance(arguments_str, str) else arguments_str - except json.JSONDecodeError: - arguments_obj = {} - langfuse_tool_call = { - "id": response_obj.get("id", ""), - "name": function.get("name", ""), - "call_id": tool_call.get("id", ""), - "type": "function_call", - "arguments": arguments_obj, - } - transformed_tool_calls.append(langfuse_tool_call) - safe_set_attribute( - span, - LangfuseSpanAttributes.OBSERVATION_OUTPUT.value, - safe_dumps(transformed_tool_calls), - ) - else: - output_data: Final = {} - if message.get("role"): - output_data["role"] = message.get("role") - if message.get("content") is not None: - output_data["content"] = message.get("content") - if output_data: - safe_set_attribute( - span, - LangfuseSpanAttributes.OBSERVATION_OUTPUT.value, - safe_dumps(output_data), - ) + @staticmethod + def _trace_tags(kwargs: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[str, ...]: + caller_tags: Final = metadata.get("tags") + request_tags: Final = (_as_gettable(kwargs.get("standard_logging_object")) or {}).get("request_tags") + default_tags: Final = litellm.langfuse_default_tags - output: Final = response_obj.get("output", []) - if output: - output_items_data: Final[list[dict]] = [] - for item in output: - if hasattr(item, "type"): - item_type = item.type - if item_type == "reasoning" and hasattr(item, "summary"): - for summary in item.summary: - if hasattr(summary, "text"): - output_items_data.append( - { - "role": "reasoning_summary", - "content": summary.text, - } - ) - elif item_type == "message": - item_content: Sequence[object] = getattr(item, "content", [{}]) - output_items_data.append( - { - "role": getattr(item, "role", "assistant"), - "content": getattr(item_content[0], "text", ""), - } - ) - elif item_type == "function_call": - item_arguments: object = getattr(item, "arguments", "{}") - arguments_obj = ( - safe_json_loads(item_arguments, default={}) - if isinstance(item_arguments, str) - else item_arguments - ) - langfuse_tool_call = { - "id": getattr(item, "id", ""), - "name": getattr(item, "name", ""), - "call_id": getattr(item, "call_id", ""), - "type": "function_call", - "arguments": arguments_obj, - } - output_items_data.append(langfuse_tool_call) - if output_items_data: - safe_set_attribute( - span, - LangfuseSpanAttributes.OBSERVATION_OUTPUT.value, - safe_dumps(output_items_data), + def _default_tag(key: str) -> str | None: + if key == "cache_hit": + return f"cache_hit:{kwargs.get('cache_hit', False)}" + if key == "cache_key": + hidden_params: Final = _as_gettable(metadata.get("hidden_params", {})) or {} + cache_key: Final = ( + hidden_params.get("cache_key") + if hidden_params.get("cache_key") is not None + else _preset_cache_key(kwargs) ) + return f"cache_key:{cache_key}" + if key == "proxy_base_url": + proxy_base_url: Final = os.environ.get("PROXY_BASE_URL") + return f"proxy_base_url:{proxy_base_url}" if proxy_base_url is not None else None + if key in metadata and metadata[key] is not None: + return f"{key}:{metadata[key]}" + return None + + expanded: Final = tuple(_default_tag(key) for key in default_tags) if isinstance(default_tags, list) else () + candidates: Final = ( + _caller_tag_list(caller_tags) + + (tuple(tag for tag in request_tags if isinstance(tag, str)) if isinstance(request_tags, list) else ()) + + tuple(tag for tag in expanded if tag is not None) + ) + return tuple(dict.fromkeys(candidates)) @staticmethod def _set_langfuse_specific_attributes(span: Span, kwargs: Mapping[str, object], response_obj): @@ -249,13 +286,12 @@ class LangfuseOtelLogger(OpenTelemetry): metadata: Final = LangfuseOtelLogger._extract_langfuse_metadata(kwargs) LangfuseOtelLogger._set_metadata_attributes(span=span, metadata=metadata) - messages: Final = kwargs.get("messages") - if messages: - safe_set_attribute( - span, - LangfuseSpanAttributes.OBSERVATION_INPUT.value, - safe_dumps(messages), - ) + joins_existing_trace: Final = metadata.get("existing_trace_id") is not None or not _starts_its_own_trace( + span, kwargs + ) + input_json: Final = safe_dumps(kwargs.get("messages")) if kwargs.get("messages") else None + if input_json is not None: + safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_INPUT.value, input_json) completion_start_time: Final = kwargs.get("completion_start_time") if isinstance(completion_start_time, datetime): @@ -265,7 +301,64 @@ class LangfuseOtelLogger(OpenTelemetry): serialize_langfuse_datetime(completion_start_time), ) - LangfuseOtelLogger._set_observation_output(span=span, response_obj=response_obj) + output_json: Final = LangfuseOtelLogger._observation_output(response_obj) + if output_json is not None: + safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_OUTPUT.value, output_json) + + if not joins_existing_trace: + LangfuseOtelLogger._set_trace_fields( + span=span, kwargs=kwargs, metadata=metadata, input_json=input_json, output_json=output_json + ) + + @staticmethod + def _set_trace_fields( + span: Span, + kwargs: Mapping[str, object], + metadata: Mapping[str, object], + input_json: str | None, + output_json: str | None, + ) -> None: + from litellm.integrations.otel.emitter import carried_keys, span_attribute_limit + + tags: Final = LangfuseOtelLogger._trace_tags(kwargs, metadata) + fields: Final = ( + *( + ((LangfuseSpanAttributes.TRACE_NAME.value, f"litellm-{kwargs.get('call_type') or 'completion'}"),) + if metadata.get("trace_name") is None + else () + ), + *(((LangfuseSpanAttributes.TAGS.value, json.dumps(list(tags))),) if tags else ()), + *( + ( + ( + LangfuseSpanAttributes.TRACE_INPUT.value, + REDACTED_BY_LITELLM if _mask_flag(metadata.get("mask_input")) else input_json, + ), + ) + if input_json is not None + else () + ), + *( + ( + ( + LangfuseSpanAttributes.TRACE_OUTPUT.value, + REDACTED_BY_LITELLM if _mask_flag(metadata.get("mask_output")) else output_json, + ), + ) + if output_json is not None + else () + ), + ) + limit: Final = span_attribute_limit(span) + carried: Final = carried_keys(span) + new_keys: Final = tuple(key for key, _value in fields if key not in carried) + room: Final = ( + len(new_keys) if limit is None else max(limit - len(carried) - len(_ERROR_ATTRIBUTES - carried), 0) + ) + written: Final = carried | frozenset(new_keys[:room]) + for key, value in fields: + if key in written: + _utils.safe_set_attribute(span, key, value) @staticmethod def _get_langfuse_otel_host() -> str | None: @@ -457,3 +550,101 @@ class LangfuseOtelLogger(OpenTelemetry): """ Langfuse should not receive service failure logs. """ + + +def _extract_choices_output(response_obj: _Gettable) -> str | None: + choices: Final = response_obj.get("choices", []) + if not isinstance(choices, list) or not choices: + return None + first_choice: Final = _as_gettable(choices[0]) + if first_choice is None: + return None + message: Final = _as_gettable(first_choice.get("message", {})) + if message is None: + return None + tool_calls: Final = message.get("tool_calls") + if isinstance(tool_calls, list) and tool_calls: + transformed_tool_calls: Final = [ + entry for tool_call in tool_calls if (entry := _transformed_tool_call(response_obj, tool_call)) is not None + ] + return safe_dumps(transformed_tool_calls) + output_data: Final = { + key: value + for key, value in ( + ("role", message.get("role") or None), + ("content", message.get("content")), + ) + if value is not None + } + return safe_dumps(output_data) if output_data else None + + +def _transformed_tool_call(response_obj: _Gettable, tool_call: object) -> dict[str, object] | None: + call: Final = _as_gettable(tool_call) + if call is None: + return None + function: Final = _as_gettable(call.get("function", {})) + if function is None: + return None + return { + "id": response_obj.get("id", ""), + "name": function.get("name", ""), + "call_id": call.get("id", ""), + "type": "function_call", + "arguments": _tool_call_arguments(function.get("arguments", "{}")), + } + + +def _tool_call_arguments(arguments: object) -> object: + if not isinstance(arguments, str): + return arguments + try: + return json.loads(arguments) + except json.JSONDecodeError: + return {} + + +def _extract_output_items(response_obj: _Gettable) -> str | None: + output: Final = response_obj.get("output", []) + if not isinstance(output, list) or not output: + return None + rendered: Final = tuple(chain.from_iterable(map(_output_items, output))) + return safe_dumps(list(rendered)) if rendered else None + + +def _output_items(item: object) -> tuple[dict[str, object], ...]: + item_type: Final = _attr(item, "type") + if item_type == "reasoning": + summaries: Final = _attr(item, "summary") + if not isinstance(summaries, Iterable): + return () + return tuple( + {"role": "reasoning_summary", "content": _attr(summary, "text")} + for summary in summaries + if hasattr(summary, "text") + ) + if item_type == "message": + content_items: Final = _attr(item, "content", [{}]) + first_content: Final = ( + content_items[0] + if isinstance(content_items, Sequence) and not isinstance(content_items, (str, bytes)) and content_items + else {} + ) + return ( + { + "role": _attr(item, "role", "assistant"), + "content": _attr(first_content, "text", ""), + }, + ) + if item_type == "function_call": + arguments: Final = _attr(item, "arguments", "{}") + return ( + { + "id": _attr(item, "id", ""), + "name": _attr(item, "name", ""), + "call_id": _attr(item, "call_id", ""), + "type": "function_call", + "arguments": safe_json_loads(arguments, default={}) if isinstance(arguments, str) else arguments, + }, + ) + return () diff --git a/litellm/integrations/otel/langfuse_logger.py b/litellm/integrations/otel/langfuse_logger.py index c869474f532..15b0f3f0cc2 100644 --- a/litellm/integrations/otel/langfuse_logger.py +++ b/litellm/integrations/otel/langfuse_logger.py @@ -1,18 +1,25 @@ from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping from typing import TYPE_CHECKING, Final +from opentelemetry.sdk.trace import ReadableSpan + from litellm._logging import verbose_logger from litellm.integrations.otel.logger import OpenTelemetryV2 from litellm.integrations.otel.mappers.langfuse import ( LANGFUSE_OBSERVATION_INPUT, LANGFUSE_OBSERVATION_OUTPUT, + LANGFUSE_TRACE_NAME, LangfuseMapper, ) +from litellm.integrations.otel.model.metadata import internal_call_origin from litellm.integrations.otel.model.request_io import request_input, response_output, stream_output from litellm.integrations.otel.model.trace_controls import caller_trace_controls +from litellm.integrations.otel.model.utils import as_str from litellm.integrations.otel.plumbing.context import request_root_span if TYPE_CHECKING: + from opentelemetry.trace import Span + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.types.utils import ModelResponseStream @@ -24,9 +31,23 @@ class LangfuseOpenTelemetryV2(OpenTelemetryV2): def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None: root: Final = request_root_span() if root is not None and root.is_recording(): - root.set_attributes(LangfuseMapper.trace_attributes(caller_trace_controls(kwargs))) + root.set_attributes( + LangfuseMapper.trace_attributes( + caller_trace_controls(kwargs), + as_str(kwargs.get("call_type")) if self._derives_trace_name(root, kwargs) else None, + ) + ) super().log_pre_api_call(model, messages, kwargs) + def _derives_trace_name(self, root: "Span", kwargs: Mapping[str, object]) -> bool: + return ( + internal_call_origin(None, kwargs) is None + and isinstance(root, ReadableSpan) + and root.parent is None + and root.resource is self.tracer_provider.resource + and LANGFUSE_TRACE_NAME not in (root.attributes or {}) + ) + class LangfuseContentOpenTelemetryV2(LangfuseOpenTelemetryV2): """Stamps the request's input and output on the root observation while it is still recording. diff --git a/litellm/integrations/otel/mappers/langfuse.py b/litellm/integrations/otel/mappers/langfuse.py index e60ea539188..230a2e261fe 100644 --- a/litellm/integrations/otel/mappers/langfuse.py +++ b/litellm/integrations/otel/mappers/langfuse.py @@ -96,10 +96,10 @@ class LangfuseMapper: return {} @staticmethod - def trace_attributes(trace: TraceControls) -> AttributeMap: + def trace_attributes(trace: TraceControls, call_type: str | None = None) -> AttributeMap: return drop_none_pairs( ( - (LANGFUSE_TRACE_NAME, trace.name or None), + (LANGFUSE_TRACE_NAME, trace.name or (f"litellm-{call_type}" if call_type else None)), (LANGFUSE_TRACE_USER_ID, trace.user_id or None), (LANGFUSE_TRACE_SESSION_ID, trace.session_id or None), (LANGFUSE_TRACE_TAGS, trace.tags or None), diff --git a/litellm/types/integrations/langfuse_otel.py b/litellm/types/integrations/langfuse_otel.py index 5647f28c757..459f2e984a4 100644 --- a/litellm/types/integrations/langfuse_otel.py +++ b/litellm/types/integrations/langfuse_otel.py @@ -32,6 +32,8 @@ class LangfuseSpanAttributes(str, Enum): OBSERVATION_COMPLETION_START_TIME = "langfuse.observation.completion_start_time" # ---- Trace-level metadata ---- + TRACE_INPUT = "langfuse.trace.input" + TRACE_OUTPUT = "langfuse.trace.output" TRACE_USER_ID = "user.id" SESSION_ID = "session.id" TAGS = "langfuse.trace.tags" diff --git a/tests/integration/observability/test_langfuse_otel_trace_fields.py b/tests/integration/observability/test_langfuse_otel_trace_fields.py new file mode 100644 index 00000000000..2b78b402926 --- /dev/null +++ b/tests/integration/observability/test_langfuse_otel_trace_fields.py @@ -0,0 +1,660 @@ +import json +import threading +import uuid +from itertools import product +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import pytest +import yaml +from _s3_v2_support import SURFACES, call_surface, mixed_burst, surface_reply +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.proto.common.v1.common_pb2 import AnyValue +from opentelemetry.proto.trace.v1.trace_pb2 import Span + +REDACTED: Final = "redacted-by-litellm" +TRACE_FIELDS: Final = ("langfuse.trace.name", "langfuse.trace.input", "langfuse.trace.output", "langfuse.trace.tags") +SURFACE_CALL_TYPES: Final = { + "chat": "acompletion", + "chat_stream": "acompletion", + "messages": "anthropic_messages", + "messages_stream": "anthropic_messages", + "responses": "aresponses", + "responses_stream": "aresponses", +} +ANTHROPIC_MODEL: Final = "anthropic/claude-sonnet-4-5-20250929" +JUDGE_MODEL: Final = "lit8281-judge-model" +JUDGE_GUARDRAIL: Final = "lit8281-judge" + + +@dataclass(frozen=True, slots=True) +class SpanRecord: + name: str + trace_id: str + parent_span_id: str + attributes: Mapping[str, object] + dropped_attributes: int = 0 + + def tags(self) -> tuple[object, ...]: + raw: Final = self.attributes.get("langfuse.trace.tags") + if isinstance(raw, tuple): + return raw + if not isinstance(raw, str) or not raw: + return () + if raw.strip().startswith("["): + return tuple(json.loads(raw)) + return tuple(tag.strip() for tag in raw.split(",")) if "," in raw else (raw,) + + +def _attribute_value(value: AnyValue) -> object: + kind: Final = value.WhichOneof("value") + if kind == "array_value": + return tuple(_attribute_value(item) for item in value.array_value.values) + return getattr(value, kind) if kind is not None else None + + +def _exported_spans(body: bytes) -> Iterator[Span]: + for resource in ExportTraceServiceRequest.FromString(body).resource_spans: + for scope in resource.scope_spans: + yield from scope.spans + + +def _span_records(body: bytes) -> tuple[SpanRecord, ...]: + return tuple( + SpanRecord( + name=span.name, + trace_id=span.trace_id.hex(), + parent_span_id=span.parent_span_id.hex(), + attributes={pair.key: _attribute_value(pair.value) for pair in span.attributes}, + dropped_attributes=span.dropped_attributes_count, + ) + for span in _exported_spans(body) + ) + + +class LangfuseSink: + def __init__(self) -> None: + self.lock: Final = threading.Lock() + self.outage: Final = threading.Event() + self.accepted: Final[list[SpanRecord]] = [] # mutable-ok: spans accumulate as the exporter delivers them + self.roots: Final[list[SpanRecord]] = [] # mutable-ok: root spans accumulate as the exporter delivers them + self.spans: Final[list[SpanRecord]] = [] # mutable-ok: every span accumulates as the exporter delivers it + self.refusals: Final[list[int]] = [] # mutable-ok: one entry per batch refused during the outage + + def respond(self, request: Request) -> Reply: + if not request.target.endswith("/v1/traces"): + return Reply(body=b"") + assert request.headers.get("x-langfuse-ingestion-version") == "4", request.headers + if self.outage.is_set(): + with self.lock: + self.refusals.append(1) + return Reply(status=503, body=b'{"error": "scripted outage"}') + records: Final = _span_records(request.body) + with self.lock: + self.accepted.extend( + record for record in records if record.attributes.get("langfuse.observation.type") == "generation" + ) + self.roots.extend(record for record in records if not record.parent_span_id) + self.spans.extend(records) + return Reply(body=b"") + + def matching(self, marker: str) -> tuple[SpanRecord, ...]: + with self.lock: + snapshot: Final = tuple(self.accepted) + return tuple(record for record in snapshot if marker in str(tuple(record.attributes.values()))) + + def one(self, marker: str) -> SpanRecord: + return eventually(lambda: self.matching(marker), lambda found: len(found) == 1, seconds=70)[0] + + def in_trace(self, trace_id: str) -> tuple[SpanRecord, ...]: + with self.lock: + return tuple(record for record in self.spans if record.trace_id == trace_id) + + def root_of(self, span: SpanRecord) -> SpanRecord: + def roots() -> tuple[SpanRecord, ...]: + with self.lock: + return tuple(root for root in self.roots if root.trace_id == span.trace_id) + + return eventually(roots, lambda found: len(found) == 1, seconds=70)[0] + + +class OtlpSink: + def __init__(self) -> None: + self.lock: Final = threading.Lock() + self.spans: Final[list[SpanRecord]] = [] # mutable-ok: spans accumulate as the exporter delivers them + + def respond(self, request: Request) -> Reply: + if request.target.endswith("/v1/traces"): + records: Final = _span_records(request.body) + with self.lock: + self.spans.extend(records) + return Reply(body=b"") + + def root_of_request(self, marker: str) -> SpanRecord: + + def roots() -> tuple[SpanRecord, ...]: + with self.lock: + snapshot: Final = tuple(self.spans) + traces: Final = {record.trace_id for record in snapshot if marker in str(tuple(record.attributes.values()))} + return tuple(record for record in snapshot if record.trace_id in traces and not record.parent_span_id) + + return eventually(roots, lambda found: len(found) == 1, seconds=70)[0] + + +def _chat_completion(marker: str, text: str = "langfuse-otel scripted reply") -> bytes: + return json.dumps( + { + "id": marker, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 4, "total_tokens": 9}, + } + ).encode() + + +def _provider(request: Request) -> Reply: + if request.method != "POST" or not request.body: + return Reply(body=b'{"object": "list", "data": []}') + body: Final = json.loads(request.body) + if request.target.endswith("/embeddings"): + return Reply( + body=json.dumps( + { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.25, 0.5]}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 3, "total_tokens": 3}, + } + ).encode() + ) + if body.get("response_format") == {"type": "json_object"}: + return Reply(body=_chat_completion("lit8281-judge-verdict", json.dumps({"overall_score": 100}))) + messages: Final = body.get("messages") + first: Final = messages[0]["content"] if isinstance(messages, list) and messages else None + if isinstance(first, str) and first.startswith("fail-"): + return Reply(status=500, body=b'{"error": {"message": "scripted provider failure", "type": "server_error"}}') + return surface_reply(request) + + +def _langfuse_config(directory: Path, name: str, settings: Mapping[str, object], provider_url: str) -> Path: + base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + judge: Final = { + "model_name": JUDGE_MODEL, + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": provider_url + "/v1", + "api_key": "synthetic-provider-key", + }, + } + config: Final = { + **base, + "model_list": [*base["model_list"], judge], + "guardrails": [ + { + "guardrail_name": JUDGE_GUARDRAIL, + "litellm_params": { + "guardrail": "llm_as_a_judge", + "mode": "post_call", + "judge_model": JUDGE_MODEL, + "on_failure": "log", + "criteria": [{"name": "helpful", "description": "The reply answers the user", "weight": 100}], + }, + } + ], + "litellm_settings": {**base["litellm_settings"], "callbacks": ["langfuse_otel"], **settings}, + } + path: Final = directory / f"{name}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@dataclass(frozen=True, slots=True) +class Rig: + proxy: Gateway + sink: LangfuseSink + openai_model: str + anthropic_model: str + embedding_model: str + key: str + + +def _rig( + tmp_path_factory: pytest.TempPathFactory, name: str, environment: Mapping[str, str], settings: Mapping[str, object] +) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp(name) + sink: Final = LangfuseSink() + with ( + wire_server(sink.respond) as destination, + wire_server(_provider) as provider, + gateway_from_environment() as gateway, + owned_proxy( + gateway, + directory, + { + "LANGFUSE_PUBLIC_KEY": "pk-lf-test", + "LANGFUSE_SECRET_KEY": "sk-lf-test", + "LANGFUSE_HOST": destination.url, + **environment, + }, + config=_langfuse_config(directory, name, settings, provider.url), + workers=2, + ) as proxy, + proxy.scenario() as scenario, + ): + openai_model: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-provider-key") + anthropic_model: Final = scenario.model( + model=ANTHROPIC_MODEL, api_base=provider.url, api_key="synthetic-provider-key" + ) + embedding_model: Final = scenario.model( + model="openai/text-embedding-3-small", + api_base=provider.url + "/v1", + api_key="synthetic-provider-key", + model_info={"mode": "embedding"}, + ) + key: Final = scenario.key(models=[openai_model, anthropic_model, embedding_model]) + yield Rig(proxy, sink, openai_model, anthropic_model, embedding_model, key) + + +SPAN_ATTRIBUTE_LIMIT: Final = 128 + + +@pytest.fixture(scope="module") +def v1(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + yield from _rig( + tmp_path_factory, + "langfuse-v1", + {"OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT": str(SPAN_ATTRIBUTE_LIMIT)}, + {"langfuse_default_tags": ["cache_hit"]}, + ) + + +V2_ENVIRONMENT: Final = {"LITELLM_OTEL_V2": "1", "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "span_only"} + + +@pytest.fixture(scope="module") +def v2(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + yield from _rig(tmp_path_factory, "langfuse-v2", V2_ENVIRONMENT, {}) + + +@pytest.fixture(scope="module") +def v2_llm_only(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + yield from _rig( + tmp_path_factory, "langfuse-v2-llm-only", {**V2_ENVIRONMENT, "LITELLM_OTEL_LANGFUSE_SPAN_SCOPE": "llm_only"}, {} + ) + + +@dataclass(frozen=True, slots=True) +class SharedRig: + rig: Rig + otel: OtlpSink + + +@pytest.fixture(scope="module") +def v2_after_otel(tmp_path_factory: pytest.TempPathFactory) -> Iterator[SharedRig]: + otel: Final = OtlpSink() + with wire_server(otel.respond) as destination: + environment: Final = { + **V2_ENVIRONMENT, + "OTEL_EXPORTER": "otlp_http", + "OTEL_ENDPOINT": destination.url + "/v1/traces", + } + for rig in _rig( + tmp_path_factory, "langfuse-v2-after-otel", environment, {"callbacks": ["otel", "langfuse_otel"]} + ): + yield SharedRig(rig, otel) + + +def _chat( + rig: Rig, + content: str, + metadata: Mapping[str, object] | None = None, + headers: Mapping[str, str] | None = None, + guardrails: Sequence[str] = (), +) -> str: + response: Final = rig.proxy.request( + "POST", + "/v1/chat/completions", + { + "model": rig.openai_model, + "messages": [{"role": "user", "content": content}], + **({"metadata": dict(metadata)} if metadata is not None else {}), + **({"guardrails": list(guardrails)} if guardrails else {}), + }, + key=rig.key, + headers=headers, + ) + assert response.status_code == 200, response.text + return response.text + + +def _trace_fields(spans: tuple[SpanRecord, ...]) -> Iterator[str]: + for record in spans: + yield from (key for key in record.attributes if key in TRACE_FIELDS) + + +def _assert_derived(span: SpanRecord, call_type: str) -> None: + attributes: Final = span.attributes + assert attributes.get("langfuse.trace.name") == f"litellm-{call_type}", attributes + assert attributes.get("langfuse.trace.input") == attributes.get("langfuse.observation.input"), attributes + assert attributes.get("langfuse.trace.output") == attributes.get("langfuse.observation.output"), attributes + tags: Final = span.tags() + assert any(str(tag).startswith("User-Agent: ") for tag in tags), tags + assert len(tags) == len(set(map(str, tags))), tags + + +@pytest.mark.parametrize("surface", SURFACES) +def test_every_surface_derives_trace_name_io_and_tags(v1: Rig, surface: str) -> None: + marker: Final = f"lit8281-{surface}-{uuid.uuid4().hex}" + call_surface(v1.proxy, surface, v1.openai_model, v1.anthropic_model, v1.key, marker) + span: Final = v1.sink.one(marker) + _assert_derived(span, SURFACE_CALL_TYPES[surface]) + assert marker in str(span.attributes["langfuse.trace.input"]), span.attributes + assert span.attributes.get("langfuse.trace.output") is not None, span.attributes + assert any(str(tag).startswith("cache_hit:") for tag in span.tags()), span.tags() + + +def test_inbound_traceparent_joins_the_caller_trace_without_derived_trace_fields(v1: Rig) -> None: + marker: Final = "lit8281-parent-" + uuid.uuid4().hex + trace_id: Final = uuid.uuid4().hex + parent_id: Final = uuid.uuid4().hex[:16] + _chat(v1, marker, headers={"traceparent": f"00-{trace_id}-{parent_id}-01"}) + span: Final = v1.sink.one(marker) + assert (span.trace_id, span.parent_span_id) == (trace_id, parent_id), span + assert not set(TRACE_FIELDS) & set(span.attributes), span.attributes + assert marker in str(span.attributes.get("langfuse.observation.input")), span.attributes + assert span.attributes.get("langfuse.observation.output") is not None, span.attributes + + +def test_inbound_traceparent_still_carries_the_caller_trace_name_and_tags(v1: Rig) -> None: + marker: Final = "lit8281-parent-caller-" + uuid.uuid4().hex + trace_id: Final = uuid.uuid4().hex + _chat( + v1, + marker, + metadata={"trace_name": "caller-trace", "tags": ["caller-tag"]}, + headers={"traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01"}, + ) + span: Final = v1.sink.one(marker) + assert span.trace_id == trace_id, span + assert span.attributes.get("langfuse.trace.name") == "caller-trace", span.attributes + assert span.tags() == ("caller-tag",), span.attributes + assert not {"langfuse.trace.input", "langfuse.trace.output"} & set(span.attributes), span.attributes + + +def test_caller_trace_name_and_tags_win_over_derived_values(v1: Rig) -> None: + marker: Final = "lit8281-caller-" + uuid.uuid4().hex + _chat(v1, marker, metadata={"trace_name": "caller-trace", "tags": ["caller-tag", "second-tag"]}) + span: Final = v1.sink.one(marker) + assert span.attributes["langfuse.trace.name"] == "caller-trace", span.attributes + assert span.tags()[:2] == ("caller-tag", "second-tag"), span.tags() + assert span.attributes.get("langfuse.trace.input") == span.attributes.get("langfuse.observation.input") + + +@pytest.mark.parametrize( + ("metadata", "headers", "expected"), + [ + pytest.param({"tags": "c1, c2"}, None, ("c1", "c2"), id="body_comma_string"), + pytest.param({"tags": '["j1", "j2"]'}, None, ("j1", "j2"), id="body_json_array_string"), + pytest.param(None, {"langfuse_tags": "h1,h2"}, ("h1", "h2"), id="header_comma_string"), + ], +) +def test_string_caller_tags_keep_the_tags_langfuse_split_them_into( + v1: Rig, metadata: Mapping[str, object] | None, headers: Mapping[str, str] | None, expected: tuple[str, ...] +) -> None: + marker: Final = "lit8281-strtags-" + uuid.uuid4().hex + _chat(v1, marker, metadata=metadata, headers=headers) + span: Final = v1.sink.one(marker) + assert span.tags()[: len(expected)] == expected, span.attributes + + +def test_trace_name_header_wins_over_the_derived_name(v1: Rig) -> None: + marker: Final = "lit8281-header-" + uuid.uuid4().hex + _chat(v1, marker, headers={"langfuse_trace_name": "header-trace"}) + span: Final = v1.sink.one(marker) + assert span.attributes["langfuse.trace.name"] == "header-trace", span.attributes + + +def test_joined_trace_keeps_caller_tags_and_gets_no_derived_trace_fields(v1: Rig) -> None: + marker: Final = "lit8281-join-" + uuid.uuid4().hex + existing: Final = uuid.uuid4().hex + _chat(v1, marker, metadata={"existing_trace_id": existing, "tags": ["join-tag"]}) + span: Final = v1.sink.one(marker) + attributes: Final = span.attributes + assert attributes.get("langfuse.trace.existing_id") == existing, attributes + assert span.tags() == ("join-tag",), attributes + assert "langfuse.trace.name" not in attributes, attributes + assert "langfuse.trace.input" not in attributes, attributes + assert "langfuse.trace.output" not in attributes, attributes + assert marker in str(attributes.get("langfuse.observation.input")), attributes + + +def test_joined_trace_without_caller_tags_gets_no_trace_tags(v1: Rig) -> None: + marker: Final = "lit8281-join-bare-" + uuid.uuid4().hex + _chat(v1, marker, metadata={"existing_trace_id": uuid.uuid4().hex}) + attributes: Final = v1.sink.one(marker).attributes + assert not { + key for key in attributes if key.startswith("langfuse.trace.") and key != "langfuse.trace.existing_id" + }, attributes + + +def test_mask_flags_redact_the_derived_trace_io(v1: Rig) -> None: + marker: Final = "lit8281-mask-" + uuid.uuid4().hex + _chat(v1, marker, metadata={"mask_input": True, "mask_output": True}) + attributes: Final = v1.sink.one(marker).attributes + assert attributes.get("langfuse.trace.input") == REDACTED, attributes + assert attributes.get("langfuse.trace.output") == REDACTED, attributes + assert attributes.get("langfuse.trace.name") == "litellm-acompletion", attributes + + +@pytest.mark.parametrize( + ("headers", "masked"), + [ + pytest.param({"langfuse_mask_input": "true", "langfuse_mask_output": "true"}, True, id="true"), + pytest.param({"langfuse_mask_input": "false", "langfuse_mask_output": "false"}, False, id="false"), + ], +) +def test_mask_flag_headers_parse_as_booleans(v1: Rig, headers: Mapping[str, str], masked: bool) -> None: + marker: Final = "lit8281-mask-header-" + uuid.uuid4().hex + _chat(v1, marker, headers=headers) + attributes: Final = v1.sink.one(marker).attributes + expected: Final = ( + (REDACTED, REDACTED) + if masked + else (attributes.get("langfuse.observation.input"), attributes.get("langfuse.observation.output")) + ) + assert (attributes.get("langfuse.trace.input"), attributes.get("langfuse.trace.output")) == expected, attributes + assert marker in str(attributes.get("langfuse.observation.input")), attributes + + +@pytest.mark.parametrize( + "tags", + [ + pytest.param(5, id="int"), + pytest.param(["ok-tag", 7, None], id="mixed_list"), + pytest.param("", id="empty_string"), + pytest.param("x" * 5000, id="five_kb_string"), + ], +) +def test_malformed_caller_tags_still_log_the_generation_with_derived_fields(v1: Rig, tags: object) -> None: + marker: Final = "lit8281-badtags-" + uuid.uuid4().hex + _chat(v1, marker, metadata={"tags": tags}) + span: Final = v1.sink.one(marker) + assert span.attributes.get("langfuse.trace.name") == "litellm-acompletion", span.attributes + assert span.attributes.get("langfuse.trace.input") == span.attributes.get("langfuse.observation.input") + assert all(isinstance(tag, str) for tag in span.tags()), span.tags() + + +def test_provider_failure_span_derives_name_and_input_without_output(v1: Rig) -> None: + marker: Final = "fail-lit8281-" + uuid.uuid4().hex + response: Final = v1.proxy.request( + "POST", + "/v1/chat/completions", + {"model": v1.openai_model, "messages": [{"role": "user", "content": marker}]}, + key=v1.key, + ) + assert response.status_code >= 500, response.text + span: Final = v1.sink.one(marker) + assert span.attributes.get("langfuse.trace.name") == "litellm-acompletion", span.attributes + assert span.attributes.get("langfuse.trace.input") == span.attributes.get("langfuse.observation.input") + assert "langfuse.trace.output" not in span.attributes, span.attributes + + +def test_embedding_call_derives_the_trace_name(v1: Rig) -> None: + marker: Final = "lit8281-embed-" + uuid.uuid4().hex + response: Final = v1.proxy.request( + "POST", "/v1/embeddings", {"model": v1.embedding_model, "input": marker, "user": marker}, key=v1.key + ) + assert response.status_code == 200, response.text + span: Final = v1.sink.one(marker) + assert span.attributes.get("langfuse.trace.name") == "litellm-aembedding", span.attributes + assert span.attributes.get("langfuse.trace.output") == span.attributes.get("langfuse.observation.output") + + +def test_identical_requests_each_log_exactly_one_span_with_derived_fields(v1: Rig) -> None: + marker: Final = "lit8281-twice-" + uuid.uuid4().hex + _chat(v1, marker) + _chat(v1, marker) + spans: Final = eventually(lambda: v1.sink.matching(marker), lambda found: len(found) == 2, seconds=70) + for span in spans: + _assert_derived(span, "acompletion") + + +def test_otel_v2_names_the_request_root_and_leaves_trace_io_to_the_root_observation(v2: Rig) -> None: + derived: Final = "lit8281-v2-" + uuid.uuid4().hex + caller: Final = "lit8281-v2-caller-" + uuid.uuid4().hex + _chat(v2, derived) + _chat(v2, caller, metadata={"trace_name": "caller-trace", "tags": ["caller-tag"]}) + derived_span: Final = v2.sink.one(derived) + assert not set(_trace_fields((derived_span,))), derived_span.attributes + root: Final = v2.sink.root_of(derived_span) + assert root.attributes.get("langfuse.trace.name") == "litellm-acompletion", root.attributes + assert derived in str(root.attributes.get("langfuse.observation.input")), root.attributes + assert root.attributes.get("langfuse.observation.output") is not None, root.attributes + caller_span: Final = v2.sink.one(caller) + assert caller_span.attributes.get("langfuse.trace.name") == "caller-trace", caller_span.attributes + assert caller_span.tags() == ("caller-tag",), caller_span.attributes + + +def test_otel_v2_inbound_traceparent_leaves_the_trace_name_to_the_caller(v2: Rig) -> None: + marker: Final = "lit8281-v2-parent-" + uuid.uuid4().hex + trace_id: Final = uuid.uuid4().hex + _chat(v2, marker, headers={"traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01"}) + span: Final = v2.sink.one(marker) + assert span.trace_id == trace_id, span + spans: Final = eventually( + lambda: v2.sink.in_trace(trace_id), + lambda found: any(record.parent_span_id != span.parent_span_id for record in found), + seconds=70, + ) + assert not set(_trace_fields(spans)), spans + + +@pytest.mark.parametrize( + ("metadata", "headers"), + [({"trace_name": "caller-trace"}, None), (None, {"langfuse_trace_name": "caller-trace"})], + ids=["body", "header"], +) +def test_otel_v2_post_call_judge_keeps_the_caller_trace_name_on_the_root( + v2: Rig, metadata: Mapping[str, object] | None, headers: Mapping[str, str] | None +) -> None: + marker: Final = "lit8281-v2-judge-" + uuid.uuid4().hex + _chat(v2, marker, metadata=metadata, headers=headers, guardrails=[JUDGE_GUARDRAIL]) + generations: Final = eventually(lambda: v2.sink.matching(marker), lambda found: len(found) == 2, seconds=70) + assert len({generation.trace_id for generation in generations}) == 1, generations + root: Final = v2.sink.root_of(generations[0]) + assert root.attributes.get("langfuse.trace.name") == "caller-trace", root.attributes + + +def test_otel_v2_generic_otel_root_gets_no_derived_langfuse_trace_name(v2_after_otel: SharedRig) -> None: + derived: Final = "lit8281-v2-after-otel-" + uuid.uuid4().hex + caller: Final = "lit8281-v2-after-otel-caller-" + uuid.uuid4().hex + _chat(v2_after_otel.rig, derived) + _chat(v2_after_otel.rig, caller, metadata={"trace_name": "caller-trace"}) + root: Final = v2_after_otel.otel.root_of_request(derived) + assert "langfuse.trace.name" not in root.attributes, root.attributes + caller_root: Final = v2_after_otel.otel.root_of_request(caller) + assert caller_root.attributes.get("langfuse.trace.name") == "caller-trace", caller_root.attributes + + +def test_otel_v2_llm_only_root_generation_keeps_its_span_name_as_the_trace_name(v2_llm_only: Rig) -> None: + marker: Final = "lit8281-v2-llm-only-" + uuid.uuid4().hex + _chat(v2_llm_only, marker) + span: Final = v2_llm_only.sink.one(marker) + assert span.parent_span_id == "", span + assert span.attributes.get("langfuse.trace.name") == span.name, span + assert marker in str(span.attributes.get("langfuse.observation.input")), span.attributes + + +def test_destination_outage_mid_burst_lands_every_span_once_with_derived_fields(v1: Rig) -> None: + marker: Final = "lit8281-chaos-" + uuid.uuid4().hex[:12] + v1.sink.outage.set() + try: + answered: Final = mixed_burst(v1.proxy, v1.openai_model, v1.anthropic_model, v1.key, marker, per_surface=4) + eventually(lambda: len(v1.sink.refusals), lambda refused: refused >= 1, seconds=70) + finally: + v1.sink.outage.clear() + assert len(answered) == 4 * len(SURFACES) + expected: Final = {f"{marker}-{surface}-{index}": surface for surface, index in product(SURFACES, range(4))} + + def landed() -> Mapping[str, tuple[SpanRecord, ...]]: + return {identity: v1.sink.matching(identity) for identity in expected} + + spans: Final = eventually(landed, lambda found: all(len(records) >= 1 for records in found.values()), seconds=120) + assert {identity: len(records) for identity, records in spans.items()} == dict.fromkeys(expected, 1) + for identity, records in spans.items(): + _assert_derived(records[0], SURFACE_CALL_TYPES[expected[identity]]) + + +def _tool(index: int) -> Mapping[str, object]: + return { + "type": "function", + "function": { + "name": f"lookup_{index}", + "description": f"Looks up record {index}", + "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}, + }, + } + + +def _chat_with_tools( + rig: Rig, + content: str, + tools: int, + metadata: Mapping[str, object] | None = None, + headers: Mapping[str, str] | None = None, +) -> SpanRecord: + response: Final = rig.proxy.request( + "POST", + "/v1/chat/completions", + { + "model": rig.openai_model, + "messages": [{"role": "user", "content": content}], + "tools": [_tool(index) for index in range(tools)], + **({"metadata": dict(metadata)} if metadata is not None else {}), + }, + key=rig.key, + headers=headers, + ) + assert response.status_code == (500 if content.startswith("fail-") else 200), response.text + return rig.sink.one(content) + + +@pytest.mark.parametrize(("tools", "prefix"), product((30, 33, 34, 35, 36, 40), ("", "fail-"))) +def test_trace_fields_never_evict_or_drop_another_attribute_of_a_full_span(v1: Rig, tools: int, prefix: str) -> None: + run: Final = uuid.uuid4().hex + derived: Final = _chat_with_tools(v1, f"{prefix}lit8281-full-{tools}-{run}", tools, {"trace_id": uuid.uuid4().hex}) + joined: Final = _chat_with_tools( + v1, + f"{prefix}lit8281-full-joined-{tools}-{run}", + tools, + headers={"traceparent": f"00-{uuid.uuid4().hex}-{uuid.uuid4().hex[:16]}-01"}, + ) + assert set(joined.attributes) <= set(derived.attributes), set(joined.attributes) - set(derived.attributes) + assert derived.dropped_attributes == joined.dropped_attributes, (derived.dropped_attributes, joined.attributes) diff --git a/tests/integration/sdk/test_langfuse_otel_trace_fields_sdk.py b/tests/integration/sdk/test_langfuse_otel_trace_fields_sdk.py new file mode 100644 index 00000000000..33846960ab2 --- /dev/null +++ b/tests/integration/sdk/test_langfuse_otel_trace_fields_sdk.py @@ -0,0 +1,362 @@ +import json +import os +import subprocess +import sys +import threading +import uuid +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from typing import Final + +import pytest +from integration._support.client import eventually +from integration._support.wire import Reply, Request, wire_server +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.proto.trace.v1.trace_pb2 import Span + +CALLS: Final = { + "sync": "completion", + "sync_stream": "completion", + "async": "acompletion", + "async_stream": "acompletion", +} +SDK_CALLS: Final = """ +import asyncio +import sys + +import litellm + +litellm.callbacks = ["langfuse_otel"] +api_base, marker = sys.argv[1], sys.argv[2] + + +def request(call): + return { + "model": "openai/gpt-4o-mini", + "api_base": api_base, + "api_key": "synthetic-provider-key", + "messages": [{"role": "user", "content": f"{marker}-{call}"}], + "stream": call.endswith("_stream"), + } + + +async def async_calls(): + await litellm.acompletion(**request("async")) + async for _ in await litellm.acompletion(**request("async_stream")): + pass + + +litellm.completion(**request("sync")) +for _ in litellm.completion(**request("sync_stream")): + pass +asyncio.run(async_calls()) +print("calls done", flush=True) +sys.stdin.read() +""" + + +APP_TRACE_CALLS: Final = """ +import asyncio +import json +import sys +import threading +import time + +import litellm +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor + +litellm.callbacks = ["langfuse_otel"] +api_base, marker, sink, writes_onto_app_span = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] == "1" +app = TracerProvider() +app.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter(endpoint=sink + "/api/public/otel/v1/traces"))) +tracer = app.get_tracer("checkout-app") + + +def request(call, **extra): + return { + "model": "openai/gpt-4o-mini", + "api_base": api_base, + "api_key": "synthetic-provider-key", + "messages": [{"role": "user", "content": f"{marker}-{call}"}], + **extra, + } + + +def name_trace(root, call): + root.set_attribute("app.call", f"{marker}-{call}") + root.set_attribute("langfuse.trace.name", f"checkout-{marker}-{call}") + root.set_attribute("langfuse.trace.input", json.dumps({"cart": f"{marker}-{call}"})) + + +def close_trace(root, call): + root.set_attribute("langfuse.trace.output", json.dumps({"order": f"{marker}-{call}"})) + root.end() + + +def litellm_wrote_onto(root): + return not writes_onto_app_span or "langfuse.observation.type" in root.attributes + + +def ambient_sync(): + with tracer.start_as_current_span("checkout", end_on_exit=False) as root: + name_trace(root, "ambient_sync") + litellm.completion(**request("ambient_sync")) + deadline = time.monotonic() + 30 + while not litellm_wrote_onto(root) and time.monotonic() < deadline: + threading.Event().wait(0.05) + close_trace(root, "ambient_sync") + + +async def ambient_async(): + with tracer.start_as_current_span("checkout", end_on_exit=False) as root: + name_trace(root, "ambient_async") + await litellm.acompletion(**request("ambient_async")) + deadline = time.monotonic() + 30 + while not litellm_wrote_onto(root) and time.monotonic() < deadline: + await asyncio.sleep(0.05) + close_trace(root, "ambient_async") + + +def caller_span_sync(): + root = tracer.start_span("checkout") + name_trace(root, "caller_span_sync") + litellm.completion(**request("caller_span_sync", metadata={"litellm_parent_otel_span": root})) + close_trace(root, "caller_span_sync") + + +async def caller_span_async(): + root = tracer.start_span("checkout") + name_trace(root, "caller_span_async") + await litellm.acompletion(**request("caller_span_async", metadata={"litellm_parent_otel_span": root})) + close_trace(root, "caller_span_async") + + +async def async_calls(): + await ambient_async() + await caller_span_async() + + +ambient_sync() +caller_span_sync() +asyncio.run(async_calls()) +print("calls done", flush=True) +sys.stdin.read() +""" +APP_CALLS: Final = ("ambient_sync", "ambient_async", "caller_span_sync", "caller_span_async") + + +def _exported_spans(body: bytes) -> Iterator[Span]: + for resource in ExportTraceServiceRequest.FromString(body).resource_spans: + for scope in resource.scope_spans: + yield from scope.spans + + +def _attributes(span: Span) -> Mapping[str, str]: + return {pair.key: pair.value.string_value for pair in span.attributes} + + +@dataclass(frozen=True, slots=True) +class TraceSpan: + trace_id: str + parent_span_id: str + attributes: Mapping[str, str] + + +class TraceSink: + def __init__(self) -> None: + self.lock: Final = threading.Lock() + self.spans: Final[list[TraceSpan]] = [] # mutable-ok: spans accumulate as both exporters deliver + + def respond(self, request: Request) -> Reply: + if not request.target.endswith("/v1/traces"): + return Reply(body=b"") + spans: Final = tuple( + TraceSpan(span.trace_id.hex(), span.parent_span_id.hex(), _attributes(span)) + for span in _exported_spans(request.body) + ) + with self.lock: + self.spans.extend(spans) + return Reply(body=b"") + + def traces(self, marker: str, calls: tuple[str, ...]) -> Mapping[str, tuple[TraceSpan, ...]]: + with self.lock: + snapshot: Final = tuple(self.spans) + roots: Final = { + call: next( + ( + span + for span in snapshot + if not span.parent_span_id and span.attributes.get("app.call") == f"{marker}-{call}" + ), + None, + ) + for call in calls + } + return { + call: tuple(span for span in snapshot if span.trace_id == root.trace_id) + for call, root in roots.items() + if root is not None + } + + +def _trace_values(spans: tuple[TraceSpan, ...], keys: tuple[str, ...]) -> Iterator[tuple[str, str]]: + for span in spans: + yield from ((key, span.attributes[key]) for key in keys if key in span.attributes) + + +def _logged(spans: tuple[TraceSpan, ...]) -> bool: + return any(span.attributes.get("langfuse.observation.type") == "generation" for span in spans) + + +def _chat_reply(request: Request) -> Reply: + body: Final = json.loads(request.body) + content: Final = body["messages"][0]["content"] + if body.get("stream"): + delta: Final = {"id": content, "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini"} + return Reply( + content_type="text/event-stream", + chunks=( + f"data: {json.dumps({**delta, 'choices': [{'index': 0, 'delta': {'role': 'assistant', 'content': 'ok'}}]})}\n\n".encode(), + f"data: {json.dumps({**delta, 'choices': [{'index': 0, 'delta': {}, 'finish_reason': 'stop'}]})}\n\n".encode(), + b"data: [DONE]\n\n", + ), + ) + return Reply( + body=json.dumps( + { + "id": content, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, + } + ).encode() + ) + + +class GenerationSink: + def __init__(self) -> None: + self.lock: Final = threading.Lock() + self.generations: Final[list[Mapping[str, str]]] = [] # mutable-ok: spans accumulate as the exporter delivers + + def respond(self, request: Request) -> Reply: + if not request.target.endswith("/v1/traces"): + return Reply(body=b"") + spans: Final = tuple(_attributes(span) for span in _exported_spans(request.body)) + with self.lock: + self.generations.extend(span for span in spans if span.get("langfuse.observation.type") == "generation") + return Reply(body=b"") + + def by_call(self, marker: str) -> Mapping[str, tuple[Mapping[str, str], ...]]: + with self.lock: + snapshot: Final = tuple(self.generations) + return { + call: tuple(span for span in snapshot if f'"{marker}-{call}"' in span.get("langfuse.observation.input", "")) + for call in CALLS + } + + +def test_sdk_sync_and_async_calls_derive_trace_name_and_io_from_their_call_type() -> None: + marker: Final = "lit8281-sdk-" + uuid.uuid4().hex + sink: Final = GenerationSink() + with wire_server(sink.respond) as destination, wire_server(_chat_reply) as provider: + child: Final = subprocess.Popen( + [sys.executable, "-P", "-c", SDK_CALLS, provider.url + "/v1", marker], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env={ + **os.environ, + "LANGFUSE_PUBLIC_KEY": "pk-lf-sdk", + "LANGFUSE_SECRET_KEY": "sk-lf-sdk", + "LANGFUSE_HOST": destination.url, + "OTEL_BSP_SCHEDULE_DELAY": "200", + }, + ) + try: + spans: Final = eventually( + lambda: sink.by_call(marker), + lambda found: all(len(records) == 1 for records in found.values()) or child.poll() is not None, + seconds=70, + ) + finally: + output: Final = child.communicate(input=b"", timeout=60)[0].decode() + assert {call: len(records) for call, records in spans.items()} == dict.fromkeys(CALLS, 1), output + for call, call_type in CALLS.items(): + attributes = spans[call][0] + assert attributes["langfuse.trace.name"] == f"litellm-{call_type}", (call, attributes) + assert attributes["langfuse.trace.input"] == attributes["langfuse.observation.input"], (call, attributes) + assert attributes["langfuse.trace.output"] == attributes["langfuse.observation.output"], (call, attributes) + assert json.loads(attributes["langfuse.trace.output"]) == {"role": "assistant", "content": "ok"}, call + + +@pytest.mark.parametrize( + ("environment", "writes_onto_app_span", "calls"), + [ + pytest.param({}, True, APP_CALLS, id="litellm_writes_onto_the_app_span"), + pytest.param({"USE_OTEL_LITELLM_REQUEST_SPAN": "true"}, False, APP_CALLS, id="litellm_request_child_span"), + pytest.param( + {"LITELLM_OTEL_V2": "1", "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "span_only"}, + False, + ("ambient_async",), + id="otel_v2_llm_call_child_span", + ), + ], +) +def test_sdk_call_inside_an_app_trace_leaves_the_app_trace_name_and_io_alone( + environment: Mapping[str, str], writes_onto_app_span: bool, calls: tuple[str, ...] +) -> None: + marker: Final = "lit8281-app-" + uuid.uuid4().hex + sink: Final = TraceSink() + with wire_server(sink.respond) as destination, wire_server(_chat_reply) as provider: + child: Final = subprocess.Popen( + [ + sys.executable, + "-P", + "-c", + APP_TRACE_CALLS, + provider.url + "/v1", + marker, + destination.url, + "1" if writes_onto_app_span else "0", + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env={ + **os.environ, + "LANGFUSE_PUBLIC_KEY": "pk-lf-sdk", + "LANGFUSE_SECRET_KEY": "sk-lf-sdk", + "LANGFUSE_HOST": destination.url, + "OTEL_BSP_SCHEDULE_DELAY": "200", + **environment, + }, + ) + try: + traces: Final = eventually( + lambda: sink.traces(marker, calls), + lambda found: ( + (len(found) == len(calls) and all(map(_logged, found.values()))) or child.poll() is not None + ), + seconds=70, + ) + finally: + output: Final = child.communicate(input=b"", timeout=60)[0].decode() + assert sorted(call for call, spans in traces.items() if _logged(spans)) == sorted(calls), output + for call, spans in traces.items(): + _assert_the_app_trace_fields_stand(marker, call, spans) + + +def _assert_the_app_trace_fields_stand(marker: str, call: str, spans: tuple[TraceSpan, ...]) -> None: + app: Final = { + "langfuse.trace.name": f"checkout-{marker}-{call}", + "langfuse.trace.input": json.dumps({"cart": f"{marker}-{call}"}), + "langfuse.trace.output": json.dumps({"order": f"{marker}-{call}"}), + } + root: Final = next(span for span in spans if not span.parent_span_id) + assert {key: root.attributes.get(key) for key in app} == app, (call, root.attributes) + written: Final = set(_trace_values(spans, (*app, "langfuse.trace.tags"))) + assert written == set(app.items()), (call, written) diff --git a/tests/unit/integrations/otel/test_langfuse_logger.py b/tests/unit/integrations/otel/test_langfuse_logger.py index d55f0577e03..1ac6635b8ee 100644 --- a/tests/unit/integrations/otel/test_langfuse_logger.py +++ b/tests/unit/integrations/otel/test_langfuse_logger.py @@ -11,6 +11,8 @@ import pytest pytest.importorskip("opentelemetry") +from opentelemetry import trace # noqa: E402 # needs the importorskip above +from opentelemetry.context import Context # noqa: E402 # needs the importorskip above from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 from opentelemetry.trace import SpanKind # noqa: E402 @@ -64,8 +66,8 @@ def _logger(*, capture: str = "span_only", mappers: Sequence[str] = ("genai", "l return build_otel_v2_logger(config=cfg, tracer_provider=tracer_provider), exporter -def _start_root(logger): - root = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) +def _start_root(logger: OpenTelemetryV2, parent_context: Context | None = None): + root: Final = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME, parent_context) set_request_root_span(root) return root @@ -331,13 +333,28 @@ def test_unrenderable_output_never_raises_into_the_request(): assert INPUT_ATTR not in attrs and OUTPUT_ATTR not in attrs +def _inbound_traceparent() -> Context: + caller: Final = trace.SpanContext( + trace_id=0x4BF92F3577B34DA6A3CE929D0E0E4736, + span_id=0x00F067AA0BA902B7, + is_remote=True, + trace_flags=trace.TraceFlags(trace.TraceFlags.SAMPLED), + ) + return trace.set_span_in_context(trace.NonRecordingSpan(caller)) + + def _run_named_request( - logger: OpenTelemetryV2, exporter: InMemorySpanExporter, litellm_params: Mapping[str, object] + logger: OpenTelemetryV2, + exporter: InMemorySpanExporter, + litellm_params: Mapping[str, object], + parent_context: Context | None = None, ) -> tuple[Mapping[str, object], Mapping[str, object]]: response: Final = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) - root: Final = _start_root(logger) + root: Final = _start_root(logger, parent_context) logger.log_pre_api_call( - model="gpt-5.4-mini", messages=[], kwargs={"litellm_call_id": "call_1", "litellm_params": litellm_params} + model="gpt-5.4-mini", + messages=[], + kwargs={"litellm_call_id": "call_1", "litellm_params": litellm_params, "call_type": "acompletion"}, ) root.end() payload: Final = { @@ -390,14 +407,113 @@ def test_body_metadata_trace_name_names_the_root_and_the_generation(): assert generation_attrs[TRACE_NAME_ATTR] == "from-body" -def test_unnamed_request_leaves_the_trace_name_off_both_spans(): +def test_unnamed_request_derives_the_trace_name_on_the_root_only(): logger, exporter = _logger() root_attrs, generation_attrs = _run_named_request(logger, exporter, {"proxy_server_request": {"headers": {}}}) + assert root_attrs[TRACE_NAME_ATTR] == "litellm-acompletion" + assert TRACE_NAME_ATTR not in generation_attrs + + +def test_unnamed_request_under_an_inbound_traceparent_leaves_the_trace_name_to_the_caller(): + logger, exporter = _logger() + + root_attrs, generation_attrs = _run_named_request( + logger, exporter, {"proxy_server_request": {"headers": {}}}, _inbound_traceparent() + ) + assert TRACE_NAME_ATTR not in root_attrs and TRACE_NAME_ATTR not in generation_attrs +def test_named_request_under_an_inbound_traceparent_keeps_the_caller_trace_name(): + logger, exporter = _logger() + + root_attrs, generation_attrs = _run_named_request( + logger, + exporter, + {"metadata": {"trace_name": "from-body"}, "proxy_server_request": {"headers": {}}}, + _inbound_traceparent(), + ) + + assert root_attrs[TRACE_NAME_ATTR] == "from-body" + assert generation_attrs[TRACE_NAME_ATTR] == "from-body" + + +UNNAMED: Final = {"proxy_server_request": {"headers": {}}} +JUDGE_PARAMS: Final = {"metadata": {"internal_call_origin": "llm_as_a_judge_guardrail"}} +CALLER_NAMES: Final = ( + {"metadata": {"trace_name": "from-caller"}, "proxy_server_request": {"headers": {}}}, + {"proxy_server_request": {"headers": {"langfuse_trace_name": "from-caller"}}}, +) + + +def _root_trace_name_after( + logger: OpenTelemetryV2, + calls: Sequence[tuple[str, Mapping[str, object]]], + root_logger: OpenTelemetryV2 | None = None, +) -> object: + root: Final = _start_root(root_logger or logger) + for index, (call_type, litellm_params) in enumerate(calls): + logger.log_pre_api_call( + model="gpt-5.4-mini", + messages=[], + kwargs={"litellm_call_id": f"call_{index}", "litellm_params": litellm_params, "call_type": call_type}, + ) + root.end() + return dict(root.attributes or {}).get(TRACE_NAME_ATTR) + + +@pytest.mark.parametrize("later", [JUDGE_PARAMS, UNNAMED], ids=["litellm-sub-call", "untagged-call"]) +@pytest.mark.parametrize("caller", CALLER_NAMES, ids=["body", "header"]) +def test_a_later_call_in_the_request_keeps_the_caller_trace_name_on_the_root( + caller: Mapping[str, object], later: Mapping[str, object] +) -> None: + logger, _ = _logger() + + assert _root_trace_name_after(logger, (("acompletion", caller), ("acompletion", later))) == "from-caller" + + +@pytest.mark.parametrize("caller", CALLER_NAMES, ids=["body", "header"]) +def test_a_caller_trace_name_replaces_the_name_an_earlier_call_derived(caller: Mapping[str, object]) -> None: + logger, _ = _logger() + + assert _root_trace_name_after(logger, (("acompletion", JUDGE_PARAMS), ("acompletion", caller))) == "from-caller" + + +def test_a_later_call_keeps_the_trace_name_the_first_call_derived(): + logger, _ = _logger() + + assert ( + _root_trace_name_after(logger, (("anthropic_messages", UNNAMED), ("acompletion", UNNAMED))) + == "litellm-anthropic_messages" + ) + + +def test_a_litellm_sub_call_before_the_request_call_leaves_the_trace_name_to_the_request_call(): + logger, _ = _logger() + + assert ( + _root_trace_name_after(logger, (("acompletion", JUDGE_PARAMS), ("anthropic_messages", UNNAMED))) + == "litellm-anthropic_messages" + ) + + +def test_a_root_another_vendor_started_gets_no_derived_trace_name(): + logger, _ = _logger() + other_vendor, _ = _logger(mappers=("genai",)) + + assert _root_trace_name_after(logger, (("acompletion", UNNAMED),), root_logger=other_vendor) is None + + +@pytest.mark.parametrize("caller", CALLER_NAMES, ids=["body", "header"]) +def test_a_root_another_vendor_started_still_gets_the_caller_trace_name(caller: Mapping[str, object]) -> None: + logger, _ = _logger() + other_vendor, _ = _logger(mappers=("genai",)) + + assert _root_trace_name_after(logger, (("acompletion", caller),), root_logger=other_vendor) == "from-caller" + + @pytest.mark.parametrize("capture", ["span_only", "no_content"]) def test_body_metadata_user_session_and_tags_land_on_the_root_and_the_generation(capture): logger, exporter = _logger(capture=capture) @@ -420,7 +536,8 @@ def test_body_metadata_user_session_and_tags_land_on_the_root_and_the_generation assert attrs["user.id"] == "user-42" assert attrs["session.id"] == "session-7" assert tuple(attrs["langfuse.trace.tags"]) == ("prod", "eval", "nightly") - assert TRACE_NAME_ATTR not in attrs + assert root_attrs[TRACE_NAME_ATTR] == "litellm-acompletion" + assert TRACE_NAME_ATTR not in generation_attrs def test_langfuse_user_and_session_headers_beat_body_metadata_on_both_spans(): @@ -489,7 +606,9 @@ def test_a_request_without_trace_controls_stamps_none_of_them(): {"metadata": {"user_api_key_team_id": "t1", "tags": []}, "proxy_server_request": {"headers": {}}}, ) - assert set(TRACE_CONTROL_ATTRS).isdisjoint(root_attrs) + caller_only: Final = tuple(attr for attr in TRACE_CONTROL_ATTRS if attr != TRACE_NAME_ATTR) + assert set(caller_only).isdisjoint(root_attrs) + assert root_attrs[TRACE_NAME_ATTR] == "litellm-acompletion" assert set(TRACE_CONTROL_ATTRS).isdisjoint(generation_attrs) diff --git a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py index 7e112fcf6a4..98401fd17d3 100644 --- a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py @@ -802,3 +802,29 @@ def test_resolve_mappers_composition_layers_vocabularies(): def test_resolve_mappers_rejects_unknown_name(): with pytest.raises(ValueError, match="unknown mapper name 'nope'"): resolve_mappers(["genai", "nope"]) + + +def test_langfuse_mapper_leaves_the_trace_name_off_an_unnamed_generation() -> None: + attrs: Final = LangfuseMapper().map(_llm_call(call_type="acompletion", trace=TraceControls())) + + assert "langfuse.trace.name" not in attrs + + +def test_langfuse_mapper_carries_the_caller_trace_name_on_the_generation() -> None: + attrs: Final = LangfuseMapper().map(_llm_call(call_type="acompletion", trace=TraceControls(name="caller-trace"))) + + assert attrs["langfuse.trace.name"] == "caller-trace" + + +@pytest.mark.parametrize( + ("trace", "call_type", "expected"), + [ + (TraceControls(), "aresponses", {"langfuse.trace.name": "litellm-aresponses"}), + (TraceControls(name="caller-trace"), "aresponses", {"langfuse.trace.name": "caller-trace"}), + (TraceControls(), None, {}), + ], +) +def test_langfuse_root_trace_attributes_derive_the_name_only_when_the_caller_gave_none( + trace: TraceControls, call_type: str | None, expected: Mapping[str, object] +) -> None: + assert LangfuseMapper.trace_attributes(trace, call_type) == expected diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index c7608159515..5d37a145de6 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -1,17 +1,25 @@ import base64 import json import os +from collections.abc import Mapping, Sequence +from dataclasses import dataclass from datetime import datetime, timezone from typing import Final from unittest.mock import MagicMock, patch import pytest -from opentelemetry.sdk.trace import TracerProvider +from openai.types.responses import ResponseFunctionToolCall +from opentelemetry import trace +from opentelemetry.context import Context +from opentelemetry.sdk.trace import SpanLimits, TracerProvider +from opentelemetry.trace import Span +import litellm +from litellm.constants import REDACTED_BY_LITELLM from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger -from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig +from litellm.integrations.opentelemetry import LITELLM_PROXY_REQUEST_SPAN_NAME, OpenTelemetry, OpenTelemetryConfig from litellm.types.llms.openai import ResponsesAPIResponse -from litellm.types.utils import StandardCallbackDynamicParams +from litellm.types.utils import Choices, ModelResponse, StandardCallbackDynamicParams class TestLangfuseOtelIntegration: @@ -138,7 +146,7 @@ class TestLangfuseOtelIntegration: ) # safe_set_attribute(span, key, value) → positional args - mock_safe_set_attribute.assert_called_once_with( + mock_safe_set_attribute.assert_any_call( mock_span, "langfuse.environment", test_env ) @@ -386,6 +394,7 @@ class TestLangfuseOtelIntegration: actual = { call.args[1]: json.loads(call.args[2]) for call in mock_safe_set_attribute.call_args_list + if call.args[1] in expect_output } assert ( @@ -449,6 +458,7 @@ class TestLangfuseOtelIntegration: actual = { call.args[1]: json.loads(call.args[2]) for call in mock_safe_set_attribute.call_args_list + if call.args[1] in expected } assert ( actual == expected @@ -879,7 +889,11 @@ class TestLangfuseOtelResponsesAPI: Summary( text="Let me analyze this problem step by step...", type="summary_text", - ) + ), + Summary( + text="Now checking the forecast data.", + type="summary_text", + ), ], ), ResponseOutputMessage( @@ -929,22 +943,31 @@ class TestLangfuseOtelResponsesAPI: # Verify output contains reasoning and message assert isinstance(output_data, list) - assert len(output_data) == 2 + assert len(output_data) == 3 - # Verify reasoning summary assert output_data[0]["role"] == "reasoning_summary" assert ( output_data[0]["content"] == "Let me analyze this problem step by step..." ) + assert output_data[1]["role"] == "reasoning_summary" + assert output_data[1]["content"] == "Now checking the forecast data." # Verify message - assert output_data[1]["role"] == "assistant" + assert output_data[2]["role"] == "assistant" assert ( - output_data[1]["content"] + output_data[2]["content"] == "The weather in San Francisco is sunny, 20°C." ) + trace_output_calls: Final = [ + call + for call in mock_safe_set_attribute.call_args_list + if call.args[1] == LangfuseSpanAttributes.TRACE_OUTPUT.value + ] + assert len(trace_output_calls) > 0, "trace.output should be set" + assert trace_output_calls[0].args[2] == output_json + def test_responses_api_with_function_calls(self): """Test Langfuse OTEL logger with Responses API function_call output.""" from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes @@ -1054,6 +1077,381 @@ class TestLangfuseOtelResponsesAPI: assert output_data[0]["arguments"] == {} +_TRACER: Final = TracerProvider().get_tracer("langfuse-otel-tests") +TRACE_FIELD_KEYS: Final = ( + "langfuse.trace.name", + "langfuse.trace.input", + "langfuse.trace.output", + "langfuse.trace.tags", +) +CALLER: Final = trace.SpanContext( + trace_id=0x4BF92F3577B34DA6A3CE929D0E0E4736, + span_id=0x00F067AA0BA902B7, + is_remote=True, + trace_flags=trace.TraceFlags(trace.TraceFlags.SAMPLED), +) +CHAT_KWARGS: Final = { + "call_type": "acompletion", + "messages": [{"role": "user", "content": "weather?"}], +} + + +def _chat_response(content: str) -> ModelResponse: + return ModelResponse( + id="chatcmpl-derived", + model="gpt-4o", + choices=[Choices(finish_reason="stop", message={"role": "assistant", "content": content})], + ) + + +def _stamped(span: Span, kwargs: Mapping[str, object], response_obj: object = None) -> Mapping[str, object]: + LangfuseOtelLogger._set_langfuse_specific_attributes(span, kwargs, response_obj) + span.end() + return dict(getattr(span, "attributes", None) or {}) + + +def _emitted(kwargs: Mapping[str, object], response_obj: object = None) -> Mapping[str, object]: + return _stamped(_TRACER.start_span("litellm_request"), kwargs, response_obj) + + +def _trace_fields(attributes: Mapping[str, object]) -> Mapping[str, object]: + return {key: attributes[key] for key in TRACE_FIELD_KEYS if key in attributes} + + +class _PresetCache: + @staticmethod + def get_preset_cache_key_from_kwargs(**kwargs: object) -> str: # kwargs-ok: mirrors Cache's signature + return "preset-abc" + + +class TestDerivedTraceFields: + def test_chat_call_derives_trace_name_input_and_output(self) -> None: + attributes: Final = _emitted({**CHAT_KWARGS, "litellm_params": {"metadata": {}}}, _chat_response("Sunny.")) + + assert attributes["langfuse.trace.name"] == "litellm-acompletion" + assert attributes["langfuse.trace.input"] == attributes["langfuse.observation.input"] + assert attributes["langfuse.trace.output"] == attributes["langfuse.observation.output"] + assert json.loads(str(attributes["langfuse.trace.input"])) == [{"role": "user", "content": "weather?"}] + assert json.loads(str(attributes["langfuse.trace.output"])) == {"role": "assistant", "content": "Sunny."} + + def test_missing_call_type_defaults_to_completion(self) -> None: + attributes: Final = _emitted( + {"messages": [{"role": "user", "content": "hi"}], "litellm_params": {"metadata": {}}} + ) + + assert attributes["langfuse.trace.name"] == "litellm-completion" + + def test_caller_trace_name_wins_and_is_the_only_name(self) -> None: + attributes: Final = _emitted( + {"call_type": "acompletion", "litellm_params": {"metadata": {"trace_name": "caller-trace"}}} + ) + + assert attributes["langfuse.trace.name"] == "caller-trace" + + def test_existing_trace_id_keeps_caller_tags_and_leaves_derived_trace_fields_off( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "langfuse_default_tags", ["cache_hit"]) + attributes: Final = _emitted( + { + **CHAT_KWARGS, + "cache_hit": True, + "litellm_params": {"metadata": {"existing_trace_id": "abc123", "tags": ["experiment"]}}, + "standard_logging_object": {"request_tags": ["injected"]}, + }, + _chat_response("hi"), + ) + + assert _trace_fields(attributes) == {"langfuse.trace.tags": json.dumps(["experiment"])} + assert attributes["langfuse.trace.existing_id"] == "abc123" + assert "langfuse.observation.input" in attributes + assert "langfuse.observation.output" in attributes + + @pytest.mark.parametrize( + ("mask_flags", "masked_trace_keys"), + [ + ({"mask_input": True}, frozenset({"langfuse.trace.input"})), + ({"mask_output": True}, frozenset({"langfuse.trace.output"})), + ({"mask_input": True, "mask_output": True}, frozenset({"langfuse.trace.input", "langfuse.trace.output"})), + ( + {"mask_input": "true", "mask_output": "TRUE"}, + frozenset({"langfuse.trace.input", "langfuse.trace.output"}), + ), + ({"mask_input": "false", "mask_output": "False"}, frozenset()), + ({"mask_input": False, "mask_output": None}, frozenset()), + ], + ) + def test_mask_flags_redact_the_derived_trace_io( + self, mask_flags: Mapping[str, object], masked_trace_keys: frozenset[str] + ) -> None: + attributes: Final = _emitted( + { + "call_type": "acompletion", + "messages": [{"role": "user", "content": "secret question"}], + "litellm_params": {"metadata": mask_flags}, + }, + _chat_response("secret answer"), + ) + observation_of: Final = { + "langfuse.trace.input": attributes["langfuse.observation.input"], + "langfuse.trace.output": attributes["langfuse.observation.output"], + } + + assert {key: attributes.get(key) for key in observation_of} == { + key: REDACTED_BY_LITELLM if key in masked_trace_keys else observation + for key, observation in observation_of.items() + } + + def test_responses_api_output_mirrors_observation_output_items(self) -> None: + response_obj: Final = ResponsesAPIResponse( + id="response-derived", + created_at=1625247700, + output=[ + ResponseFunctionToolCall( + id="fc-1", + type="function_call", + name="get_weather", + call_id="call-1", + arguments='{"city":"sf"}', + status="completed", + ) + ], + ) + attributes: Final = _emitted({"call_type": "aresponses", "litellm_params": {"metadata": {}}}, response_obj) + + assert attributes["langfuse.trace.name"] == "litellm-aresponses" + assert attributes["langfuse.trace.output"] == attributes["langfuse.observation.output"] + assert json.loads(str(attributes["langfuse.trace.output"])) == [ + { + "id": "fc-1", + "name": "get_weather", + "call_id": "call-1", + "type": "function_call", + "arguments": {"city": "sf"}, + } + ] + + def test_tags_merge_caller_request_and_default_tags_in_order(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "langfuse_default_tags", ["cache_hit", "user_api_key_alias"]) + attributes: Final = _emitted( + { + "call_type": "acompletion", + "cache_hit": True, + "litellm_params": {"metadata": {"tags": ["a", "b"], "user_api_key_alias": "k1"}}, + "standard_logging_object": {"request_tags": ["b", "c"]}, + } + ) + + assert json.loads(str(attributes["langfuse.trace.tags"])) == [ + "a", + "b", + "c", + "cache_hit:True", + "user_api_key_alias:k1", + ] + + @pytest.mark.parametrize( + ("metadata", "expected"), + [ + ({}, ["cache_key:preset-abc"]), + ({"hidden_params": {"cache_key": "explicit-key"}}, ["cache_key:explicit-key"]), + ], + ids=["falls_back_to_the_preset_cache_key", "prefers_hidden_params_over_the_preset"], + ) + def test_cache_key_default_tag( + self, monkeypatch: pytest.MonkeyPatch, metadata: Mapping[str, object], expected: Sequence[str] + ) -> None: + monkeypatch.setattr(litellm, "langfuse_default_tags", ["cache_key"]) + monkeypatch.setattr(litellm, "cache", _PresetCache()) + attributes: Final = _emitted({"call_type": "acompletion", "litellm_params": {"metadata": metadata}}) + + assert json.loads(str(attributes["langfuse.trace.tags"])) == expected + + @pytest.mark.parametrize( + ("caller_tags", "expected"), + [ + ("single", ["single", "req"]), + ("h1, h2", ["h1", "h2", "req"]), + ('["j1", "j2"]', ["j1", "j2", "req"]), + (["x", 1], ["x", "1", "req"]), + ("", ["req"]), + ], + ) + def test_caller_tags_are_read_the_way_langfuse_reads_a_raw_tags_value( + self, caller_tags: object, expected: Sequence[str] + ) -> None: + attributes: Final = _emitted( + { + "call_type": "acompletion", + "litellm_params": {"metadata": {"tags": caller_tags}}, + "standard_logging_object": {"request_tags": ["req"]}, + } + ) + + assert json.loads(str(attributes["langfuse.trace.tags"])) == expected + + def test_no_tags_anywhere_leaves_trace_tags_unset(self) -> None: + attributes: Final = _emitted({"call_type": "acompletion", "litellm_params": {"metadata": {}}}) + + assert "langfuse.trace.tags" not in attributes + + def test_empty_string_role_is_omitted_from_output(self) -> None: + response_obj: Final = {"id": "chatcmpl-norole", "choices": [{"message": {"role": "", "content": "hi"}}]} + attributes: Final = _emitted({"call_type": "acompletion", "litellm_params": {"metadata": {}}}, response_obj) + + assert json.loads(str(attributes["langfuse.observation.output"])) == {"content": "hi"} + + def test_no_messages_leaves_trace_input_unset_but_emits_output(self) -> None: + attributes: Final = _emitted( + {"call_type": "acompletion", "litellm_params": {"metadata": {}}}, _chat_response("hi") + ) + + assert "langfuse.trace.input" not in attributes + assert "langfuse.observation.input" not in attributes + assert attributes["langfuse.trace.output"] == attributes["langfuse.observation.output"] + + +def _proxy_span(parent: Context | None = None) -> Span: + return _TRACER.start_span(LITELLM_PROXY_REQUEST_SPAN_NAME, context=parent) + + +def _child_of(parent: Span) -> Span: + return _TRACER.start_span("litellm_request", context=trace.set_span_in_context(parent)) + + +def _carrying(parent: Span, key: str = "metadata") -> Mapping[str, object]: + return {**CHAT_KWARGS, "litellm_params": {key: {"litellm_parent_otel_span": parent}}} + + +class TestDerivedTraceFieldsOnlyOnTracesLitellmStarts: + def test_a_span_under_an_inbound_traceparent_joins_the_caller_trace(self) -> None: + span: Final = _TRACER.start_span( + "litellm_request", context=trace.set_span_in_context(trace.NonRecordingSpan(CALLER)) + ) + + attributes: Final = _stamped(span, {**CHAT_KWARGS, "litellm_params": {"metadata": {}}}, _chat_response("ok")) + + assert _trace_fields(attributes) == {} + assert "langfuse.observation.input" in attributes and "langfuse.observation.output" in attributes + + def test_the_active_app_span_litellm_writes_onto_keeps_its_trace_fields(self) -> None: + with _TRACER.start_as_current_span("checkout") as app_span: + app_span.set_attribute("langfuse.trace.name", "checkout") + + attributes: Final = _stamped( + app_span, {**CHAT_KWARGS, "litellm_params": {"metadata": {}}}, _chat_response("ok") + ) + + assert _trace_fields(attributes) == {"langfuse.trace.name": "checkout"} + + def test_a_litellm_root_span_derives_while_an_app_span_is_current(self) -> None: + with _TRACER.start_as_current_span("checkout"): + span: Final = _TRACER.start_span("litellm_request", context=Context()) + + attributes: Final = _stamped( + span, {**CHAT_KWARGS, "litellm_params": {"metadata": {}}}, _chat_response("ok") + ) + + assert attributes["langfuse.trace.name"] == "litellm-acompletion" + + def test_a_span_under_a_caller_provided_parent_span_joins_the_caller_trace(self) -> None: + caller_span: Final = _TRACER.start_span("caller-root") + + attributes: Final = _stamped(_child_of(caller_span), _carrying(caller_span), _chat_response("ok")) + + assert _trace_fields(attributes) == {} + + @pytest.mark.parametrize("key", ["metadata", "litellm_metadata"]) + def test_a_span_under_litellms_own_unparented_proxy_span_derives(self, key: str) -> None: + proxy_span: Final = _proxy_span() + + attributes: Final = _stamped(_child_of(proxy_span), _carrying(proxy_span, key), _chat_response("ok")) + + assert attributes["langfuse.trace.name"] == "litellm-acompletion" + assert attributes["langfuse.trace.input"] == attributes["langfuse.observation.input"] + assert attributes["langfuse.trace.output"] == attributes["langfuse.observation.output"] + + def test_a_span_under_a_proxy_span_with_an_inbound_traceparent_joins_the_caller_trace(self) -> None: + proxy_span: Final = _proxy_span(trace.set_span_in_context(trace.NonRecordingSpan(CALLER))) + + attributes: Final = _stamped(_child_of(proxy_span), _carrying(proxy_span), _chat_response("ok")) + + assert _trace_fields(attributes) == {} + + def test_a_span_whose_parent_is_not_the_carried_proxy_span_joins_that_parent_trace(self) -> None: + other_parent: Final = _TRACER.start_span("caller-root") + + attributes: Final = _stamped(_child_of(other_parent), _carrying(_proxy_span()), _chat_response("ok")) + + assert _trace_fields(attributes) == {} + + def test_an_inbound_traceparent_still_carries_the_caller_trace_name_and_tags(self) -> None: + span: Final = _TRACER.start_span( + "litellm_request", context=trace.set_span_in_context(trace.NonRecordingSpan(CALLER)) + ) + + attributes: Final = _stamped( + span, + {**CHAT_KWARGS, "litellm_params": {"metadata": {"trace_name": "caller-trace", "tags": ["caller"]}}}, + _chat_response("ok"), + ) + + assert _trace_fields(attributes) == { + "langfuse.trace.name": "caller-trace", + "langfuse.trace.tags": json.dumps(["caller"]), + } + + +ERROR_KEYS: Final = ( + "error.code", + "http.response.status_code", + "error.type", + "error.message", + "error.llm_provider", + "error.stack_trace", +) +SPAN_ATTRIBUTE_LIMIT: Final = 128 +_LIMITED_TRACER: Final = TracerProvider(span_limits=SpanLimits(max_span_attributes=SPAN_ATTRIBUTE_LIMIT)).get_tracer( + "langfuse-otel-limit-tests" +) + + +@dataclass(frozen=True, slots=True) +class ExportedSpan: + keys: frozenset[str] + dropped: int + + +def _exported_full_span(filler: int, *, joins_caller: bool, failed: bool) -> ExportedSpan: + span: Final = _LIMITED_TRACER.start_span( + "litellm_request", + context=trace.set_span_in_context(trace.NonRecordingSpan(CALLER)) if joins_caller else Context(), + ) + span.set_attributes({f"main.{index}": index for index in range(filler)}) + LangfuseOtelLogger._set_langfuse_specific_attributes( + span, {**CHAT_KWARGS, "litellm_params": {"metadata": {}}}, _chat_response("ok") + ) + span.set_attributes(dict.fromkeys(ERROR_KEYS, "scripted failure") if failed else {}) + span.end() + return ExportedSpan(frozenset(getattr(span, "attributes", None) or {}), getattr(span, "dropped_attributes", 0)) + + +class TestDerivedTraceFieldsAtTheSpanAttributeLimit: + @pytest.mark.parametrize("failed", [False, True]) + @pytest.mark.parametrize("filler", range(SPAN_ATTRIBUTE_LIMIT - 30, SPAN_ATTRIBUTE_LIMIT + 1)) + def test_trace_fields_never_evict_or_drop_another_attribute(self, filler: int, failed: bool) -> None: + derived: Final = _exported_full_span(filler, joins_caller=False, failed=failed) + without_trace_fields: Final = _exported_full_span(filler, joins_caller=True, failed=failed) + + assert without_trace_fields.keys <= derived.keys + assert derived.dropped == without_trace_fields.dropped + + def test_a_span_with_room_to_spare_carries_every_trace_field(self) -> None: + derived: Final = _exported_full_span(SPAN_ATTRIBUTE_LIMIT - 30, joins_caller=False, failed=False) + + assert {"langfuse.trace.name", "langfuse.trace.input", "langfuse.trace.output"} <= derived.keys + + LANGFUSE_ENV_ONLY: Final = { key: value for key, value in os.environ.items() From 0a1dbc0fb4f6b52aec97700dddb457adf91d1cbd Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:47:50 -0700 Subject: [PATCH 003/102] refactor(mcp): drop the experimental label from the MCP server and client paths (#48513) * refactor(mcp): drop the experimental label from the MCP server and client paths Move litellm/proxy/_experimental/mcp_server to litellm/proxy/mcp_server and litellm/experimental_mcp_client to litellm/mcp_client, with the mirrored tests moved under tests/unit/proxy/mcp_server and tests/unit/mcp_client. Both old packages stay as shims that warn with DeprecationWarning, re-export the new package's public names and alias every submodule in sys.modules so deep imports keep resolving to the same objects. The OpenAPI docs gate now puts the merge base's enterprise package on PYTHONPATH so its base counts do not mix the checked-out enterprise code with the base litellm tree Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci(mcp): keep tests/unit/proxy/_experimental as a sharded root for the shim tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(mcp): alias namespace subpackages and bind attributes in the deprecated MCP shims pkgutil.walk_packages skipped mcp_server/auth because it has no __init__.py, so old imports of litellm.proxy._experimental.mcp_server.auth.* failed, and sys.modules entries alone left litellm.experimental_mcp_client.tools unreachable as an attribute. Walk the package on disk and setattr each alias on its old parent. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-unit.yml | 54 +- .gitignore | 2 +- Makefile | 4 +- README.md | 4 +- .../litellm_enterprise/proxy/liteadmin.py | 4 +- litellm/cost_calculator.py | 2 +- litellm/experimental_mcp_client/__init__.py | 11 +- .../litellm_core_utils/deprecated_package.py | 53 + litellm/llms/__init__.py | 2 +- .../anthropic/messages/native/mcp_handler.py | 6 +- .../Readme.md | 6 +- litellm/mcp_client/__init__.py | 8 + .../client.py | 20 +- .../tools.py | 0 .../_experimental/mcp_server/__init__.py | 4 + litellm/proxy/_lazy_features.py | 8 +- litellm/proxy/_types.py | 2 +- .../auth/agent_permission_handler.py | 6 +- .../anthropic_endpoints/gateway_endpoints.py | 26 +- .../ui_discovery_endpoints.py | 2 +- .../guardrails/guardrail_hooks/akto/akto.py | 2 +- .../mcp_end_user_permission.py | 2 +- .../mcp_security/mcp_security_guardrail.py | 2 +- .../panw_prisma_airs/panw_prisma_airs.py | 2 +- .../proxy/hooks/mcp_semantic_filter/hook.py | 8 +- litellm/proxy/litellm_pre_call_utils.py | 2 +- .../access_group_endpoints.py | 2 +- .../key_management_endpoints.py | 16 +- .../mcp_management_endpoints.py | 182 +- .../management_endpoints/sso/saml_sso.py | 8 +- litellm/proxy/management_endpoints/ui_sso.py | 22 +- .../object_permission_utils.py | 8 +- .../{_experimental => }/mcp_server/AGENTS.md | 4 +- .../{_experimental => }/mcp_server/README.md | 0 .../mcp_server/auth/litellm_auth_handler.py | 0 .../mcp_server/auth/managed_agent_access.py | 8 +- .../mcp_server/auth/token_endpoint_auth.py | 0 .../mcp_server/auth/user_api_key_auth_mcp.py | 142 +- .../mcp_server/bridge_token_flow.py | 54 +- .../mcp_server/byok_credential_cache.py | 0 .../mcp_server/byok_oauth_endpoints.py | 12 +- .../mcp_server/capabilities.py | 0 .../{_experimental => }/mcp_server/catalog.py | 92 +- .../mcp_server/client_allowlist.py | 0 .../mcp_server/contracts.py | 4 +- .../mcp_server/cost_calculator.py | 0 .../{_experimental => }/mcp_server/db.py | 22 +- .../mcp_server/discoverable_endpoints.py | 90 +- .../mcp_server/elicitation_handler.py | 0 .../mcp_server/exceptions.py | 2 +- .../mcp_server/faults/__init__.py | 8 +- .../mcp_server/faults/classify.py | 2 +- .../mcp_server/faults/list_outcomes.py | 6 +- .../mcp_server/faults/render_oauth.py | 4 +- .../mcp_server/faults/traversal.py | 0 .../mcp_server/faults/types.py | 0 .../mcp_server/gateway_dcr_flow.py | 28 +- .../guardrail_translation/__init__.py | 2 +- .../guardrail_translation/handler.py | 6 +- .../mcp_server/id_jag_grant.py | 14 +- .../mcp_server/idp_token_exchange.py | 2 +- .../mcp_server/interactions.py | 6 +- .../mcp_server/legacy_callbacks.py | 16 +- .../mcp_server/mcp_context.py | 0 .../mcp_server/mcp_debug.py | 18 +- .../mcp_server/mcp_server_manager.py | 140 +- .../mcp_server/oauth2_flow_backfill.py | 2 +- .../mcp_server/oauth2_token_cache.py | 14 +- .../mcp_server/oauth_identity_binding.py | 4 +- .../mcp_server/oauth_issuer_stamp_backfill.py | 2 +- .../mcp_server/oauth_utils.py | 6 +- .../mcp_server/openapi_to_mcp_generator.py | 12 +- .../mcp_server/operations.py | 104 +- .../outbound_credentials/__init__.py | 8 +- .../outbound_credentials/adapter.py | 10 +- .../authz_code_refresher.py | 8 +- .../bridge_credentials.py | 2 +- .../client_credentials.py | 10 +- .../dual_cache_token_backend.py | 6 +- .../outbound_credentials/envelope.py | 0 .../outbound_credentials/httpx_auth.py | 0 .../login_assertion_exchange.py | 20 +- .../outbound_credentials/oauth_token_store.py | 0 .../per_user_oauth_store.py | 18 +- .../presented_token_store.py | 2 +- .../redis_distributed_lock.py | 2 +- .../redis_refresh_coordinator.py | 2 +- .../outbound_credentials/resolver.py | 18 +- .../mcp_server/outbound_credentials/result.py | 0 .../runtime_refresh_coordinator.py | 6 +- .../session_credentials.py | 2 +- .../outbound_credentials/session_token.py | 0 .../sso_assertion_refresher.py | 10 +- .../sso_assertion_store.py | 2 +- .../outbound_credentials/token_cache_codec.py | 2 +- .../outbound_credentials/token_endpoint.py | 4 +- .../token_exchange_provider.py | 4 +- .../outbound_credentials/token_exchanger.py | 8 +- .../mcp_server/outbound_credentials/types.py | 2 +- .../outbound_credentials/v2_token_store.py | 2 +- .../mcp_server/proxy_api_credentials.py | 14 +- .../mcp_server/rest_endpoints.py | 106 +- .../mcp_server/result_conversion.py | 2 +- .../mcp_server/sampling_handler.py | 2 +- .../mcp_server/scope_challenge.py | 2 +- .../mcp_server/semantic_tool_filter.py | 8 +- .../{_experimental => }/mcp_server/server.py | 66 +- .../mcp_server/server_resolution.py | 4 +- .../mcp_server/sse_transport.py | 0 .../mcp_server/state_tokens.py | 2 +- .../mcp_server/stdio_gate.py | 0 .../mcp_server/subscriptions.py | 18 +- .../mcp_server/tool_catalog_guard.py | 2 +- .../mcp_server/tool_outcome.py | 0 .../mcp_server/tool_registry.py | 0 .../mcp_server/tool_search.py | 8 +- .../mcp_server/toolset_db.py | 2 +- .../mcp_server/ui_session_utils.py | 6 +- .../mcp_server/upstream.py | 20 +- .../{_experimental => }/mcp_server/utils.py | 0 litellm/proxy/proxy_server.py | 50 +- .../public_endpoints/public_endpoints.py | 2 +- .../proxy_setting_endpoints.py | 2 +- litellm/proxy/vector_store_endpoints/utils.py | 8 +- .../mcp/litellm_proxy_mcp_handler.py | 28 +- .../responses/mcp/mcp_streaming_iterator.py | 2 +- litellm/responses/mcp/tool_precheck.py | 2 +- litellm/responses/utils.py | 2 +- scripts/check_mcp_operation_boundary.py | 2 +- scripts/check_mcp_sdk_install.py | 12 +- scripts/openapi_docs_gate.py | 5 +- scripts/pre_commit_lint.sh | 2 +- .../base_sdk_tests/check_base_sdk_install.py | 4 +- tests/benchmarks/test_mcp_benchmarks.py | 4 +- .../ensure_async_clients_test.py | 4 +- .../unbounded_in_baseline.txt | 12 +- tests/e2e/coverage_registry/mcp.yaml | 2 +- tests/integration/mcp/test_interactions.py | 18 +- .../mcp/test_mcp_protocol_errors.py | 6 +- tests/integration/mcp/test_mcp_transports.py | 15 +- tests/integration/mcp/test_pagination.py | 15 +- tests/integration/mcp/test_proxy_mcp_e2e.py | 12 +- tests/integration/mcp_coverage.toml | 4 +- .../unit/experimental_mcp_client/test_init.py | 29 + .../litellm_proxy/skills/test_skill_search.py | 4 +- .../auth => mcp_client}/__init__.py | 0 .../test_mcp_client.py | 83 +- .../test_tools.py | 22 +- .../_experimental/mcp_server/test_init.py | 30 + .../test_gateway_endpoints.py | 30 +- tests/unit/proxy/auth/test_login_utils.py | 6 +- .../unit/proxy/auth/test_mcp_ip_filtering.py | 2 +- tests/unit/proxy/db/mcp_server/test_db.py | 3 +- .../guardrail_hooks/akto/test_akto.py | 5 +- .../guardrail_hooks/test_mcp_security.py | 11 +- .../guardrail_hooks/test_panw_prisma_airs.py | 2 +- .../guardrail_hooks/test_presidio.py | 10 +- .../test_unified_guardrail.py | 6 +- .../hooks/test_parallel_request_limiter_v3.py | 6 +- .../test_key_management_endpoints.py | 58 +- .../test_mcp_management_endpoints.py | 212 ++- .../management_endpoints/test_mcp_servers.py | 4 +- .../management_endpoints/test_saml_sso.py | 2 +- .../proxy/management_endpoints/test_ui_sso.py | 20 +- .../test_object_permission_utils.py | 104 +- .../faults => mcp_server}/__init__.py | 0 .../auth}/__init__.py | 0 .../auth/test_managed_agent_access.py | 16 +- .../auth/test_token_endpoint_auth.py | 2 +- .../auth/test_user_api_key_auth_mcp.py | 570 +++---- .../mcp_server/conftest.py | 10 +- .../faults}/__init__.py | 0 .../mcp_server/faults/test_classify.py | 8 +- .../mcp_server/faults/test_list_outcomes.py | 40 +- .../mcp_server/faults/test_render_oauth.py | 6 +- .../mcp_server/faults/test_traversal.py | 2 +- .../guardrail_translation}/__init__.py | 0 .../test_mcp_guardrail_handler.py | 10 +- .../outbound_credentials}/__init__.py | 0 .../outbound_credentials/test_adapter.py | 20 +- .../test_authz_code_refresher.py | 6 +- .../test_bridge_credentials.py | 6 +- .../test_client_credentials.py | 11 +- .../test_dual_cache_token_backend.py | 6 +- .../outbound_credentials/test_envelope.py | 8 +- .../outbound_credentials/test_httpx_auth.py | 2 +- .../test_login_assertion_exchange.py | 14 +- .../test_oauth_token_store.py | 2 +- .../test_per_user_oauth_store.py | 20 +- .../test_presented_token_store.py | 4 +- .../test_redis_distributed_lock.py | 4 +- .../test_redis_refresh_coordinator.py | 4 +- .../outbound_credentials/test_resolver.py | 20 +- .../outbound_credentials/test_result.py | 2 +- .../test_session_credentials.py | 6 +- .../test_session_token.py | 2 +- .../test_sso_assertion_refresher.py | 8 +- .../test_sso_assertion_store.py | 20 +- .../test_token_cache_codec.py | 4 +- .../test_token_endpoint.py | 12 +- .../test_token_exchange_provider.py | 4 +- .../test_token_exchanger.py | 14 +- .../outbound_credentials/test_types.py | 2 +- .../test_v2_token_store.py | 2 +- .../mcp_server/test_byok_credential_cache.py | 6 +- .../mcp_server/test_byok_oauth_endpoints.py | 159 +- .../test_callback_oauth_error_responses.py | 10 +- .../mcp_server/test_capabilities.py | 10 +- .../mcp_server/test_catalog.py | 40 +- .../mcp_server/test_client_allowlist.py | 2 +- .../mcp_server/test_contracts.py | 2 +- .../mcp_server/test_db_credentials.py | 99 +- .../mcp_server/test_discoverable_endpoints.py | 1518 ++++++++--------- .../mcp_server/test_gateway_dcr_flow.py | 52 +- .../mcp_server/test_id_jag_grant.py | 20 +- .../mcp_server/test_idp_token_exchange.py | 8 +- .../mcp_server/test_interactions.py | 22 +- .../mcp_server/test_is_tool_name_prefixed.py | 3 +- .../mcp_server/test_jwt_mcp_enforcement.py | 9 +- .../mcp_server/test_jwt_mcp_simple.py | 15 +- .../test_mcp_auth_header_extraction.py | 7 +- .../mcp_server/test_mcp_auth_priority.py | 7 +- .../mcp_server/test_mcp_block_recording.py | 7 +- .../mcp_server/test_mcp_chat_completions.py | 32 +- .../mcp_server/test_mcp_client_unit.py | 16 +- .../mcp_server/test_mcp_cost_calculator.py | 3 +- .../mcp_server/test_mcp_custom_fields.py | 11 +- .../mcp_server/test_mcp_debug.py | 17 +- .../mcp_server/test_mcp_discovery.py | 2 - .../test_mcp_elicitation_handler.py | 11 +- .../mcp_server/test_mcp_env_vars.py | 139 +- .../test_mcp_guardrail_usage_monitor.py | 2 +- .../mcp_server/test_mcp_header_alias_utils.py | 2 +- .../mcp_server/test_mcp_hook_extra_headers.py | 46 +- .../mcp_server/test_mcp_logging.py | 24 +- .../test_mcp_max_concurrent_requests.py | 10 +- .../test_mcp_metadata_preservation.py | 3 +- .../mcp_server/test_mcp_oauth_passthrough.py | 24 +- .../test_mcp_oauth_passthrough_cold_start.py | 13 +- .../test_mcp_oauth_passthrough_tools.py | 59 +- .../mcp_server/test_mcp_partial_update.py | 23 +- .../mcp_server/test_mcp_proxy_mode.py | 12 +- .../test_mcp_sampling_completion_flow.py | 29 +- .../test_mcp_sampling_model_access.py | 24 +- .../test_mcp_sampling_model_resolution.py | 2 +- .../test_mcp_sampling_priority_selection.py | 2 +- .../test_mcp_sampling_request_builder.py | 2 +- .../test_mcp_sampling_response_conversion.py | 2 +- .../test_mcp_sampling_tool_conversion.py | 3 +- .../mcp_server/test_mcp_server.py | 320 ++-- .../test_mcp_server_identity_env.py | 4 +- .../mcp_server/test_mcp_server_manager.py | 746 ++++---- .../test_mcp_server_tool_calls_and_headers.py | 1075 ++++++------ .../mcp_server/test_mcp_session_logging.py | 2 +- .../mcp_server/test_mcp_sigv4_auth.py | 84 +- .../mcp_server/test_mcp_stale_session.py | 256 +-- .../mcp_server/test_mcp_tool_search.py | 147 +- .../mcp_server/test_mcp_toolset_scope.py | 112 +- .../mcp_server/test_oauth2_flow_backfill.py | 2 +- .../mcp_server/test_oauth2_mcp_config.yaml | 0 .../mcp_server/test_oauth2_token_cache.py | 42 +- .../mcp_server/test_oauth_identity_binding.py | 16 +- .../test_oauth_issuer_stamp_backfill.py | 4 +- .../mcp_server/test_openapi_spec_path_url.py | 2 +- .../test_openapi_to_mcp_generator.py | 52 +- .../mcp_server/test_openapi_tool_auth.py | 83 +- .../mcp_server/test_operations.py | 154 +- .../mcp_server/test_per_user_oauth_cache.py | 40 +- .../mcp_server/test_proxy_api_credentials.py | 16 +- .../mcp_server/test_rest_endpoints.py | 105 +- .../mcp_server/test_result_conversion.py | 8 +- .../mcp_server/test_semantic_tool_filter.py | 75 +- .../mcp_server/test_server_resolution.py | 17 +- .../mcp_server/test_short_mcp_tool_prefix.py | 10 +- .../mcp_server/test_state_tokens.py | 4 +- .../mcp_server/test_subscriptions.py | 28 +- .../mcp_server/test_tool_registry.py | 5 +- .../mcp_server/test_ui_session_utils.py | 38 +- .../mcp_server/test_utils.py | 10 +- .../public_endpoints/test_public_endpoints.py | 8 +- tests/unit/proxy/test_dynamic_mcp_route.py | 13 +- .../unit/proxy/test_litellm_pre_call_utils.py | 6 +- ...test_proxy_server_endpoints_and_startup.py | 19 +- .../test_get_instance_fn_runtime_gate.py | 2 +- tests/unit/responses/mcp/conftest.py | 2 +- .../mcp/test_aresponses_api_with_mcp.py | 21 +- .../mcp/test_chat_completions_handler.py | 43 +- .../mcp/test_litellm_proxy_mcp_handler.py | 52 +- .../mcp/test_mcp_streaming_iterator.py | 2 +- .../unit/responses/mcp/test_tool_precheck.py | 2 +- .../unit/test_check_mcp_operation_boundary.py | 8 +- tests/unit/test_circleci_path_filter.py | 2 +- tests/unit/test_internal_context.py | 14 +- tests/unit/test_private_usage_aliases.py | 146 +- .../src/utils/mcpHeaderUtils.ts | 2 +- 295 files changed, 4954 insertions(+), 4799 deletions(-) create mode 100644 litellm/litellm_core_utils/deprecated_package.py rename litellm/{experimental_mcp_client => mcp_client}/Readme.md (89%) create mode 100644 litellm/mcp_client/__init__.py rename litellm/{experimental_mcp_client => mcp_client}/client.py (98%) rename litellm/{experimental_mcp_client => mcp_client}/tools.py (100%) create mode 100644 litellm/proxy/_experimental/mcp_server/__init__.py rename litellm/proxy/{_experimental => }/mcp_server/AGENTS.md (98%) rename litellm/proxy/{_experimental => }/mcp_server/README.md (100%) rename litellm/proxy/{_experimental => }/mcp_server/auth/litellm_auth_handler.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/auth/managed_agent_access.py (89%) rename litellm/proxy/{_experimental => }/mcp_server/auth/token_endpoint_auth.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/auth/user_api_key_auth_mcp.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/bridge_token_flow.py (95%) rename litellm/proxy/{_experimental => }/mcp_server/byok_credential_cache.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/byok_oauth_endpoints.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/capabilities.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/catalog.py (93%) rename litellm/proxy/{_experimental => }/mcp_server/client_allowlist.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/contracts.py (96%) rename litellm/proxy/{_experimental => }/mcp_server/cost_calculator.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/db.py (99%) rename litellm/proxy/{_experimental => }/mcp_server/discoverable_endpoints.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/elicitation_handler.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/exceptions.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/faults/__init__.py (81%) rename litellm/proxy/{_experimental => }/mcp_server/faults/classify.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/faults/list_outcomes.py (99%) rename litellm/proxy/{_experimental => }/mcp_server/faults/render_oauth.py (96%) rename litellm/proxy/{_experimental => }/mcp_server/faults/traversal.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/faults/types.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/gateway_dcr_flow.py (99%) rename litellm/proxy/{_experimental => }/mcp_server/guardrail_translation/__init__.py (88%) rename litellm/proxy/{_experimental => }/mcp_server/guardrail_translation/handler.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/id_jag_grant.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/idp_token_exchange.py (99%) rename litellm/proxy/{_experimental => }/mcp_server/interactions.py (97%) rename litellm/proxy/{_experimental => }/mcp_server/legacy_callbacks.py (87%) rename litellm/proxy/{_experimental => }/mcp_server/mcp_context.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/mcp_debug.py (96%) rename litellm/proxy/{_experimental => }/mcp_server/mcp_server_manager.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/oauth2_flow_backfill.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/oauth2_token_cache.py (97%) rename litellm/proxy/{_experimental => }/mcp_server/oauth_identity_binding.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/oauth_issuer_stamp_backfill.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/oauth_utils.py (99%) rename litellm/proxy/{_experimental => }/mcp_server/openapi_to_mcp_generator.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/operations.py (97%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/__init__.py (85%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/adapter.py (97%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/authz_code_refresher.py (94%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/bridge_credentials.py (99%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/client_credentials.py (96%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/dual_cache_token_backend.py (92%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/envelope.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/httpx_auth.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/login_assertion_exchange.py (92%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/oauth_token_store.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/per_user_oauth_store.py (92%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/presented_token_store.py (90%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/redis_distributed_lock.py (97%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/redis_refresh_coordinator.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/resolver.py (97%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/result.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/runtime_refresh_coordinator.py (85%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/session_credentials.py (99%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/session_token.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/sso_assertion_refresher.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/sso_assertion_store.py (99%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/token_cache_codec.py (95%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/token_endpoint.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/token_exchange_provider.py (96%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/token_exchanger.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/types.py (99%) rename litellm/proxy/{_experimental => }/mcp_server/outbound_credentials/v2_token_store.py (97%) rename litellm/proxy/{_experimental => }/mcp_server/proxy_api_credentials.py (96%) rename litellm/proxy/{_experimental => }/mcp_server/rest_endpoints.py (96%) rename litellm/proxy/{_experimental => }/mcp_server/result_conversion.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/sampling_handler.py (99%) rename litellm/proxy/{_experimental => }/mcp_server/scope_challenge.py (97%) rename litellm/proxy/{_experimental => }/mcp_server/semantic_tool_filter.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/server.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/server_resolution.py (96%) rename litellm/proxy/{_experimental => }/mcp_server/sse_transport.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/state_tokens.py (97%) rename litellm/proxy/{_experimental => }/mcp_server/stdio_gate.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/subscriptions.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/tool_catalog_guard.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/tool_outcome.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/tool_registry.py (100%) rename litellm/proxy/{_experimental => }/mcp_server/tool_search.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/toolset_db.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/ui_session_utils.py (98%) rename litellm/proxy/{_experimental => }/mcp_server/upstream.py (94%) rename litellm/proxy/{_experimental => }/mcp_server/utils.py (100%) create mode 100644 tests/unit/experimental_mcp_client/test_init.py rename tests/unit/{proxy/_experimental/mcp_server/auth => mcp_client}/__init__.py (100%) rename tests/unit/{experimental_mcp_client => mcp_client}/test_mcp_client.py (98%) rename tests/unit/{experimental_mcp_client => mcp_client}/test_tools.py (96%) create mode 100644 tests/unit/proxy/_experimental/mcp_server/test_init.py rename tests/unit/proxy/{_experimental/mcp_server/faults => mcp_server}/__init__.py (100%) rename tests/unit/proxy/{_experimental/mcp_server/guardrail_translation => mcp_server/auth}/__init__.py (100%) rename tests/unit/proxy/{_experimental => }/mcp_server/auth/test_managed_agent_access.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/auth/test_token_endpoint_auth.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/auth/test_user_api_key_auth_mcp.py (95%) rename tests/unit/proxy/{_experimental => }/mcp_server/conftest.py (93%) rename tests/unit/proxy/{_experimental/mcp_server/outbound_credentials => mcp_server/faults}/__init__.py (100%) rename tests/unit/proxy/{_experimental => }/mcp_server/faults/test_classify.py (98%) rename tests/unit/proxy/{_experimental => }/mcp_server/faults/test_list_outcomes.py (93%) rename tests/unit/proxy/{_experimental => }/mcp_server/faults/test_render_oauth.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/faults/test_traversal.py (95%) rename tests/unit/proxy/{experimental => mcp_server/guardrail_translation}/__init__.py (100%) rename tests/unit/proxy/{_experimental => }/mcp_server/guardrail_translation/test_mcp_guardrail_handler.py (99%) rename tests/unit/proxy/{experimental/mcp_server => mcp_server/outbound_credentials}/__init__.py (100%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_adapter.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_authz_code_refresher.py (98%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_bridge_credentials.py (98%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_client_credentials.py (98%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_dual_cache_token_backend.py (95%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_envelope.py (99%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_httpx_auth.py (95%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_login_assertion_exchange.py (96%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_oauth_token_store.py (99%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_per_user_oauth_store.py (92%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_presented_token_store.py (73%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_redis_distributed_lock.py (96%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_redis_refresh_coordinator.py (98%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_resolver.py (98%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_result.py (90%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_session_credentials.py (96%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_session_token.py (99%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_sso_assertion_refresher.py (99%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_sso_assertion_store.py (96%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_token_cache_codec.py (93%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_token_endpoint.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_token_exchange_provider.py (96%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_token_exchanger.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_types.py (99%) rename tests/unit/proxy/{_experimental => }/mcp_server/outbound_credentials/test_v2_token_store.py (95%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_byok_credential_cache.py (96%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_byok_oauth_endpoints.py (92%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_callback_oauth_error_responses.py (94%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_capabilities.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_catalog.py (95%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_client_allowlist.py (99%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_contracts.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_db_credentials.py (96%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_discoverable_endpoints.py (90%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_gateway_dcr_flow.py (98%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_id_jag_grant.py (99%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_idp_token_exchange.py (98%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_interactions.py (94%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_is_tool_name_prefixed.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_jwt_mcp_enforcement.py (99%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_jwt_mcp_simple.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_auth_header_extraction.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_auth_priority.py (91%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_block_recording.py (96%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_chat_completions.py (99%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_client_unit.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_cost_calculator.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_custom_fields.py (98%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_debug.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_discovery.py (99%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_elicitation_handler.py (98%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_env_vars.py (93%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_guardrail_usage_monitor.py (99%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_header_alias_utils.py (98%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_hook_extra_headers.py (96%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_logging.py (95%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_max_concurrent_requests.py (98%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_metadata_preservation.py (96%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_oauth_passthrough.py (96%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_oauth_passthrough_cold_start.py (90%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_oauth_passthrough_tools.py (93%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_partial_update.py (98%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_proxy_mode.py (96%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_sampling_completion_flow.py (85%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_sampling_model_access.py (92%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_sampling_model_resolution.py (98%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_sampling_priority_selection.py (99%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_sampling_request_builder.py (98%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_sampling_response_conversion.py (99%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_sampling_tool_conversion.py (99%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_server.py (92%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_server_identity_env.py (95%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_server_manager.py (96%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_server_tool_calls_and_headers.py (91%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_session_logging.py (91%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_sigv4_auth.py (93%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_stale_session.py (84%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_tool_search.py (90%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_mcp_toolset_scope.py (88%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_oauth2_flow_backfill.py (99%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_oauth2_mcp_config.yaml (100%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_oauth2_token_cache.py (90%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_oauth_identity_binding.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_oauth_issuer_stamp_backfill.py (98%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_openapi_spec_path_url.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_openapi_to_mcp_generator.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_openapi_tool_auth.py (91%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_operations.py (94%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_per_user_oauth_cache.py (91%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_proxy_api_credentials.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_rest_endpoints.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_result_conversion.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_semantic_tool_filter.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_server_resolution.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_short_mcp_tool_prefix.py (97%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_state_tokens.py (95%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_subscriptions.py (95%) rename tests/unit/proxy/{experimental => }/mcp_server/test_tool_registry.py (92%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_ui_session_utils.py (91%) rename tests/unit/proxy/{_experimental => }/mcp_server/test_utils.py (98%) diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 76af64c8ccd..9658e834c42 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -579,13 +579,13 @@ jobs: - shard: mcp-elicitation test-path: |- - tests/unit/experimental_mcp_client/test_mcp_client.py - tests/unit/proxy/_experimental/mcp_server/test_capabilities.py - tests/unit/proxy/_experimental/mcp_server/test_interactions.py - tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py - tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py - tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py - tests/unit/proxy/_experimental/mcp_server/test_operations.py + tests/unit/mcp_client/test_mcp_client.py + tests/unit/proxy/mcp_server/test_capabilities.py + tests/unit/proxy/mcp_server/test_interactions.py + tests/unit/proxy/mcp_server/test_mcp_elicitation_handler.py + tests/unit/proxy/mcp_server/test_mcp_server_manager.py + tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py + tests/unit/proxy/mcp_server/test_operations.py python-version: "3.12" workers: 4 reruns: 0 @@ -603,7 +603,7 @@ jobs: tests/unit/proxy/spend_tracking --ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py tests/unit/proxy/pass_through_endpoints - tests/unit/proxy/experimental + tests/unit/proxy/mcp_server/test_tool_registry.py tests/unit/proxy/common_utils --ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py --ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -811,14 +811,14 @@ jobs: - shard: mcp-oauth test-path: |- - tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py - tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py - tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py - tests/unit/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py - tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py - tests/unit/proxy/_experimental/mcp_server/test_mcp_max_concurrent_requests.py - tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py - tests/unit/proxy/_experimental/mcp_server/outbound_credentials + tests/unit/proxy/mcp_server/test_discoverable_endpoints.py + tests/unit/proxy/mcp_server/test_mcp_server_manager.py + tests/unit/proxy/mcp_server/test_db_credentials.py + tests/unit/proxy/mcp_server/test_gateway_dcr_flow.py + tests/unit/proxy/mcp_server/test_mcp_oauth_passthrough_tools.py + tests/unit/proxy/mcp_server/test_mcp_max_concurrent_requests.py + tests/unit/proxy/mcp_server/faults/test_list_outcomes.py + tests/unit/proxy/mcp_server/outbound_credentials python-version: "3.12" workers: 4 reruns: 2 @@ -828,19 +828,21 @@ jobs: - shard: mcp-server test-path: |- + tests/unit/mcp_client tests/unit/experimental_mcp_client + tests/unit/proxy/mcp_server tests/unit/proxy/_experimental/mcp_server tests/unit/responses/mcp - --ignore=tests/unit/experimental_mcp_client/test_mcp_client.py - --ignore=tests/unit/proxy/_experimental/mcp_server/test_capabilities.py - --ignore=tests/unit/proxy/_experimental/mcp_server/test_interactions.py - --ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py - --ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py - --ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py - --ignore=tests/unit/proxy/_experimental/mcp_server/test_operations.py - --ignore=tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py - --ignore=tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py - --ignore=tests/unit/proxy/_experimental/mcp_server/outbound_credentials + --ignore=tests/unit/mcp_client/test_mcp_client.py + --ignore=tests/unit/proxy/mcp_server/test_capabilities.py + --ignore=tests/unit/proxy/mcp_server/test_interactions.py + --ignore=tests/unit/proxy/mcp_server/test_mcp_elicitation_handler.py + --ignore=tests/unit/proxy/mcp_server/test_mcp_server_manager.py + --ignore=tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py + --ignore=tests/unit/proxy/mcp_server/test_operations.py + --ignore=tests/unit/proxy/mcp_server/test_discoverable_endpoints.py + --ignore=tests/unit/proxy/mcp_server/test_db_credentials.py + --ignore=tests/unit/proxy/mcp_server/outbound_credentials python-version: "3.12" workers: 4 reruns: 0 diff --git a/.gitignore b/.gitignore index 5ca20f02274..b08ee92d6af 100644 --- a/.gitignore +++ b/.gitignore @@ -105,7 +105,7 @@ litellm_config.yaml .cursor litellm/proxy/to_delete_loadtest_work/* update_model_cost_map.py -tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py +tests/unit/proxy/mcp_server/test_mcp_server_manager.py scripts/test_vertex_ai_search.py LAZY_LOADING_IMPROVEMENTS.md STABILIZATION_TODO.md diff --git a/Makefile b/Makefile index dec853515b9..937873f8835 100644 --- a/Makefile +++ b/Makefile @@ -313,7 +313,7 @@ test-unit-proxy-core: install-test-deps $(UV_RUN) pytest tests/unit/proxy/auth tests/unit/proxy/client tests/unit/proxy/db tests/unit/proxy/hooks tests/unit/proxy/policy_engine --ignore=tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py --ignore=tests/unit/proxy/db/test_update_daily_tag_spend.py --tb=short -vv -n 4 --durations=20 test-unit-proxy-misc: install-test-deps - $(UV_RUN) pytest tests/unit/proxy/agent_endpoints tests/unit/proxy/anthropic_endpoints tests/unit/proxy/common_utils --ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py --ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py --ignore=tests/unit/proxy/common_utils/test_check_responses_cost.py --ignore=tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py --ignore=tests/unit/proxy/common_utils/test_realtime_cache.py tests/unit/proxy/discovery_endpoints tests/unit/proxy/experimental tests/unit/proxy/google_endpoints tests/unit/proxy/health_endpoints tests/unit/proxy/image_endpoints tests/unit/proxy/middleware --ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py tests/unit/proxy/openai_files_endpoint tests/unit/proxy/pass_through_endpoints tests/unit/proxy/prompts tests/unit/proxy/public_endpoints tests/unit/proxy/response_api_endpoints tests/unit/proxy/shutdown tests/unit/proxy/spend_tracking --ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py tests/unit/proxy/ui_crud_endpoints tests/unit/proxy/vector_store_endpoints tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py --ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py --ignore=tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py --ignore=tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py --ignore=tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py --tb=short -vv -n 4 --durations=20 + $(UV_RUN) pytest tests/unit/proxy/agent_endpoints tests/unit/proxy/anthropic_endpoints tests/unit/proxy/common_utils --ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py --ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py --ignore=tests/unit/proxy/common_utils/test_check_responses_cost.py --ignore=tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py --ignore=tests/unit/proxy/common_utils/test_realtime_cache.py tests/unit/proxy/discovery_endpoints tests/unit/proxy/mcp_server/test_tool_registry.py tests/unit/proxy/google_endpoints tests/unit/proxy/health_endpoints tests/unit/proxy/image_endpoints tests/unit/proxy/middleware --ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py tests/unit/proxy/openai_files_endpoint tests/unit/proxy/pass_through_endpoints tests/unit/proxy/prompts tests/unit/proxy/public_endpoints tests/unit/proxy/response_api_endpoints tests/unit/proxy/shutdown tests/unit/proxy/spend_tracking --ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py tests/unit/proxy/ui_crud_endpoints tests/unit/proxy/vector_store_endpoints tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py --ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py --ignore=tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py --ignore=tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py --ignore=tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py --tb=short -vv -n 4 --durations=20 test-unit-proxy-root: install-test-deps $(UV_RUN) pytest tests/unit/proxy/test_*.py --ignore=tests/unit/proxy/test_aproxy_startup.py --ignore=tests/unit/proxy/test_credential_slot_registry.py --ignore=tests/unit/proxy/test_custom_callback_input.py --ignore=tests/unit/proxy/test_custom_logger_s3_gcs.py --ignore=tests/unit/proxy/test_custom_tokenizer_bug.py --ignore=tests/unit/proxy/test_db_schema_changes.py --ignore=tests/unit/proxy/test_deprecated_key_grace_period.py --ignore=tests/unit/proxy/test_get_favicon.py --ignore=tests/unit/proxy/test_get_image.py --ignore=tests/unit/proxy/test_prisma_client_backoff_retry.py --ignore=tests/unit/proxy/test_prompt_test_endpoint.py --ignore=tests/unit/proxy/test_proxy_config_unit_test.py --ignore=tests/unit/proxy/test_proxy_custom_auth.py --ignore=tests/unit/proxy/test_proxy_reject_logging.py --ignore=tests/unit/proxy/test_proxy_server.py --ignore=tests/unit/proxy/test_proxy_setting_guardrails.py --ignore=tests/unit/proxy/test_proxy_token_counter.py --ignore=tests/unit/proxy/test_proxy_utils.py --ignore=tests/unit/proxy/test_reducto_ocr_route.py --ignore=tests/unit/proxy/test_response_polling_pre_call_checks.py --ignore=tests/unit/proxy/test_server_root_path.py --ignore=tests/unit/proxy/test_ui_path_detection.py --ignore=tests/unit/proxy/test_unit_test_proxy_hooks.py --ignore=tests/unit/proxy/test_update_spend.py --ignore=tests/unit/proxy/test_zero_cost_model_budget_bypass.py --tb=short -vv -n 4 --durations=20 @@ -325,7 +325,7 @@ test-unit-core-utils: install-test-deps $(UV_RUN) pytest tests/unit/litellm_core_utils --tb=short -vv -n 2 --durations=20 test-unit-other: install-test-deps - $(UV_RUN) pytest tests/unit/caching tests/unit/responses tests/unit/secret_managers tests/unit/vector_stores tests/unit/a2a_protocol tests/unit/completion_extras tests/unit/containers tests/unit/enterprise tests/unit/experimental_mcp_client tests/unit/google_genai tests/unit/images tests/unit/interactions tests/unit/router_strategy tests/unit/router_utils tests/unit/types --tb=short -vv -n 4 --durations=20 + $(UV_RUN) pytest tests/unit/caching tests/unit/responses tests/unit/secret_managers tests/unit/vector_stores tests/unit/a2a_protocol tests/unit/completion_extras tests/unit/containers tests/unit/enterprise tests/unit/mcp_client tests/unit/google_genai tests/unit/images tests/unit/interactions tests/unit/router_strategy tests/unit/router_utils tests/unit/types --tb=short -vv -n 4 --durations=20 test-unit-root: install-test-deps $(UV_RUN) pytest tests/unit/test_*.py --tb=short -vv -n 4 --durations=20 diff --git a/README.md b/README.md index 4feb6e00ffa..5dfa8d01fc4 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,7 @@ async with httpx.AsyncClient(headers=headers, timeout=60.0) as http_client: ```python from mcp import ClientSession, StdioServerParameters from mcp.client.stdio import stdio_client -from litellm import experimental_mcp_client +from litellm import mcp_client import litellm server_params = StdioServerParameters(command="python", args=["mcp_server.py"]) @@ -233,7 +233,7 @@ async with stdio_client(server_params) as (read, write): await session.initialize() # Load MCP tools in OpenAI format - tools = await experimental_mcp_client.load_mcp_tools(session=session, format="openai") + tools = await mcp_client.load_mcp_tools(session=session, format="openai") # Use with any LiteLLM model response = await litellm.acompletion( diff --git a/enterprise/litellm_enterprise/proxy/liteadmin.py b/enterprise/litellm_enterprise/proxy/liteadmin.py index e89bf11bd6e..bbf42829f47 100644 --- a/enterprise/litellm_enterprise/proxy/liteadmin.py +++ b/enterprise/litellm_enterprise/proxy/liteadmin.py @@ -18,8 +18,8 @@ from fastapi.responses import HTMLResponse, RedirectResponse, Response from pydantic import ConfigDict, Field, SecretStr, TypeAdapter, ValidationError from litellm.llms.custom_httpx.http_handler import get_async_httpx_client -from litellm.proxy._experimental.mcp_server.oauth_utils import get_request_base_url from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.mcp_server.oauth_utils import get_request_base_url from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.proxy.auth.auth_checks import UserNotFoundError @@ -122,7 +122,7 @@ def _cookie_name(token: str) -> str: async def _session_user(request: Request) -> str | None: - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( + from litellm.proxy.mcp_server.byok_oauth_endpoints import ( get_authenticated_browser_user_id, ) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 8331c68fe24..86b22069a39 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1980,7 +1980,7 @@ def completion_cost( base_pricing_model=(selected_model if base_model is not None and not custom_pricing else None), ) elif call_type == _MCP_CALL_TYPE: - from litellm.proxy._experimental.mcp_server.cost_calculator import ( + from litellm.proxy.mcp_server.cost_calculator import ( MCPCostCalculator, ) diff --git a/litellm/experimental_mcp_client/__init__.py b/litellm/experimental_mcp_client/__init__.py index 7a3917a50ea..f2905c0f95c 100644 --- a/litellm/experimental_mcp_client/__init__.py +++ b/litellm/experimental_mcp_client/__init__.py @@ -1,8 +1,7 @@ -try: - from .tools import call_openai_tool, load_mcp_tools -except ModuleNotFoundError as exc: - if exc.name not in ("mcp", "httpx2"): - raise - raise ImportError("MCP client dependencies are missing. Install them with: pip install 'litellm[mcp]'") from exc +import litellm.mcp_client as _mcp_client +from litellm.litellm_core_utils.deprecated_package import alias_deprecated_package +from litellm.mcp_client import call_openai_tool, load_mcp_tools + +alias_deprecated_package(__name__, _mcp_client) __all__ = ["call_openai_tool", "load_mcp_tools"] diff --git a/litellm/litellm_core_utils/deprecated_package.py b/litellm/litellm_core_utils/deprecated_package.py new file mode 100644 index 00000000000..542686ecf19 --- /dev/null +++ b/litellm/litellm_core_utils/deprecated_package.py @@ -0,0 +1,53 @@ +import importlib +import sys +import warnings +from itertools import chain +from pathlib import Path +from types import ModuleType +from typing import Final + + +def _dotted_name(root: Path, path: Path, package_name: str) -> str: + parts: Final = path.relative_to(root).with_suffix("").parts + return ".".join((package_name, *parts)).removesuffix(".__init__") + + +def _parents_and_self(dotted: str, package_name: str) -> tuple[str, ...]: + parts: Final = dotted.removeprefix(f"{package_name}.").split(".") + return tuple(f"{package_name}.{'.'.join(parts[:depth])}" for depth in range(1, len(parts) + 1)) + + +def _module_names(package: ModuleType) -> tuple[str, ...]: + roots: Final = tuple(Path(root) for root in package.__path__) + files: Final = tuple( + (root, path) for root in roots for path in root.rglob("*.py") + ) # comprehension-ok: nested walk over each package root + leaves: Final = tuple( + name + for name in (_dotted_name(root, path, package.__name__) for root, path in files) + if name != package.__name__ + ) + return tuple( + sorted( + set(chain.from_iterable(_parents_and_self(leaf, package.__name__) for leaf in leaves)) - {package.__name__} + ) + ) + + +def _alias(old_name: str, new_package_name: str, new_dotted: str) -> None: + module: Final = importlib.import_module(new_dotted) + old_dotted: Final = f"{old_name}{new_dotted.removeprefix(new_package_name)}" + old_parent, _, child = old_dotted.rpartition(".") + sys.modules[old_dotted] = module + setattr(sys.modules[old_parent], child, module) + + +def alias_deprecated_package(old_name: str, new_package: ModuleType) -> None: + """Make every module under ``new_package`` importable and attribute-reachable under ``old_name`` and warn once.""" + warnings.warn( + f"{old_name} is deprecated, import {new_package.__name__} instead", + DeprecationWarning, + stacklevel=3, + ) + for new_dotted in _module_names(new_package): + _alias(old_name, new_package.__name__, new_dotted) diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index 91f33635658..c18ce8cea68 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -144,7 +144,7 @@ def discover_guardrail_translation_mappings() -> dict[CallTypes, type["BaseTrans continue try: - from litellm.proxy._experimental.mcp_server.guardrail_translation import ( + from litellm.proxy.mcp_server.guardrail_translation import ( guardrail_translation_mappings as mcp_guardrail_translation_mappings, ) diff --git a/litellm/llms/anthropic/messages/native/mcp_handler.py b/litellm/llms/anthropic/messages/native/mcp_handler.py index 9df3c2c32e5..8fe18d79f24 100644 --- a/litellm/llms/anthropic/messages/native/mcp_handler.py +++ b/litellm/llms/anthropic/messages/native/mcp_handler.py @@ -76,12 +76,12 @@ async def anthropic_messages_with_mcp( upstream provider as a url it cannot reach. """ import litellm - from litellm.experimental_mcp_client.tools import ( - transform_mcp_tool_to_anthropic_tool, - ) from litellm.llms.anthropic.messages.adapters.handler import request_safeguards_evaluator from litellm.llms.anthropic.messages.native.handler import proxy_classifies_tool_uses from litellm.llms.anthropic.messages.safeguards import with_safeguard_results + from litellm.mcp_client.tools import ( + transform_mcp_tool_to_anthropic_tool, + ) from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) diff --git a/litellm/experimental_mcp_client/Readme.md b/litellm/mcp_client/Readme.md similarity index 89% rename from litellm/experimental_mcp_client/Readme.md rename to litellm/mcp_client/Readme.md index 3385a37cf69..b6008591dd7 100644 --- a/litellm/experimental_mcp_client/Readme.md +++ b/litellm/mcp_client/Readme.md @@ -5,13 +5,13 @@ LiteLLM MCP Client allows you to use MCP tools with LiteLLM Install the optional dependencies with `pip install 'litellm[mcp]'`, then use the existing public imports: ```python -from litellm.experimental_mcp_client import call_openai_tool, load_mcp_tools -from litellm.experimental_mcp_client.client import MCPClient +from litellm.mcp_client import call_openai_tool, load_mcp_tools +from litellm.mcp_client.client import MCPClient client = MCPClient(server_url="https://mcp.example.com/mcp") ``` -Core `import litellm` works without the MCP extra. Importing the experimental MCP client without its MCP or HTTPX2 dependency raises an error with this installation command +`litellm.experimental_mcp_client` is a deprecated alias of `litellm.mcp_client` that warns on import. Core `import litellm` works without the MCP extra. Importing the MCP client without its MCP or HTTPX2 dependency raises an error with this installation command ## MCP Python SDK compatibility diff --git a/litellm/mcp_client/__init__.py b/litellm/mcp_client/__init__.py new file mode 100644 index 00000000000..7a3917a50ea --- /dev/null +++ b/litellm/mcp_client/__init__.py @@ -0,0 +1,8 @@ +try: + from .tools import call_openai_tool, load_mcp_tools +except ModuleNotFoundError as exc: + if exc.name not in ("mcp", "httpx2"): + raise + raise ImportError("MCP client dependencies are missing. Install them with: pip install 'litellm[mcp]'") from exc + +__all__ = ["call_openai_tool", "load_mcp_tools"] diff --git a/litellm/experimental_mcp_client/client.py b/litellm/mcp_client/client.py similarity index 98% rename from litellm/experimental_mcp_client/client.py rename to litellm/mcp_client/client.py index 79887e4ab99..107b7155303 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/mcp_client/client.py @@ -100,11 +100,11 @@ from litellm.constants import ( MCP_TOOL_LISTING_MAX_PAGES, MCP_TOOL_LISTING_TIMEOUT, ) -from litellm.experimental_mcp_client.tools import list_tools_with_pagination from litellm.llms.custom_httpx.http_handler import get_ssl_configuration -from litellm.proxy._experimental.mcp_server.mcp_context import get_active_mcp_request_ctx -from litellm.proxy._experimental.mcp_server.mcp_debug import capture_upstream_error_response -from litellm.proxy._experimental.mcp_server.result_conversion import ( +from litellm.mcp_client.tools import list_tools_with_pagination +from litellm.proxy.mcp_server.mcp_context import get_active_mcp_request_ctx +from litellm.proxy.mcp_server.mcp_debug import capture_upstream_error_response +from litellm.proxy.mcp_server.result_conversion import ( age_freshness, aggregate_freshness, error_text_result, @@ -125,14 +125,14 @@ from litellm.types.mcp import ( ) if TYPE_CHECKING: - from litellm.proxy._experimental.mcp_server.contracts import CatalogListRequest, CatalogListResult - from litellm.proxy._experimental.mcp_server.legacy_callbacks import ElicitationCallback + from litellm.proxy.mcp_server.contracts import CatalogListRequest, CatalogListResult + from litellm.proxy.mcp_server.legacy_callbacks import ElicitationCallback def _mcp_response_hooks() -> tuple[Callable[[httpx2.Response], Awaitable[None]], ...]: if get_active_mcp_request_ctx() is None: return (capture_upstream_error_response,) - from litellm.proxy._experimental.mcp_server.legacy_callbacks import ( + from litellm.proxy.mcp_server.legacy_callbacks import ( get_scope_response, record_upstream_tool_authorization, ) @@ -564,7 +564,7 @@ class MCPClient: return self._hash_discovery_auth(await self.prepare_request_auth(), self._aws_identity()) async def subscription_auth_fingerprint(self) -> str: - from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.client_credentials import ( ClientCredentialsBearerAuth, ) @@ -1125,8 +1125,8 @@ class MCPClient: request_state: str | None, allow_input_required: bool, ) -> TSessionResult | InputRequiredResult: - from litellm.proxy._experimental.mcp_server.contracts import ClientInteraction - from litellm.proxy._experimental.mcp_server.interactions import LegacyClientInteraction, ModernClientInteraction + from litellm.proxy.mcp_server.contracts import ClientInteraction + from litellm.proxy.mcp_server.interactions import LegacyClientInteraction, ModernClientInteraction with anyio.fail_after(self.timeout): first: Final = await request(input_responses, request_state) diff --git a/litellm/experimental_mcp_client/tools.py b/litellm/mcp_client/tools.py similarity index 100% rename from litellm/experimental_mcp_client/tools.py rename to litellm/mcp_client/tools.py diff --git a/litellm/proxy/_experimental/mcp_server/__init__.py b/litellm/proxy/_experimental/mcp_server/__init__.py new file mode 100644 index 00000000000..cf217531dc4 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/__init__.py @@ -0,0 +1,4 @@ +import litellm.proxy.mcp_server as _mcp_server +from litellm.litellm_core_utils.deprecated_package import alias_deprecated_package + +alias_deprecated_package(__name__, _mcp_server) diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 4e54d6ac733..db01475154b 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -163,7 +163,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( # /.well-known/oauth-authorization-server: the root document is the # gateway's own authorization server metadata (token endpoint /token). name="mcp_discoverable", - module_path="litellm.proxy._experimental.mcp_server.discoverable_endpoints", + module_path="litellm.proxy.mcp_server.discoverable_endpoints", path_prefixes=( "/oauth/client-metadata.json", "/.well-known/oauth-", @@ -185,19 +185,19 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( # No /mcp/oauth prefix here: the mounted /mcp sub-app would # shadow it, and there are no actual routes there anyway. name="mcp_byok_oauth", - module_path="litellm.proxy._experimental.mcp_server.byok_oauth_endpoints", + module_path="litellm.proxy.mcp_server.byok_oauth_endpoints", path_prefixes=("/v1/mcp/oauth", "/.well-known/oauth-"), ), LazyFeature( name="mcp_rest", - module_path="litellm.proxy._experimental.mcp_server.rest_endpoints", + module_path="litellm.proxy.mcp_server.rest_endpoints", path_prefixes=("/mcp-rest",), ), LazyFeature( # Hardcoded /mcp matches BASE_MCP_ROUTE; importing the constant # here would defeat lazy loading. name="mcp_app", - module_path="litellm.proxy._experimental.mcp_server.server", + module_path="litellm.proxy.mcp_server.server", path_prefixes=("/mcp",), register_fn=_mount_app("/mcp", attr_name="app"), persistent_swagger_stub=True, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 49d23d82aa9..dd91bb6eef3 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -37,7 +37,7 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( validate_langfuse_span_scope_value, validate_no_callback_env_reference, ) -from litellm.proxy._experimental.mcp_server.stdio_gate import MCP_STDIO_DISABLED_MESSAGE, is_mcp_stdio_enabled +from litellm.proxy.mcp_server.stdio_gate import MCP_STDIO_DISABLED_MESSAGE, is_mcp_stdio_enabled from litellm.types.agents import ( AgentCaller as AgentCaller, # noqa: PLC0414 # public re-export ) diff --git a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py index dd002a2763e..9be9d4c7121 100644 --- a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py +++ b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py @@ -16,7 +16,6 @@ from fastapi import HTTPException from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger -from litellm.proxy._experimental.mcp_server.ui_session_utils import build_effective_auth_contexts from litellm.proxy._types import ( UI_TEAM_ID, LiteLLM_ObjectPermissionTable, @@ -30,6 +29,7 @@ from litellm.proxy.agent_endpoints.auth.agent_access_groups import ( ) from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_auth from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_agent_policy +from litellm.proxy.mcp_server.ui_session_utils import build_effective_auth_contexts from litellm.repositories.table_repositories import AgentsRepository from litellm.types.agents import AgentResponse @@ -244,7 +244,7 @@ class AgentRequestHandler: or user_api_key_auth is None ): return False - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler key_hash: Final = user_api_key_auth.api_key or user_api_key_auth.token authority: Final = ( @@ -905,7 +905,7 @@ async def _managed_actor_agent_access(auth: UserAPIKeyAuth) -> AgentAccess: async def _verified_human_agent_sources( user_id: str | None, *, allowed_team_ids: frozenset[str] | None = None ) -> tuple[tuple[str | None, frozenset[str]], ...]: - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler if user_id is None: return () diff --git a/litellm/proxy/anthropic_endpoints/gateway_endpoints.py b/litellm/proxy/anthropic_endpoints/gateway_endpoints.py index ba3307bc1ad..1911cb574ef 100644 --- a/litellm/proxy/anthropic_endpoints/gateway_endpoints.py +++ b/litellm/proxy/anthropic_endpoints/gateway_endpoints.py @@ -37,18 +37,6 @@ from litellm.constants import ( CLI_SSO_SESSION_TTL_SECONDS, LITELLM_CLI_SOURCE_IDENTIFIER, ) -from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ( - PROXY_API_AUDIENCE, - MintedProxyCredential, - MintProxyCredential, - SessionSigning, - proxy_credential_response, - refresh_proxy_credential, - resolve_session_signing, - revoke_session_refresh_token, -) -from litellm.proxy._experimental.mcp_server.oauth_utils import TOKEN_NO_CACHE_HEADERS -from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import SessionPrincipal from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles from litellm.proxy.anthropic_endpoints.endpoints import anthropic_response, count_tokens from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -58,6 +46,18 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg ) from litellm.proxy.management_endpoints.sso_helper_utils import CLI_SSO_SESSIONS_TARGET from litellm.proxy.management_endpoints.ui_sso import CliSsoTeamDetail +from litellm.proxy.mcp_server.gateway_dcr_flow import ( + PROXY_API_AUDIENCE, + MintedProxyCredential, + MintProxyCredential, + SessionSigning, + proxy_credential_response, + refresh_proxy_credential, + resolve_session_signing, + revoke_session_refresh_token, +) +from litellm.proxy.mcp_server.oauth_utils import TOKEN_NO_CACHE_HEADERS +from litellm.proxy.mcp_server.outbound_credentials.session_token import SessionPrincipal from litellm.proxy.proxy_server import model_list from litellm.types.llms.base import LiteLLMBaseModel @@ -340,7 +340,7 @@ async def _claim_device_code(login_id: str, cache: DualCache) -> bool: def _proxy_credential_minter() -> MintProxyCredential: - from litellm.proxy._experimental.mcp_server.proxy_api_credentials import mint_proxy_credential + from litellm.proxy.mcp_server.proxy_api_credentials import mint_proxy_credential return mint_proxy_credential diff --git a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py index 86e77596e70..8225984a964 100644 --- a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py +++ b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py @@ -4,8 +4,8 @@ from typing import Final from fastapi import APIRouter -from litellm.proxy._experimental.mcp_server.stdio_gate import is_mcp_stdio_enabled from litellm.proxy.common_utils.html_forms.default_credentials_hint import should_hide_default_credentials_hint +from litellm.proxy.mcp_server.stdio_gate import is_mcp_stdio_enabled from litellm.types.proxy.discovery_endpoints.ui_discovery_endpoints import ( UiDiscoveryEndpoints, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py b/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py index 224f1f78270..da131931aec 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py +++ b/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py @@ -38,9 +38,9 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.proxy._experimental.mcp_server.utils import JSONLeafPath, json_string_leaves from litellm.proxy._types import SpecialHeaders from litellm.proxy.litellm_pre_call_utils import get_chain_id_from_headers +from litellm.proxy.mcp_server.utils import JSONLeafPath, json_string_leaves from litellm.types.guardrails import GuardrailEventHooks, LitellmParams, Mode from litellm.types.proxy.guardrails.guardrail_hooks.akto import AktoGuardrailConfigModelOptionalParams from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/mcp_end_user_permission.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/mcp_end_user_permission.py index d7ce3438fca..ab35ead03c7 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/mcp_end_user_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/mcp_end_user_permission.py @@ -189,7 +189,7 @@ class MCPEndUserPermissionGuardrail(CustomGuardrail): if not direct_mcp_servers and not mcp_access_groups: return None # Both empty → no restrictions - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py index 3cb20a00e2f..9c0a574ae3b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py @@ -104,7 +104,7 @@ class MCPSecurityGuardrail(CustomGuardrail): if not requested_servers: return frozenset() - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index 6426bfb8788..d50001ba636 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -577,7 +577,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): if request_data.get("server_name"): return request_data["server_name"] try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) diff --git a/litellm/proxy/hooks/mcp_semantic_filter/hook.py b/litellm/proxy/hooks/mcp_semantic_filter/hook.py index d33811d3dea..c6e26999766 100644 --- a/litellm/proxy/hooks/mcp_semantic_filter/hook.py +++ b/litellm/proxy/hooks/mcp_semantic_filter/hook.py @@ -18,16 +18,16 @@ from litellm.constants import ( DEFAULT_MCP_SEMANTIC_FILTER_TOP_K, ) from litellm.integrations.custom_logger import CustomLogger -from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( +from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticToolFilterContextWindowError, ) if TYPE_CHECKING: from litellm.caching.caching import DualCache - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) - from litellm.proxy._types import UserAPIKeyAuth from litellm.router import Router @@ -487,7 +487,7 @@ class SemanticToolFilterHook(CustomLogger): Returns: SemanticToolFilterHook instance if enabled, None otherwise """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 5f1948d1c1c..baa150efc86 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2525,7 +2525,7 @@ async def add_litellm_data_to_request( _headers, allow_client_message_redaction_opt_out=_allow_client_message_redaction_opt_out, ) - from litellm.proxy._experimental.mcp_server.utils import upstream_credential_headers + from litellm.proxy.mcp_server.utils import upstream_credential_headers _mcp_credential_headers: Final = upstream_credential_headers(_headers) _logging_safe_headers: Final = redact_credential_headers( diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index a87455e5a60..6558c82f80c 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -10,7 +10,6 @@ from typing_extensions import ReadOnly, TypedDict from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.core_helpers import iter_items -from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy._types import ( CommonProxyErrors, LiteLLM_AccessGroupTable, @@ -37,6 +36,7 @@ from litellm.proxy.management_helpers.resource_display_names import ( key_display_names, mcp_server_display_names, ) +from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy.utils import PrismaClient, get_prisma_client_or_throw from litellm.repositories.table_repositories import AccessGroupRepository, TeamRepository from litellm.types.access_group import ( diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 0ad4258fb04..b97d409a6c3 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -44,14 +44,6 @@ from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.models.credentials import CredentialItem from litellm.models.project import LiteLLM_ProjectTable -from litellm.proxy._experimental.mcp_server.db import ( - rotate_mcp_server_credentials_master_key, - rotate_mcp_user_credentials_master_key, - rotate_mcp_user_env_vars_master_key, -) -from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( - rotate_sso_identity_assertions_master_key, -) from litellm.proxy._types import * from litellm.proxy._types import Litellm_EntityType, LiteLLM_VerificationToken, hash_token from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module exports @@ -153,6 +145,14 @@ from litellm.proxy.management_helpers.team_member_permission_checks import ( TeamMemberPermissionChecks, ) from litellm.proxy.management_helpers.utils import management_endpoint_wrapper +from litellm.proxy.mcp_server.db import ( + rotate_mcp_server_credentials_master_key, + rotate_mcp_user_credentials_master_key, + rotate_mcp_user_env_vars_master_key, +) +from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store import ( + rotate_sso_identity_assertions_master_key, +) from litellm.proxy.search_endpoints.search_tool_registry import rotate_search_tools_master_key from litellm.proxy.spend_tracking.budget_reservation import get_budget_window_start from litellm.proxy.spend_tracking.spend_counter_batch import SPEND_COUNTERS_TARGET diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 776072c69be..f6b836bc4be 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -49,7 +49,7 @@ from fastapi.responses import JSONResponse from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict -from litellm.proxy._experimental.mcp_server.catalog import catalog_operation, public_catalog_operation +from litellm.proxy.mcp_server.catalog import catalog_operation, public_catalog_operation try: from prisma.errors import RecordNotFoundError, UniqueViolationError @@ -63,19 +63,6 @@ from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._uuid import uuid from litellm.constants import LITELLM_PROXY_ADMIN_NAME, MCP_GATEWAY_SESSION_ID_PREFIX_LENGTH from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.proxy._experimental.mcp_server.utils import ( - LITELLM_MCP_SERVER_DESCRIPTION, - LITELLM_MCP_SERVER_NAME, - MCP_SERVERS_TARGET, - McpServerPayloadLike, - build_env_var_setup_url, - collect_env_var_references, - get_server_prefix, - parse_admin_env_vars, -) -from litellm.proxy._experimental.mcp_server.utils import ( - validate_and_normalize_mcp_server_payload as _base_validate_and_normalize_mcp_server_payload, -) from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, @@ -87,6 +74,19 @@ from litellm.proxy.management_helpers.audit_logs import ( get_audit_log_changed_by, is_audit_logging_enabled, ) +from litellm.proxy.mcp_server.utils import ( + LITELLM_MCP_SERVER_DESCRIPTION, + LITELLM_MCP_SERVER_NAME, + MCP_SERVERS_TARGET, + McpServerPayloadLike, + build_env_var_setup_url, + collect_env_var_references, + get_server_prefix, + parse_admin_env_vars, +) +from litellm.proxy.mcp_server.utils import ( + validate_and_normalize_mcp_server_payload as _base_validate_and_normalize_mcp_server_payload, +) from litellm.repositories.table_repositories import ( MCPServerRepository, MCPUserCredentialsRepository, @@ -159,63 +159,6 @@ if MCP_AVAILABLE: def validate_tool_name(name: str) -> _ToolNameValidationResult: return _ToolNameValidationResult() - from litellm.proxy._experimental.mcp_server.contracts import TargetCatalog - from litellm.proxy._experimental.mcp_server.db import ( - McpIdentifierConflict, - approve_mcp_server, - create_draft_mcp_server, - create_mcp_server_if_identifier_free, - delete_mcp_server, - delete_user_credential, - delete_user_env_vars, - get_all_mcp_servers, - get_draft_mcp_server, - get_mcp_server, - get_mcp_servers, - get_mcp_submissions, - get_user_env_vars, - get_user_env_vars_bulk, - get_user_oauth_credential, - is_resubmitted_oauth_client, - list_server_user_credentials, - list_user_oauth_credentials, - mcp_oauth_token_identity, - merge_user_env_vars, - oauth_credentials_for_upstream_edit, - purge_user_oauth_credentials_for_server, - reject_mcp_server, - set_mcp_server_pinned_tools, - stale_mcp_auth_fields, - store_user_credential, - store_user_oauth_credential, - update_mcp_server, - ) - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( # noqa: F401 # legacy module exports - _raise_if_not_oauth2, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - authorize_with_server, - client_supplied_application_type, - client_supplied_redirect_uris, - exchange_token_with_server, - get_request_base_url, - raise_if_not_oauth2, - redeem_passthrough_authorization_code, - register_client_with_server, - resolve_ephemeral_dcr_client, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - from litellm.proxy._experimental.mcp_server.server_resolution import ( - MCPServerTargetCatalog, - authorize_mcp_server, - resolve_mcp_server, - ) - from litellm.proxy._experimental.mcp_server.ui_session_utils import ( - admitted_user_context, - build_effective_auth_contexts, - granted_toolset_ids, - is_ui_session_credential, - ) from litellm.proxy._types import ( LiteLLM_MCPServerTable, LiteLLM_ObjectPermissionTable, @@ -266,6 +209,63 @@ if MCP_AVAILABLE: convert_connector_entries, ) from litellm.proxy.management_helpers.utils import management_endpoint_wrapper + from litellm.proxy.mcp_server.contracts import TargetCatalog + from litellm.proxy.mcp_server.db import ( + McpIdentifierConflict, + approve_mcp_server, + create_draft_mcp_server, + create_mcp_server_if_identifier_free, + delete_mcp_server, + delete_user_credential, + delete_user_env_vars, + get_all_mcp_servers, + get_draft_mcp_server, + get_mcp_server, + get_mcp_servers, + get_mcp_submissions, + get_user_env_vars, + get_user_env_vars_bulk, + get_user_oauth_credential, + is_resubmitted_oauth_client, + list_server_user_credentials, + list_user_oauth_credentials, + mcp_oauth_token_identity, + merge_user_env_vars, + oauth_credentials_for_upstream_edit, + purge_user_oauth_credentials_for_server, + reject_mcp_server, + set_mcp_server_pinned_tools, + stale_mcp_auth_fields, + store_user_credential, + store_user_oauth_credential, + update_mcp_server, + ) + from litellm.proxy.mcp_server.discoverable_endpoints import ( # noqa: F401 # legacy module exports + _raise_if_not_oauth2, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + authorize_with_server, + client_supplied_application_type, + client_supplied_redirect_uris, + exchange_token_with_server, + get_request_base_url, + raise_if_not_oauth2, + redeem_passthrough_authorization_code, + register_client_with_server, + resolve_ephemeral_dcr_client, + ) + from litellm.proxy.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.proxy.mcp_server.server_resolution import ( + MCPServerTargetCatalog, + authorize_mcp_server, + resolve_mcp_server, + ) + from litellm.proxy.mcp_server.ui_session_utils import ( + admitted_user_context, + build_effective_auth_contexts, + granted_toolset_ids, + is_ui_session_credential, + ) from litellm.types.mcp import ( MCP_ADMIN_CONFIG_CREDENTIAL_KEYS, MCPAuth, @@ -1097,7 +1097,7 @@ if MCP_AVAILABLE: """ Get all MCP tools available for the current key, including those from access groups """ - from litellm.proxy._experimental.mcp_server.server import _list_mcp_tools + from litellm.proxy.mcp_server.server import _list_mcp_tools listing: Final = await _list_mcp_tools( user_api_key_auth=user_api_key_dict, @@ -1122,7 +1122,7 @@ if MCP_AVAILABLE: """ Get all available MCP access groups from the database AND config """ - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) from litellm.proxy.proxy_server import prisma_client @@ -1678,7 +1678,7 @@ if MCP_AVAILABLE: status_code=status.HTTP_403_FORBIDDEN, detail={"error": "Admin access required to view MCP gateway sessions."}, ) - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( get_mcp_gateway_sessions_report, ) @@ -1713,7 +1713,7 @@ if MCP_AVAILABLE: "error": "Provide session_id_prefix and/or user_id to select the sessions to terminate.", }, ) - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( terminate_mcp_gateway_sessions, ) @@ -1777,7 +1777,7 @@ if MCP_AVAILABLE: status_code=status.HTTP_404_NOT_FOUND, detail={"error": f"MCP server '{server_id}' not found in the database."}, ) - from litellm.proxy._experimental.mcp_server.rest_endpoints import fetch_pinnable_tool_catalog + from litellm.proxy.mcp_server.rest_endpoints import fetch_pinnable_tool_catalog snapshot: Final = await fetch_pinnable_tool_catalog(server, request, user_api_key_dict) if not snapshot: @@ -2342,12 +2342,12 @@ if MCP_AVAILABLE: # issued to this client could mint fresh upstream access tokens through # us. Require normal LiteLLM auth for those. if not api_key: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 - global_mcp_server_manager, - ) from litellm.proxy.auth.auth_utils import ( # noqa: PLC0415 get_request_route, ) + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 + global_mcp_server_manager, + ) server_id: Final[str] = request.path_params.get("server_id", "") if server_id: @@ -2677,7 +2677,7 @@ if MCP_AVAILABLE: ) if payload.save: await store_user_credential(prisma_client, user_id, server_id, payload.credential) - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _invalidate_byok_cred_cache, ) @@ -2708,7 +2708,7 @@ if MCP_AVAILABLE: await delete_user_credential(prisma_client, target_user_id, server_id) except RecordNotFoundError: pass # Already deleted or didn't exist - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _invalidate_byok_cred_cache, ) @@ -2732,7 +2732,7 @@ if MCP_AVAILABLE: """Persist the OAuth2 access token obtained by the calling user.""" prisma_client: Final = get_prisma_client_or_throw("Database not connected. Connect a database to your proxy") await _authorize_and_fetch_mcp_server(prisma_client, user_api_key_dict, server_id) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # keep manager import lazy + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # keep manager import lazy global_mcp_server_manager as _manager, ) @@ -2769,7 +2769,7 @@ if MCP_AVAILABLE: expires_in=payload.expires_in, scopes=payload.scopes, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 global_mcp_server_manager, ) @@ -2813,7 +2813,7 @@ if MCP_AVAILABLE: await delete_user_credential(prisma_client, target_user_id, server_id) except RecordNotFoundError: pass # Already gone — treat as a successful delete - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 global_mcp_server_manager, ) @@ -3107,7 +3107,7 @@ if MCP_AVAILABLE: allowed_names: Final = {spec["name"] for spec in user_specs} updates: Final = {k: v for k, v in payload.values.items() if k in allowed_names and v != ""} merged: Final = await merge_user_env_vars(prisma_client, user_id, server_id, updates, allowed_names) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( invalidate_user_env_vars_cache, ) @@ -3134,7 +3134,7 @@ if MCP_AVAILABLE: ) server: Final = await _authorize_and_fetch_mcp_server(prisma_client, user_api_key_dict, server_id) await delete_user_env_vars(prisma_client, user_id, server_id) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( invalidate_user_env_vars_cache, ) @@ -3369,7 +3369,7 @@ if MCP_AVAILABLE: try: # Update the public model groups import litellm - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) from litellm.proxy.proxy_server import proxy_config @@ -3532,7 +3532,7 @@ if MCP_AVAILABLE: # MCP Toolset endpoints # --------------------------------------------------------------------------- - from litellm.proxy._experimental.mcp_server.toolset_db import ( + from litellm.proxy.mcp_server.toolset_db import ( create_mcp_toolset, delete_mcp_toolset, get_mcp_toolset, @@ -3577,7 +3577,7 @@ if MCP_AVAILABLE: status_code=status.HTTP_409_CONFLICT, detail={"error": f"A toolset named '{payload.toolset_name}' already exists."}, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -3673,7 +3673,7 @@ if MCP_AVAILABLE: status_code=status.HTTP_404_NOT_FOUND, detail={"error": f"Toolset '{payload.toolset_id}' not found."}, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -3703,7 +3703,7 @@ if MCP_AVAILABLE: status_code=status.HTTP_404_NOT_FOUND, detail={"error": f"Toolset '{toolset_id}' not found."}, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) diff --git a/litellm/proxy/management_endpoints/sso/saml_sso.py b/litellm/proxy/management_endpoints/sso/saml_sso.py index ba265acfe4a..d9fe9fcf8f4 100644 --- a/litellm/proxy/management_endpoints/sso/saml_sso.py +++ b/litellm/proxy/management_endpoints/sso/saml_sso.py @@ -38,13 +38,13 @@ from pydantic import ValidationError from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.caching.dual_cache import DualCache -from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( - SSOIdentityAssertion, - assertion_from_saml_login, -) from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.proxy.management_endpoints.sso_helper_utils import SSO_SESSIONS_TARGET from litellm.proxy.management_endpoints.types import CustomOpenID, get_litellm_user_role +from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store import ( + SSOIdentityAssertion, + assertion_from_saml_login, +) from litellm.proxy.utils import get_custom_url try: diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 02494de8c87..22c4f27d94a 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -72,17 +72,6 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.login_assertion_exchange import ( - exchange_login_assertion_if_configured, -) -from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( - GOOGLE_REFRESH_TOKEN_PARAMS, - MICROSOFT_REFRESH_TOKEN_SCOPE, - SSOIdentityAssertion, - assertion_from_sso_login, - ema_assertion_retention_enabled, - retain_sso_identity_assertion_for_ema, -) from litellm.proxy._types import ( CommonProxyErrors, LiteLLM_UserTable, @@ -143,6 +132,17 @@ from litellm.proxy.management_endpoints.types import ( get_litellm_user_role, is_valid_litellm_user_role, ) +from litellm.proxy.mcp_server.outbound_credentials.login_assertion_exchange import ( + exchange_login_assertion_if_configured, +) +from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store import ( + GOOGLE_REFRESH_TOKEN_PARAMS, + MICROSOFT_REFRESH_TOKEN_SCOPE, + SSOIdentityAssertion, + assertion_from_sso_login, + ema_assertion_retention_enabled, + retain_sso_identity_assertion_for_ema, +) from litellm.proxy.utils import ( PrismaClient, ProxyLogging, diff --git a/litellm/proxy/management_helpers/object_permission_utils.py b/litellm/proxy/management_helpers/object_permission_utils.py index 920970427ba..a719a5c67d4 100644 --- a/litellm/proxy/management_helpers/object_permission_utils.py +++ b/litellm/proxy/management_helpers/object_permission_utils.py @@ -300,7 +300,7 @@ async def _resolve_mcp_server_identifiers_to_ids( if not identifiers: return {} - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -450,7 +450,7 @@ async def _resolve_team_allowed_mcp_servers( - Servers from mcp_access_groups - Server IDs referenced in mcp_tool_permissions keys """ - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) @@ -475,7 +475,7 @@ async def _resolve_team_allowed_mcp_servers( def get_allow_all_keys_server_ids() -> set[str]: """Return the set of MCP server IDs marked with allow_all_keys=True.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -487,7 +487,7 @@ _get_allow_all_keys_server_ids: Final = get_allow_all_keys_server_ids def _get_all_mcp_server_ids() -> set[str]: """Return every MCP server id registered on the proxy (config + DB union).""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) diff --git a/litellm/proxy/_experimental/mcp_server/AGENTS.md b/litellm/proxy/mcp_server/AGENTS.md similarity index 98% rename from litellm/proxy/_experimental/mcp_server/AGENTS.md rename to litellm/proxy/mcp_server/AGENTS.md index 8c53216f77a..25150c11f1d 100644 --- a/litellm/proxy/_experimental/mcp_server/AGENTS.md +++ b/litellm/proxy/mcp_server/AGENTS.md @@ -12,7 +12,7 @@ wiring must change with it. Respect the current package boundaries: ```text -litellm/proxy/_experimental/mcp_server/ +litellm/proxy/mcp_server/ AGENTS.md server.py # ASGI/MCP route handling, sessions, tool calls [PR7: 7-arm only — move BYOK/OAuth pre-fetch into resolver] mcp_server_manager.py # upstream server registry, clients, tool routing [PR7: _create_mcp_client swaps resolve_mcp_auth -> resolve_credentials] @@ -89,7 +89,7 @@ module materially harder to understand. ## Tests -Mirror this package under `tests/unit/proxy/_experimental/mcp_server/`. +Mirror this package under `tests/unit/proxy/mcp_server/`. For regressions, extend the existing mapped test file instead of creating a new one. Use subdirectories that match the implementation path, such as `auth/test_token_endpoint_auth.py` for `auth/token_endpoint_auth.py` and diff --git a/litellm/proxy/_experimental/mcp_server/README.md b/litellm/proxy/mcp_server/README.md similarity index 100% rename from litellm/proxy/_experimental/mcp_server/README.md rename to litellm/proxy/mcp_server/README.md diff --git a/litellm/proxy/_experimental/mcp_server/auth/litellm_auth_handler.py b/litellm/proxy/mcp_server/auth/litellm_auth_handler.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/auth/litellm_auth_handler.py rename to litellm/proxy/mcp_server/auth/litellm_auth_handler.py diff --git a/litellm/proxy/_experimental/mcp_server/auth/managed_agent_access.py b/litellm/proxy/mcp_server/auth/managed_agent_access.py similarity index 89% rename from litellm/proxy/_experimental/mcp_server/auth/managed_agent_access.py rename to litellm/proxy/mcp_server/auth/managed_agent_access.py index 096b7eb3c77..a01db0db6d8 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/managed_agent_access.py +++ b/litellm/proxy/mcp_server/auth/managed_agent_access.py @@ -8,15 +8,15 @@ from litellm.types.proxy.agent_identity import AgentIdentityFailure async def _delegated_resource_subject(user_id: str) -> UserAPIKeyAuth: - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler human: Final = await MCPRequestHandler.reload_admitted_user(user_id, requires_fresh_policy=True) return human.model_copy(update=MappingProxyType({"mcp_explicit_grants_only": True})) async def managed_agent_servers(auth: UserAPIKeyAuth) -> tuple[str, ...]: - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager agent: Final = auth.managed_agent_policy if agent is None: @@ -49,7 +49,7 @@ async def managed_agent_servers(auth: UserAPIKeyAuth) -> tuple[str, ...]: async def managed_agent_tools(server_id: str, auth: UserAPIKeyAuth) -> list[str] | None: - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler if server_id not in await managed_agent_servers(auth): return [] diff --git a/litellm/proxy/_experimental/mcp_server/auth/token_endpoint_auth.py b/litellm/proxy/mcp_server/auth/token_endpoint_auth.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/auth/token_endpoint_auth.py rename to litellm/proxy/mcp_server/auth/token_endpoint_auth.py diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py rename to litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py index f883031a529..cc7080738d0 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py @@ -21,33 +21,7 @@ import litellm from litellm._internal_context import with_service_target from litellm._logging import verbose_logger from litellm.constants import MCP_ALL_TOOLS_WILDCARD, MCP_PEEKED_BODY_SCOPE_KEY -from litellm.experimental_mcp_client.client import strip_auth_scheme -from litellm.proxy._experimental.mcp_server.catalog import global_manager -from litellm.proxy._experimental.mcp_server.oauth_utils import ( - get_passthrough_resource_metadata_url, - get_passthrough_www_authenticate, - get_request_base_url, - get_route_relative_request_path, - well_known_root_suffix, -) -from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( - BridgeEnvelopeAdmitted, - BridgeEnvelopeInvalid, - NotBridgeEnvelope, - envelope_keys_from_master_key, - is_bridge_envelope_shaped, - resolve_bridge_envelope, -) -from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( - EnvelopeIdentity, -) -from litellm.proxy._experimental.mcp_server.outbound_credentials.session_credentials import ( - is_session_bearer_shaped, -) -from litellm.proxy._experimental.mcp_server.utils import ( - asgi_scope_fields, - strip_known_server_prefix, -) +from litellm.mcp_client.client import strip_auth_scheme from litellm.proxy._types import ( UI_TEAM_ID, LiteLLM_AccessGroupTable, @@ -87,6 +61,32 @@ from litellm.proxy.common_utils.user_api_key_cache import ( get_management_object_ttl, user_object_permission_id_cache_key, ) +from litellm.proxy.mcp_server.catalog import global_manager +from litellm.proxy.mcp_server.oauth_utils import ( + get_passthrough_resource_metadata_url, + get_passthrough_www_authenticate, + get_request_base_url, + get_route_relative_request_path, + well_known_root_suffix, +) +from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( + BridgeEnvelopeAdmitted, + BridgeEnvelopeInvalid, + NotBridgeEnvelope, + envelope_keys_from_master_key, + is_bridge_envelope_shaped, + resolve_bridge_envelope, +) +from litellm.proxy.mcp_server.outbound_credentials.envelope import ( + EnvelopeIdentity, +) +from litellm.proxy.mcp_server.outbound_credentials.session_credentials import ( + is_session_bearer_shaped, +) +from litellm.proxy.mcp_server.utils import ( + asgi_scope_fields, + strip_known_server_prefix, +) from litellm.repositories.table_repositories import ( AgentsRepository, MCPServerRepository, @@ -201,7 +201,7 @@ def _is_mcp_passthrough_cold_start(mcp_servers: list[str] | None, client_ip: str cannot be resolved.""" if not mcp_servers: return False - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -216,7 +216,7 @@ def _is_legacy_delegate_cold_start(mcp_servers: list[str] | None, client_ip: str """Allow only credential-free legacy delegates to reach the route's OAuth challenge.""" if not mcp_servers: return False - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, global_mcp_server_manager, ) @@ -308,7 +308,7 @@ def _gateway_dcr_challenge_target( client_ip: str | None, ) -> str | None: """Resolve a single path target whose sign-in metadata advertises the gateway.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -328,8 +328,8 @@ def gateway_dcr_challenge_scope( ) -> str | None: if MCPRequestHandler.extract_target_server_names_from_path(route) or not mcp_servers: return None - from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import gateway_server_scope - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.gateway_dcr_flow import gateway_server_scope + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -919,7 +919,7 @@ class MCPRequestHandler: transparent proxy and the caller's ``Authorization`` is an upstream token, never a LiteLLM key. A mixed-target request keeps normal auth. """ - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) from litellm.types.mcp import MCPAuth @@ -946,7 +946,7 @@ class MCPRequestHandler: envelope admission arm never fires for an aggregate scope or a server that did not opt into the bridge. Mirrors :meth:`_target_servers_are_true_passthrough`. """ - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -1163,7 +1163,7 @@ class MCPRequestHandler: mint-time snapshot. Pre-DB gates (size, IP, route allowlist) run first, mirroring the standard pipeline. Fails closed with the requested scope's ``invalid_token`` challenge on an expired, tampered, foreign, or refresh token, or a missing/deactivated/policy-rejected user.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.session_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.session_credentials import ( NotSessionBearer, SessionBearerAdmitted, SessionBearerInvalid, @@ -1332,7 +1332,7 @@ class MCPRequestHandler: so each is resolved to an id via ``expand_permission_list`` before the membership check. Returns None (no descriptors) when nothing applies; a lookup failure narrows to None rather than raising, since rate limiting must not deny a request authorization already allowed.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -1394,7 +1394,7 @@ class MCPRequestHandler: else: return refreshed if refreshed.mcp_toolset_id is not None: - from litellm.proxy._experimental.mcp_server.server import apply_toolset_scope + from litellm.proxy.mcp_server.server import apply_toolset_scope return await apply_toolset_scope(refreshed, refreshed.mcp_toolset_id) return refreshed @@ -1416,10 +1416,10 @@ class MCPRequestHandler: project, org, and budget state are NOT re-checked here; the caller runs the admitted identity through ``_enforce_admitted_live_policy`` for those. """ - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( + from litellm.proxy.auth.auth_checks import get_key_object + from litellm.proxy.mcp_server.bridge_token_flow import ( master_key_admin_auth, # noqa: PLC0415 # inline import avoids a module-load circular import ) - from litellm.proxy.auth.auth_checks import get_key_object from litellm.proxy.proxy_server import prisma_client, user_api_key_cache admin: Final = master_key_admin_auth(key_hash) @@ -1661,7 +1661,7 @@ class MCPRequestHandler: @staticmethod def get_guardrail_bearer_token(headers: Headers, auth: UserAPIKeyAuth) -> SecretStr | None: - from litellm.proxy._experimental.mcp_server.utils import ( + from litellm.proxy.mcp_server.utils import ( _custom_litellm_key_header_name, # pyright: ignore[reportPrivateUsage] # reuse the shared credential-header setting ) @@ -1884,7 +1884,7 @@ class MCPRequestHandler: from litellm.proxy.proxy_server import general_settings if managed_agent_policy(user_api_key_auth) is not None: - from litellm.proxy._experimental.mcp_server.auth.managed_agent_access import managed_agent_servers + from litellm.proxy.mcp_server.auth.managed_agent_access import managed_agent_servers return MCPServerAccess(server_ids=await managed_agent_servers(user_api_key_auth), scope="scoped") @@ -2257,7 +2257,7 @@ class MCPRequestHandler: intersected with it, so the route narrows a team grant exactly as it narrows the user's own.""" if auth.mcp_toolset_id is None: return None - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -2308,8 +2308,8 @@ class MCPRequestHandler: if not is_mcp_admitted_user_subject(auth) and not personal_key_inherits_team_grants(auth): return auth try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager - from litellm.proxy._experimental.mcp_server.operations import ( + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.operations import ( _get_allowed_mcp_servers, _resolve_display_name_to_original, resolve_scoped_mcp_server, @@ -2473,7 +2473,7 @@ class MCPRequestHandler: onto a server reached through a different source. A source that grants the server with no tool restriction means the user can use every tool on it, so allow-all wins the union. When no source grants the server the result is ``[]`` — deny all, fail closed.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -2620,8 +2620,8 @@ class MCPRequestHandler: async def personal_key_team_rpm_limits( auth: UserAPIKeyAuth, server: MCPServer, tool_name: str ) -> Mapping[str, Mapping[str, int]] | None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy.auth.auth_checks import get_team_object + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache try: @@ -2672,7 +2672,7 @@ class MCPRequestHandler: then carries the key's OWN tool rules, never a team's; otherwise nothing is allowed. Each team resolves through the raising ``_single_credential_tools`` so one team's fault reaches the caller's fail-closed handler instead of becoming an allow-all ``None`` in the union.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -2773,7 +2773,7 @@ class MCPRequestHandler: nothing (deleted or unknown ids, a swallowed DB fault, or a toolset with no tools): that is a KNOWN restriction with unknown contents, and every caller already turns this error into deny rather than letting the level read as unrestricted.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -2827,7 +2827,7 @@ class MCPRequestHandler: ``mcp_tool_permissions`` entry for it. A granting group with no entry grants every tool, so ``None`` wins. A group that does not grant the server contributes nothing, and an explicit ``[]`` entry on a granting group grants no tools. ``None`` when no group grants the server.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -2848,7 +2848,7 @@ class MCPRequestHandler: @staticmethod def _group_grants_server(group: LiteLLM_AccessGroupTable, server_id: str) -> bool: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -2892,7 +2892,7 @@ class MCPRequestHandler: """Servers the key's object permission grants on its own: ``mcp_servers`` by id, name or alias and legacy ``mcp_access_groups`` by the tag the server carries, exactly as server admission resolves them. A tag never doubles as a server name, so a tag that collides with one grants nothing by itself.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager if key_obj_perm is None: return frozenset() @@ -3054,7 +3054,7 @@ class MCPRequestHandler: return None if managed_agent_policy(user_api_key_auth) is not None: - from litellm.proxy._experimental.mcp_server.auth.managed_agent_access import managed_agent_tools + from litellm.proxy.mcp_server.auth.managed_agent_access import managed_agent_tools return await managed_agent_tools(server_id, user_api_key_auth) @@ -3111,7 +3111,7 @@ class MCPRequestHandler: # server_ids OR names/aliases; normalize to server_id-keyed form # before lookup so a name-based key does not silently drop its # tool restrictions when server_id is the resolved uuid. - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -3203,7 +3203,7 @@ class MCPRequestHandler: fail-open (skip the org step, keep the key/team/agent restrictions; letting the raise escape would collapse them to allow-all, WIDER than before the fault), while a keyless source re-raises so the outer handler denies that one source (its only org bound is this ceiling).""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -3317,12 +3317,12 @@ class MCPRequestHandler: if user_api_key_auth is None: return [] try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) from litellm.proxy.auth.auth_checks import ( get_mcp_server_ids_from_access_groups, ) + from litellm.proxy.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) from litellm.proxy.proxy_server import ( prisma_client, proxy_logging_obj, @@ -3374,12 +3374,12 @@ class MCPRequestHandler: if user_api_key_auth is None: return [] try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) from litellm.proxy.auth.auth_checks import ( get_object_permission, ) + from litellm.proxy.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) from litellm.proxy.proxy_server import ( prisma_client, proxy_logging_obj, @@ -3538,7 +3538,7 @@ class MCPRequestHandler: ``mcp_servers``, the ``all_proxy_servers`` sentinel → the full registry, legacy access groups, tool-perm-referenced servers, toolset-referenced servers) unioned with its unified ``access_group_ids`` servers.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -3750,7 +3750,7 @@ class MCPRequestHandler: if object_permissions is None: return [] - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -3855,7 +3855,7 @@ class MCPRequestHandler: try: # Permission entries may be server_ids OR names/aliases — expand to ids. - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -3990,7 +3990,7 @@ class MCPRequestHandler: ``mcp_tool_permissions`` or reached through ``mcp_toolsets`` count as entitled, exactly as they do for a key or a team, so granting one tool never requires naming its server twice. """ - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -4130,7 +4130,7 @@ class MCPRequestHandler: restriction. Returns ``[]`` (deny every tool on this server) when the entitlement cannot be resolved, because the caller's own except-handler treats a raise as allow-all for key auth. """ - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -4173,7 +4173,7 @@ class MCPRequestHandler: axis twin of ``apply_agent_caller_ceiling``, so the headers only ever narrow. Denies every tool on the server when the caller's team cannot be loaded, since a caller we cannot resolve must not read as unrestricted.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -4217,7 +4217,7 @@ class MCPRequestHandler: user_api_key_auth: UserAPIKeyAuth | None = None, ) -> Sequence[str] | None: """Narrow a key/team tool allowlist by the end user's (customer's) tool entitlement.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) from litellm.proxy.proxy_server import prisma_client @@ -4353,7 +4353,7 @@ class MCPRequestHandler: return [] try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -4390,7 +4390,7 @@ class MCPRequestHandler: allow, or None when the agent has none attached. Unlike the object_permission path above, an attached group set that names no servers is an empty ceiling and denies every server. """ - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -4469,7 +4469,7 @@ class MCPRequestHandler: ) try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -4545,7 +4545,7 @@ class MCPRequestHandler: try: # Import here to avoid circular import - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) diff --git a/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py b/litellm/proxy/mcp_server/bridge_token_flow.py similarity index 95% rename from litellm/proxy/_experimental/mcp_server/bridge_token_flow.py rename to litellm/proxy/mcp_server/bridge_token_flow.py index 9f393f25e49..63428dfb28a 100644 --- a/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py +++ b/litellm/proxy/mcp_server/bridge_token_flow.py @@ -13,24 +13,24 @@ from pydantic import SecretStr from typing_extensions import assert_never from litellm._logging import verbose_logger -from litellm.proxy._experimental.mcp_server.oauth_utils import TOKEN_NO_CACHE_HEADERS from litellm.proxy.common_utils.encrypt_decrypt_utils import ( # noqa: F401 # legacy module exports _V2_GCM_PREFIX, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export V2_GCM_PREFIX, # pyright: ignore[reportPrivateUsage] # reuse the encrypted credential's format discriminator ) +from litellm.proxy.mcp_server.oauth_utils import TOKEN_NO_CACHE_HEADERS from litellm.types.mcp_server.mcp_server_manager import MCPServer if TYPE_CHECKING: from litellm.models.user import LiteLLM_UserTable - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import _BridgeAuthorizationCode - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.handle_jwt import JWTIdentity + from litellm.proxy.mcp_server.discoverable_endpoints import _BridgeAuthorizationCode + from litellm.proxy.mcp_server.outbound_credentials.envelope import ( EnvelopeIdentity, EnvelopeKeys, RefreshCredential, UpstreamTokenGrant, ) - from litellm.proxy._types import UserAPIKeyAuth - from litellm.proxy.auth.handle_jwt import JWTIdentity def _litellm_key_from_request(request: Request) -> str | None: @@ -87,14 +87,14 @@ async def oauth_authorization_uses_gateway_credential(request: Request) -> bool: async def _opaque_bearer_is_gateway_credential(token: str) -> bool: - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # envelope imports bridge types - is_envelope, - is_refresh_envelope, - ) from litellm.proxy._types import hash_token # noqa: PLC0415 # proxy import cycle from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken # noqa: PLC0415 # proxy import cycle from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError # noqa: PLC0415 # proxy import cycle from litellm.proxy.auth.resolvers.store import IdentityStore # noqa: PLC0415 # proxy import cycle + from litellm.proxy.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # envelope imports bridge types + is_envelope, + is_refresh_envelope, + ) from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the identity store dependencies prisma_client, user_api_key_cache, @@ -447,16 +447,16 @@ async def _resolve_request_auth( async def can_store_oauth_credential(request: Request, auth: "UserAPIKeyAuth", server_id: str) -> bool: """Apply the same write policy to request credentials and verified signed-callback users.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # registry imports auth helpers - global_mcp_server_manager, - ) - from litellm.proxy._experimental.mcp_server.ui_session_utils import ( - can_access_mcp_server, # noqa: PLC0415 # proxy import cycle - ) from litellm.proxy.auth.route_checks import RouteChecks # noqa: PLC0415 # proxy import cycle from litellm.proxy.auth.user_api_key_auth import ( # noqa: PLC0415 # proxy import cycle run_centralized_common_checks, # pyright: ignore[reportPrivateUsage] # reuse admission policy for the credential-write action ) + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # registry imports auth helpers + global_mcp_server_manager, + ) + from litellm.proxy.mcp_server.ui_session_utils import ( + can_access_mcp_server, # noqa: PLC0415 # proxy import cycle + ) write_route: Final = f"/v1/mcp/server/{server_id}/oauth-user-credential" try: @@ -584,7 +584,7 @@ def _bridge_grant_from_token_response(token_response: object) -> "UpstreamTokenG lifetime leaves the grant ``expires_in`` ``None`` for the envelope fallback, a positive value is honoured, and an explicit already-elapsed value is a rejection rather than a silent fall-through to the cap.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # inline import avoids a module-load circular import + from litellm.proxy.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # inline import avoids a module-load circular import UpstreamTokenGrant, ) @@ -790,18 +790,18 @@ async def prepare_bridge_mint( presented key keeps its resolution origin so the mapper statuses it truthfully; neither source present is ``no_identity``. The refresh_token grant has its own phase-1 (:func:`_prepare_bridge_refresh`), which recovers identity from the presented refresh envelope.""" - from litellm.proxy._experimental.mcp_server.client_allowlist import ( # noqa: PLC0415 # keep mint policy dependencies local + from litellm.proxy._types import UserAPIKeyAuth # noqa: PLC0415 # proxy import cycle + from litellm.proxy.auth.handle_jwt import JWTHandler # noqa: PLC0415 # proxy import cycle + from litellm.proxy.mcp_server.client_allowlist import ( # noqa: PLC0415 # keep mint policy dependencies local load_mcp_client_allowlist, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( # noqa: PLC0415 # inline import avoids a module-load circular import + from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( # noqa: PLC0415 # inline import avoids a module-load circular import envelope_keys_from_master_key, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # inline import avoids a module-load circular import + from litellm.proxy.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # inline import avoids a module-load circular import key_hash_identity, user_identity, ) - from litellm.proxy._types import UserAPIKeyAuth # noqa: PLC0415 # proxy import cycle - from litellm.proxy.auth.handle_jwt import JWTHandler # noqa: PLC0415 # proxy import cycle from litellm.proxy.proxy_server import ( # noqa: PLC0415 # inline import avoids a module-load circular import general_settings, master_key, @@ -877,7 +877,7 @@ async def prepare_bridge_refresh( never a raw upstream refresh token, so a missing value, a non-envelope, an unopenable envelope, or one minted for another server is ``invalid_grant``. Running before the exchange means a rejected refresh never consumes or rotates the upstream refresh token.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( # noqa: PLC0415 # inline import avoids a module-load circular import + from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( # noqa: PLC0415 # inline import avoids a module-load circular import BridgeRefreshOpened, envelope_keys_from_master_key, open_bridge_refresh_envelope, @@ -917,10 +917,10 @@ def finish_bridge_mint( fresh refresh envelope. The only hard failures here are properties of the upstream access token (no usable token, an already-expired lifetime, or a token too large to seal); a refresh token that cannot be sealed degrades to an access-only response rather than failing the whole exchange.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( # noqa: PLC0415 # inline import avoids a module-load circular import + from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( # noqa: PLC0415 # inline import avoids a module-load circular import build_bridge_token_response, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # inline import avoids a module-load circular import + from litellm.proxy.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # inline import avoids a module-load circular import EnvelopeLifetimeUnrepresentable, SealedEnvelope, UpstreamTokenGrant, @@ -961,7 +961,7 @@ def _upstream_refresh_credential(token_response: object) -> "RefreshCredential | token would hand the client a full-TTL-capped envelope the IdP will reject, so the exchange degrades to an access-only response (the client re-authenticates at access expiry), mirroring how :func:`_bridge_grant_from_token_response` refuses an already-elapsed access token instead of capping it.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # inline import avoids a module-load circular import + from litellm.proxy.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # inline import avoids a module-load circular import RefreshCredential, ) @@ -988,10 +988,10 @@ def _mint_refresh_envelope_value( ``None`` when the upstream returned no refresh token or the refresh token is too large to seal. A too-large refresh token degrades to an access-only response (logged) rather than failing an exchange that already succeeded upstream: the client simply re-authenticates when the access envelope expires.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( # noqa: PLC0415 # inline import avoids a module-load circular import + from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( # noqa: PLC0415 # inline import avoids a module-load circular import build_bridge_refresh_token_response, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # inline import avoids a module-load circular import + from litellm.proxy.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # inline import avoids a module-load circular import SealedEnvelope, ) diff --git a/litellm/proxy/_experimental/mcp_server/byok_credential_cache.py b/litellm/proxy/mcp_server/byok_credential_cache.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/byok_credential_cache.py rename to litellm/proxy/mcp_server/byok_credential_cache.py diff --git a/litellm/proxy/_experimental/mcp_server/byok_oauth_endpoints.py b/litellm/proxy/mcp_server/byok_oauth_endpoints.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/byok_oauth_endpoints.py rename to litellm/proxy/mcp_server/byok_oauth_endpoints.py index 2181e92baa9..868b73e39f1 100644 --- a/litellm/proxy/_experimental/mcp_server/byok_oauth_endpoints.py +++ b/litellm/proxy/mcp_server/byok_oauth_endpoints.py @@ -25,16 +25,16 @@ from fastapi import APIRouter, Depends, Form, HTTPException, Request from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse from litellm._logging import verbose_proxy_logger -from litellm.proxy._experimental.mcp_server.catalog import public_catalog_operation -from litellm.proxy._experimental.mcp_server.db import store_user_credential -from litellm.proxy._experimental.mcp_server.oauth_utils import ( +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.mcp_server.catalog import public_catalog_operation +from litellm.proxy.mcp_server.db import store_user_credential +from litellm.proxy.mcp_server.oauth_utils import ( BYOK_RESOURCE_METADATA_PATH, TOKEN_NO_CACHE_HEADERS, get_request_base_url, validate_loopback_redirect_uri, well_known_root_suffix, ) -from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.middleware.per_request_root_path_middleware import get_server_root_paths # --------------------------------------------------------------------------- @@ -779,7 +779,7 @@ async def byok_authorize_get( help_url = "" if server_id: try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -955,7 +955,7 @@ async def byok_token( ) # Invalidate any cached negative result so the user isn't blocked # for up to the TTL period after completing the OAuth flow. - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _invalidate_byok_cred_cache, ) diff --git a/litellm/proxy/_experimental/mcp_server/capabilities.py b/litellm/proxy/mcp_server/capabilities.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/capabilities.py rename to litellm/proxy/mcp_server/capabilities.py diff --git a/litellm/proxy/_experimental/mcp_server/catalog.py b/litellm/proxy/mcp_server/catalog.py similarity index 93% rename from litellm/proxy/_experimental/mcp_server/catalog.py rename to litellm/proxy/mcp_server/catalog.py index 2db548f5863..cfa620c978e 100644 --- a/litellm/proxy/_experimental/mcp_server/catalog.py +++ b/litellm/proxy/mcp_server/catalog.py @@ -23,15 +23,15 @@ from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter from litellm._logging import verbose_logger from litellm.caching.in_memory_cache import InMemoryCache -from litellm.proxy._experimental.mcp_server.result_conversion import age_freshness, aggregate_freshness +from litellm.proxy.mcp_server.result_conversion import age_freshness, aggregate_freshness if TYPE_CHECKING: from mcp.types import ListToolsResult, PaginatedRequestParams, PaginatedResult - from litellm.proxy._experimental.mcp_server.contracts import CatalogListRequest, CatalogListResult, OperationContext - from litellm.proxy._experimental.mcp_server.db import OAuthCredentialPayload - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing, ServerOutcome - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.contracts import CatalogListRequest, CatalogListResult, OperationContext + from litellm.proxy.mcp_server.db import OAuthCredentialPayload + from litellm.proxy.mcp_server.faults.list_outcomes import AggregateToolListing, ServerOutcome + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager from litellm.types.mcp_server.mcp_server_manager import MCPServer from litellm.types.mcp_server.tool_registry import MCPTool @@ -90,7 +90,7 @@ def _check_oauth_revision(selected: MCPServer, candidate: MCPServer | None) -> N def _snapshot(manager: MCPServerManager, database_identity: str) -> CatalogSnapshot: - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry servers: Final = manager.config_mcp_servers | manager.registry detached: Final = MappingProxyType({key: value.model_copy(deep=True) for key, value in servers.items()}) @@ -154,7 +154,7 @@ class CatalogSnapshots: if not should_load_db_object("mcp"): return _snapshot(self.manager, self._database_identity) - from litellm.proxy._experimental.mcp_server.db import get_mcp_catalog_revision + from litellm.proxy.mcp_server.db import get_mcp_catalog_revision try: revision: Final = await get_mcp_catalog_revision(prisma_client) @@ -198,7 +198,7 @@ class CatalogSnapshots: if current is not None: yield current return - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry shared: Final = await self._fresh_snapshot() initial_routing: Final = self._unchanged_routing(shared.servers, self.manager.published_tool_routes) @@ -227,7 +227,7 @@ class CatalogSnapshots: def _unchanged_routing( self, servers: Mapping[str, MCPServer], routing: Mapping[str, str] ) -> MappingProxyType[str, str]: - from litellm.proxy._experimental.mcp_server.utils import normalize_server_name + from litellm.proxy.mcp_server.utils import normalize_server_name current: Final = self.manager.config_mcp_servers | self.manager.registry unchanged: Final = ( @@ -252,7 +252,7 @@ class CatalogSnapshots: server: MCPServer, resolve: Callable[[MCPServer], Awaitable[MCPServer]], ) -> MCPServer: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import oauth_endpoints_unresolved + from litellm.proxy.mcp_server.mcp_server_manager import oauth_endpoints_unresolved snapshot: Final = self.current() selected: Final = snapshot.servers.get(server.server_id) if snapshot is not None else None @@ -270,7 +270,7 @@ class CatalogSnapshots: return resolved async def reload(self) -> None: - from litellm.proxy._experimental.mcp_server.db import get_mcp_catalog_revision + from litellm.proxy.mcp_server.db import get_mcp_catalog_revision from litellm.proxy.proxy_server import prisma_client revision: Final = await get_mcp_catalog_revision(prisma_client) if prisma_client is not None else None @@ -294,8 +294,8 @@ class CatalogSnapshots: self._operation.reset(token) async def _reload_and_publish(self, *, reuse_unchanged: bool) -> None: - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry - from litellm.proxy._experimental.mcp_server.utils import normalize_server_name + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.utils import normalize_server_name previous_config: Final = MappingProxyType( {key: value.model_copy(deep=True) for key, value in self.manager.config_mcp_servers.items()} @@ -399,8 +399,8 @@ class CatalogSnapshots: async def _stage_servers( self, rows: Sequence[BaseModel], *, reuse_unchanged: bool ) -> dict[str, MCPServer]: # mutable-ok: assign_unique_short_prefix requires a dict registry - from litellm.proxy._experimental.mcp_server.db import LiteLLM_MCPServerTable - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.db import LiteLLM_MCPServerTable + from litellm.proxy.mcp_server.mcp_server_manager import ( carry_forward_resolved_oauth_endpoints, oauth_endpoints_unresolved, warn_on_server_name_fields, @@ -464,20 +464,20 @@ class CatalogSnapshots: return new_registry async def _reload(self, *, reuse_unchanged: bool) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - config_ids_capturing_db_identifiers, - warn_on_shared_identifier_prefixes, - ) from litellm.proxy.management_endpoints.mcp_management_endpoints import ( get_prisma_client_or_throw, ) + from litellm.proxy.mcp_server.mcp_server_manager import ( + config_ids_capturing_db_identifiers, + warn_on_shared_identifier_prefixes, + ) verbose_logger.debug("Loading MCP servers from database into registry...") prisma_client: Final = get_prisma_client_or_throw("Database not connected. Connect a database to your proxy") # Load only "active", legacy "approved", and NULL (no approval workflow) rows. # Pending/rejected servers are excluded at the DB level so we never load them. - from litellm.proxy._experimental.mcp_server.db import get_runtime_mcp_server_rows + from litellm.proxy.mcp_server.db import get_runtime_mcp_server_rows raw_rows: Final[Sequence[BaseModel]] = await get_runtime_mcp_server_rows(prisma_client) database_identity: Final = hashlib.sha256( @@ -584,7 +584,7 @@ def catalog_operation( def global_manager() -> MCPServerManager: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager return global_mcp_server_manager @@ -608,8 +608,8 @@ async def paginate_catalog( from pydantic import ValidationError from litellm.constants import MCP_TOOL_LISTING_MAX_PAGES - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error - from litellm.proxy._experimental.mcp_server.state_tokens import open_state, seal_state + from litellm.proxy.mcp_server.outbound_credentials.result import Error + from litellm.proxy.mcp_server.state_tokens import open_state, seal_state ordered: Final = tuple(sorted(server_ids)) purpose: Final = "mcp.catalog.list.v1:" + method @@ -673,7 +673,7 @@ async def paginate_catalog( for task in tasks: task.cancel() await asyncio.gather(*tasks, return_exceptions=True) - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import SERVER_OUTCOMES_META_KEY + from litellm.proxy.mcp_server.faults.list_outcomes import SERVER_OUTCOMES_META_KEY elapsed: Final = time.monotonic() - started pages: Final = tuple( @@ -719,7 +719,7 @@ async def list_tools_page( ) -> ListToolsResult: from mcp.types import ListToolsResult - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import SERVER_OUTCOMES_META_KEY + from litellm.proxy.mcp_server.faults.list_outcomes import SERVER_OUTCOMES_META_KEY pages, next_cursor, outcomes = await paginate_catalog( method="tools/list", @@ -773,9 +773,9 @@ async def get_filtered_server_tools( ) -> tuple[ListToolsResult, ServerOutcome]: from mcp.types import ListToolsResult - from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ServerListOk, classify_list_exception - from litellm.proxy._experimental.mcp_server.operations import ( + from litellm.proxy.mcp_server.exceptions import MCPUpstreamAuthError + from litellm.proxy.mcp_server.faults.list_outcomes import ServerListOk, classify_list_exception + from litellm.proxy.mcp_server.operations import ( _get_byok_credential, _get_user_oauth_extra_headers_from_db, _prepare_mcp_server_headers, @@ -806,7 +806,7 @@ async def get_filtered_server_tools( # Prefer server-stored per-user OAuth when configured, so a stale # Authorization header from the MCP client cannot override Redis/DB # (same issue as call_tool in mcp_server_manager: VS Code caches tokens). - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.outbound_credentials.adapter import ( # noqa: PLC0415 to_server_spec, ) @@ -885,8 +885,8 @@ async def get_filtered_server_tools( user_api_key_auth=user_api_key_auth, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ListedToolsCaller - from litellm.proxy._experimental.mcp_server.utils import strip_known_server_prefix + from litellm.proxy.mcp_server.mcp_server_manager import ListedToolsCaller + from litellm.proxy.mcp_server.utils import strip_known_server_prefix record: Final = partial( global_mcp_server_manager.record_listed_tools, @@ -908,7 +908,7 @@ async def get_filtered_server_tools( listing_updates.callback(record) if mcp_proxy_mode: - from litellm.proxy._experimental.mcp_server.tool_search import with_mcp_proxy_identity + from litellm.proxy.mcp_server.tool_search import with_mcp_proxy_identity filtered_tools = [with_mcp_proxy_identity(tool, server.server_id) for tool in filtered_tools] else: @@ -936,7 +936,7 @@ async def get_filtered_server_tools( def _caller_scope(context: OperationContext, servers: Sequence[MCPServer]) -> str: - from litellm.proxy._experimental.mcp_server.utils import upstream_credential_headers + from litellm.proxy.mcp_server.utils import upstream_credential_headers caller: Final = context.user_api_key_auth headers: Final = context.raw_headers or {} @@ -975,18 +975,18 @@ async def aggregate_gateway_tools( from mcp.types import ListToolsResult, PaginatedRequestParams from pydantic import TypeAdapter - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + from litellm.proxy.mcp_server.faults.list_outcomes import ( SERVER_OUTCOMES_META_KEY, AggregateToolListing, ServerOutcome, classify_list_exception, ) - from litellm.proxy._experimental.mcp_server.operations import ( + from litellm.proxy.mcp_server.operations import ( _aggregate_server_key, _mcp_server_rate_limit_rejection, global_mcp_server_manager, ) - from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError async with global_mcp_server_manager.catalog.operation() as snapshot: servers: Final = {server.server_id: server for server in allowed} @@ -1065,12 +1065,12 @@ async def list_gateway_tools( ) -> ListToolsResult: from mcp.types import INTERNAL_ERROR, ListToolsResult - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( + from litellm.proxy.mcp_server.faults.list_outcomes import ( SERVER_OUTCOMES_META_KEY, listing_failed_outright, outcome_wire_value, ) - from litellm.proxy._experimental.mcp_server.operations import _list_mcp_tools + from litellm.proxy.mcp_server.operations import _list_mcp_tools caller, auth, servers, server_headers, oauth_headers, headers, client_ip = context.legacy_auth() listing: Final = await _list_mcp_tools( @@ -1115,13 +1115,13 @@ async def list_gateway_catalog( PaginatedRequestParams, ) - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler - from litellm.proxy._experimental.mcp_server.operations import ( + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + from litellm.proxy.mcp_server.operations import ( _get_allowed_mcp_servers, global_mcp_server_manager, raise_denied_scoped_mcp_access, ) - from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError context = replace(context, _caller=await MCPRequestHandler.refresh_catalog_authority(context.user_api_key_auth)) params: Final = request.params or PaginatedRequestParams() @@ -1146,11 +1146,11 @@ async def list_gateway_catalog( from mcp.shared.exceptions import MCPError from mcp.types import INVALID_PARAMS - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( + from litellm.proxy.mcp_server.faults.list_outcomes import ( SERVER_OUTCOMES_META_KEY, classify_list_exception, ) - from litellm.proxy._experimental.mcp_server.operations import ( + from litellm.proxy.mcp_server.operations import ( _aggregate_server_key, _mcp_server_rate_limit_rejection, ) @@ -1204,7 +1204,7 @@ async def list_gateway_catalog( ) if params.cursor is None and servers and len(rejections) == len(servers): raise rejections[0] - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( + from litellm.proxy.mcp_server.faults.list_outcomes import ( SERVER_OUTCOMES_META_KEY, ServerOutcome, outcome_wire_value, @@ -1232,7 +1232,7 @@ async def fetch_optional_catalog_page( ) -> CatalogListResult: from mcp.types import PaginatedRequestParams - from litellm.proxy._experimental.mcp_server.operations import _prepare_mcp_server_headers, global_mcp_server_manager + from litellm.proxy.mcp_server.operations import _prepare_mcp_server_headers, global_mcp_server_manager caller, auth, _, server_headers, oauth_headers, raw_headers, client_ip = context.legacy_auth() auth_header, extra_headers = _prepare_mcp_server_headers( @@ -1269,7 +1269,7 @@ def combine_optional_catalog( ListResourceTemplatesResult, ) - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import SERVER_OUTCOMES_META_KEY + from litellm.proxy.mcp_server.faults.list_outcomes import SERVER_OUTCOMES_META_KEY outcomes: Final = (meta or {}).get(SERVER_OUTCOMES_META_KEY) incomplete: Final = isinstance(outcomes, dict) and any( diff --git a/litellm/proxy/_experimental/mcp_server/client_allowlist.py b/litellm/proxy/mcp_server/client_allowlist.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/client_allowlist.py rename to litellm/proxy/mcp_server/client_allowlist.py diff --git a/litellm/proxy/_experimental/mcp_server/contracts.py b/litellm/proxy/mcp_server/contracts.py similarity index 96% rename from litellm/proxy/_experimental/mcp_server/contracts.py rename to litellm/proxy/mcp_server/contracts.py index 84cd2751444..43c42d3b0df 100644 --- a/litellm/proxy/_experimental/mcp_server/contracts.py +++ b/litellm/proxy/mcp_server/contracts.py @@ -10,8 +10,8 @@ from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias from mcp.types import ErrorData, InputRequest, InputResponse, InputResponses from pydantic import SecretStr -from litellm.proxy._experimental.mcp_server.tool_outcome import WireCompat from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.mcp_server.tool_outcome import WireCompat from litellm.types.mcp_server.mcp_server_manager import MCPServer if TYPE_CHECKING: @@ -33,7 +33,7 @@ if TYPE_CHECKING: ListToolsResult | ListPromptsResult | ListResourcesResult | ListResourceTemplatesResult ) - from litellm.proxy._experimental.mcp_server.server_resolution import ResolvedMCPServer + from litellm.proxy.mcp_server.server_resolution import ResolvedMCPServer class TargetCatalog(Protocol): diff --git a/litellm/proxy/_experimental/mcp_server/cost_calculator.py b/litellm/proxy/mcp_server/cost_calculator.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/cost_calculator.py rename to litellm/proxy/mcp_server/cost_calculator.py diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/mcp_server/db.py similarity index 99% rename from litellm/proxy/_experimental/mcp_server/db.py rename to litellm/proxy/mcp_server/db.py index 0666cc7e6c0..27f8b90a586 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/mcp_server/db.py @@ -15,14 +15,6 @@ from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.constants import MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS from litellm.llms.custom_httpx.http_handler import get_async_httpx_client -from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( - RefreshTokenPresented, - credential_binding_matches, - enforce_oauth_identity_binding, -) -from litellm.proxy._experimental.mcp_server.oauth_utils import ( - build_upstream_oauth2_token_request, -) from litellm.proxy._types import ( LiteLLM_MCPServerTable, MCPApprovalStatus, @@ -42,6 +34,14 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( # noqa: F401 # encrypt_value_helper, get_salt_key, ) +from litellm.proxy.mcp_server.oauth_identity_binding import ( + RefreshTokenPresented, + credential_binding_matches, + enforce_oauth_identity_binding, +) +from litellm.proxy.mcp_server.oauth_utils import ( + build_upstream_oauth2_token_request, +) from litellm.proxy.utils import PrismaClient from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.object_permission_repository import ObjectPermissionRepository @@ -1066,7 +1066,7 @@ async def delete_mcp_server( ) if credential_user_ids: if invalidate_token_cache is None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -2031,7 +2031,7 @@ async def purge_user_oauth_credentials_for_server( where={"server_id": server_id, "user_id": {"in": [row.user_id for row in oauth_rows]}} ) if invalidate_token_cache is None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -2238,7 +2238,7 @@ async def resolve_user_oauth_access_token( if not user_id or not server_id: return None try: - from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( + from litellm.proxy.mcp_server.oauth2_token_cache import ( compute_per_user_token_ttl, mcp_per_user_token_cache, ) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/mcp_server/discoverable_endpoints.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py rename to litellm/proxy/mcp_server/discoverable_endpoints.py index ca7ad9e3384..eb12b15cdae 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/mcp_server/discoverable_endpoints.py @@ -22,11 +22,21 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( +from litellm.proxy.auth.ip_address_utils import IPAddressUtils +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + decrypt_value_helper, + encrypt_value_helper, +) +from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports + _read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + read_request_body, +) +from litellm.proxy.mcp_server.auth.token_endpoint_auth import ( TokenEndpointAuthConfigError, normalize_token_endpoint_auth_method, ) -from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( # noqa: F401 # legacy module exports +from litellm.proxy.mcp_server.bridge_token_flow import ( # noqa: F401 # legacy module exports _bridge_mint_error_response, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _BridgeMintReady, _BridgeRefreshReady, @@ -45,8 +55,8 @@ from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( # noqa: prepare_bridge_refresh, reload_active_user_by_id, ) -from litellm.proxy._experimental.mcp_server.catalog import public_catalog_operation -from litellm.proxy._experimental.mcp_server.faults import ( +from litellm.proxy.mcp_server.catalog import public_catalog_operation +from litellm.proxy.mcp_server.faults import ( CallerRejected, CredentialSource, UpstreamProtocolFault, @@ -55,7 +65,7 @@ from litellm.proxy._experimental.mcp_server.faults import ( dcr_fault_detail, render_token_fault, ) -from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ( +from litellm.proxy.mcp_server.gateway_dcr_flow import ( JWT_BEARER_GRANT_TYPE, VendorCredentialState, aggregate_authorize, @@ -72,21 +82,21 @@ from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ( revoke_refresh_token, supported_grant_types, ) -from litellm.proxy._experimental.mcp_server.id_jag_grant import ( +from litellm.proxy.mcp_server.id_jag_grant import ( basic_client_id, jwt_bearer_available, redeem_id_jag_assertion, ) -from litellm.proxy._experimental.mcp_server.idp_token_exchange import ( +from litellm.proxy.mcp_server.idp_token_exchange import ( exchange_idp_subject_token, token_exchange_available, ) -from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( +from litellm.proxy.mcp_server.oauth_identity_binding import ( RefreshOwnershipProven, RefreshTokenPresented, enforce_oauth_identity_binding, ) -from litellm.proxy._experimental.mcp_server.oauth_utils import ( +from litellm.proxy.mcp_server.oauth_utils import ( CIMD_METADATA_PATH, TOKEN_NO_CACHE_HEADERS, build_upstream_authorize_url, @@ -101,27 +111,17 @@ from litellm.proxy._experimental.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, well_known_root_suffix, ) -from litellm.proxy._experimental.mcp_server.proxy_api_credentials import ( +from litellm.proxy.mcp_server.proxy_api_credentials import ( lookup_consent_teams, mint_proxy_credential, ) -from litellm.proxy.auth.ip_address_utils import IPAddressUtils -from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.common_utils.encrypt_decrypt_utils import ( - decrypt_value_helper, - encrypt_value_helper, -) -from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports - _read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - read_request_body, -) from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.mcp import MCPAuth, MCPCredentials from litellm.types.mcp_server.mcp_server_manager import MCPServer, MCPTokenEndpointAuthMethod if TYPE_CHECKING: - from litellm.proxy._experimental.mcp_server.db import OAuthCredentialPayload from litellm.proxy._types import LiteLLM_MCPServerTable + from litellm.proxy.mcp_server.db import OAuthCredentialPayload # TTL cache for upstream OAuth metadata fetched from pass-through MCP servers. # Keeps us from hammering the upstream IdP on each discovery request. @@ -476,7 +476,7 @@ async def _session_cookie_user_id(request: Request) -> str | None: """The signed-in litellm user for a browser request, or ``None``. Thin wrapper so the aggregate DCR flow's verbs receive the identity as a plain value instead of parsing cookies themselves.""" - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( # noqa: PLC0415 # circular import at module load + from litellm.proxy.mcp_server.byok_oauth_endpoints import ( # noqa: PLC0415 # circular import at module load get_authenticated_browser_user_id, ) @@ -578,7 +578,7 @@ def _append_query_params(url: str, params: dict[str, str]) -> str: def _resolve_mcp_server_by_name_or_id(lookup: str, client_ip: str | None) -> MCPServer | None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -598,7 +598,7 @@ def _resolve_oauth2_server_for_root_endpoints( without a server name prefix, we try to find the right server automatically. Returns the server if exactly one OAuth2 server is configured, else None. """ - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -684,7 +684,7 @@ async def _store_per_user_token_server_side( Errors are logged but NOT re-raised — the token is always returned to the client even when server-side storage fails. """ - from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.oauth2_token_cache import ( # noqa: PLC0415 compute_per_user_token_ttl, mcp_per_user_token_cache, ) @@ -706,7 +706,7 @@ async def _store_per_user_token_server_side( try: prisma_client: Final = get_prisma_client_or_throw("Database not connected. Cannot store per-user OAuth token.") - from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.db import ( # noqa: PLC0415 store_user_oauth_credential, ) @@ -735,7 +735,7 @@ async def _store_per_user_token_server_side( ) return # Don't warm Redis if DB write failed - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 global_mcp_server_manager, ) @@ -762,7 +762,7 @@ def raise_if_not_oauth2(mcp_server: MCPServer) -> None: modes (``_persist_dcr_client_registration`` skips them unconditionally, so even the admin Authorize path with ``replace_stale_client`` enabled writes nothing to the server row). """ - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # circular import with mcp_server_manager at module load + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # circular import with mcp_server_manager at module load UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES, ) @@ -825,7 +825,7 @@ async def _server_with_oauth_endpoints( """ if needed_endpoint(mcp_server) is not None and not needs_cimd_discovery(mcp_server): return mcp_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # circular import with mcp_server_manager at module load + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # circular import with mcp_server_manager at module load global_mcp_server_manager, ) @@ -972,10 +972,10 @@ def _bridge_access_denied_redirect(redirect_uri: str, state: str, mcp_server: MC async def _user_can_reach_mcp_server(user_id: str, server_id: str) -> bool: - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -1013,7 +1013,7 @@ async def _resolve_oauth_authorization_user( async def _managed_authorization_scope( request: Request, server: MCPServer, scope: str | None, authorized_user_id: str | None ) -> tuple[str | None, str | None]: - from litellm.proxy._experimental.mcp_server.db import get_user_oauth_credential + from litellm.proxy.mcp_server.db import get_user_oauth_credential from litellm.proxy.proxy_server import prisma_client gateway_credential: Final = authorized_user_id is None and await oauth_authorization_uses_gateway_credential( @@ -1214,7 +1214,7 @@ async def _saved_refresh_credential( if not user_id or not refresh_token or not server.needs_user_oauth_token or server.auth_type != MCPAuth.oauth2: return None from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.db import get_user_oauth_credential + from litellm.proxy.mcp_server.db import get_user_oauth_credential if proxy_server.prisma_client is None: return None @@ -1524,7 +1524,7 @@ async def exchange_token_with_server( try: # Identity binding above must retain the verified caller even when a write is # denied. Authorize persistence separately, immediately before its side effect. - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler # A sealed code delegates a verified user for this authorized server. Raw # request credentials retain their own JWT/key restrictions during resolution. @@ -1683,7 +1683,7 @@ async def _load_store_dcr_credentials( (which has no LiteLLM_MCPServerTable row). Returns None when the store has no usable client_id or the DB is unreachable. Registration writes request strict reads so a failed lookup cannot be mistaken for an absent client.""" - from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 # avoids circular import + from litellm.proxy.mcp_server.db import ( # noqa: PLC0415 # avoids circular import get_mcp_server_oauth_client_credentials, ) from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 # avoids circular import @@ -1737,8 +1737,8 @@ async def _resolve_persisted_dcr_client( the server-scoped store, while a rowless non-config server is a throwaway temp/session server with no persisted client. Returns (row_or_None, credentials_or_None); the row is only needed by the reuse path to refresh the registry for a DB-declared server.""" - from litellm.proxy._experimental.mcp_server.db import get_mcp_server # noqa: PLC0415 # avoids circular import - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # avoids circular import + from litellm.proxy.mcp_server.db import get_mcp_server # noqa: PLC0415 # avoids circular import + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # avoids circular import global_mcp_server_manager, ) from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 # avoids circular import @@ -1793,7 +1793,7 @@ async def _reuse_persisted_dcr_client_if_available( async def _refresh_persisted_dcr_server(persisted_server: Optional["LiteLLM_MCPServerTable"]) -> None: if persisted_server is not None and persisted_server.approval_status != "draft": - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # avoids circular import + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # avoids circular import global_mcp_server_manager, ) @@ -1910,16 +1910,16 @@ async def _persist_dcr_client_registration( "dcr_server_url": mcp_server.url, } - from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 # avoids circular import + from litellm.proxy._types import UpdateMCPServerRequest # noqa: PLC0415 + from litellm.proxy.mcp_server.db import ( # noqa: PLC0415 # avoids circular import McpIdentifierConflict, create_mcp_server_oauth_client_credentials_if_absent, update_mcp_server, upsert_mcp_server_oauth_client_credentials, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 global_mcp_server_manager, ) - from litellm.proxy._types import UpdateMCPServerRequest # noqa: PLC0415 from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 try: @@ -2403,7 +2403,7 @@ async def authorize( scope=scope, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager lookup_name: Final[str | None] = mcp_server_name or client_id client_ip: Final = IPAddressUtils.get_mcp_client_ip(request) @@ -2532,7 +2532,7 @@ async def _vendor_credential_state( The one reading of "authorized" the connect page displays and the finish step enforces, so the button a user sees and the grant they get cannot disagree. A read fault is neither, and fails the scoped grant closed.""" - from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 # circular import at module load + from litellm.proxy.mcp_server.db import ( # noqa: PLC0415 # circular import at module load get_user_oauth_credential, oauth_grant_state, ) @@ -3184,7 +3184,7 @@ def _aggregate_header_scopes(request: Request) -> tuple[str, ...]: discovery request, so a client that skips the 401 challenge (Claude Code re-authenticating a server it already knows needs auth) still carries the selection to ``/authorize`` through ``scopes_supported``. Empty when the header is absent or any entry does not resolve.""" - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( # noqa: PLC0415 # proxy import cycle + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( # noqa: PLC0415 # proxy import cycle MCPRequestHandler, gateway_dcr_challenge_scope, ) @@ -3489,7 +3489,7 @@ async def register_client(request: Request, mcp_server_name: str | None = None): request=request, request_body=data, token_exchange_available=token_exchange_available() ) client_application_type: Final = client_supplied_application_type(data.get("application_type")) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager async with global_mcp_server_manager.catalog.operation(): dummy_return: Final = { diff --git a/litellm/proxy/_experimental/mcp_server/elicitation_handler.py b/litellm/proxy/mcp_server/elicitation_handler.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/elicitation_handler.py rename to litellm/proxy/mcp_server/elicitation_handler.py diff --git a/litellm/proxy/_experimental/mcp_server/exceptions.py b/litellm/proxy/mcp_server/exceptions.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/exceptions.py rename to litellm/proxy/mcp_server/exceptions.py index ca2f04bb96a..8be3ae78d15 100644 --- a/litellm/proxy/_experimental/mcp_server/exceptions.py +++ b/litellm/proxy/mcp_server/exceptions.py @@ -4,7 +4,7 @@ from typing import Final from fastapi import HTTPException -from litellm.proxy._experimental.mcp_server.oauth_utils import encode_upstream_scope +from litellm.proxy.mcp_server.oauth_utils import encode_upstream_scope TOKEN_EXCHANGE_SUBJECT_ERROR: Final = ( "Missing or invalid subject token. Authenticate with your identity provider and send its token " diff --git a/litellm/proxy/_experimental/mcp_server/faults/__init__.py b/litellm/proxy/mcp_server/faults/__init__.py similarity index 81% rename from litellm/proxy/_experimental/mcp_server/faults/__init__.py rename to litellm/proxy/mcp_server/faults/__init__.py index de7ee5c866a..bcaa6314869 100644 --- a/litellm/proxy/_experimental/mcp_server/faults/__init__.py +++ b/litellm/proxy/mcp_server/faults/__init__.py @@ -7,16 +7,16 @@ a server-fault status) unrepresentable, and gives the trust-boundary rule one en spec-defined machine fields may cross to callers, upstream prose and raw bodies go to server logs. """ -from litellm.proxy._experimental.mcp_server.faults.classify import ( +from litellm.proxy.mcp_server.faults.classify import ( classify_upstream_dcr_rejection, classify_upstream_token_rejection, ) -from litellm.proxy._experimental.mcp_server.faults.render_oauth import ( +from litellm.proxy.mcp_server.faults.render_oauth import ( dcr_fault_detail, render_token_fault, ) -from litellm.proxy._experimental.mcp_server.faults.traversal import iter_exception_tree -from litellm.proxy._experimental.mcp_server.faults.types import ( +from litellm.proxy.mcp_server.faults.traversal import iter_exception_tree +from litellm.proxy.mcp_server.faults.types import ( CallerRejected, CredentialSource, GatewayRejected, diff --git a/litellm/proxy/_experimental/mcp_server/faults/classify.py b/litellm/proxy/mcp_server/faults/classify.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/faults/classify.py rename to litellm/proxy/mcp_server/faults/classify.py index 7826fc34eac..b5e5cbcd27d 100644 --- a/litellm/proxy/_experimental/mcp_server/faults/classify.py +++ b/litellm/proxy/mcp_server/faults/classify.py @@ -13,7 +13,7 @@ import httpx from pydantic import TypeAdapter from litellm._logging import verbose_logger -from litellm.proxy._experimental.mcp_server.faults.types import ( +from litellm.proxy.mcp_server.faults.types import ( GATEWAY_CAPABILITY_CODES, GATEWAY_CREDENTIAL_CODES, MAX_WIRE_FIELD_CHARS, diff --git a/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py b/litellm/proxy/mcp_server/faults/list_outcomes.py similarity index 99% rename from litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py rename to litellm/proxy/mcp_server/faults/list_outcomes.py index e148e28e826..b8af3ead8fb 100644 --- a/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py +++ b/litellm/proxy/mcp_server/faults/list_outcomes.py @@ -33,12 +33,12 @@ from litellm.constants import ( MCP_UPSTREAM_RATE_LIMIT_MAX_DELAY_SECONDS, MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES, ) -from litellm.proxy._experimental.mcp_server.exceptions import ( +from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError +from litellm.proxy.mcp_server.exceptions import ( MCPServerListError, MCPUpstreamAuthError, ) -from litellm.proxy._experimental.mcp_server.faults.traversal import iter_exception_tree -from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError +from litellm.proxy.mcp_server.faults.traversal import iter_exception_tree from litellm.types.llms.base import LiteLLMBaseModel ListFaultCategory: TypeAlias = Literal[ diff --git a/litellm/proxy/_experimental/mcp_server/faults/render_oauth.py b/litellm/proxy/mcp_server/faults/render_oauth.py similarity index 96% rename from litellm/proxy/_experimental/mcp_server/faults/render_oauth.py rename to litellm/proxy/mcp_server/faults/render_oauth.py index 3ecf5310482..92fa457e476 100644 --- a/litellm/proxy/_experimental/mcp_server/faults/render_oauth.py +++ b/litellm/proxy/mcp_server/faults/render_oauth.py @@ -11,8 +11,8 @@ from typing import Final from fastapi.responses import JSONResponse from typing_extensions import assert_never -from litellm.proxy._experimental.mcp_server.faults.types import CallerRejected, UpstreamOAuthFault -from litellm.proxy._experimental.mcp_server.oauth_utils import TOKEN_NO_CACHE_HEADERS +from litellm.proxy.mcp_server.faults.types import CallerRejected, UpstreamOAuthFault +from litellm.proxy.mcp_server.oauth_utils import TOKEN_NO_CACHE_HEADERS def _gateway_rejected_description(code: str) -> str: diff --git a/litellm/proxy/_experimental/mcp_server/faults/traversal.py b/litellm/proxy/mcp_server/faults/traversal.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/faults/traversal.py rename to litellm/proxy/mcp_server/faults/traversal.py diff --git a/litellm/proxy/_experimental/mcp_server/faults/types.py b/litellm/proxy/mcp_server/faults/types.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/faults/types.py rename to litellm/proxy/mcp_server/faults/types.py diff --git a/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py b/litellm/proxy/mcp_server/gateway_dcr_flow.py similarity index 99% rename from litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py rename to litellm/proxy/mcp_server/gateway_dcr_flow.py index 70e4bac01b5..ae74d878b36 100644 --- a/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py +++ b/litellm/proxy/mcp_server/gateway_dcr_flow.py @@ -58,8 +58,15 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict, assert_never from litellm._internal_context import with_service_target from litellm._logging import verbose_logger from litellm.caching.caching import DualCache -from litellm.proxy._experimental.mcp_server.catalog import catalog_operation, global_manager -from litellm.proxy._experimental.mcp_server.oauth_utils import ( +from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + decrypt_value_helper, + encrypt_value_helper, +) +from litellm.proxy.common_utils.html_forms.native_client_consent import ( + render_native_client_consent_page, +) +from litellm.proxy.mcp_server.catalog import catalog_operation, global_manager +from litellm.proxy.mcp_server.oauth_utils import ( TOKEN_NO_CACHE_HEADERS, UPSTREAM_SCOPE_PREFIX, UpstreamScope, @@ -70,13 +77,13 @@ from litellm.proxy._experimental.mcp_server.oauth_utils import ( is_loopback_redirect_host, validate_redirect_uri_shape, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.session_credentials import ( +from litellm.proxy.mcp_server.outbound_credentials.session_credentials import ( SessionRefreshOpened, SessionSigningConfigError, active_session_signing_keys, open_session_refresh_bearer, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import ( +from litellm.proxy.mcp_server.outbound_credentials.session_token import ( SESSION_ISSUER, SESSION_REFRESH_TTL_SECONDS, MintedSessionToken, @@ -91,13 +98,6 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token i open_session_refresh_token, open_session_token, ) -from litellm.proxy.common_utils.encrypt_decrypt_utils import ( - decrypt_value_helper, - encrypt_value_helper, -) -from litellm.proxy.common_utils.html_forms.native_client_consent import ( - render_native_client_consent_page, -) from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -624,7 +624,7 @@ def _is_aggregate_resource(request: Request, resource: str) -> bool: def _gateway_flow_server(name: str) -> MCPServer | None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # proxy import cycle + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # proxy import cycle global_mcp_server_manager, ) @@ -656,7 +656,7 @@ def resolve_scoped_resource_server(request: Request, resource: str | None) -> MC base: Final = canonicalize_url_identity(get_request_base_url(request)) if _is_aggregate_resource(request, resource) or not canonical.startswith(f"{base}/"): return None - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( # noqa: PLC0415 # proxy import cycle + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( # noqa: PLC0415 # proxy import cycle MCPRequestHandler, ) @@ -992,7 +992,7 @@ async def _flow_target( ) -> tuple[Literal["unscoped", "interactive", "m2m", "stale"], MCPServer | None]: if server_id is None: return "unscoped", None - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # import cycle + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # import cycle MCPServerManager, global_mcp_server_manager, ) diff --git a/litellm/proxy/_experimental/mcp_server/guardrail_translation/__init__.py b/litellm/proxy/mcp_server/guardrail_translation/__init__.py similarity index 88% rename from litellm/proxy/_experimental/mcp_server/guardrail_translation/__init__.py rename to litellm/proxy/mcp_server/guardrail_translation/__init__.py index ac06aa93c96..c4b52b745a3 100644 --- a/litellm/proxy/_experimental/mcp_server/guardrail_translation/__init__.py +++ b/litellm/proxy/mcp_server/guardrail_translation/__init__.py @@ -2,7 +2,7 @@ from typing import Final -from litellm.proxy._experimental.mcp_server.guardrail_translation.handler import ( +from litellm.proxy.mcp_server.guardrail_translation.handler import ( MCPGuardrailTranslationHandler, ) from litellm.types.utils import CallTypes diff --git a/litellm/proxy/_experimental/mcp_server/guardrail_translation/handler.py b/litellm/proxy/mcp_server/guardrail_translation/handler.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/guardrail_translation/handler.py rename to litellm/proxy/mcp_server/guardrail_translation/handler.py index d1a8084f4b2..28c51148da5 100644 --- a/litellm/proxy/_experimental/mcp_server/guardrail_translation/handler.py +++ b/litellm/proxy/mcp_server/guardrail_translation/handler.py @@ -13,7 +13,7 @@ every ``description`` string in the schema lead ``texts``, so a guardrail that blocks or masks them decides what the client gets to see in ``tools/list``. Note: For MCP tool definitions (schema) -> OpenAI tools=[], see -litellm.experimental_mcp_client.tools.transform_mcp_tool_to_openai_tool +litellm.mcp_client.tools.transform_mcp_tool_to_openai_tool when you have a full MCP Tool from list_tools. Here we only have the call payload (name + arguments) so we just build the tool definition. """ @@ -25,9 +25,9 @@ from fastapi import HTTPException from mcp.types import Tool as MCPTool from litellm._logging import verbose_proxy_logger -from litellm.experimental_mcp_client.tools import transform_mcp_tool_to_openai_tool from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation -from litellm.proxy._experimental.mcp_server.utils import ( +from litellm.mcp_client.tools import transform_mcp_tool_to_openai_tool +from litellm.proxy.mcp_server.utils import ( MAX_STRUCTURED_CONTENT_SCAN_DEPTH, JSONLeafPath, json_string_leaves, diff --git a/litellm/proxy/_experimental/mcp_server/id_jag_grant.py b/litellm/proxy/mcp_server/id_jag_grant.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/id_jag_grant.py rename to litellm/proxy/mcp_server/id_jag_grant.py index f2177df5a26..96f212e3bd7 100644 --- a/litellm/proxy/_experimental/mcp_server/id_jag_grant.py +++ b/litellm/proxy/mcp_server/id_jag_grant.py @@ -27,20 +27,20 @@ from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger from litellm.models.user import LiteLLM_UserTable -from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ( +from litellm.proxy._types import IdJagRequestingApp +from litellm.proxy.auth.handle_jwt import ID_JAG_TYP, AssertionRejection, JWTHandler +from litellm.proxy.mcp_server.gateway_dcr_flow import ( AssertionIdentity, ConsentTeam, SubjectIdentity, SubjectTokenRefusal, ) -from litellm.proxy._experimental.mcp_server.idp_token_exchange import ( +from litellm.proxy.mcp_server.idp_token_exchange import ( gateway_outage_refusal, log_line, read_token_exchange_prerequisites, ) -from litellm.proxy._experimental.mcp_server.oauth_utils import proxy_base_url_from_env -from litellm.proxy._types import IdJagRequestingApp -from litellm.proxy.auth.handle_jwt import ID_JAG_TYP, AssertionRejection, JWTHandler +from litellm.proxy.mcp_server.oauth_utils import proxy_base_url_from_env ID_JAG_REQUIRED_CLAIMS: Final = ("iss", "sub", "aud", "exp", "iat", "jti", "client_id") REJECTED_ASSERTION: Final = "assertion was rejected" @@ -395,10 +395,10 @@ class AssertionRedemption: async def redeem_id_jag_assertion( assertion: str, client_id: str, client_secret: str | None, request: Request ) -> AssertionIdentity | SubjectTokenRefusal: - from litellm.proxy._experimental.mcp_server.proxy_api_credentials import ( # noqa: PLC0415 # circular import at module load + from litellm.proxy.auth.auth_checks import get_user_object # noqa: PLC0415 # circular import at module load + from litellm.proxy.mcp_server.proxy_api_credentials import ( # noqa: PLC0415 # circular import at module load lookup_consent_teams_from_database, ) - from litellm.proxy.auth.auth_checks import get_user_object # noqa: PLC0415 # circular import at module load from litellm.proxy.proxy_server import ( # noqa: PLC0415 # rebound after startup, so read them per call jwt_handler, prisma_client, diff --git a/litellm/proxy/_experimental/mcp_server/idp_token_exchange.py b/litellm/proxy/mcp_server/idp_token_exchange.py similarity index 99% rename from litellm/proxy/_experimental/mcp_server/idp_token_exchange.py rename to litellm/proxy/mcp_server/idp_token_exchange.py index 5044a5bd3dc..c8ccdc9bca6 100644 --- a/litellm/proxy/_experimental/mcp_server/idp_token_exchange.py +++ b/litellm/proxy/mcp_server/idp_token_exchange.py @@ -13,10 +13,10 @@ from fastapi import HTTPException, Request from typing_extensions import assert_never from litellm._logging import verbose_proxy_logger -from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import SubjectIdentity, SubjectTokenRefusal from litellm.proxy._types import JWTAuthBuilderResult, ProxyException from litellm.proxy.auth.handle_jwt import JWTAuthManager, JWTHandler from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler +from litellm.proxy.mcp_server.gateway_dcr_flow import SubjectIdentity, SubjectTokenRefusal EXCHANGE_ROUTE: Final = "/token" REJECTED_SUBJECT_TOKEN: Final = "subject_token was rejected by the gateway's JWT auth" diff --git a/litellm/proxy/_experimental/mcp_server/interactions.py b/litellm/proxy/mcp_server/interactions.py similarity index 97% rename from litellm/proxy/_experimental/mcp_server/interactions.py rename to litellm/proxy/mcp_server/interactions.py index 42addcfb65b..24cedb57ef8 100644 --- a/litellm/proxy/_experimental/mcp_server/interactions.py +++ b/litellm/proxy/mcp_server/interactions.py @@ -25,9 +25,9 @@ from mcp.types import ( ) from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError -from litellm.proxy._experimental.mcp_server.contracts import OperationContext -from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error -from litellm.proxy._experimental.mcp_server.state_tokens import StateTokenError, open_state, seal_state +from litellm.proxy.mcp_server.contracts import OperationContext +from litellm.proxy.mcp_server.outbound_credentials.result import Error +from litellm.proxy.mcp_server.state_tokens import StateTokenError, open_state, seal_state from litellm.types.mcp_server.mcp_server_manager import MCPServer diff --git a/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py b/litellm/proxy/mcp_server/legacy_callbacks.py similarity index 87% rename from litellm/proxy/_experimental/mcp_server/legacy_callbacks.py rename to litellm/proxy/mcp_server/legacy_callbacks.py index 440b184e6ee..3d2f97492bb 100644 --- a/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py +++ b/litellm/proxy/mcp_server/legacy_callbacks.py @@ -13,14 +13,14 @@ from mcp.types import ( ) from litellm.constants import MCP_CLIENT_TIMEOUT -from litellm.proxy._experimental.mcp_server.contracts import OperationContext -from litellm.proxy._experimental.mcp_server.mcp_context import get_active_mcp_request_ctx -from litellm.proxy._experimental.mcp_server.scope_challenge import ( +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.mcp_server.contracts import OperationContext +from litellm.proxy.mcp_server.mcp_context import get_active_mcp_request_ctx +from litellm.proxy.mcp_server.scope_challenge import ( OAuthScopeResponse, is_tool_call_request, scope_response_from_request, ) -from litellm.proxy._types import UserAPIKeyAuth class SamplingCallback(Protocol): @@ -43,7 +43,7 @@ def create_sampling_callback( client_ip: str | None = None, operation_context: OperationContext | None = None, ) -> SamplingCallback: - from litellm.proxy._experimental.mcp_server.server import get_active_auth_context + from litellm.proxy.mcp_server.server import get_active_auth_context auth: Final = get_active_auth_context() if operation_context is None and user_api_key_auth is None else None captured: Final = ( @@ -60,7 +60,7 @@ def create_sampling_callback( context: ClientRequestContext, params: CreateMessageRequestParams ) -> CreateMessageResult | CreateMessageResultWithTools | ErrorData: import litellm - from litellm.proxy._experimental.mcp_server.sampling_handler import handle_sampling_create_message + from litellm.proxy.mcp_server.sampling_handler import handle_sampling_create_message return await handle_sampling_create_message( context=context, @@ -75,7 +75,7 @@ def create_sampling_callback( def create_elicitation_callback(timeout: float | None = None) -> ElicitationCallback: - from litellm.proxy._experimental.mcp_server.server import get_active_mcp_session + from litellm.proxy.mcp_server.server import get_active_mcp_session downstream_session: Final = get_active_mcp_session() request: Final = get_active_mcp_request_ctx() @@ -94,7 +94,7 @@ def create_elicitation_callback(timeout: float | None = None) -> ElicitationCall code=-32602, message="A legacy upstream cannot resume input for a modern client; the operation may have partially completed", ) - from litellm.proxy._experimental.mcp_server.elicitation_handler import handle_elicitation_request + from litellm.proxy.mcp_server.elicitation_handler import handle_elicitation_request return await handle_elicitation_request( context=context, diff --git a/litellm/proxy/_experimental/mcp_server/mcp_context.py b/litellm/proxy/mcp_server/mcp_context.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/mcp_context.py rename to litellm/proxy/mcp_server/mcp_context.py diff --git a/litellm/proxy/_experimental/mcp_server/mcp_debug.py b/litellm/proxy/mcp_server/mcp_debug.py similarity index 96% rename from litellm/proxy/_experimental/mcp_server/mcp_debug.py rename to litellm/proxy/mcp_server/mcp_debug.py index bf74ea99e55..94b3d96a359 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_debug.py +++ b/litellm/proxy/mcp_server/mcp_debug.py @@ -124,7 +124,7 @@ from litellm.litellm_core_utils.secret_redaction import REDACTED, redact_string from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker if TYPE_CHECKING: - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import AuthResolution + from litellm.proxy.mcp_server.outbound_credentials.types import AuthResolution # Header the client sends to opt into debug mode MCP_DEBUG_REQUEST_HEADER: Final = "x-litellm-mcp-debug" @@ -137,7 +137,7 @@ MCP_AUTH_DIAGNOSTICS_SCOPE_KEY: Final = "litellm.mcp.auth_diagnostics" def record_auth_resolution(server_id: str, source: AuthResolution) -> None: - from litellm.proxy._experimental.mcp_server.mcp_context import get_active_mcp_request_ctx + from litellm.proxy.mcp_server.mcp_context import get_active_mcp_request_ctx context: Final[object] = get_active_mcp_request_ctx() request: Final[object] = getattr(context, "request", None) @@ -155,7 +155,7 @@ class MCPAuthDiagnostics: self._outcomes = tuple(item for item in self._outcomes if item[0] != server_id) + ((server_id, resolution),) def resolution(self) -> str: - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import AuthResolution + from litellm.proxy.mcp_server.outbound_credentials.types import AuthResolution match self._outcomes: case (): @@ -166,7 +166,7 @@ class MCPAuthDiagnostics: return AuthResolution.multiple.value def headers(self) -> Mapping[str, str]: - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import AuthResolution + from litellm.proxy.mcp_server.outbound_credentials.types import AuthResolution if len(self._outcomes) <= 1: return MappingProxyType({"x-mcp-debug-auth-resolution": self.resolution()}) @@ -370,16 +370,16 @@ class MCPDebug: if not raw_headers or not MCPDebug.is_debug_enabled(raw_headers): return {} - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) server_url: str | None = None server_auth_type: str | None = None - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import AuthResolution + from litellm.proxy.mcp_server.outbound_credentials.types import AuthResolution auth_resolution: Final = AuthResolution.unresolved.value @@ -471,7 +471,7 @@ def _body_secret_values(request: httpx.Request | httpx2.Request) -> tuple[str, . body: Final = _JSON_BODY.validate_json(raw) except ValueError: return None - from litellm.proxy._experimental.mcp_server.utils import ( # noqa: PLC0415 # MCP utils imports clients; inspect bodies only after initialization + from litellm.proxy.mcp_server.utils import ( # noqa: PLC0415 # MCP utils imports clients; inspect bodies only after initialization json_string_leaves, ) @@ -613,7 +613,7 @@ def describe_upstream_response(response: httpx.Response | httpx2.Response) -> st def describe_upstream_http_failure(exc: BaseException) -> str | None: - from litellm.proxy._experimental.mcp_server.faults.traversal import ( # noqa: PLC0415 # fault package initialization imports the credential resolver + from litellm.proxy.mcp_server.faults.traversal import ( # noqa: PLC0415 # fault package initialization imports the credential resolver iter_exception_tree, ) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/mcp_server/mcp_server_manager.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/mcp_server_manager.py rename to litellm/proxy/mcp_server/mcp_server_manager.py index db70c7d77de..8e613a8a6e8 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/mcp_server/mcp_server_manager.py @@ -73,7 +73,12 @@ from litellm.constants import ( MCP_UPSTREAM_SESSION_SHUTDOWN_TIMEOUT_SECONDS, ) from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException -from litellm.experimental_mcp_client.client import ( +from litellm.integrations.custom_guardrail import ( + _sync_guardrail_info_to_logging_obj, # pyright: ignore[reportPrivateUsage] - the same bridge @log_guardrail_information uses; reimplementing it here would fork the metadata-key logic +) +from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get +from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.mcp_client.client import ( SESSION_CLEANUP_TIMEOUT_MESSAGE, MCPClient, PersistentMCPSession, @@ -81,102 +86,114 @@ from litellm.experimental_mcp_client.client import ( strip_auth_scheme, to_basic_credentials, ) -from litellm.integrations.custom_guardrail import ( - _sync_guardrail_info_to_logging_obj, # pyright: ignore[reportPrivateUsage] - the same bridge @log_guardrail_information uses; reimplementing it here would fork the metadata-key logic +from litellm.proxy._types import ( + LiteLLM_MCPServerTable, + MCPAuthType, + MCPEnvVar, + MCPTransport, + MCPTransportType, + SpecialMCPServerNames, + UserAPIKeyAuth, + is_per_server_oauth_discovery_eligible, ) -from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get -from litellm.llms.custom_httpx.http_handler import get_async_httpx_client -from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( # noqa: F401 # legacy module exports +from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_agent_policy +from litellm.proxy.auth.ip_address_utils import IPAddressUtils +from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper +from litellm.proxy.common_utils.user_api_key_cache import get_management_object_ttl +from litellm.proxy.management_endpoints.sso.id_jag_assertion_capture import ( + id_jag_assertion_capture_gap_at_startup, +) +from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( # noqa: F401 # legacy module exports MCPRequestHandler, MCPServerAccess, _is_mcp_admitted_user_subject, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export is_mcp_admitted_user_subject, ) -from litellm.proxy._experimental.mcp_server.catalog import _configuration_identity, _DiscoveryCache, _DiscoveryKey -from litellm.proxy._experimental.mcp_server.contracts import OperationContext -from litellm.proxy._experimental.mcp_server.elicitation_handler import ( +from litellm.proxy.mcp_server.catalog import _configuration_identity, _DiscoveryCache, _DiscoveryKey +from litellm.proxy.mcp_server.contracts import OperationContext +from litellm.proxy.mcp_server.elicitation_handler import ( MCP_ELICITATION_AVAILABLE, ) -from litellm.proxy._experimental.mcp_server.exceptions import ( +from litellm.proxy.mcp_server.exceptions import ( MCPServerListError, MCPUpstreamAuthError, ) -from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( +from litellm.proxy.mcp_server.faults.list_outcomes import ( ServerListFault, raise_classified_list_failure, retry_upstream_rate_limit, upstream_auth_challenge, upstream_insufficient_scope, ) -from litellm.proxy._experimental.mcp_server.interactions import bind_target -from litellm.proxy._experimental.mcp_server.legacy_callbacks import get_scope_response, has_active_mcp_request_context -from litellm.proxy._experimental.mcp_server.mcp_debug import describe_upstream_http_failure, record_auth_resolution -from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( +from litellm.proxy.mcp_server.interactions import bind_target +from litellm.proxy.mcp_server.legacy_callbacks import get_scope_response, has_active_mcp_request_context +from litellm.proxy.mcp_server.mcp_debug import describe_upstream_http_failure, record_auth_resolution +from litellm.proxy.mcp_server.oauth2_token_cache import ( MCPPerUserTokenCache, mcp_per_user_token_cache, ) -from litellm.proxy._experimental.mcp_server.oauth_utils import ( # noqa: F401 # legacy module exports +from litellm.proxy.mcp_server.oauth_utils import ( # noqa: F401 # legacy module exports _redact_mcp_resource_url, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export canonicalize_url_identity, get_byok_www_authenticate, needs_cimd_discovery, redact_mcp_resource_url, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials import ( +from litellm.proxy.mcp_server.outbound_credentials import ( Error, Ok, UpstreamCredentialProvider, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( +from litellm.proxy.mcp_server.outbound_credentials.adapter import ( raise_public, raise_token_exchange_challenge, to_server_spec, to_subject, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( InvalidatableOAuthTokenStore, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import ( +from litellm.proxy.mcp_server.outbound_credentials.per_user_oauth_store import ( LazyPerUserOAuthTokenStore, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchange_provider import ( +from litellm.proxy.mcp_server.outbound_credentials.token_exchange_provider import ( build_token_exchanger, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( +from litellm.proxy.mcp_server.outbound_credentials.types import ( DEFAULT_CREDENTIAL_HEADER, AuthResolution, IdJagConfig, PassthroughConfig, TokenExchangeConfig, ) -from litellm.proxy._experimental.mcp_server.result_conversion import ( +from litellm.proxy.mcp_server.result_conversion import ( WireCompat, complete_call_tool_result, handler_outcome, to_gateway_tool, ) -from litellm.proxy._experimental.mcp_server.sampling_handler import ( +from litellm.proxy.mcp_server.sampling_handler import ( MCP_SAMPLING_AVAILABLE, ) -from litellm.proxy._experimental.mcp_server.stdio_gate import ( +from litellm.proxy.mcp_server.stdio_gate import ( MCP_STDIO_DISABLED_MESSAGE, is_mcp_stdio_blocked, warn_if_mcp_stdio_blocked, ) -from litellm.proxy._experimental.mcp_server.tool_catalog_guard import ( +from litellm.proxy.mcp_server.tool_catalog_guard import ( CatalogAlert, apply_description_overrides, pin_tool_catalog, scan_tool_descriptions, ) -from litellm.proxy._experimental.mcp_server.upstream import ( +from litellm.proxy.mcp_server.upstream import ( passthrough_token_from_mcp_auth_header, prepare_upstream_client, resolve_upstream_auth, take_forwarded_authorization, to_server_spec_fail_closed, ) -from litellm.proxy._experimental.mcp_server.utils import ( +from litellm.proxy.mcp_server.utils import ( MCP_SERVERS_TARGET, MCP_TOOL_PREFIX_SEPARATOR, MCPMissingUserEnvVarsError, @@ -201,23 +218,6 @@ from litellm.proxy._experimental.mcp_server.utils import ( strip_known_server_prefix, validate_mcp_server_name, ) -from litellm.proxy._types import ( - LiteLLM_MCPServerTable, - MCPAuthType, - MCPEnvVar, - MCPTransport, - MCPTransportType, - SpecialMCPServerNames, - UserAPIKeyAuth, - is_per_server_oauth_discovery_eligible, -) -from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_agent_policy -from litellm.proxy.auth.ip_address_utils import IPAddressUtils -from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper -from litellm.proxy.common_utils.user_api_key_cache import get_management_object_ttl -from litellm.proxy.management_endpoints.sso.id_jag_assertion_capture import ( - id_jag_assertion_capture_gap_at_startup, -) from litellm.proxy.middleware.per_request_root_path_middleware import ( get_request_root_path, ) @@ -242,7 +242,7 @@ from litellm.types.utils import CallTypes if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.proxy._experimental.mcp_server.contracts import CatalogListRequest, CatalogListResult + from litellm.proxy.mcp_server.contracts import CatalogListRequest, CatalogListResult from litellm.types.mcp_server.mcp_toolset import MCPToolset try: @@ -1032,7 +1032,7 @@ async def _openapi_spec_health( ) -> tuple[Literal["healthy", "unhealthy", "unknown"], str | None]: """Check specification availability, not upstream operations or user credentials.""" from litellm.llms.custom_httpx.http_handler import HTTPResponseLimitError - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import load_openapi_spec_async + from litellm.proxy.mcp_server.openapi_to_mcp_generator import load_openapi_spec_async if not spec_path.startswith(("http://", "https://")): return "unknown", "OpenAPI servers have no protocol-level health probe" @@ -1464,7 +1464,7 @@ async def _resolve_byok_mcp_auth_header( if not mcp_server.is_byok: return mcp_auth_header - from litellm.proxy._experimental.mcp_server.operations import ( + from litellm.proxy.mcp_server.operations import ( _check_byok_credential, _get_byok_credential, ) @@ -1835,7 +1835,7 @@ def _create_sampling_callback( ): if not MCP_SAMPLING_AVAILABLE: return None - from litellm.proxy._experimental.mcp_server.legacy_callbacks import create_sampling_callback + from litellm.proxy.mcp_server.legacy_callbacks import create_sampling_callback return create_sampling_callback(user_api_key_auth, raw_headers, client_ip, operation_context) @@ -1843,7 +1843,7 @@ def _create_sampling_callback( def _create_elicitation_callback(timeout: float | None = None): if not MCP_ELICITATION_AVAILABLE: return None - from litellm.proxy._experimental.mcp_server.legacy_callbacks import create_elicitation_callback + from litellm.proxy.mcp_server.legacy_callbacks import create_elicitation_callback return create_elicitation_callback(timeout=timeout) @@ -2035,7 +2035,7 @@ class MCPServerManager: self._template_discovery_cache = _DiscoveryCache[ListResourceTemplatesResult]( discovery_ttl, discovery_clock, TypeAdapter(ListResourceTemplatesResult) ) - from litellm.proxy._experimental.mcp_server.catalog import CatalogSnapshots + from litellm.proxy.mcp_server.catalog import CatalogSnapshots self.catalog = CatalogSnapshots(self) self.registry: dict[str, MCPServer] = {} @@ -2887,7 +2887,7 @@ class MCPServerManager: """Overlay each config-declared server's persisted DCR client (from the server-scoped store) onto its in-memory object so token refresh authenticates after a restart. A best-effort no-op when the DB is unreachable at config-load time.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( # noqa: PLC0415 # circular import + from litellm.proxy.mcp_server.discoverable_endpoints import ( # noqa: PLC0415 # circular import hydrate_config_server_dcr_client, ) @@ -2919,16 +2919,16 @@ class MCPServerManager: server: The MCPServer instance to register tools for base_url: Base URL for API calls """ - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( build_input_schema, create_tool_function, load_openapi_spec_async, resolve_operation_params, ) - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( get_base_url as get_openapi_base_url, ) - from litellm.proxy._experimental.mcp_server.tool_registry import ( + from litellm.proxy.mcp_server.tool_registry import ( global_mcp_tool_registry, ) @@ -3061,7 +3061,7 @@ class MCPServerManager: self.remove_server_tool_routing(server) def remove_server_tool_routing(self, server: MCPServer) -> None: - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry prefix_root: Final = normalize_server_name(get_server_prefix(server)) if server.spec_path and prefix_root: @@ -3128,7 +3128,7 @@ class MCPServerManager: ) -> _EnvVarList | None: env_vars_list: Final = _deserialize_json_list(getattr(mcp_server, "env_vars", None)) if env_vars_are_encrypted: - from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.db import ( # noqa: PLC0415 decrypt_global_env_var_values, ) @@ -3578,7 +3578,7 @@ class MCPServerManager: return [] try: - from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.db import ( # noqa: PLC0415 get_active_submitted_mcp_server_ids_for_user, ) from litellm.proxy.proxy_server import prisma_client, user_api_key_cache @@ -3835,7 +3835,7 @@ class MCPServerManager: one silently renames the tool when a native name happens to begin with it (``greyhound_internal_events`` on a server prefixed ``greyhound``). """ - from litellm.proxy._experimental.mcp_server.toolset_db import list_mcp_toolsets + from litellm.proxy.mcp_server.toolset_db import list_mcp_toolsets from litellm.proxy.proxy_server import prisma_client, user_api_key_cache if not toolset_ids or prisma_client is None: @@ -3933,7 +3933,7 @@ class MCPServerManager: return MCPToolset(**cached) return cached - from litellm.proxy._experimental.mcp_server.toolset_db import ( + from litellm.proxy.mcp_server.toolset_db import ( get_mcp_toolset_by_name, ) @@ -4291,7 +4291,7 @@ class MCPServerManager: "is configured. Connect a database to your proxy to use per-user " "MCP env vars." ) - from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.db import ( # noqa: PLC0415 get_user_env_vars, ) @@ -4455,7 +4455,7 @@ class MCPServerManager: record_listing: bool = False, listing_updates: ExitStack | None = None, ) -> ListToolsResult: - from litellm.proxy._experimental.mcp_server.tool_registry import ( + from litellm.proxy.mcp_server.tool_registry import ( global_mcp_tool_registry, ) @@ -4662,7 +4662,7 @@ class MCPServerManager: self._template_discovery_cache.invalidate(server_id) def invalidate_server_definition_caches(self, server_id: str) -> None: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( # noqa: PLC0415 # lazy: discoverable_endpoints lazily imports this module's manager singleton + from litellm.proxy.mcp_server.discoverable_endpoints import ( # noqa: PLC0415 # lazy: discoverable_endpoints lazily imports this module's manager singleton invalidate_oauth_metadata_cache, ) @@ -5830,7 +5830,7 @@ class MCPServerManager: Returns: List of tools with prefixed names """ - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry if register_bare_names and listing_updates is not None: listing_updates.callback(self._create_prefixed_tools, tools, server, add_prefix=add_prefix) @@ -5942,7 +5942,7 @@ class MCPServerManager: deriving the spellings routing accepts, never by stripping the entry, which would cut a second boundary out of a native name that opens with the server prefix. """ - from litellm.proxy._experimental.mcp_server.utils import ( + from litellm.proxy.mcp_server.utils import ( server_applies_tool_allowlist, ) @@ -6005,7 +6005,7 @@ class MCPServerManager: Raises: HTTPException: If tool is not allowed for this key/team """ - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) @@ -6050,7 +6050,7 @@ class MCPServerManager: """ from mcp.types import TextContent - from litellm.proxy._experimental.mcp_server.tool_registry import ( + from litellm.proxy.mcp_server.tool_registry import ( global_mcp_tool_registry, ) @@ -6868,7 +6868,7 @@ class MCPServerManager: return oauth2_headers try: - from litellm.proxy._experimental.mcp_server.operations import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.operations import ( # noqa: PLC0415 _get_user_oauth_extra_headers_from_db, ) @@ -7108,7 +7108,7 @@ class MCPServerManager: ) async def _call_openapi_via_handler(): - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( request_auth_header, request_extra_headers, request_resolved_auth_headers, @@ -7207,7 +7207,7 @@ class MCPServerManager: Returns: MCPServer if found, None otherwise """ - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry registered_tool: Final = global_mcp_tool_registry.get_tool(tool_name) if registered_tool is not None: diff --git a/litellm/proxy/_experimental/mcp_server/oauth2_flow_backfill.py b/litellm/proxy/mcp_server/oauth2_flow_backfill.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/oauth2_flow_backfill.py rename to litellm/proxy/mcp_server/oauth2_flow_backfill.py index 71281b276fa..1ef67df498f 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth2_flow_backfill.py +++ b/litellm/proxy/mcp_server/oauth2_flow_backfill.py @@ -42,7 +42,7 @@ from typing import Final, Literal, Protocol from pydantic import JsonValue from litellm._logging import verbose_proxy_logger -from litellm.proxy._experimental.mcp_server.db import ( # noqa: F401 # legacy module exports +from litellm.proxy.mcp_server.db import ( # noqa: F401 # legacy module exports _decode_oauth_payload, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export decode_oauth_payload, decrypt_credentials, diff --git a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py b/litellm/proxy/mcp_server/oauth2_token_cache.py similarity index 97% rename from litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py rename to litellm/proxy/mcp_server/oauth2_token_cache.py index cc92b8cce9b..1e6e83c0f4b 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py +++ b/litellm/proxy/mcp_server/oauth2_token_cache.py @@ -26,17 +26,17 @@ from litellm.constants import ( MCP_PER_USER_TOKEN_REDIS_KEY_PREFIX, ) from litellm.llms.custom_httpx.http_handler import get_async_httpx_client -from litellm.proxy._experimental.mcp_server.oauth_utils import ( - build_upstream_oauth2_token_request, - resolve_upstream_resource, -) -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import OAuthToken -from litellm.proxy._experimental.mcp_server.outbound_credentials.token_cache_codec import OAuthTokenCacheCodec -from litellm.proxy._experimental.mcp_server.utils import MCP_OAUTH_TOKENS_TARGET from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) +from litellm.proxy.mcp_server.oauth_utils import ( + build_upstream_oauth2_token_request, + resolve_upstream_resource, +) +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import OAuthToken +from litellm.proxy.mcp_server.outbound_credentials.token_cache_codec import OAuthTokenCacheCodec +from litellm.proxy.mcp_server.utils import MCP_OAUTH_TOKENS_TARGET from litellm.types.llms.custom_http import httpxSpecialProvider if TYPE_CHECKING: diff --git a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py b/litellm/proxy/mcp_server/oauth_identity_binding.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py rename to litellm/proxy/mcp_server/oauth_identity_binding.py index b3aa040c507..6fdda498d50 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py +++ b/litellm/proxy/mcp_server/oauth_identity_binding.py @@ -184,7 +184,7 @@ def _upstream_principal( async def _load_caller_principal(litellm_user_id: str, binding: MCPOAuthIdentityBinding) -> str | None: if binding.caller_field == "user_id": return litellm_user_id - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( # noqa: PLC0415 # inline import avoids a module-load circular import + from litellm.proxy.mcp_server.bridge_token_flow import ( # noqa: PLC0415 # inline import avoids a module-load circular import load_active_user_by_id, ) @@ -198,7 +198,7 @@ async def _load_stored_refresh_token( litellm_user_id: str, server_id: str, binding: MCPOAuthIdentityBinding ) -> VerifiedRefreshToken | None: try: - from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 # keep database imports lazy + from litellm.proxy.mcp_server.db import ( # noqa: PLC0415 # keep database imports lazy get_user_oauth_credential, ) from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 # keep database imports lazy diff --git a/litellm/proxy/_experimental/mcp_server/oauth_issuer_stamp_backfill.py b/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/oauth_issuer_stamp_backfill.py rename to litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py index eb02ed64e91..c417187396f 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_issuer_stamp_backfill.py +++ b/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py @@ -40,8 +40,8 @@ from typing import Final, Protocol from urllib.parse import urlparse from litellm._logging import verbose_proxy_logger -from litellm.proxy._experimental.mcp_server.oauth_utils import canonicalize_url_identity from litellm.proxy.db.db_span import db_span +from litellm.proxy.mcp_server.oauth_utils import canonicalize_url_identity from litellm.proxy.utils import PrismaClient # The actor the removed discovery write-back stamped rows with. diff --git a/litellm/proxy/_experimental/mcp_server/oauth_utils.py b/litellm/proxy/mcp_server/oauth_utils.py similarity index 99% rename from litellm/proxy/_experimental/mcp_server/oauth_utils.py rename to litellm/proxy/mcp_server/oauth_utils.py index 133bad780bf..76343f77e32 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_utils.py +++ b/litellm/proxy/mcp_server/oauth_utils.py @@ -14,15 +14,15 @@ from fastapi import HTTPException, Request from starlette.types import Scope from litellm._logging import verbose_logger -from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( +from litellm.proxy.auth.ip_address_utils import IPAddressUtils +from litellm.proxy.mcp_server.auth.token_endpoint_auth import ( TokenEndpointClientAuth, build_token_endpoint_client_auth, normalize_token_endpoint_auth_method, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( +from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store import ( GOOGLE_REFRESH_TOKEN_PARAMS, ) -from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.proxy.middleware.per_request_root_path_middleware import get_request_root_path if TYPE_CHECKING: diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/mcp_server/openapi_to_mcp_generator.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py rename to litellm/proxy/mcp_server/openapi_to_mcp_generator.py index f0353f309a9..96d178dbeb1 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/mcp_server/openapi_to_mcp_generator.py @@ -16,7 +16,7 @@ import httpx from typing_extensions import ReadOnly, Required from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError -from litellm.proxy._experimental.mcp_server.exceptions import ( +from litellm.proxy.mcp_server.exceptions import ( MCPOpenApiUpstreamError, MCPUpstreamAuthError, ) @@ -54,8 +54,8 @@ from litellm.llms.custom_httpx.http_handler import ( header_value, httpxSpecialProvider, ) -from litellm.proxy._experimental.mcp_server.tool_outcome import JsonResult, TextResult, parse_http_body -from litellm.proxy._experimental.mcp_server.tool_registry import ( +from litellm.proxy.mcp_server.tool_outcome import JsonResult, TextResult, parse_http_body +from litellm.proxy.mcp_server.tool_registry import ( global_mcp_tool_registry, ) from litellm.types.mcp import MCPAuthType, credential_redirect_hook, custom_credential_slot @@ -442,7 +442,7 @@ def _merge_openapi_tool_request_headers( the MCP caller (allowlisted by ``MCPServer.extra_headers``) This matches the existing MCP invariant in - :func:`litellm.proxy._experimental.mcp_server.utils.merge_mcp_headers` + :func:`litellm.proxy.mcp_server.utils.merge_mcp_headers` and the managed MCP path, where ``static_headers`` always wins over caller-forwarded headers. Keeping the same precedence here prevents an authenticated caller from overriding an operator-configured value @@ -542,11 +542,11 @@ def create_tool_function( """ effective_headers: Final = _merge_openapi_tool_request_headers(headers) if auth_type is not None: - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( + from litellm.proxy.mcp_server.outbound_credentials.adapter import ( raise_public, validate_static_credential, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Ok + from litellm.proxy.mcp_server.outbound_credentials.result import Error, Ok match validate_static_credential(auth_type, effective_headers, upstream_token_header, headers or ()): case Error(error): diff --git a/litellm/proxy/_experimental/mcp_server/operations.py b/litellm/proxy/mcp_server/operations.py similarity index 97% rename from litellm/proxy/_experimental/mcp_server/operations.py rename to litellm/proxy/mcp_server/operations.py index 39e882fcab0..763e3578a96 100644 --- a/litellm/proxy/_experimental/mcp_server/operations.py +++ b/litellm/proxy/mcp_server/operations.py @@ -56,40 +56,51 @@ from litellm.constants import ( MCP_TOOL_LISTING_TIMEOUT, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( +from litellm.proxy._types import ( + UserAPIKeyAuth, +) +from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( + publish_auth_cache_invalidation, +) +from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError +from litellm.proxy.litellm_pre_call_utils import ( + LiteLLMProxyRequestSetup, + get_chain_id_from_headers, +) +from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) -from litellm.proxy._experimental.mcp_server.byok_credential_cache import ( +from litellm.proxy.mcp_server.byok_credential_cache import ( byok_credential_cache, byok_credential_cache_key, cache_byok_credential, get_cached_byok_credential, ) -from litellm.proxy._experimental.mcp_server.capabilities import ( +from litellm.proxy.mcp_server.capabilities import ( build_discovery, configured_versions, disabled_capabilities, enabled_gateway_operations, ) -from litellm.proxy._experimental.mcp_server.catalog import catalog_operation, global_manager -from litellm.proxy._experimental.mcp_server.contracts import ( +from litellm.proxy.mcp_server.catalog import catalog_operation, global_manager +from litellm.proxy.mcp_server.contracts import ( AuthorizedToolCall, OperationContext, ProgressCallback, ) -from litellm.proxy._experimental.mcp_server.db import OAuthCredentialPayload -from litellm.proxy._experimental.mcp_server.exceptions import ( +from litellm.proxy.mcp_server.db import OAuthCredentialPayload +from litellm.proxy.mcp_server.exceptions import ( MCPToolResultError, MCPUpstreamAuthError, ) -from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( +from litellm.proxy.mcp_server.faults.list_outcomes import ( AggregateToolListing, ServerListOk, ServerOutcome, classify_list_exception, outcome_wire_value, ) -from litellm.proxy._experimental.mcp_server.interactions import ( +from litellm.proxy.mcp_server.interactions import ( BoundInputRequiredResult, ContinuationState, InteractionOperation, @@ -97,8 +108,8 @@ from litellm.proxy._experimental.mcp_server.interactions import ( seal_continuation, target_digest, ) -from litellm.proxy._experimental.mcp_server.legacy_callbacks import get_scope_response -from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: F401 # legacy module exports +from litellm.proxy.mcp_server.legacy_callbacks import get_scope_response +from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: F401 # legacy module exports MCPServerManager, _caller_authorization_fans_out, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _client_forwarded_authorization_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -111,12 +122,12 @@ from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: resolve_openapi_tool_auth, should_strip_caller_authorization, ) -from litellm.proxy._experimental.mcp_server.oauth_utils import ( # noqa: F401 # legacy module exports +from litellm.proxy.mcp_server.oauth_utils import ( # noqa: F401 # legacy module exports _redact_mcp_resource_url, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export get_byok_www_authenticate, redact_mcp_resource_url, ) -from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( # noqa: F401 # legacy module exports +from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( # noqa: F401 # legacy module exports _request_auth_header, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _request_extra_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _request_resolved_auth_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -124,16 +135,16 @@ from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( # request_extra_headers, request_resolved_auth_headers, ) -from litellm.proxy._experimental.mcp_server.result_conversion import ( +from litellm.proxy.mcp_server.result_conversion import ( WireCompat, complete_call_tool_result, handler_outcome, to_call_tool_result, ) -from litellm.proxy._experimental.mcp_server.tool_registry import ( +from litellm.proxy.mcp_server.tool_registry import ( global_mcp_tool_registry, ) -from litellm.proxy._experimental.mcp_server.utils import ( +from litellm.proxy.mcp_server.utils import ( MCP_TOOL_PREFIX_SEPARATOR, MCPMissingUserEnvVarsError, add_server_prefix_to_name, @@ -148,20 +159,9 @@ from litellm.proxy._experimental.mcp_server.utils import ( split_server_prefix_from_name, strip_known_server_prefix, ) -from litellm.proxy._experimental.mcp_server.utils import ( +from litellm.proxy.mcp_server.utils import ( server_answers_to as _server_answers_to, ) -from litellm.proxy._types import ( - UserAPIKeyAuth, -) -from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( - publish_auth_cache_invalidation, -) -from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError -from litellm.proxy.litellm_pre_call_utils import ( - LiteLLMProxyRequestSetup, - get_chain_id_from_headers, -) from litellm.types.mcp import ( DEFAULT_CREDENTIAL_HEADER, MCP_LEGACY_VERSIONS, @@ -426,7 +426,7 @@ async def _dispatch_virtual_mcp_tool( the caller falls through to normal tool routing. """ from litellm.llms.litellm_proxy.skills.skill_search import DEFAULT_SKILL_SEARCH_TOP_K - from litellm.proxy._experimental.mcp_server.tool_search import ( + from litellm.proxy.mcp_server.tool_search import ( AGENT_SEARCH_TOOL_NAME, DEFAULT_AGENT_SEARCH_TOP_K, MCP_PROXY_CALL_TOOL_NAME, @@ -799,7 +799,7 @@ def filter_tools_by_allowed_tools( Returns: Filtered list of tools """ - from litellm.proxy._experimental.mcp_server.utils import ( + from litellm.proxy.mcp_server.utils import ( server_applies_tool_allowlist, ) @@ -911,7 +911,7 @@ def _client_has_per_server_auth_header( """ if not mcp_server_auth_headers: return False - from litellm.proxy._experimental.mcp_server.utils import ( + from litellm.proxy.mcp_server.utils import ( lookup_mcp_server_auth_in_headers, ) @@ -960,7 +960,7 @@ async def _get_user_oauth_extra_headers_from_db( """ if server.auth_type != MCPAuth.oauth2 or user_api_key_auth is None: return None - from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.db import ( # noqa: PLC0415 resolve_user_oauth_access_token, ) @@ -981,7 +981,7 @@ async def _prefetch_oauth_creds_for_user( if not user_id: return {} try: - from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.db import ( # noqa: PLC0415 list_user_oauth_credentials, ) from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 @@ -1015,7 +1015,7 @@ def _prepare_mcp_server_headers( """ server_auth_header: dict[str, str] | str | None = None if mcp_server_auth_headers: - from litellm.proxy._experimental.mcp_server.utils import ( + from litellm.proxy.mcp_server.utils import ( lookup_mcp_server_auth_in_headers, ) @@ -1182,8 +1182,8 @@ async def find_mcp_proxy_tool( ) -> MCPTool | None: """Resolve an /mcp/proxy tool_id against the caller's catalog, returning as soon as the server that owns it has listed instead of waiting for every allowed server.""" - from litellm.proxy._experimental.mcp_server.catalog import get_filtered_server_tools - from litellm.proxy._experimental.mcp_server.tool_search import mcp_proxy_tool_id + from litellm.proxy.mcp_server.catalog import get_filtered_server_tools + from litellm.proxy.mcp_server.tool_search import mcp_proxy_tool_id allowed_mcp_servers: Final = await _get_allowed_mcp_servers( user_api_key_auth=user_api_key_auth, @@ -1366,7 +1366,7 @@ async def _get_tools_from_mcp_servers( await _prefetch_oauth_creds_for_user(user_api_key_auth) if _has_oauth2_server else {} ) - from litellm.proxy._experimental.mcp_server.catalog import get_filtered_server_tools + from litellm.proxy.mcp_server.catalog import get_filtered_server_tools context: Final = OperationContext( _caller=user_api_key_auth, @@ -1413,7 +1413,7 @@ async def _get_tools_from_mcp_servers( }, ) else: - from litellm.proxy._experimental.mcp_server.catalog import aggregate_gateway_tools + from litellm.proxy.mcp_server.catalog import aggregate_gateway_tools aggregated = await aggregate_gateway_tools( context, @@ -1912,7 +1912,7 @@ async def _get_byok_credential( if cached is not None: return cached.credential - from litellm.proxy._experimental.mcp_server.db import get_user_credential + from litellm.proxy.mcp_server.db import get_user_credential from litellm.proxy.proxy_server import prisma_client if prisma_client is None: @@ -1970,7 +1970,7 @@ async def _check_byok_credential( ) return - from litellm.proxy._experimental.mcp_server.db import get_user_credential + from litellm.proxy.mcp_server.db import get_user_credential from litellm.proxy.proxy_server import prisma_client if prisma_client is None: @@ -2032,7 +2032,7 @@ def _challenge_missing_token_exchange_subject( return if global_mcp_server_manager.extract_subject_token(oauth2_headers, raw_headers, user_api_key_auth) is not None: return - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( # noqa: PLC0415 # lazy: adapter pulls MCP subgraph + from litellm.proxy.mcp_server.outbound_credentials.adapter import ( # noqa: PLC0415 # lazy: adapter pulls MCP subgraph raise_token_exchange_challenge, ) from litellm.proxy.middleware.per_request_root_path_middleware import ( # noqa: PLC0415 # lazy: middleware imports proxy utils @@ -3262,7 +3262,7 @@ async def _execute_handle_list_tools( ) from mcp.types import Tool - from litellm.proxy._experimental.mcp_server.tool_search import ( + from litellm.proxy.mcp_server.tool_search import ( get_mcp_proxy_tool_definitions, get_virtual_tool_definitions, ) @@ -3280,8 +3280,8 @@ async def _execute_handle_list_tools( raise MCPError(code=INVALID_PARAMS, message="Invalid pagination cursor; start a fresh listing") return ListToolsResult(tools=[Tool.model_validate(d) for d in get_virtual_tool_definitions()]) - from litellm.proxy._experimental.mcp_server.catalog import list_gateway_catalog - from litellm.proxy._experimental.mcp_server.server_resolution import MCPServerTargetCatalog + from litellm.proxy.mcp_server.catalog import list_gateway_catalog + from litellm.proxy.mcp_server.server_resolution import MCPServerTargetCatalog catalog: Final = MCPServerTargetCatalog( global_mcp_server_manager, @@ -3310,7 +3310,7 @@ async def _execute_handle_list_tools( async def _discover_tools_listing( context: OperationContext, host_progress_callback: ProgressCallback | None ) -> ListToolsResult: - from litellm.proxy._experimental.mcp_server.catalog import ToolListingFailedOutright + from litellm.proxy.mcp_server.catalog import ToolListingFailedOutright try: return await _execute_handle_list_tools( @@ -3487,8 +3487,8 @@ async def _execute_list_prompts( from mcp.shared.exceptions import MCPError from mcp.types import INVALID_PARAMS, INVALID_REQUEST - from litellm.proxy._experimental.mcp_server.catalog import list_gateway_catalog - from litellm.proxy._experimental.mcp_server.server_resolution import MCPServerTargetCatalog + from litellm.proxy.mcp_server.catalog import list_gateway_catalog + from litellm.proxy.mcp_server.server_resolution import MCPServerTargetCatalog if context.mcp_proxy_mode: _reject_mcp_proxy_operation() @@ -3551,8 +3551,8 @@ async def _execute_list_resources( from mcp.shared.exceptions import MCPError from mcp.types import INVALID_PARAMS, INVALID_REQUEST - from litellm.proxy._experimental.mcp_server.catalog import list_gateway_catalog - from litellm.proxy._experimental.mcp_server.server_resolution import MCPServerTargetCatalog + from litellm.proxy.mcp_server.catalog import list_gateway_catalog + from litellm.proxy.mcp_server.server_resolution import MCPServerTargetCatalog if context.mcp_proxy_mode: _reject_mcp_proxy_operation() @@ -3581,8 +3581,8 @@ async def _execute_list_resource_templates( from mcp.shared.exceptions import MCPError from mcp.types import INVALID_PARAMS, INVALID_REQUEST - from litellm.proxy._experimental.mcp_server.catalog import list_gateway_catalog - from litellm.proxy._experimental.mcp_server.server_resolution import MCPServerTargetCatalog + from litellm.proxy.mcp_server.catalog import list_gateway_catalog + from litellm.proxy.mcp_server.server_resolution import MCPServerTargetCatalog if context.mcp_proxy_mode: _reject_mcp_proxy_operation() @@ -3809,7 +3809,7 @@ class GatewayOperations: ) try: results: Final = await asyncio.gather(*tasks) - from litellm.proxy._experimental.mcp_server.subscriptions import subscription_capabilities + from litellm.proxy.mcp_server.subscriptions import subscription_capabilities async def discover_notifications() -> tuple[frozenset[str], ServerCapabilities]: remaining: Final = deadline - time.monotonic() diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/__init__.py b/litellm/proxy/mcp_server/outbound_credentials/__init__.py similarity index 85% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/__init__.py rename to litellm/proxy/mcp_server/outbound_credentials/__init__.py index 3260603c3ca..3d5fcf08d8e 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/__init__.py +++ b/litellm/proxy/mcp_server/outbound_credentials/__init__.py @@ -8,19 +8,19 @@ CredError]``) rather than raised, so every seam is total. Nothing here is wired request path yet. """ -from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( +from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( NoOpAuth, StaticHeaderAuth, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import ( +from litellm.proxy.mcp_server.outbound_credentials.resolver import ( UpstreamCredentialProvider, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.result import ( +from litellm.proxy.mcp_server.outbound_credentials.result import ( Error, Ok, Result, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( +from litellm.proxy.mcp_server.outbound_credentials.types import ( DEFAULT_CREDENTIAL_HEADER, Ambient, ApiKeyConfig, diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py b/litellm/proxy/mcp_server/outbound_credentials/adapter.py similarity index 97% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py rename to litellm/proxy/mcp_server/outbound_credentials/adapter.py index 37404ceece8..1aac70cd847 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py +++ b/litellm/proxy/mcp_server/outbound_credentials/adapter.py @@ -20,11 +20,11 @@ from fastapi import HTTPException from pydantic import SecretStr from typing_extensions import assert_never -from litellm.experimental_mcp_client.client import MCPClient, strip_auth_scheme, to_basic_credentials -from litellm.proxy._experimental.mcp_server.exceptions import TOKEN_EXCHANGE_SUBJECT_ERROR, MCPServerURLCredentialsError -from litellm.proxy._experimental.mcp_server.oauth_utils import resolve_upstream_resource -from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Ok, Result -from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( +from litellm.mcp_client.client import MCPClient, strip_auth_scheme, to_basic_credentials +from litellm.proxy.mcp_server.exceptions import TOKEN_EXCHANGE_SUBJECT_ERROR, MCPServerURLCredentialsError +from litellm.proxy.mcp_server.oauth_utils import resolve_upstream_resource +from litellm.proxy.mcp_server.outbound_credentials.result import Error, Ok, Result +from litellm.proxy.mcp_server.outbound_credentials.types import ( DEFAULT_CREDENTIAL_HEADER, ApiKeyConfig, AuthorizationCodeConfig, diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py b/litellm/proxy/mcp_server/outbound_credentials/authz_code_refresher.py similarity index 94% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py rename to litellm/proxy/mcp_server/outbound_credentials/authz_code_refresher.py index a653ef5e5ff..286b761b969 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py +++ b/litellm/proxy/mcp_server/outbound_credentials/authz_code_refresher.py @@ -17,16 +17,16 @@ from typing import TYPE_CHECKING, Final, Protocol from fastapi import HTTPException from litellm._logging import verbose_logger -from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( +from litellm.proxy.mcp_server.auth.token_endpoint_auth import ( TokenEndpointAuthConfigError, ) -from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( +from litellm.proxy.mcp_server.oauth_identity_binding import ( BindingValidator, RefreshTokenPresented, enforce_oauth_identity_binding, ) -from litellm.proxy._experimental.mcp_server.oauth_utils import build_upstream_oauth2_token_request, get_cimd_client_id -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.oauth_utils import build_upstream_oauth2_token_request, get_cimd_client_id +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/bridge_credentials.py b/litellm/proxy/mcp_server/outbound_credentials/bridge_credentials.py similarity index 99% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/bridge_credentials.py rename to litellm/proxy/mcp_server/outbound_credentials/bridge_credentials.py index 1441d9662be..2326bcebdc3 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/bridge_credentials.py +++ b/litellm/proxy/mcp_server/outbound_credentials/bridge_credentials.py @@ -16,7 +16,7 @@ from typing import Final, Literal, TypeAlias from pydantic import ConfigDict, SecretStr -from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( +from litellm.proxy.mcp_server.outbound_credentials.envelope import ( EnvelopeIdentity, EnvelopeKeys, EnvelopeMintError, diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/client_credentials.py b/litellm/proxy/mcp_server/outbound_credentials/client_credentials.py similarity index 96% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/client_credentials.py rename to litellm/proxy/mcp_server/outbound_credentials/client_credentials.py index 1652f5d8bd4..d130de4f405 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/client_credentials.py +++ b/litellm/proxy/mcp_server/outbound_credentials/client_credentials.py @@ -40,17 +40,17 @@ from pydantic import ConfigDict, Field, SecretStr, TypeAdapter, ValidationError from typing_extensions import assert_never from litellm._logging import verbose_logger -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( InMemoryTokenCacheBackend, OAuthToken, TokenCacheBackend, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.result import ( +from litellm.proxy.mcp_server.outbound_credentials.result import ( Error, Ok, Result, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( +from litellm.proxy.mcp_server.outbound_credentials.types import ( ClientCredentialsConfig, CredError, HeaderCarrier, @@ -105,7 +105,7 @@ async def post_client_credentials_grant( from litellm.llms.custom_httpx.http_handler import ( # noqa: PLC0415 # defer heavy handler import to call time get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # handler factory params are coarsely typed ) - from litellm.proxy._experimental.mcp_server.mcp_debug import ( # noqa: PLC0415 # diagnostics import credential enums through this package + from litellm.proxy.mcp_server.mcp_debug import ( # noqa: PLC0415 # diagnostics import credential enums through this package describe_upstream_http_failure, describe_upstream_response, safe_upstream_url, @@ -298,7 +298,7 @@ def _prepare_grant(config: ClientCredentialsConfig) -> Result[_PreparedGrant, Cr ) return Error(CredError.of_misconfigured(f"client_credentials config is missing: {missing}")) - from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( # noqa: PLC0415 # keep package v1-free at import time + from litellm.proxy.mcp_server.auth.token_endpoint_auth import ( # noqa: PLC0415 # keep package v1-free at import time build_token_endpoint_client_auth, ) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/dual_cache_token_backend.py b/litellm/proxy/mcp_server/outbound_credentials/dual_cache_token_backend.py similarity index 92% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/dual_cache_token_backend.py rename to litellm/proxy/mcp_server/outbound_credentials/dual_cache_token_backend.py index 69ff7cda2b7..b360b787962 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/dual_cache_token_backend.py +++ b/litellm/proxy/mcp_server/outbound_credentials/dual_cache_token_backend.py @@ -14,13 +14,13 @@ from typing import Final, Protocol from litellm._internal_context import with_service_target from litellm._logging import verbose_logger -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.token_cache_codec import ( +from litellm.proxy.mcp_server.outbound_credentials.token_cache_codec import ( OAuthTokenCacheCodec, ) -from litellm.proxy._experimental.mcp_server.utils import MCP_OAUTH_TOKENS_TARGET +from litellm.proxy.mcp_server.utils import MCP_OAUTH_TOKENS_TARGET class AsyncCache(Protocol): diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/envelope.py b/litellm/proxy/mcp_server/outbound_credentials/envelope.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/envelope.py rename to litellm/proxy/mcp_server/outbound_credentials/envelope.py diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/httpx_auth.py b/litellm/proxy/mcp_server/outbound_credentials/httpx_auth.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/httpx_auth.py rename to litellm/proxy/mcp_server/outbound_credentials/httpx_auth.py diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/login_assertion_exchange.py b/litellm/proxy/mcp_server/outbound_credentials/login_assertion_exchange.py similarity index 92% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/login_assertion_exchange.py rename to litellm/proxy/mcp_server/outbound_credentials/login_assertion_exchange.py index fcd8ae22ce3..6280a313f1a 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/login_assertion_exchange.py +++ b/litellm/proxy/mcp_server/outbound_credentials/login_assertion_exchange.py @@ -27,24 +27,24 @@ from datetime import datetime, timezone from typing import Final, Protocol from litellm._logging import verbose_proxy_logger -from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Ok, Result -from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_refresher import ( +from litellm.proxy._types import LiteLLM_MCPServerTable +from litellm.proxy.mcp_server.outbound_credentials.result import Error, Ok, Result +from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_refresher import ( SSOClientConfig, sso_client_config, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( +from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store import ( SSOIdentityAssertion, with_login_refresh_token, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.token_endpoint import ExchangedToken -from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( +from litellm.proxy.mcp_server.outbound_credentials.token_endpoint import ExchangedToken +from litellm.proxy.mcp_server.outbound_credentials.types import ( ClientAuth, CredError, IdJagConfig, IdJagSubjectSource, ServerSpec, ) -from litellm.proxy._types import LiteLLM_MCPServerTable from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -236,14 +236,14 @@ async def exchange_login_assertion_if_configured( exchange when one of them presents a login refresh token. Nothing here can fail the login.""" if assertion is None: return None - from litellm.proxy._experimental.mcp_server.db import get_all_mcp_servers # noqa: PLC0415 # import cycle - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # avoids import cycle + from litellm.proxy.mcp_server.db import get_all_mcp_servers # noqa: PLC0415 # import cycle + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # avoids import cycle global_mcp_server_manager, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( # noqa: PLC0415 # import cycle + from litellm.proxy.mcp_server.outbound_credentials.adapter import ( # noqa: PLC0415 # import cycle to_server_spec, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.token_endpoint import ( # noqa: PLC0415 # cycle + from litellm.proxy.mcp_server.outbound_credentials.token_endpoint import ( # noqa: PLC0415 # cycle TokenEndpointClient, ) from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 # runtime global diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/oauth_token_store.py b/litellm/proxy/mcp_server/outbound_credentials/oauth_token_store.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/oauth_token_store.py rename to litellm/proxy/mcp_server/outbound_credentials/oauth_token_store.py diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py b/litellm/proxy/mcp_server/outbound_credentials/per_user_oauth_store.py similarity index 92% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py rename to litellm/proxy/mcp_server/outbound_credentials/per_user_oauth_store.py index 2e9f3232c8d..f3748f99ba4 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py +++ b/litellm/proxy/mcp_server/outbound_credentials/per_user_oauth_store.py @@ -16,15 +16,15 @@ from functools import partial from typing import TYPE_CHECKING, Final from litellm._logging import verbose_logger -from litellm.proxy._experimental.mcp_server.oauth_identity_binding import credential_binding_matches -from litellm.proxy._experimental.mcp_server.outbound_credentials.authz_code_refresher import ( +from litellm.proxy.mcp_server.oauth_identity_binding import credential_binding_matches +from litellm.proxy.mcp_server.outbound_credentials.authz_code_refresher import ( AuthorizationCodeRefresher, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.dual_cache_token_backend import ( +from litellm.proxy.mcp_server.outbound_credentials.dual_cache_token_backend import ( AsyncCache, DualCacheTokenCacheBackend, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( CachedOAuthTokenStore, InvalidatableOAuthTokenStore, OAuthToken, @@ -33,13 +33,13 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_sto TokenCacheBackend, TokenStoreUnavailable, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.runtime_refresh_coordinator import ( +from litellm.proxy.mcp_server.outbound_credentials.runtime_refresh_coordinator import ( runtime_refresh_coordinator, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.token_cache_codec import ( +from litellm.proxy.mcp_server.outbound_credentials.token_cache_codec import ( OAuthTokenCacheCodec, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.v2_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.v2_token_store import ( V2PerUserTokenStore, ) @@ -54,7 +54,7 @@ StoreBuilder = Callable[[ServerLookup], tuple[InvalidatableOAuthTokenStore, bool async def _read_credential(user_id: str, server_id: str) -> Mapping[str, object] | None: - from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.db import ( # noqa: PLC0415 get_user_oauth_credential, ) from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 @@ -74,7 +74,7 @@ async def _persist_credential( identity_binding_proof: str | None = None, cimd_client_id: str | None = None, ) -> None: - from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.db import ( # noqa: PLC0415 store_user_oauth_credential, ) from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/presented_token_store.py b/litellm/proxy/mcp_server/outbound_credentials/presented_token_store.py similarity index 90% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/presented_token_store.py rename to litellm/proxy/mcp_server/outbound_credentials/presented_token_store.py index c88d31dd6bc..a7078676528 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/presented_token_store.py +++ b/litellm/proxy/mcp_server/outbound_credentials/presented_token_store.py @@ -11,7 +11,7 @@ from __future__ import annotations from dataclasses import dataclass -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/redis_distributed_lock.py b/litellm/proxy/mcp_server/outbound_credentials/redis_distributed_lock.py similarity index 97% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/redis_distributed_lock.py rename to litellm/proxy/mcp_server/outbound_credentials/redis_distributed_lock.py index c7eefed75ed..7262cb6f100 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/redis_distributed_lock.py +++ b/litellm/proxy/mcp_server/outbound_credentials/redis_distributed_lock.py @@ -21,7 +21,7 @@ from dataclasses import KW_ONLY, dataclass from typing import Final, Protocol from litellm._logging import verbose_logger -from litellm.proxy._experimental.mcp_server.outbound_credentials.redis_refresh_coordinator import ( +from litellm.proxy.mcp_server.outbound_credentials.redis_refresh_coordinator import ( LockAcquisition, ) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/redis_refresh_coordinator.py b/litellm/proxy/mcp_server/outbound_credentials/redis_refresh_coordinator.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/redis_refresh_coordinator.py rename to litellm/proxy/mcp_server/outbound_credentials/redis_refresh_coordinator.py index d2c660eb2af..98446f2b66e 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/redis_refresh_coordinator.py +++ b/litellm/proxy/mcp_server/outbound_credentials/redis_refresh_coordinator.py @@ -28,7 +28,7 @@ from dataclasses import KW_ONLY, dataclass from enum import Enum from typing import Final, Protocol -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py b/litellm/proxy/mcp_server/outbound_credentials/resolver.py similarity index 97% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py rename to litellm/proxy/mcp_server/outbound_credentials/resolver.py index ea0cdee5c1c..dff578ef274 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py +++ b/litellm/proxy/mcp_server/outbound_credentials/resolver.py @@ -30,28 +30,28 @@ import httpx2 from typing_extensions import assert_never from litellm._logging import verbose_proxy_logger -from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import ( +from litellm.proxy.mcp_server.outbound_credentials.client_credentials import ( ClientCredentialsBearerAuth, ClientCredentialsTokenSource, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( +from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( NoOpAuth, StaticHeaderAuth, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, OAuthTokenStore, TokenStoreUnavailable, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.result import ( +from litellm.proxy.mcp_server.outbound_credentials.result import ( Error, Ok, Result, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_refresher import ( +from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_refresher import ( default_sso_assertion_store, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( +from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store import ( AssertionStoreUnavailable, SSOAssertionStore, SSOIdentityAssertion, @@ -59,15 +59,15 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_s exchanged_refresh_token_issued_for, login_refresh_token_expired, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.token_endpoint import ( +from litellm.proxy.mcp_server.outbound_credentials.token_endpoint import ( ExchangedToken, ExchangedTokenCache, TokenEndpointClient, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger import ( +from litellm.proxy.mcp_server.outbound_credentials.token_exchanger import ( TokenExchanger, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( +from litellm.proxy.mcp_server.outbound_credentials.types import ( ApiKeyConfig, AuthorizationCodeConfig, AuthResolution, diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/result.py b/litellm/proxy/mcp_server/outbound_credentials/result.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/result.py rename to litellm/proxy/mcp_server/outbound_credentials/result.py diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/runtime_refresh_coordinator.py b/litellm/proxy/mcp_server/outbound_credentials/runtime_refresh_coordinator.py similarity index 85% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/runtime_refresh_coordinator.py rename to litellm/proxy/mcp_server/outbound_credentials/runtime_refresh_coordinator.py index e799838b5b7..9376d1bda50 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/runtime_refresh_coordinator.py +++ b/litellm/proxy/mcp_server/outbound_credentials/runtime_refresh_coordinator.py @@ -14,13 +14,13 @@ from __future__ import annotations from typing import Final -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( RefreshCoordinator, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.redis_distributed_lock import ( +from litellm.proxy.mcp_server.outbound_credentials.redis_distributed_lock import ( RedisDistributedLock, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.redis_refresh_coordinator import ( +from litellm.proxy.mcp_server.outbound_credentials.redis_refresh_coordinator import ( RedisRefreshCoordinator, ) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/session_credentials.py b/litellm/proxy/mcp_server/outbound_credentials/session_credentials.py similarity index 99% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/session_credentials.py rename to litellm/proxy/mcp_server/outbound_credentials/session_credentials.py index b216f463824..6e3d09f4dd2 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/session_credentials.py +++ b/litellm/proxy/mcp_server/outbound_credentials/session_credentials.py @@ -22,7 +22,7 @@ from typing import Final, Literal, TypeAlias from pydantic import ConfigDict, Field, SecretStr, ValidationError -from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import ( +from litellm.proxy.mcp_server.outbound_credentials.session_token import ( AsymmetricSessionKeys, OpenedSessionToken, SessionExpired, diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/session_token.py b/litellm/proxy/mcp_server/outbound_credentials/session_token.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/session_token.py rename to litellm/proxy/mcp_server/outbound_credentials/session_token.py diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_refresher.py b/litellm/proxy/mcp_server/outbound_credentials/sso_assertion_refresher.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_refresher.py rename to litellm/proxy/mcp_server/outbound_credentials/sso_assertion_refresher.py index ae5e9d0417e..84a20bf4f5c 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_refresher.py +++ b/litellm/proxy/mcp_server/outbound_credentials/sso_assertion_refresher.py @@ -58,22 +58,22 @@ from litellm.exceptions import Timeout from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # litellm http handler is untyped ) -from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( +from litellm.proxy.mcp_server.auth.token_endpoint_auth import ( build_token_endpoint_client_auth, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( InProcessRefreshCoordinator, RefreshCoordinator, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.result import ( +from litellm.proxy.mcp_server.outbound_credentials.result import ( Error, Ok, Result, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.runtime_refresh_coordinator import ( +from litellm.proxy.mcp_server.outbound_credentials.runtime_refresh_coordinator import ( runtime_refresh_coordinator, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( +from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store import ( AssertionStoreUnavailable, DbSSOAssertionStore, SSOAssertionStore, diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py b/litellm/proxy/mcp_server/outbound_credentials/sso_assertion_store.py similarity index 99% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py rename to litellm/proxy/mcp_server/outbound_credentials/sso_assertion_store.py index 19b3bc294d5..fb4d838aff9 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py +++ b/litellm/proxy/mcp_server/outbound_credentials/sso_assertion_store.py @@ -294,7 +294,7 @@ async def ema_assertion_retention_enabled() -> bool: snapshot of the DB state that can be stale both ways (a server added on another pod would silently drop the write, one removed on another pod would keep retaining bearer material), and a gate guarding a shared-DB write must judge against that storage's authority.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # avoids import cycle + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # avoids import cycle global_mcp_server_manager, ) from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 # runtime global diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_cache_codec.py b/litellm/proxy/mcp_server/outbound_credentials/token_cache_codec.py similarity index 95% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/token_cache_codec.py rename to litellm/proxy/mcp_server/outbound_credentials/token_cache_codec.py index 56dc3f19cdd..4576e925939 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_cache_codec.py +++ b/litellm/proxy/mcp_server/outbound_credentials/token_cache_codec.py @@ -14,7 +14,7 @@ from typing import Final from pydantic import TypeAdapter, ValidationError -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_endpoint.py b/litellm/proxy/mcp_server/outbound_credentials/token_endpoint.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/token_endpoint.py rename to litellm/proxy/mcp_server/outbound_credentials/token_endpoint.py index 1ee1b7ecf2f..6fd63450b26 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_endpoint.py +++ b/litellm/proxy/mcp_server/outbound_credentials/token_endpoint.py @@ -39,12 +39,12 @@ from litellm.exceptions import Timeout from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # litellm http handler is untyped ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.result import ( +from litellm.proxy.mcp_server.outbound_credentials.result import ( Error, Ok, Result, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( +from litellm.proxy.mcp_server.outbound_credentials.types import ( ClientAuth, ClientSecretAuth, CredError, diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchange_provider.py b/litellm/proxy/mcp_server/outbound_credentials/token_exchange_provider.py similarity index 96% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchange_provider.py rename to litellm/proxy/mcp_server/outbound_credentials/token_exchange_provider.py index d84a65fee02..0beab749076 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchange_provider.py +++ b/litellm/proxy/mcp_server/outbound_credentials/token_exchange_provider.py @@ -21,10 +21,10 @@ from litellm.constants import ( MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS, MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( InMemoryTokenCacheBackend, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger import ( +from litellm.proxy.mcp_server.outbound_credentials.token_exchanger import ( OboTokenExchanger, SubjectTokenRejected, TokenExchangeClientError, diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchanger.py b/litellm/proxy/mcp_server/outbound_credentials/token_exchanger.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchanger.py rename to litellm/proxy/mcp_server/outbound_credentials/token_exchanger.py index 2e72d98c77d..83457d5cef1 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchanger.py +++ b/litellm/proxy/mcp_server/outbound_credentials/token_exchanger.py @@ -27,22 +27,22 @@ from typing import Final, Literal, Protocol from typing_extensions import assert_never from litellm._logging import verbose_logger -from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( +from litellm.proxy.mcp_server.auth.token_endpoint_auth import ( build_token_endpoint_client_auth, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( InMemoryTokenCacheBackend, InProcessRefreshCoordinator, OAuthToken, RefreshCoordinator, TokenCacheBackend, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.result import ( +from litellm.proxy.mcp_server.outbound_credentials.result import ( Error, Ok, Result, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( +from litellm.proxy.mcp_server.outbound_credentials.types import ( CredError, ServerSpec, TokenExchangeConfig, diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py b/litellm/proxy/mcp_server/outbound_credentials/types.py similarity index 99% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py rename to litellm/proxy/mcp_server/outbound_credentials/types.py index 45c53af9349..5a30fafd7d9 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py +++ b/litellm/proxy/mcp_server/outbound_credentials/types.py @@ -35,7 +35,7 @@ from expression import case, tag, tagged_union from pydantic import ConfigDict, Field, SecretStr, field_validator from typing_extensions import assert_never -from litellm.proxy._experimental.mcp_server.outbound_credentials.result import ( +from litellm.proxy.mcp_server.outbound_credentials.result import ( Error, Ok, Result, diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/v2_token_store.py b/litellm/proxy/mcp_server/outbound_credentials/v2_token_store.py similarity index 97% rename from litellm/proxy/_experimental/mcp_server/outbound_credentials/v2_token_store.py rename to litellm/proxy/mcp_server/outbound_credentials/v2_token_store.py index 56ab8e6bbdc..a1a3b88259e 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/v2_token_store.py +++ b/litellm/proxy/mcp_server/outbound_credentials/v2_token_store.py @@ -14,7 +14,7 @@ from collections.abc import Awaitable, Callable, Mapping from datetime import datetime, timezone from typing import Final -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) diff --git a/litellm/proxy/_experimental/mcp_server/proxy_api_credentials.py b/litellm/proxy/mcp_server/proxy_api_credentials.py similarity index 96% rename from litellm/proxy/_experimental/mcp_server/proxy_api_credentials.py rename to litellm/proxy/mcp_server/proxy_api_credentials.py index fd637a2f187..53fe4ce457d 100644 --- a/litellm/proxy/_experimental/mcp_server/proxy_api_credentials.py +++ b/litellm/proxy/mcp_server/proxy_api_credentials.py @@ -8,13 +8,6 @@ from collections.abc import Sequence from typing import Final from litellm.constants import CLI_JWT_EXPIRATION_HOURS -from litellm.proxy._experimental.mcp_server.bridge_token_flow import load_active_user_by_id -from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ( - ConsentTeam, - MintedProxyCredential, - ProxyCredentialMintFailure, - ReloadUserFailure, -) from litellm.proxy._types import LiteLLM_UserTable from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken, effective_user_role from litellm.proxy.management_endpoints.ui_sso import ( @@ -22,6 +15,13 @@ from litellm.proxy.management_endpoints.ui_sso import ( fetch_cli_sso_team_details, selected_cli_sso_team_detail, ) +from litellm.proxy.mcp_server.bridge_token_flow import load_active_user_by_id +from litellm.proxy.mcp_server.gateway_dcr_flow import ( + ConsentTeam, + MintedProxyCredential, + ProxyCredentialMintFailure, + ReloadUserFailure, +) async def lookup_consent_teams(user_id: str) -> tuple[ConsentTeam, ...] | ReloadUserFailure: diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/mcp_server/rest_endpoints.py similarity index 96% rename from litellm/proxy/_experimental/mcp_server/rest_endpoints.py rename to litellm/proxy/mcp_server/rest_endpoints.py index 62387b7cc28..2737b567985 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/mcp_server/rest_endpoints.py @@ -24,35 +24,6 @@ from litellm.exceptions import ( GuardrailRaisedException, ModifyResponseException, ) -from litellm.proxy._experimental.mcp_server.catalog import catalog_operation, global_manager -from litellm.proxy._experimental.mcp_server.exceptions import ( - MCPServerListError, - MCPServerURLCredentialsError, - MCPUpstreamAuthError, -) -from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( - ServerListOk, - ServerOutcome, - classify_list_exception, - list_fault_http_status, - outcome_wire_value, -) -from litellm.proxy._experimental.mcp_server.faults.traversal import iter_exception_tree -from litellm.proxy._experimental.mcp_server.oauth_utils import ( # noqa: F401 # legacy module exports - _redact_mcp_resource_url, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - redact_mcp_resource_url, -) -from litellm.proxy._experimental.mcp_server.result_conversion import WireCompat, complete_call_tool_result -from litellm.proxy._experimental.mcp_server.ui_session_utils import ( - acting_user_auth, - build_effective_auth_contexts, -) -from litellm.proxy._experimental.mcp_server.utils import ( - MCPMissingUserEnvVarsError, - add_server_prefix_to_name, - get_server_prefix, - merge_mcp_headers, -) from litellm.proxy._types import ( LitellmUserRoles, UserAPIKeyAuth, @@ -61,13 +32,42 @@ from litellm.proxy._types import ( from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError +from litellm.proxy.mcp_server.catalog import catalog_operation, global_manager +from litellm.proxy.mcp_server.exceptions import ( + MCPServerListError, + MCPServerURLCredentialsError, + MCPUpstreamAuthError, +) +from litellm.proxy.mcp_server.faults.list_outcomes import ( + ServerListOk, + ServerOutcome, + classify_list_exception, + list_fault_http_status, + outcome_wire_value, +) +from litellm.proxy.mcp_server.faults.traversal import iter_exception_tree +from litellm.proxy.mcp_server.oauth_utils import ( # noqa: F401 # legacy module exports + _redact_mcp_resource_url, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + redact_mcp_resource_url, +) +from litellm.proxy.mcp_server.result_conversion import WireCompat, complete_call_tool_result +from litellm.proxy.mcp_server.ui_session_utils import ( + acting_user_auth, + build_effective_auth_contexts, +) +from litellm.proxy.mcp_server.utils import ( + MCPMissingUserEnvVarsError, + add_server_prefix_to_name, + get_server_prefix, + merge_mcp_headers, +) from litellm.responses.mcp.request_context import MCPRequestContext if TYPE_CHECKING: from mcp.types import CallToolResult, InputRequiredResult from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.proxy._experimental.mcp_server.db import OAuthCredentialPayload + from litellm.proxy.mcp_server.db import OAuthCredentialPayload from litellm.proxy.utils import ProxyLogging from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -82,7 +82,7 @@ class _MCPModelMetadata(TypedDict): def _stamp_mcp_tool_metadata(logging_obj: "LiteLLMLoggingObj | None", server_id: str, tool_name: str) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager if logging_obj is None: return @@ -227,22 +227,22 @@ if MCP_AVAILABLE: from mcp.shared.exceptions import MCPError from mcp.types import Tool as MCPTool - from litellm.experimental_mcp_client.client import MCPClient, as_mcp_read_timeout from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.litellm_proxy.skills.skill_search import ( DEFAULT_SKILL_SEARCH_TOP_K, ) - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: F401 # legacy module exports + from litellm.mcp_client.client import MCPClient, as_mcp_read_timeout + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: F401 # legacy module exports _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES, ListedToolsCaller, global_mcp_server_manager, ) - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( get_request_base_url, ) - from litellm.proxy._experimental.mcp_server.operations import ( + from litellm.proxy.mcp_server.operations import ( ListMCPToolsRestAPIResponseObject, MCPInfo, MCPServer, @@ -254,12 +254,12 @@ if MCP_AVAILABLE: fire_mcp_tool_call_failure_logging, resolve_native_tool_name_for_server, ) - from litellm.proxy._experimental.mcp_server.server import ( # noqa: F401 # legacy module exports + from litellm.proxy.mcp_server.server import ( # noqa: F401 # legacy module exports _apply_toolset_scope, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export apply_toolset_scope, reject_disallowed_mcp_client, ) - from litellm.proxy._experimental.mcp_server.tool_catalog_guard import ( + from litellm.proxy.mcp_server.tool_catalog_guard import ( apply_description_overrides, scan_tool_descriptions, ) @@ -337,7 +337,8 @@ if MCP_AVAILABLE: ``mcp_tool_search_enabled``). Kept out of ``call_tool_rest_api`` so that endpoint stays a single dispatch. An upstream 401 raised by the virtual ``mcp_tool_call`` propagates unhandled to the caller's ``except MCPUpstreamAuthError`` relay, the same as the direct call path.""" - from litellm.proxy._experimental.mcp_server.tool_search import ( + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.mcp_server.tool_search import ( AGENT_SEARCH_TOOL_NAME, DEFAULT_AGENT_SEARCH_TOP_K, MCP_TOOL_SEARCH_TOOL_NAME, @@ -348,7 +349,6 @@ if MCP_AVAILABLE: handle_mcp_tool_search, handle_skill_search, ) - from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.proxy_server import general_settings, proxy_config, proxy_logging_obj if not getattr(getattr(user_api_key_dict, "object_permission", None), "mcp_tool_search_enabled", False): @@ -443,7 +443,7 @@ if MCP_AVAILABLE: mcp_auth_header: str | None, ) -> dict[str, str] | str | None: """Helper function to get server-specific auth header with case-insensitive matching.""" - from litellm.proxy._experimental.mcp_server.utils import ( + from litellm.proxy.mcp_server.utils import ( lookup_mcp_server_auth_in_headers, ) @@ -466,7 +466,7 @@ if MCP_AVAILABLE: round-trip whose result is discarded. Mirrors the same guard on the protocol listing path and in ``_resolve_oauth2_headers_for_tool_call``. """ - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( + from litellm.proxy.mcp_server.outbound_credentials.adapter import ( to_server_spec, ) @@ -511,7 +511,7 @@ if MCP_AVAILABLE: if not user_id or not server_id: return None try: - from litellm.proxy._experimental.mcp_server.db import ( + from litellm.proxy.mcp_server.db import ( get_user_oauth_credential, resolve_valid_user_oauth_token, ) @@ -555,7 +555,7 @@ if MCP_AVAILABLE: if not user_id: return {} try: - from litellm.proxy._experimental.mcp_server.db import ( + from litellm.proxy.mcp_server.db import ( list_user_oauth_credentials, ) from litellm.proxy.utils import get_prisma_client_or_throw @@ -603,7 +603,7 @@ if MCP_AVAILABLE: Returns: Tuple of (mcp_auth_header, mcp_server_auth_headers, raw_headers, oauth2_headers) """ - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( OPTIONAL_STRING_ADAPTER, MCPRequestHandler, ) @@ -1062,7 +1062,7 @@ if MCP_AVAILABLE: False, ) ): - from litellm.proxy._experimental.mcp_server.tool_search import ( + from litellm.proxy.mcp_server.tool_search import ( get_virtual_tool_definitions, ) @@ -1344,7 +1344,7 @@ if MCP_AVAILABLE: tool_name: Final[str | None] = data.get("name") tool_arguments: Final[dict[str, object]] = data.get("arguments") or {} - from litellm.proxy._experimental.mcp_server.tool_search import VIRTUAL_TOOL_NAMES + from litellm.proxy.mcp_server.tool_search import VIRTUAL_TOOL_NAMES if tool_name in VIRTUAL_TOOL_NAMES: return await _handle_virtual_mcp_tool(request, data, tool_name, user_api_key_dict) @@ -1594,7 +1594,7 @@ if MCP_AVAILABLE: temporary client the same credentials, or a server that the saved connection reaches fine fails one of them. """ - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) @@ -1725,16 +1725,16 @@ if MCP_AVAILABLE: # becomes the upstream credential, so it is not also forwarded as a caller header. Gated # to the v2-mapped oauth2 case (to_server_spec non-None); M2M (client_credentials), # delegate/passthrough, and token-exchange are unaffected. - from litellm.proxy._experimental.mcp_server.outbound_credentials import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.outbound_credentials import ( # noqa: PLC0415 UpstreamCredentialProvider, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.outbound_credentials.adapter import ( # noqa: PLC0415 to_server_spec, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( # noqa: PLC0415 OAuthToken, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.presented_token_store import ( # noqa: PLC0415 + from litellm.proxy.mcp_server.outbound_credentials.presented_token_store import ( # noqa: PLC0415 PresentedOAuthTokenStore, ) @@ -1795,7 +1795,7 @@ if MCP_AVAILABLE: async def _preview_openapi_tools(spec_path: str) -> dict: """Generate tool previews from an OpenAPI spec without creating a server.""" - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( OPENAPI_TOOL_NAME_MAX_LEN, build_input_schema, load_openapi_spec_async, diff --git a/litellm/proxy/_experimental/mcp_server/result_conversion.py b/litellm/proxy/mcp_server/result_conversion.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/result_conversion.py rename to litellm/proxy/mcp_server/result_conversion.py index fdb9a01fea6..8955381add1 100644 --- a/litellm/proxy/_experimental/mcp_server/result_conversion.py +++ b/litellm/proxy/mcp_server/result_conversion.py @@ -18,7 +18,7 @@ from typing import Final, Protocol, TypeAlias, TypeVar from mcp.types import CacheableResult, CallToolResult, ContentBlock, InputRequiredResult, TextContent, Tool from typing_extensions import ReadOnly, TypedDict, assert_never -from litellm.proxy._experimental.mcp_server.tool_outcome import ( +from litellm.proxy.mcp_server.tool_outcome import ( JsonResult, TextResult, WireCompat, diff --git a/litellm/proxy/_experimental/mcp_server/sampling_handler.py b/litellm/proxy/mcp_server/sampling_handler.py similarity index 99% rename from litellm/proxy/_experimental/mcp_server/sampling_handler.py rename to litellm/proxy/mcp_server/sampling_handler.py index d049ada4dfd..ec14f81aef2 100644 --- a/litellm/proxy/_experimental/mcp_server/sampling_handler.py +++ b/litellm/proxy/mcp_server/sampling_handler.py @@ -1047,7 +1047,7 @@ def _build_sampling_request( ) -> "Request": """The synthetic FastAPI Request for sampling sub-calls, carrying the original MCP connection's headers and client IP.""" - from litellm.proxy._experimental.mcp_server.utils import build_synthetic_mcp_request + from litellm.proxy.mcp_server.utils import build_synthetic_mcp_request return build_synthetic_mcp_request( path="/mcp/sampling/createMessage", diff --git a/litellm/proxy/_experimental/mcp_server/scope_challenge.py b/litellm/proxy/mcp_server/scope_challenge.py similarity index 97% rename from litellm/proxy/_experimental/mcp_server/scope_challenge.py rename to litellm/proxy/mcp_server/scope_challenge.py index 121234e305d..128b27b0f91 100644 --- a/litellm/proxy/_experimental/mcp_server/scope_challenge.py +++ b/litellm/proxy/mcp_server/scope_challenge.py @@ -10,7 +10,7 @@ from starlette.requests import Request from starlette.responses import JSONResponse from starlette.types import Message, Send -from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError +from litellm.proxy.mcp_server.exceptions import MCPUpstreamAuthError if TYPE_CHECKING: from mcp.server.context import CallNext, HandlerResult, ServerRequestContext diff --git a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py b/litellm/proxy/mcp_server/semantic_tool_filter.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py rename to litellm/proxy/mcp_server/semantic_tool_filter.py index a7748b9ca88..4fe723d6b24 100644 --- a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py +++ b/litellm/proxy/mcp_server/semantic_tool_filter.py @@ -11,9 +11,9 @@ from typing import TYPE_CHECKING, Final, TypeVar from litellm._logging import verbose_logger from litellm.exceptions import ContextWindowExceededError from litellm.litellm_core_utils.exception_mapping_utils import ExceptionCheckers -from litellm.proxy._experimental.mcp_server.catalog import catalog_operation, global_manager -from litellm.proxy._experimental.mcp_server.faults import iter_exception_tree -from litellm.proxy._experimental.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR +from litellm.proxy.mcp_server.catalog import catalog_operation, global_manager +from litellm.proxy.mcp_server.faults import iter_exception_tree +from litellm.proxy.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR if TYPE_CHECKING: from semantic_router.routers import SemanticRouter @@ -84,7 +84,7 @@ class SemanticMCPToolFilter: @catalog_operation(global_manager) async def build_router_from_mcp_registry(self) -> None: """Build semantic router from all MCP tools in the registry (no auth checks).""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/mcp_server/server.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/server.py rename to litellm/proxy/mcp_server/server.py index 7207c4150ad..32f790a8e81 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/mcp_server/server.py @@ -35,23 +35,29 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( # noqa: F401 # legacy module exports +from litellm.proxy._types import ( + ProxyException, + SpecialMCPServerNames, + UserAPIKeyAuth, +) +from litellm.proxy.auth.ip_address_utils import IPAddressUtils +from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( # noqa: F401 # legacy module exports MCPRequestHandler, _is_mcp_admitted_user_subject, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export is_mcp_admitted_user_subject, ) -from litellm.proxy._experimental.mcp_server.client_allowlist import ( +from litellm.proxy.mcp_server.client_allowlist import ( MCPClientAllowlist, check_mcp_client_allowed, load_mcp_client_allowlist, ) -from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( +from litellm.proxy.mcp_server.discoverable_endpoints import ( get_request_base_url, ) -from litellm.proxy._experimental.mcp_server.exceptions import ( +from litellm.proxy.mcp_server.exceptions import ( MCPUpstreamAuthError, ) -from litellm.proxy._experimental.mcp_server.mcp_context import ( # noqa: F401 # legacy module exports +from litellm.proxy.mcp_server.mcp_context import ( # noqa: F401 # legacy module exports _mcp_active_toolset_id, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _mcp_gateway_initialize_instructions, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _mcp_gateway_server_name, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -62,43 +68,37 @@ from litellm.proxy._experimental.mcp_server.mcp_context import ( # noqa: F401 mcp_gateway_initialize_instructions, mcp_gateway_server_name, ) -from litellm.proxy._experimental.mcp_server.mcp_debug import ( +from litellm.proxy.mcp_server.mcp_debug import ( MCP_AUTH_DIAGNOSTICS_SCOPE_KEY, MCPAuthDiagnostics, MCPDebug, ) -from litellm.proxy._experimental.mcp_server.oauth_utils import ( +from litellm.proxy.mcp_server.oauth_utils import ( get_passthrough_www_authenticate, get_route_relative_request_path, redact_mcp_resource_url, well_known_root_suffix, ) -from litellm.proxy._experimental.mcp_server.scope_challenge import ( +from litellm.proxy.mcp_server.scope_challenge import ( SCOPE_RESPONSE_KEY, OAuthScopeResponse, finish_scope_response, is_tool_call_request, ) -from litellm.proxy._experimental.mcp_server.ui_session_utils import ( +from litellm.proxy.mcp_server.ui_session_utils import ( ActingUser, GrantedToolsetIds, acting_user_auth, granted_toolset_ids, is_ui_session_credential, ) -from litellm.proxy._experimental.mcp_server.utils import ( +from litellm.proxy.mcp_server.utils import ( LITELLM_MCP_SERVER_DESCRIPTION, LITELLM_MCP_SERVER_NAME, LITELLM_MCP_SERVER_VERSION, asgi_scope_fields, merge_mcp_headers, ) -from litellm.proxy._types import ( - ProxyException, - SpecialMCPServerNames, - UserAPIKeyAuth, -) -from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.types.mcp import ( MCPAuth, MCPGatewaySession, @@ -162,7 +162,7 @@ def _raw_header_pairs(source: _RawHeaderPairsSource) -> Sequence[tuple[bytes | s def unsupported_protocol_version(scope: Scope) -> str | None: """Admit configured HTTP revisions while keeping SSE on the legacy protocol.""" - from litellm.proxy._experimental.mcp_server.capabilities import configured_versions + from litellm.proxy.mcp_server.capabilities import configured_versions headers: Final[Iterable[tuple[bytes, bytes]]] = scope.get("headers") or () values: Final = tuple( @@ -547,14 +547,14 @@ if MCP_AVAILABLE: ReadResourceRequest, ) - from litellm.proxy._experimental.mcp_server import operations - from litellm.proxy._experimental.mcp_server.contracts import OperationContext - from litellm.proxy._experimental.mcp_server.interactions import InteractionOperation - from litellm.proxy._experimental.mcp_server.operations import ( + from litellm.proxy.mcp_server import operations + from litellm.proxy.mcp_server.contracts import OperationContext + from litellm.proxy.mcp_server.interactions import InteractionOperation + from litellm.proxy.mcp_server.operations import ( _invalidate_byok_cred_cache, _mcp_session_id_from_headers, ) - from litellm.proxy._experimental.mcp_server.result_conversion import wire_compat_for + from litellm.proxy.mcp_server.result_conversion import wire_compat_for try: from mcp.server.streamable_http_manager import StreamableHTTPSessionManager @@ -576,16 +576,16 @@ if MCP_AVAILABLE: ReadResourceRequestParams, ) - from litellm.proxy._experimental.mcp_server.auth.litellm_auth_handler import ( + from litellm.proxy.mcp_server.auth.litellm_auth_handler import ( MCPAuthenticatedUser, ) - from litellm.proxy._experimental.mcp_server.capabilities import ( + from litellm.proxy.mcp_server.capabilities import ( GatewayVersionPolicy, configured_versions, disabled_capabilities, disabled_operations, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, global_mcp_server_manager, ) @@ -595,10 +595,10 @@ if MCP_AVAILABLE: # Defined here because we don't want to add `mcp` as a # required dependency for `litellm` pip package ###################################################### - from litellm.proxy._experimental.mcp_server.operations import ( + from litellm.proxy.mcp_server.operations import ( ListMCPToolsRestAPIResponseObject, ) - from litellm.proxy._experimental.mcp_server.sse_transport import SseServerTransport + from litellm.proxy.mcp_server.sse_transport import SseServerTransport def _gateway_create_initialization_options( self, @@ -946,7 +946,7 @@ if MCP_AVAILABLE: raise MCPError(code=METHOD_NOT_FOUND, message="Operation unavailable on /mcp/proxy") - from litellm.proxy._experimental.mcp_server.operations import ( + from litellm.proxy.mcp_server.operations import ( _build_virtual_call_logging_obj, _dispatch_virtual_mcp_tool, ) @@ -1036,7 +1036,7 @@ if MCP_AVAILABLE: UnsubscribeRequestParams, ) - from litellm.proxy._experimental.mcp_server.subscriptions import GatewaySubscriptions, SubscriptionAuthorization + from litellm.proxy.mcp_server.subscriptions import GatewaySubscriptions, SubscriptionAuthorization subscriptions: Final = GatewaySubscriptions() legacy_subscription_request: Final[contextvars.ContextVar[StarletteRequest | None]] = contextvars.ContextVar( @@ -1126,7 +1126,7 @@ if MCP_AVAILABLE: ############ Helper Functions ########################## ######################################################## - from litellm.proxy._experimental.mcp_server.operations import ( + from litellm.proxy.mcp_server.operations import ( _client_has_passthrough_authorization, _client_has_per_server_auth_header, _get_allowed_mcp_servers, @@ -1189,7 +1189,7 @@ if MCP_AVAILABLE: mcp_gateway_initialize_instructions.reset(instructions_token) mcp_gateway_server_name.reset(server_name_token) - from litellm.proxy._experimental.mcp_server.operations import ( + from litellm.proxy.mcp_server.operations import ( _MCP_CREDENTIAL_REQUEST_FIELDS, _aggregate_server_key, _check_byok_credential, @@ -1836,7 +1836,7 @@ if MCP_AVAILABLE: await operations.global_mcp_server_manager.resolve_toolset_tool_permissions(toolset_ids=[toolset_id]) ) - from litellm.proxy._experimental.mcp_server.catalog import catalog_operation + from litellm.proxy.mcp_server.catalog import catalog_operation @catalog_operation(lambda: operations.global_mcp_server_manager) async def _raise_preemptive_401_for_unauthenticated_servers( @@ -1971,7 +1971,7 @@ if MCP_AVAILABLE: ) is None ): - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( # noqa: PLC0415 # lazy: adapter pulls MCP subgraph + from litellm.proxy.mcp_server.outbound_credentials.adapter import ( # noqa: PLC0415 # lazy: adapter pulls MCP subgraph raise_token_exchange_challenge, ) from litellm.proxy.middleware.per_request_root_path_middleware import ( # noqa: PLC0415 # lazy: middleware imports proxy utils diff --git a/litellm/proxy/_experimental/mcp_server/server_resolution.py b/litellm/proxy/mcp_server/server_resolution.py similarity index 96% rename from litellm/proxy/_experimental/mcp_server/server_resolution.py rename to litellm/proxy/mcp_server/server_resolution.py index 31bcd897e72..96c485c0d3c 100644 --- a/litellm/proxy/_experimental/mcp_server/server_resolution.py +++ b/litellm/proxy/mcp_server/server_resolution.py @@ -6,12 +6,12 @@ from typing import TYPE_CHECKING, Final, Literal, Protocol from fastapi import HTTPException, status -from litellm.proxy._experimental.mcp_server.ui_session_utils import can_access_mcp_server from litellm.proxy._types import LiteLLM_MCPServerTable, UserAPIKeyAuth +from litellm.proxy.mcp_server.ui_session_utils import can_access_mcp_server from litellm.types.mcp_server.mcp_server_manager import MCPServer if TYPE_CHECKING: - from litellm.proxy._experimental.mcp_server.contracts import CatalogListRequest, CatalogListResult, OperationContext + from litellm.proxy.mcp_server.contracts import CatalogListRequest, CatalogListResult, OperationContext class MCPServerRegistry(Protocol): diff --git a/litellm/proxy/_experimental/mcp_server/sse_transport.py b/litellm/proxy/mcp_server/sse_transport.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/sse_transport.py rename to litellm/proxy/mcp_server/sse_transport.py diff --git a/litellm/proxy/_experimental/mcp_server/state_tokens.py b/litellm/proxy/mcp_server/state_tokens.py similarity index 97% rename from litellm/proxy/_experimental/mcp_server/state_tokens.py rename to litellm/proxy/mcp_server/state_tokens.py index 88f98bef6b8..b65a8ca4e88 100644 --- a/litellm/proxy/_experimental/mcp_server/state_tokens.py +++ b/litellm/proxy/mcp_server/state_tokens.py @@ -12,7 +12,7 @@ from cryptography.hazmat.primitives.ciphers.aead import AESGCM from cryptography.hazmat.primitives.kdf.hkdf import HKDF from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError -from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Ok, Result +from litellm.proxy.mcp_server.outbound_credentials.result import Error, Ok, Result _PREFIX: Final = "mcp_state_v1." _MAX_TOKEN_LENGTH: Final = 65536 diff --git a/litellm/proxy/_experimental/mcp_server/stdio_gate.py b/litellm/proxy/mcp_server/stdio_gate.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/stdio_gate.py rename to litellm/proxy/mcp_server/stdio_gate.py diff --git a/litellm/proxy/_experimental/mcp_server/subscriptions.py b/litellm/proxy/mcp_server/subscriptions.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/subscriptions.py rename to litellm/proxy/mcp_server/subscriptions.py index e9de5692d83..f133752ee32 100644 --- a/litellm/proxy/_experimental/mcp_server/subscriptions.py +++ b/litellm/proxy/mcp_server/subscriptions.py @@ -48,17 +48,17 @@ from mcp.types import ( ) from litellm.constants import MCP_TOOL_LISTING_MAX_PAGES, MCP_TOOL_LISTING_TIMEOUT -from litellm.experimental_mcp_client.client import MCPClient -from litellm.proxy._experimental.mcp_server import operations -from litellm.proxy._experimental.mcp_server.catalog import ( - ToolListingFailedOutright, -) -from litellm.proxy._experimental.mcp_server.catalog import ( - _configuration_identity as configuration_identity, # pyright: ignore[reportPrivateUsage] # reuse catalog-owned configuration identity without treating discovered metadata as configuration -) -from litellm.proxy._experimental.mcp_server.contracts import OperationContext +from litellm.mcp_client.client import MCPClient from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError +from litellm.proxy.mcp_server import operations +from litellm.proxy.mcp_server.catalog import ( + ToolListingFailedOutright, +) +from litellm.proxy.mcp_server.catalog import ( + _configuration_identity as configuration_identity, # pyright: ignore[reportPrivateUsage] # reuse catalog-owned configuration identity without treating discovered metadata as configuration +) +from litellm.proxy.mcp_server.contracts import OperationContext from litellm.types.mcp import MCP_LEGACY_VERSIONS from litellm.types.mcp_server.mcp_server_manager import MCPServer diff --git a/litellm/proxy/_experimental/mcp_server/tool_catalog_guard.py b/litellm/proxy/mcp_server/tool_catalog_guard.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/tool_catalog_guard.py rename to litellm/proxy/mcp_server/tool_catalog_guard.py index b3c70a33d0e..4bf7b584977 100644 --- a/litellm/proxy/_experimental/mcp_server/tool_catalog_guard.py +++ b/litellm/proxy/mcp_server/tool_catalog_guard.py @@ -12,7 +12,7 @@ from mcp.types import Tool as MCPTool from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict -from litellm.proxy._experimental.mcp_server.utils import logging_safe_mcp_headers, strip_known_server_prefix +from litellm.proxy.mcp_server.utils import logging_safe_mcp_headers, strip_known_server_prefix from litellm.types.mcp import MCPPreCallRequestObject from litellm.types.mcp_server.mcp_server_manager import MCPServer, PinnedMCPTool from litellm.types.utils import CallTypes diff --git a/litellm/proxy/_experimental/mcp_server/tool_outcome.py b/litellm/proxy/mcp_server/tool_outcome.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/tool_outcome.py rename to litellm/proxy/mcp_server/tool_outcome.py diff --git a/litellm/proxy/_experimental/mcp_server/tool_registry.py b/litellm/proxy/mcp_server/tool_registry.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/tool_registry.py rename to litellm/proxy/mcp_server/tool_registry.py diff --git a/litellm/proxy/_experimental/mcp_server/tool_search.py b/litellm/proxy/mcp_server/tool_search.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/tool_search.py rename to litellm/proxy/mcp_server/tool_search.py index 371a1ff6fc9..96bcf8fbacf 100644 --- a/litellm/proxy/_experimental/mcp_server/tool_search.py +++ b/litellm/proxy/mcp_server/tool_search.py @@ -20,7 +20,6 @@ from typing_extensions import ReadOnly, Required, assert_never import litellm from litellm.llms.litellm_proxy.skills.skill_search import DEFAULT_SKILL_SEARCH_TOP_K -from litellm.proxy._experimental.mcp_server.result_conversion import WireCompat, complete_call_tool_result from litellm.proxy.agent_endpoints.agent_search import DEFAULT_AGENT_SEARCH_TOP_K from litellm.proxy.common_utils.semantic_text_index import ( Embedder, @@ -28,6 +27,7 @@ from litellm.proxy.common_utils.semantic_text_index import ( SemanticTextIndex, router_embedder, ) +from litellm.proxy.mcp_server.result_conversion import WireCompat, complete_call_tool_result from litellm.types.mcp import MCPToolSearchSettings if TYPE_CHECKING: @@ -549,7 +549,7 @@ async def handle_mcp_tool_search( oauth2_headers: dict[str, str] | None = None, raw_headers: dict[str, str] | None = None, ) -> CallToolResult: - from litellm.proxy._experimental.mcp_server.operations import ( + from litellm.proxy.mcp_server.operations import ( _list_mcp_tools, ) from litellm.proxy.proxy_server import llm_router, proxy_logging_obj @@ -606,7 +606,7 @@ async def handle_mcp_proxy_tool( from fastapi import HTTPException from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.operations import ( + from litellm.proxy.mcp_server.operations import ( _list_mcp_tools, find_mcp_proxy_tool, ) @@ -713,7 +713,7 @@ async def handle_mcp_tool_call( incoming_bearer_token: SecretStr | None | EllipsisType = ..., guardrail_auth: UserAPIKeyAuth | None = None, ) -> CallToolResult: - from litellm.proxy._experimental.mcp_server.operations import ( + from litellm.proxy.mcp_server.operations import ( _get_allowed_mcp_servers, execute_mcp_tool, raise_denied_scoped_mcp_access, diff --git a/litellm/proxy/_experimental/mcp_server/toolset_db.py b/litellm/proxy/mcp_server/toolset_db.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/toolset_db.py rename to litellm/proxy/mcp_server/toolset_db.py index 35906c793b4..ae40e8af0d9 100644 --- a/litellm/proxy/_experimental/mcp_server/toolset_db.py +++ b/litellm/proxy/mcp_server/toolset_db.py @@ -117,7 +117,7 @@ async def list_mcp_toolsets( except Exception as e: if use_writer: raise - verbose_proxy_logger.warning("litellm.proxy._experimental.mcp_server.toolset_db::list_mcp_toolsets - %s", e) + verbose_proxy_logger.warning("litellm.proxy.mcp_server.toolset_db::list_mcp_toolsets - %s", e) return [] diff --git a/litellm/proxy/_experimental/mcp_server/ui_session_utils.py b/litellm/proxy/mcp_server/ui_session_utils.py similarity index 98% rename from litellm/proxy/_experimental/mcp_server/ui_session_utils.py rename to litellm/proxy/mcp_server/ui_session_utils.py index 9f60f0afbb6..308a41b8938 100644 --- a/litellm/proxy/_experimental/mcp_server/ui_session_utils.py +++ b/litellm/proxy/mcp_server/ui_session_utils.py @@ -97,7 +97,7 @@ async def admitted_user_context(user_api_key_auth: UserAPIKeyAuth) -> UserAPIKey user_id: Final = user_api_key_auth.user_id if not is_ui_session_credential(user_api_key_auth) or user_id is None: return None - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) @@ -192,7 +192,7 @@ async def toolset_grant_contexts( if not is_keyless_mcp_subject(user_api_key_auth): return (user_api_key_auth,) - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) @@ -269,7 +269,7 @@ async def granted_toolset_ids( ``require_key_mcp_access_defined``, which inherits nothing. A keyless subject's team sources always inherit. A team that cannot be read contributes nothing while every other source still counts, and an own grant that is named but cannot be read grants nothing. No grant anywhere yields the empty set.""" - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) from litellm.proxy.proxy_server import general_settings diff --git a/litellm/proxy/_experimental/mcp_server/upstream.py b/litellm/proxy/mcp_server/upstream.py similarity index 94% rename from litellm/proxy/_experimental/mcp_server/upstream.py rename to litellm/proxy/mcp_server/upstream.py index 1e39c068806..96a1c2db9ba 100644 --- a/litellm/proxy/_experimental/mcp_server/upstream.py +++ b/litellm/proxy/mcp_server/upstream.py @@ -7,12 +7,13 @@ import httpx2 from fastapi import HTTPException from litellm.constants import MCP_CLIENT_TIMEOUT, MCP_NPM_CACHE_DIR, MCP_STDIO_ALLOWED_COMMANDS -from litellm.experimental_mcp_client.client import MCPClient, MCPSigV4Auth -from litellm.proxy._experimental.mcp_server.legacy_callbacks import ElicitationCallback, SamplingCallback -from litellm.proxy._experimental.mcp_server.mcp_debug import record_auth_resolution -from litellm.proxy._experimental.mcp_server.oauth2_token_cache import resolve_mcp_auth, resolved_token_header -from litellm.proxy._experimental.mcp_server.outbound_credentials import Error, Ok, UpstreamCredentialProvider -from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( +from litellm.mcp_client.client import MCPClient, MCPSigV4Auth +from litellm.proxy._types import MCPTransport, UserAPIKeyAuth +from litellm.proxy.mcp_server.legacy_callbacks import ElicitationCallback, SamplingCallback +from litellm.proxy.mcp_server.mcp_debug import record_auth_resolution +from litellm.proxy.mcp_server.oauth2_token_cache import resolve_mcp_auth, resolved_token_header +from litellm.proxy.mcp_server.outbound_credentials import Error, Ok, UpstreamCredentialProvider +from litellm.proxy.mcp_server.outbound_credentials.adapter import ( prepare_mcp_client, raise_public, raise_token_exchange_challenge, @@ -20,8 +21,8 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import to_server_spec, to_subject, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source -from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( +from litellm.proxy.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source +from litellm.proxy.mcp_server.outbound_credentials.types import ( DEFAULT_CREDENTIAL_HEADER, AuthorizationCodeConfig, AuthResolution, @@ -32,8 +33,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( ServerSpec, TokenExchangeConfig, ) -from litellm.proxy._experimental.mcp_server.stdio_gate import MCP_STDIO_DISABLED_MESSAGE, is_mcp_stdio_enabled -from litellm.proxy._types import MCPTransport, UserAPIKeyAuth +from litellm.proxy.mcp_server.stdio_gate import MCP_STDIO_DISABLED_MESSAGE, is_mcp_stdio_enabled from litellm.types.mcp import ( MCPAuth, MCPStdioConfig, diff --git a/litellm/proxy/_experimental/mcp_server/utils.py b/litellm/proxy/mcp_server/utils.py similarity index 100% rename from litellm/proxy/_experimental/mcp_server/utils.py rename to litellm/proxy/mcp_server/utils.py diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bcba76ca1b3..bc79becf7bb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -383,9 +383,6 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.openai.live import parse_live_session_start from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS from litellm.llms.vertex_ai.vertex_llm_base import VertexBase -from litellm.proxy._experimental.mcp_server.byok_credential_cache import byok_credential_cache -from litellm.proxy._experimental.mcp_server.catalog import public_catalog_operation -from litellm.proxy._experimental.mcp_server.stdio_gate import MCP_STDIO_ENABLED_ENV_VAR, is_mcp_stdio_flag_key from litellm.proxy._lazy_features import attach_lazy_features, reserve_lazy_slot from litellm.proxy._types import * from litellm.proxy.agent_endpoints.auth.agent_access_groups import CeilingResolver, resolve_agent_access_group_ceiling @@ -786,6 +783,9 @@ from litellm.proxy.management_helpers.team_metadata_validation import ( TEAM_METADATA_VALIDATOR_REGISTRY, parse_team_metadata_schema, ) +from litellm.proxy.mcp_server.byok_credential_cache import byok_credential_cache +from litellm.proxy.mcp_server.catalog import public_catalog_operation +from litellm.proxy.mcp_server.stdio_gate import MCP_STDIO_ENABLED_ENV_VAR, is_mcp_stdio_flag_key from litellm.proxy.memory.memory_endpoints import router as memory_router from litellm.proxy.middleware.billable_request_metrics_middleware import ( BillableRequestMetricsMiddleware, @@ -7381,7 +7381,7 @@ class ProxyConfig: ## MCP TOOLS mcp_tools_config: Final = config.get("mcp_tools", None) if mcp_tools_config: - from litellm.proxy._experimental.mcp_server.tool_registry import ( + from litellm.proxy.mcp_server.tool_registry import ( global_mcp_tool_registry, ) @@ -7398,7 +7398,7 @@ class ProxyConfig: mcp_servers_config: Final = config.get("mcp_servers", None) if mcp_servers_config: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -9402,19 +9402,19 @@ class ProxyConfig: ) async def _init_mcp_servers_in_db(self): - from litellm.proxy._experimental.mcp_server.utils import is_mcp_available + from litellm.proxy.mcp_server.utils import is_mcp_available if not is_mcp_available(): verbose_proxy_logger.debug("MCP module not available, skipping MCP server initialization") return - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._experimental.mcp_server.oauth2_flow_backfill import ( + from litellm.proxy.mcp_server.oauth2_flow_backfill import ( backfill_null_oauth2_flows, ) - from litellm.proxy._experimental.mcp_server.oauth_issuer_stamp_backfill import ( + from litellm.proxy.mcp_server.oauth_issuer_stamp_backfill import ( backfill_discovery_stamped_issuers, ) @@ -9453,11 +9453,11 @@ class ProxyConfig: """ if not self._should_load_db_object(object_type="mcp"): return - from litellm.proxy._experimental.mcp_server.utils import is_mcp_available + from litellm.proxy.mcp_server.utils import is_mcp_available if not is_mcp_available(): return - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -21764,11 +21764,11 @@ async def _stream_mcp_asgi_response(handle_fn, scope: dict, receive) -> "Streami ) async def proxy_mcp_route(request: Request) -> Response: """Serve the fixed three-tool MCP proxy surface.""" - from litellm.proxy._experimental.mcp_server.mcp_context import ( # pyright: ignore[reportPrivateUsage] # route-owned mode + from litellm.proxy.mcp_server.mcp_context import ( # pyright: ignore[reportPrivateUsage] # route-owned mode _mcp_proxy_mode, # pyright: ignore[reportPrivateUsage] # route-owned mode ) - from litellm.proxy._experimental.mcp_server.server import handle_streamable_http_mcp - from litellm.proxy._experimental.mcp_server.utils import is_mcp_available + from litellm.proxy.mcp_server.server import handle_streamable_http_mcp + from litellm.proxy.mcp_server.utils import is_mcp_available if not is_mcp_available(): raise HTTPException(status_code=404, detail="Not Found") @@ -21791,12 +21791,12 @@ async def aggregate_mcp_route(request: Request): """Serve the aggregate MCP endpoint on the bare ``/mcp`` spelling: the ``/mcp`` mount cannot match its bare prefix, and the resulting 307 breaks MCP clients behind TLS-terminating proxies.""" - from litellm.proxy._experimental.mcp_server.utils import is_mcp_available + from litellm.proxy.mcp_server.utils import is_mcp_available if not is_mcp_available(): raise HTTPException(status_code=404, detail="Not Found") - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, ) @@ -21822,10 +21822,10 @@ async def toolset_mcp_route(toolset_name: str, request: Request): will be rejected with a 403. """ try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, mcp_active_toolset_id, ) @@ -21857,7 +21857,7 @@ async def toolset_mcp_route(toolset_name: str, request: Request): async def _mcp_forward_as_path(path_segment: str, request: Request): """Rewrite path to /mcp/{path_segment} and stream the response.""" - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, ) @@ -21890,11 +21890,11 @@ async def _resolve_mcp_csv_tokens(csv_segment: str, client_ip: str | None) -> li all-unmatched server filter falls back to the full ``allowed_mcp_servers`` list and silently broadens the request scope). """ - from litellm.proxy._experimental.mcp_server.catalog import global_manager + from litellm.proxy.mcp_server.catalog import global_manager async with global_manager().catalog.operation(): from litellm.constants import DEFAULT_MCP_NAMESPACE_CSV_MAX_TOKENS - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -21924,7 +21924,7 @@ async def _is_mcp_access_group_cached(name: str) -> bool: list) cannot hide a real group for long. """ from litellm.constants import DEFAULT_MCP_ACCESS_GROUP_NEGATIVE_CACHE_TTL - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) @@ -21957,10 +21957,10 @@ async def dynamic_mcp_route(mcp_server_name: str, request: Request): 4. MCP access group tag (DB lookup, cached) """ try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.auth.ip_address_utils import IPAddressUtils + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy.auth.ip_address_utils import IPAddressUtils client_ip: Final = IPAddressUtils.get_mcp_client_ip(request) @@ -21987,7 +21987,7 @@ async def dynamic_mcp_route(mcp_server_name: str, request: Request): # 3. Toolset name (cached) if prisma_client is not None: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, mcp_active_toolset_id, ) diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 351e4c10d87..9f33e0d2adc 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -342,7 +342,7 @@ async def get_agents(request: Request): response_model=list[MCPPublicServer], ) async def get_mcp_servers(): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 1bb88d5ff69..0efd4e4c619 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -24,7 +24,6 @@ import litellm from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_keys -from litellm.proxy._experimental.mcp_server.tool_search import MCP_TOOL_SEARCH_SETTINGS_KEY from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.validation_error_body import public_validation_errors @@ -40,6 +39,7 @@ from litellm.proxy.management_endpoints.team_admin_field_permissions import ( TEAM_ADMIN_EDITABLE_TEAM_FIELDS_SETTING, TEAM_ADMIN_RAISE_MAX_BUDGET_PERMISSION, ) +from litellm.proxy.mcp_server.tool_search import MCP_TOOL_SEARCH_SETTINGS_KEY from litellm.proxy.spend_tracking.ptu_feature_flag import ( PTU_COST_ATTRIBUTION_ENV_VAR, is_ptu_cost_attribution_enabled, diff --git a/litellm/proxy/vector_store_endpoints/utils.py b/litellm/proxy/vector_store_endpoints/utils.py index b97c3a949d5..8ce48fc3516 100644 --- a/litellm/proxy/vector_store_endpoints/utils.py +++ b/litellm/proxy/vector_store_endpoints/utils.py @@ -9,15 +9,15 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm._logging import verbose_proxy_logger -from litellm.proxy._experimental.mcp_server.ui_session_utils import ( - is_ui_session_credential, - resolve_ui_session_team_ids, -) from litellm.proxy._types import ( LiteLLM_ObjectPermissionTable, LitellmUserRoles, UserAPIKeyAuth, ) +from litellm.proxy.mcp_server.ui_session_utils import ( + is_ui_session_credential, + resolve_ui_session_team_ids, +) from litellm.types.utils import LlmProviders from litellm.types.vector_stores import LiteLLM_ManagedVectorStore from litellm.utils import ProviderConfigManager diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index e3c3ab671ea..c01d07fb5c3 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -11,8 +11,8 @@ from typing_extensions import Unpack from litellm._logging import verbose_logger from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.proxy._experimental.mcp_server.catalog import catalog_operation, global_manager -from litellm.proxy._experimental.mcp_server.utils import ( +from litellm.proxy.mcp_server.catalog import catalog_operation, global_manager +from litellm.proxy.mcp_server.utils import ( iter_known_server_prefixes, logging_safe_mcp_headers, split_server_prefix_from_name, @@ -48,7 +48,7 @@ if TYPE_CHECKING: from mcp.types import CallToolResult from mcp.types import Tool as MCPTool - from litellm.proxy._experimental.mcp_server.ui_session_utils import GrantedToolsetIds + from litellm.proxy.mcp_server.ui_session_utils import GrantedToolsetIds from litellm.proxy.utils import ProxyLogging from litellm.responses.mcp.mcp_streaming_iterator import MCPEnhancedStreamingIterator from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth @@ -71,7 +71,7 @@ class _AresponsesOptions(ExtraItemsTypedDict, total=False, extra_items=object): def _registered_mcp_servers() -> Collection[MCPServer]: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -89,7 +89,7 @@ def _registry_serves(name: str, servers: Collection[MCPServer]) -> bool: async def _toolset_exists(name: str) -> bool: try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) from litellm.proxy.proxy_server import prisma_client @@ -193,7 +193,7 @@ class LiteLLM_Proxy_MCP_Handler: from litellm.proxy._types import LiteLLM_ObjectPermissionTable try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -253,10 +253,10 @@ class LiteLLM_Proxy_MCP_Handler: List of MCP tools List names of allowed MCP servers """ - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_allowed_mcp_servers_from_mcp_server_names, _get_tools_from_mcp_servers, ) @@ -282,12 +282,12 @@ class LiteLLM_Proxy_MCP_Handler: toolset = await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name) if toolset is not None: if user_api_key_auth is not None: - from litellm.proxy._experimental.mcp_server.ui_session_utils import ( - granted_toolset_ids, - ) from litellm.proxy.management_endpoints.common_utils import ( user_api_key_has_admin_view, ) + from litellm.proxy.mcp_server.ui_session_utils import ( + granted_toolset_ids, + ) if not user_api_key_has_admin_view(user_api_key_auth) and toolset.toolset_id not in ( await (granted_toolsets or granted_toolset_ids)(user_api_key_auth) @@ -527,7 +527,7 @@ class LiteLLM_Proxy_MCP_Handler: target_format: Literal["responses", "chat"] = "responses", ) -> Sequence[FunctionToolParam | ChatCompletionToolParam]: """Transform MCP tools to OpenAI-compatible format.""" - from litellm.experimental_mcp_client.tools import ( + from litellm.mcp_client.tools import ( transform_mcp_tool_to_openai_responses_api_tool, transform_mcp_tool_to_openai_tool, ) @@ -720,10 +720,10 @@ class LiteLLM_Proxy_MCP_Handler: from litellm._uuid import uuid from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _resolve_display_name_to_original, ) from litellm.proxy.proxy_server import proxy_logging_obj diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index 9e51e2533ea..cc79d1c88bb 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -368,7 +368,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): from starlette.datastructures import Headers - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) diff --git a/litellm/responses/mcp/tool_precheck.py b/litellm/responses/mcp/tool_precheck.py index b76f4066e6d..a29e6895f9a 100644 --- a/litellm/responses/mcp/tool_precheck.py +++ b/litellm/responses/mcp/tool_precheck.py @@ -7,7 +7,7 @@ LITELLM_PROXY_MCP_SERVER_URL: Final = "litellm_proxy" LITELLM_PROXY_MCP_SERVER_URL_PREFIX: Final = f"{LITELLM_PROXY_MCP_SERVER_URL}/mcp/" _PROXY_MCP_PATH_RE: Final = re.compile(r"^https?://.+/mcp/([^/]+)$") -_GATEWAY_REGISTRY_MODULE: Final = "litellm.proxy._experimental.mcp_server.mcp_server_manager" +_GATEWAY_REGISTRY_MODULE: Final = "litellm.proxy.mcp_server.mcp_server_manager" def _mcp_server_url(tool: object) -> str | None: diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 2d0873a218a..a6b042a2234 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1190,7 +1190,7 @@ class ResponsesAPIRequestUtils: """ from starlette.datastructures import Headers - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) diff --git a/scripts/check_mcp_operation_boundary.py b/scripts/check_mcp_operation_boundary.py index b6c9dcefabf..65c92828fd8 100644 --- a/scripts/check_mcp_operation_boundary.py +++ b/scripts/check_mcp_operation_boundary.py @@ -3,7 +3,7 @@ import sys from pathlib import Path from typing import Final -PACKAGE: Final = Path("litellm/proxy/_experimental/mcp_server") +PACKAGE: Final = Path("litellm/proxy/mcp_server") LEGACY_ADAPTERS: Final = frozenset({"server.py", "legacy_callbacks.py", "mcp_context.py", "mcp_debug.py"}) CONFINED_NAMES: Final = frozenset( { diff --git a/scripts/check_mcp_sdk_install.py b/scripts/check_mcp_sdk_install.py index f5ab51b2f55..f2c2891bfd8 100644 --- a/scripts/check_mcp_sdk_install.py +++ b/scripts/check_mcp_sdk_install.py @@ -8,11 +8,11 @@ MINIMUM_MCP_VERSION: Final[tuple[int, int, int]] = (2, 2, 0) IMPORTED_MODULES: Final[tuple[str, ...]] = ( "litellm", - "litellm.experimental_mcp_client", - "litellm.experimental_mcp_client.client", - "litellm.proxy._experimental.mcp_server.server", - "litellm.proxy._experimental.mcp_server.mcp_server_manager", - "litellm.proxy._experimental.mcp_server.rest_endpoints", + "litellm.mcp_client", + "litellm.mcp_client.client", + "litellm.proxy.mcp_server.server", + "litellm.proxy.mcp_server.mcp_server_manager", + "litellm.proxy.mcp_server.rest_endpoints", ) @@ -50,7 +50,7 @@ def main() -> int: "path": "/mcp", "headers": [(b"mcp-protocol-version", b"2026-07-28")], } - mcp_server: Final = sys.modules["litellm.proxy._experimental.mcp_server.server"] + mcp_server: Final = sys.modules["litellm.proxy.mcp_server.server"] if mcp_server.unsupported_protocol_version(scope) != "2026-07-28": sys.stderr.write("unsupported_protocol_version accepted a modern-only version\n") return 1 diff --git a/scripts/openapi_docs_gate.py b/scripts/openapi_docs_gate.py index a2a8fad78e9..5f5d1021c22 100755 --- a/scripts/openapi_docs_gate.py +++ b/scripts/openapi_docs_gate.py @@ -76,7 +76,10 @@ def parse_violations(out: str) -> tuple[Violation, ...]: def _check(root: Path, checker: Path) -> tuple[Violation, ...]: resolved: Final = root.resolve() - env: Final = {**os.environ, "PYTHONPATH": os.pathsep.join((str(resolved), os.environ.get("PYTHONPATH", "")))} + env: Final = { + **os.environ, + "PYTHONPATH": os.pathsep.join((str(resolved), str(resolved / "enterprise"), os.environ.get("PYTHONPATH", ""))), + } return parse_violations(_run([sys.executable, str(checker)], cwd=resolved, env=env)) diff --git a/scripts/pre_commit_lint.sh b/scripts/pre_commit_lint.sh index edb69f24e29..371884fb791 100755 --- a/scripts/pre_commit_lint.sh +++ b/scripts/pre_commit_lint.sh @@ -103,7 +103,7 @@ ui_prettier_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs|json|css|s ui_eslint_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs)$' litellm_py_files=$(scope_match "$litellm_py_pattern") -if [ -n "$(scope_match '^(litellm/proxy/_experimental/mcp_server/|scripts/check_mcp_operation_boundary\.py)')" ]; then +if [ -n "$(scope_match '^(litellm/proxy/mcp_server/|scripts/check_mcp_operation_boundary\.py)')" ]; then uv run --no-sync python scripts/check_mcp_operation_boundary.py || exit 1 fi e2e_py_files=$(scope_match "$e2e_py_pattern") diff --git a/tests/base_sdk_tests/check_base_sdk_install.py b/tests/base_sdk_tests/check_base_sdk_install.py index 4abd71e39a6..e546549d25f 100644 --- a/tests/base_sdk_tests/check_base_sdk_install.py +++ b/tests/base_sdk_tests/check_base_sdk_install.py @@ -127,7 +127,7 @@ def check_completion_with_function_tool() -> str: def check_mcp_install_guidance() -> str: try: - import litellm.experimental_mcp_client + import litellm.mcp_client except ImportError as error: _require("pip install 'litellm[mcp]'" in str(error), f"missing MCP installation guidance: {error}") _require(isinstance(error.__cause__, ModuleNotFoundError), "original missing-dependency cause was lost") @@ -174,8 +174,8 @@ def check_token_counter() -> str: def check_tokenizer_dependencies() -> str: import litellm - from litellm.rust_bridge import tokenizer from litellm.litellm_core_utils.tokenizer import HuggingFace, HuggingFaceTokenizer, Tokenizer + from litellm.rust_bridge import tokenizer from litellm.utils import claude_json_str _require(isinstance(litellm.encoding, Tokenizer), "runtime alias rejects the default encoding") diff --git a/tests/benchmarks/test_mcp_benchmarks.py b/tests/benchmarks/test_mcp_benchmarks.py index 7e23ab1b4f5..d4464d803cf 100644 --- a/tests/benchmarks/test_mcp_benchmarks.py +++ b/tests/benchmarks/test_mcp_benchmarks.py @@ -10,11 +10,11 @@ request. Both are pure-CPU and deterministic. import pytest from mcp.types import Tool as MCPTool -from litellm.experimental_mcp_client.tools import ( +from litellm.mcp_client.tools import ( transform_mcp_tool_to_openai_tool, transform_openai_tool_call_request_to_mcp_tool_call_request, ) -from litellm.proxy._experimental.mcp_server.utils import ( +from litellm.proxy.mcp_server.utils import ( add_server_prefix_to_name, split_server_prefix_from_name, ) diff --git a/tests/code_coverage_tests/ensure_async_clients_test.py b/tests/code_coverage_tests/ensure_async_clients_test.py index 93ae2e2837b..f4922a11806 100644 --- a/tests/code_coverage_tests/ensure_async_clients_test.py +++ b/tests/code_coverage_tests/ensure_async_clients_test.py @@ -17,7 +17,7 @@ ALLOWED_FILES = [ "../../litellm/llms/base.py", "../../litellm/llms/custom_httpx/httpx_handler.py", "../../litellm/llms/openai/common_utils.py", - "../../litellm/experimental_mcp_client/client.py", + "../../litellm/mcp_client/client.py", # when running on ci/cd "./litellm/__init__.py", "./litellm/llms/custom_httpx/http_handler.py", @@ -27,7 +27,7 @@ ALLOWED_FILES = [ "./litellm/llms/base.py", "./litellm/llms/custom_httpx/httpx_handler.py", "./litellm/llms/openai/common_utils.py", - "./litellm/experimental_mcp_client/client.py", + "./litellm/mcp_client/client.py", ] warning_msg = "this is a serious violation that can impact latency. Creating Async clients per request can add +500ms per request" diff --git a/tests/code_coverage_tests/unbounded_in_baseline.txt b/tests/code_coverage_tests/unbounded_in_baseline.txt index 1286ea25c4e..1da964681a9 100644 --- a/tests/code_coverage_tests/unbounded_in_baseline.txt +++ b/tests/code_coverage_tests/unbounded_in_baseline.txt @@ -2,12 +2,12 @@ # Fix a site and delete its line; regenerate with `check_unbounded_in_lists.py --update-baseline`. litellm/integrations/shadow_eval_logger.py ShadowEvalLogger._active_jobs prisma job_id.in `[str(record.id) for record in records]` 0 litellm/llms/litellm_proxy/skills/handler.py LiteLLMSkillsHandler.list_skills prisma created_by.in `owner_scopes` 0 -litellm/proxy/_experimental/mcp_server/db.py get_mcp_servers prisma server_id.in `server_ids` 0 -litellm/proxy/_experimental/mcp_server/db.py get_user_env_vars_bulk prisma server_id.in `ids` 0 -litellm/proxy/_experimental/mcp_server/db.py purge_user_oauth_credentials_for_server prisma user_id.in `[row.user_id for row in oauth_rows]` 0 -litellm/proxy/_experimental/mcp_server/oauth2_flow_backfill.py backfill_null_oauth2_flows prisma server_id.in `server_ids_for_flow` 0 -litellm/proxy/_experimental/mcp_server/oauth2_flow_backfill.py backfill_null_oauth2_flows prisma server_id.in `server_ids` 0 -litellm/proxy/_experimental/mcp_server/toolset_db.py list_mcp_toolsets prisma toolset_id.in `toolset_ids` 0 +litellm/proxy/mcp_server/db.py get_mcp_servers prisma server_id.in `server_ids` 0 +litellm/proxy/mcp_server/db.py get_user_env_vars_bulk prisma server_id.in `ids` 0 +litellm/proxy/mcp_server/db.py purge_user_oauth_credentials_for_server prisma user_id.in `[row.user_id for row in oauth_rows]` 0 +litellm/proxy/mcp_server/oauth2_flow_backfill.py backfill_null_oauth2_flows prisma server_id.in `server_ids_for_flow` 0 +litellm/proxy/mcp_server/oauth2_flow_backfill.py backfill_null_oauth2_flows prisma server_id.in `server_ids` 0 +litellm/proxy/mcp_server/toolset_db.py list_mcp_toolsets prisma toolset_id.in `toolset_ids` 0 litellm/proxy/agent_endpoints/endpoints.py _attach_keys_to_agents prisma agent_id.in `agent_ids` 0 litellm/proxy/agent_endpoints/endpoints.py get_agents prisma agent_id.in `agent_ids` 0 litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_skill_access.py SkillVisibility.where prisma name.in `sorted(self.granted)` 0 diff --git a/tests/e2e/coverage_registry/mcp.yaml b/tests/e2e/coverage_registry/mcp.yaml index 2ad815930a9..8eac9adeb33 100644 --- a/tests/e2e/coverage_registry/mcp.yaml +++ b/tests/e2e/coverage_registry/mcp.yaml @@ -1,4 +1,4 @@ -# MCP module. Grounded in litellm/proxy/_experimental/mcp_server/. See tests/e2e/AGENTS.md for the grammar. +# MCP module. Grounded in litellm/proxy/mcp_server/. See tests/e2e/AGENTS.md for the grammar. - id: mcp.list_tools.api_key.succeeds module: mcp tier: P0 diff --git a/tests/integration/mcp/test_interactions.py b/tests/integration/mcp/test_interactions.py index 8a4bdb8e8ec..5577cf16067 100644 --- a/tests/integration/mcp/test_interactions.py +++ b/tests/integration/mcp/test_interactions.py @@ -93,6 +93,8 @@ def sdk_interaction_peer(*, sampling: bool = False): import queue from contextlib import asynccontextmanager + from integration._support.asgi import asgi_server + from integration._support.mcp import McpPeer, _capturing, _drain_sse_streams from mcp.server import Server from mcp.server.streamable_http_manager import StreamableHTTPSessionManager from mcp.server.transport_security import TransportSecuritySettings @@ -100,27 +102,25 @@ def sdk_interaction_peer(*, sampling: bool = False): CallToolResult, CreateMessageRequest, CreateMessageRequestParams, - SamplingMessage, ElicitRequest, ElicitRequestFormParams, GetPromptResult, InputRequiredResult, - ListToolsResult, ListPromptsResult, ListResourcesResult, + ListToolsResult, Prompt, PromptMessage, ReadResourceResult, Resource, + SamplingMessage, TextContent, TextResourceContents, Tool, ) + from sse_starlette.sse import AppStatus from starlette.applications import Starlette from starlette.routing import Mount - from integration._support.asgi import asgi_server - from integration._support.mcp import McpPeer, _capturing, _drain_sse_streams - from sse_starlette.sse import AppStatus async def tools(ctx, params): return ListToolsResult(tools=[Tool(name="confirm", input_schema={"type": "object"})]) @@ -666,7 +666,7 @@ def test_modern_subscription_delivers_changes_between_calls_and_refreshes_tools( " data = json.loads(Path(__file__).with_suffix('.json').read_text())\n" " change = data.pop('subscription_test_update', None)\n" " if change:\n" - " from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager as manager\n" + " from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager as manager\n" " server = manager.config_mcp_servers['subscription-server']\n" " fields = {'authorization_url': 'https://authorization.example/authorize'} if change == 'metadata' else {'url': server.url.rstrip('/') + '/changed'}\n" " manager.config_mcp_servers[server.server_id] = server.model_copy(update=fields)\n" @@ -1295,9 +1295,9 @@ def test_subscription_partial_setup_releases_upstream_capacity() -> None: from starlette.applications import Starlette from starlette.routing import Mount - from litellm.experimental_mcp_client.client import MCPClient - from litellm.proxy._experimental.mcp_server.contracts import OperationContext - from litellm.proxy._experimental.mcp_server.subscriptions import GatewaySubscriptions, SubscriptionTarget + from litellm.mcp_client.client import MCPClient + from litellm.proxy.mcp_server.contracts import OperationContext + from litellm.proxy.mcp_server.subscriptions import GatewaySubscriptions, SubscriptionTarget from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer as RegisteredServer diff --git a/tests/integration/mcp/test_mcp_protocol_errors.py b/tests/integration/mcp/test_mcp_protocol_errors.py index 4930258ec07..d6e5ac66a34 100644 --- a/tests/integration/mcp/test_mcp_protocol_errors.py +++ b/tests/integration/mcp/test_mcp_protocol_errors.py @@ -4,7 +4,6 @@ import uuid from typing import Final import pytest - from integration._support.client import Gateway from integration._support.mcp import McpPeer, call_tool, register_mcp, tool_names from integration._support.wire import Reply, Request, wire_server @@ -98,11 +97,12 @@ def test_configured_revision_blocks_unadvertised_handshake_and_keeps_allowed_con import yaml from integration._support.mcp import mcp_peer from integration._support.process import owned_proxy - from litellm.experimental_mcp_client.client import MCPClient - from litellm.types.mcp import MCPTransport from mcp import MCPError from mcp.types import CallToolRequestParams + from litellm.mcp_client.client import MCPClient + from litellm.types.mcp import MCPTransport + with mcp_peer() as upstream, gateway.scenario() as scenario: alias: Final = "restricted" + uuid.uuid4().hex[:8] identity: Final = register_mcp(scenario, upstream, alias) diff --git a/tests/integration/mcp/test_mcp_transports.py b/tests/integration/mcp/test_mcp_transports.py index ef240cba570..92fcf37994e 100644 --- a/tests/integration/mcp/test_mcp_transports.py +++ b/tests/integration/mcp/test_mcp_transports.py @@ -8,7 +8,6 @@ from typing import Final import httpx import pytest import yaml -from mcp.shared.exceptions import MCPError from integration._support.client import Gateway from integration._support.database import read_rows from integration._support.mcp import ( @@ -34,6 +33,7 @@ from integration._support.mcp import ( ) from integration._support.mcp_grants import create_toolset from integration._support.process import owned_proxy +from mcp.shared.exceptions import MCPError ADD: Final = {"http": "add", "sse": "add", "stdio": "add", "openapi": "getpet"} ARGUMENTS: Final = {"add": {"a": 3, "b": 4}, "getpet": {"petId": "7"}} @@ -571,7 +571,7 @@ def test_pinned_revision_pairs_list_and_call_through_gateway( from mcp.types import CallToolRequestParams - from litellm.experimental_mcp_client.client import MCPClient + from litellm.mcp_client.client import MCPClient from litellm.types.mcp import MCPTransport with peer_of(peer_kind) as peer, gateway.scenario() as scenario: @@ -616,7 +616,7 @@ def test_legacy_gateway_calls_modern_upstream_without_initialize( from mcp.types import CallToolRequestParams - from litellm.experimental_mcp_client.client import MCPClient + from litellm.mcp_client.client import MCPClient from litellm.types.mcp import MCPTransport with peer_of(peer_kind) as peer, gateway.scenario() as scenario: @@ -661,13 +661,16 @@ def test_sdk_progress_and_cancellation_reach_upstream_without_losing_session( ) -> None: import asyncio import os + from contextlib import asynccontextmanager from pathlib import Path from threading import Event - from contextlib import asynccontextmanager import httpx import httpx2 import yaml + from integration._support.asgi import asgi_server + from integration._support.mcp import _drain_sse_streams + from integration._support.process import owned_proxy from mcp import ClientSession, MCPError from mcp.client.streamable_http import streamable_http_client from mcp.server.mcpserver import Context, MCPServer @@ -677,10 +680,6 @@ def test_sdk_progress_and_cancellation_reach_upstream_without_losing_session( from starlette.applications import Starlette from starlette.routing import Mount - from integration._support.asgi import asgi_server - from integration._support.mcp import _drain_sse_streams - from integration._support.process import owned_proxy - upstream: Final = MCPServer("progress-cancellation-peer") started: Final = Event() stopped: Final = Event() diff --git a/tests/integration/mcp/test_pagination.py b/tests/integration/mcp/test_pagination.py index e52757ec19a..aa464eca30e 100644 --- a/tests/integration/mcp/test_pagination.py +++ b/tests/integration/mcp/test_pagination.py @@ -6,14 +6,14 @@ from typing import Final, Literal import httpx import pytest import yaml +from integration._support.client import Gateway +from integration._support.mcp import McpPeer, paginated_mcp_peer +from integration._support.process import owned_proxy from mcp import ClientSession, MCPError from mcp.client.streamable_http import streamable_http_client from mcp.types import CallToolRequest, CallToolRequestParams, CallToolResult, DiscoverResult, PaginatedRequestParams -from integration._support.client import Gateway -from integration._support.mcp import McpPeer, paginated_mcp_peer -from integration._support.process import owned_proxy -from litellm.experimental_mcp_client.client import MCPClient +from litellm.mcp_client.client import MCPClient from litellm.types.mcp import MCPTransport @@ -422,7 +422,7 @@ def test_failed_upstream_continuation_requires_restart_for_every_catalog(tmp_pat @pytest.mark.parametrize("foreign", [{"foreign": {"status": "error", "message": "foreign outcome"}}, "malformed"]) def test_optional_catalog_ignores_upstream_gateway_outcomes(tmp_path: Path, foreign): - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import SERVER_OUTCOMES_META_KEY + from litellm.proxy.mcp_server.faults.list_outcomes import SERVER_OUTCOMES_META_KEY async def exercise(gateway): async with catalog_session(gateway) as session: @@ -577,7 +577,8 @@ def test_aggregate_freshness_matches_real_upstream() -> None: ListToolsRequest, ListToolsResult, ) - from litellm.proxy._experimental.mcp_server.catalog import combine_optional_catalog, list_tools_page + + from litellm.proxy.mcp_server.catalog import combine_optional_catalog, list_tools_page async def exercise(peer: McpPeer) -> None: client: Final = MCPClient(server_url=peer.url, transport_type=MCPTransport.http, protocol_version="2026-07-28") @@ -604,8 +605,8 @@ def test_aggregate_freshness_matches_real_upstream() -> None: @pytest.mark.parametrize("ttl_ms", [0, 9000]) def test_discovery_cache_freshness_and_caller_isolation_over_http(ttl_ms: int) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager from litellm.types.mcp_server.mcp_server_manager import MCPServer async def exercise(peer: McpPeer) -> None: diff --git a/tests/integration/mcp/test_proxy_mcp_e2e.py b/tests/integration/mcp/test_proxy_mcp_e2e.py index 2f690463143..6fb8d284016 100644 --- a/tests/integration/mcp/test_proxy_mcp_e2e.py +++ b/tests/integration/mcp/test_proxy_mcp_e2e.py @@ -31,8 +31,8 @@ from starlette.requests import Request import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.proxy._experimental.mcp_server.tool_search import handle_mcp_proxy_tool from litellm.proxy._types import LiteLLM_ObjectPermissionTable, ProxyException, UserAPIKeyAuth +from litellm.proxy.mcp_server.tool_search import handle_mcp_proxy_tool from litellm.proxy.proxy_server import ( app as proxy_app, ) @@ -41,11 +41,11 @@ from litellm.proxy.proxy_server import ( initialize, ) from tests._master_key import MASTER_KEY +from tests.integration._support.ports import allocate_port from tests.integration._support.process import stop_child from tests.integration._support.timing import assert_enforced_within from tests.integration._support.wire import Reply, Wire, wire_server from tests.integration._support.wire import Request as WireRequest -from tests.integration._support.ports import allocate_port CONFIG_TEMPLATE_PATH = Path("tests/integration/mcp/_config_mcp_e2e.yaml") MCP_SERVER_SCRIPT = Path("tests/integration/mcp/_math_server.py") @@ -86,7 +86,7 @@ async def drain_logging_worker() -> typing.AsyncIterator[None]: async def test_schema_validation_stops_expensive_work_and_recovers(cancel: bool, concurrency: int) -> None: import psutil - from litellm.proxy._experimental.mcp_server.tool_search import ( + from litellm.proxy.mcp_server.tool_search import ( _VALIDATION_TIMEOUT_SECONDS, _tool_argument_validation_error, ) @@ -246,8 +246,8 @@ def _clear_proxy_database_env() -> typing.Iterator[None]: async def _initialize_proxy(config_path: str) -> None: - from litellm.proxy._experimental.mcp_server.catalog import CatalogSnapshots - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.catalog import CatalogSnapshots + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager cleanup_router_config_variables() global_mcp_server_manager.catalog = CatalogSnapshots(global_mcp_server_manager) @@ -307,7 +307,7 @@ def _start_proxy_server( failure: Final[Future[BaseException | None]] = Future() async def _serve() -> None: - from litellm.proxy._experimental.mcp_server import server as mcp_server + from litellm.proxy.mcp_server import server as mcp_server await _initialize_proxy(config_path) async with proxy_app.router.lifespan_context(proxy_app), mcp_server.lifespan(proxy_app): diff --git a/tests/integration/mcp_coverage.toml b/tests/integration/mcp_coverage.toml index daa782eade2..439fac10169 100644 --- a/tests/integration/mcp_coverage.toml +++ b/tests/integration/mcp_coverage.toml @@ -4,10 +4,10 @@ core = "sysmon" parallel = true relative_files = true include = [ - "litellm/proxy/_experimental/mcp_server/*", + "litellm/proxy/mcp_server/*", "litellm/proxy/management_endpoints/mcp_management_endpoints.py", "litellm/responses/mcp/*", - "litellm/experimental_mcp_client/*", + "litellm/mcp_client/*", "litellm/proxy/guardrails/guardrail_hooks/mcp_*", ] diff --git a/tests/unit/experimental_mcp_client/test_init.py b/tests/unit/experimental_mcp_client/test_init.py new file mode 100644 index 00000000000..9d2f1137470 --- /dev/null +++ b/tests/unit/experimental_mcp_client/test_init.py @@ -0,0 +1,29 @@ +import importlib +import sys +from collections.abc import Mapping + +import pytest + +import litellm.mcp_client +import litellm.mcp_client.client +import litellm.mcp_client.tools + +OLD_PACKAGE = "litellm.experimental_mcp_client" + + +def test_old_package_warns_and_deep_imports_resolve_to_mcp_client(monkeypatch: pytest.MonkeyPatch) -> None: + for name in tuple(sys.modules): + if name.startswith(OLD_PACKAGE): + monkeypatch.delitem(sys.modules, name) + + with pytest.warns(DeprecationWarning, match="litellm.mcp_client"): + import litellm.experimental_mcp_client as old + + assert old.__all__ == litellm.mcp_client.__all__ + assert old.load_mcp_tools is litellm.mcp_client.load_mcp_tools + assert old.call_openai_tool is litellm.mcp_client.call_openai_tool + assert importlib.import_module(f"{OLD_PACKAGE}.client") is litellm.mcp_client.client + assert importlib.import_module(f"{OLD_PACKAGE}.tools") is litellm.mcp_client.tools + attrs: Mapping[str, object] = vars(old) + assert attrs["tools"] is litellm.mcp_client.tools + assert attrs["client"] is litellm.mcp_client.client diff --git a/tests/unit/llms/litellm_proxy/skills/test_skill_search.py b/tests/unit/llms/litellm_proxy/skills/test_skill_search.py index c0754f6f1af..0647b203aff 100644 --- a/tests/unit/llms/litellm_proxy/skills/test_skill_search.py +++ b/tests/unit/llms/litellm_proxy/skills/test_skill_search.py @@ -456,7 +456,7 @@ class TestHandleSkillSearchMCP: self, accessible_skills: AsyncMock, embedding_router: MagicMock ) -> None: from litellm.llms.litellm_proxy.skills.skill_search import MAX_SKILL_SEARCH_TOP_K - from litellm.proxy._experimental.mcp_server.tool_search import handle_skill_search + from litellm.proxy.mcp_server.tool_search import handle_skill_search many_skills: Final = tuple( LiteLLM_SkillsTable( @@ -476,7 +476,7 @@ class TestHandleSkillSearchMCP: async def test_top_k_below_one_is_raised_to_one( self, accessible_skills: AsyncMock, embedding_router: MagicMock ) -> None: - from litellm.proxy._experimental.mcp_server.tool_search import handle_skill_search + from litellm.proxy.mcp_server.tool_search import handle_skill_search result = await handle_skill_search( query="language translation", top_k=0, user_api_key_dict=UserAPIKeyAuth(user_id="u") diff --git a/tests/unit/proxy/_experimental/mcp_server/auth/__init__.py b/tests/unit/mcp_client/__init__.py similarity index 100% rename from tests/unit/proxy/_experimental/mcp_server/auth/__init__.py rename to tests/unit/mcp_client/__init__.py diff --git a/tests/unit/experimental_mcp_client/test_mcp_client.py b/tests/unit/mcp_client/test_mcp_client.py similarity index 98% rename from tests/unit/experimental_mcp_client/test_mcp_client.py rename to tests/unit/mcp_client/test_mcp_client.py index 60600afab84..43021d248a9 100644 --- a/tests/unit/experimental_mcp_client/test_mcp_client.py +++ b/tests/unit/mcp_client/test_mcp_client.py @@ -49,8 +49,8 @@ from starlette.responses import Response from starlette.types import ASGIApp, Message, Receive, Scope, Send # Add the parent directory to the path so we can import litellm -import litellm.experimental_mcp_client.client as mcp_client_module -from litellm.experimental_mcp_client.client import ( +import litellm.mcp_client.client as mcp_client_module +from litellm.mcp_client.client import ( MCPClient, MCPSigV4Auth, PersistentMCPSession, @@ -61,16 +61,16 @@ from litellm.experimental_mcp_client.client import ( as_mcp_read_timeout, strip_auth_scheme, ) -from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( +from litellm.proxy.mcp_server.faults.list_outcomes import ( ServerListFault, classify_list_exception, list_fault_http_status, upstream_rate_limit, ) -from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( +from litellm.proxy.mcp_server.mcp_server_manager import ( _format_byok_openapi_auth_header, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import StaticHeaderAuth +from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import StaticHeaderAuth from litellm.types.mcp import MCPAuth, MCPStdioConfig, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer from tests._support.clock_loops import AutojumpClockLoop, ManualClockLoop @@ -163,7 +163,7 @@ async def test_modern_input_request_uses_existing_elicitation_callback( TextContent, ) - from litellm.proxy._experimental.mcp_server.interactions import BoundInputRequiredResult + from litellm.proxy.mcp_server.interactions import BoundInputRequiredResult observed: Final[SimpleQueue[str]] = SimpleQueue() sampled: Final = CreateMessageResult( @@ -379,8 +379,8 @@ class TestMCPClient: await client.run_with_session(_noop) @pytest.mark.asyncio - @patch("litellm.experimental_mcp_client.client.stdio_client") - @patch("litellm.experimental_mcp_client.client.ClientSession") + @patch("litellm.mcp_client.client.stdio_client") + @patch("litellm.mcp_client.client.ClientSession") async def test_mcp_client_stdio_connect_success(self, mock_session, mock_stdio_client): """Test successful stdio connection""" # Setup mocks - create proper async context manager @@ -432,7 +432,7 @@ class TestMCPClient: mock_streamable_http_client.return_value = mock_http_ctx # Mock the session - with patch("litellm.experimental_mcp_client.client.ClientSession") as mock_session: + with patch("litellm.mcp_client.client.ClientSession") as mock_session: mock_session_instance = AsyncMock() mock_session_instance.initialize = AsyncMock(return_value=_initialized()) mock_session_ctx = AsyncMock() @@ -478,7 +478,7 @@ class TestMCPClient: mock_sse_client.return_value = mock_sse_ctx # Mock the session - with patch("litellm.experimental_mcp_client.client.ClientSession") as mock_session: + with patch("litellm.mcp_client.client.ClientSession") as mock_session: mock_session_instance = AsyncMock() mock_session_instance.initialize = AsyncMock(return_value=_initialized()) mock_session_ctx = AsyncMock() @@ -530,7 +530,7 @@ class TestMCPClient: mock_streamable_http_client.return_value = mock_http_ctx # Mock the session - with patch("litellm.experimental_mcp_client.client.ClientSession") as mock_session: + with patch("litellm.mcp_client.client.ClientSession") as mock_session: mock_session_instance = AsyncMock() mock_session_instance.initialize = AsyncMock(return_value=_initialized()) mock_session_ctx = AsyncMock() @@ -670,7 +670,7 @@ class TestMCPClientInstructionsCapture: assert client._last_initialize_instructions is None @pytest.mark.asyncio - @patch("litellm.experimental_mcp_client.client.ClientSession") + @patch("litellm.mcp_client.client.ClientSession") async def test_captures_instructions_from_initialize(self, mock_session_cls): """Instructions from upstream initialize() are captured and stripped.""" client = MCPClient( @@ -698,7 +698,7 @@ class TestMCPClientInstructionsCapture: assert client._last_initialize_instructions == "upstream says hello" @pytest.mark.asyncio - @patch("litellm.experimental_mcp_client.client.ClientSession") + @patch("litellm.mcp_client.client.ClientSession") async def test_none_instructions_stays_none(self, mock_session_cls): """When upstream returns no instructions the field stays None.""" client = MCPClient( @@ -787,7 +787,7 @@ class TestExecuteSessionOperationSurfacesTransportError: return transport_ctx @pytest.mark.asyncio - @patch("litellm.experimental_mcp_client.client.ClientSession") + @patch("litellm.mcp_client.client.ClientSession") async def test_surfaces_connect_error_over_cancelled(self, mock_session_cls): client = MCPClient(server_url="http://example.com/mcp", transport_type="http") self._make_session( @@ -804,7 +804,7 @@ class TestExecuteSessionOperationSurfacesTransportError: await client._execute_session_operation(transport_ctx, _op) @pytest.mark.asyncio - @patch("litellm.experimental_mcp_client.client.ClientSession") + @patch("litellm.mcp_client.client.ClientSession") async def test_genuine_cancellation_is_not_replaced(self, mock_session_cls): client = MCPClient(server_url="http://example.com/mcp", transport_type="http") self._make_session(mock_session_cls, AsyncMock(side_effect=asyncio.CancelledError())) @@ -818,7 +818,7 @@ class TestExecuteSessionOperationSurfacesTransportError: @pytest.mark.asyncio @pytest.mark.parametrize("failure_phase", ("early", "late", "mixed")) - @patch("litellm.experimental_mcp_client.client.ClientSession") + @patch("litellm.mcp_client.client.ClientSession") async def test_response_close_preserves_cancellation_and_original_errors(self, session_class, failure_phase): closed: Final = asyncio.Event() close_error: Final = httpx2.ReadError("response close failed") @@ -863,7 +863,7 @@ class TestExecuteSessionOperationSurfacesTransportError: assert closed.is_set() @pytest.mark.asyncio - @patch("litellm.experimental_mcp_client.client.ClientSession") + @patch("litellm.mcp_client.client.ClientSession") async def test_cleanup_error_after_success_is_swallowed(self, mock_session_cls): client = MCPClient(server_url="http://example.com/mcp", transport_type="http") init_result = _initialized() @@ -877,7 +877,7 @@ class TestExecuteSessionOperationSurfacesTransportError: assert result == "done" @pytest.mark.asyncio - @patch("litellm.experimental_mcp_client.client.ClientSession") + @patch("litellm.mcp_client.client.ClientSession") async def test_session_entry_failure_still_closes_transport(self, session_class): failure: Final = RuntimeError("session dispatcher did not start") session_class.return_value.__aenter__ = AsyncMock(side_effect=failure) @@ -896,7 +896,7 @@ class TestExecuteSessionOperationSurfacesTransportError: @pytest.mark.asyncio @pytest.mark.parametrize("original_error", (False, True)) - @patch("litellm.experimental_mcp_client.client.ClientSession") + @patch("litellm.mcp_client.client.ClientSession") async def test_session_exit_cancellation_preserves_original_failure(self, session_class, original_error): self._make_session(session_class, AsyncMock(return_value=_initialized())) cancelled: Final = asyncio.CancelledError("cancelled while closing session") @@ -918,7 +918,7 @@ class TestExecuteSessionOperationSurfacesTransportError: @pytest.mark.asyncio @pytest.mark.parametrize("signal_type", (SystemExit, KeyboardInterrupt)) @pytest.mark.parametrize("phase", ("session", "transport")) - @patch("litellm.experimental_mcp_client.client.ClientSession") + @patch("litellm.mcp_client.client.ClientSession") async def test_cleanup_preserves_process_exit(self, session_class, phase, signal_type): self._make_session(session_class, AsyncMock(return_value=_initialized())) signal: Final = signal_type("process stopping") @@ -932,7 +932,7 @@ class TestExecuteSessionOperationSurfacesTransportError: transport.__aexit__.assert_awaited_once() @pytest.mark.asyncio - @patch("litellm.experimental_mcp_client.client.ClientSession") + @patch("litellm.mcp_client.client.ClientSession") async def test_session_and_termination_share_one_cleanup_deadline(self, session_class): self._make_session(session_class, AsyncMock(return_value=_initialized())) deleting: Final = asyncio.Event() @@ -1835,7 +1835,7 @@ def test_a_differently_cased_injected_header_cannot_shadow_the_slot() -> None: async def test_invalid_http_response_surfaces_without_waiting_for_timeout( content_type: str, body: bytes, expected_type: type[Exception] ) -> None: - from litellm.proxy._experimental.mcp_server.rest_endpoints import _connection_error_message + from litellm.proxy.mcp_server.rest_endpoints import _connection_error_message def respond(request: httpx2.Request) -> httpx2.Response: if expected_type is ValidationError: @@ -1979,7 +1979,7 @@ async def test_sse_catalog_limit_reports_the_failure_and_allows_smaller_pages(pa @pytest.mark.asyncio async def test_http_status_check_allows_auth_refresh_before_rejecting() -> None: - from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.client_credentials import ( ClientCredentialsBearerAuth, ) @@ -1993,7 +1993,7 @@ async def test_http_status_check_allows_auth_refresh_before_rejecting() -> None: seen.append(request.headers["authorization"]) return httpx2.Response(401 if len(seen) == 1 else 200, json={"ok": True}) - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ClientCredentialsConfig + from litellm.proxy.mcp_server.outbound_credentials.types import ClientCredentialsConfig auth = ClientCredentialsBearerAuth("stale", refresh, ClientCredentialsConfig()) client = MCPClient(server_url="https://example.com/mcp", resolved_auth=auth) @@ -2057,7 +2057,7 @@ async def test_http_response_handler_preserves_notifications_and_tool_listing() @pytest.mark.asyncio async def test_invalid_tool_list_schema_is_identified_as_an_upstream_response() -> None: - from litellm.proxy._experimental.mcp_server.rest_endpoints import _connection_error_message + from litellm.proxy.mcp_server.rest_endpoints import _connection_error_message def respond(request: httpx2.Request) -> httpx2.Response: if request.method == "DELETE": @@ -2249,7 +2249,7 @@ async def test_transport_completion_and_normal_messages( ) -> None: from mcp import ClientSession - from litellm.proxy._experimental.mcp_server.rest_endpoints import _connection_error_message + from litellm.proxy.mcp_server.rest_endpoints import _connection_error_message logging_callback: Final = AsyncMock() read_timeout: Final = 0.2 if mode == "silent" else 30 @@ -2847,7 +2847,7 @@ def test_standalone_client_request_does_not_require_proxy_dependencies(): for name in ("fastapi", "orjson", "email_validator", "annotated_doc"): sys.modules[name] = None import httpx2 - from litellm.experimental_mcp_client.client import MCPClient + from litellm.mcp_client.client import MCPClient async def request(): upstream = MCPClient(server_url="https://upstream.example/mcp") @@ -2872,7 +2872,7 @@ def test_client_import_before_proxy_credentials_succeeds_in_fresh_process(): [ sys.executable, "-c", - "import litellm.experimental_mcp_client.client; from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager; print(MCPServerManager.__name__)", + "import litellm.mcp_client.client; from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager; print(MCPServerManager.__name__)", ], capture_output=True, text=True, @@ -2886,7 +2886,7 @@ def test_client_import_before_proxy_credentials_succeeds_in_fresh_process(): @pytest.mark.asyncio @pytest.mark.parametrize("resolved", (False, True)) async def test_discovery_auth_fingerprint_tracks_effective_credentials(resolved: bool) -> None: - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import StaticHeaderAuth + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import StaticHeaderAuth def client(token: str) -> MCPClient: return MCPClient( @@ -2932,7 +2932,7 @@ async def test_discovery_auth_fingerprint_tracks_aws_sigv4_credential_identity() @pytest.mark.asyncio async def test_request_auth_preview_uses_the_same_effective_headers_as_egress() -> None: - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import StaticHeaderAuth + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import StaticHeaderAuth client: Final = MCPClient( server_url="https://upstream.example/mcp", @@ -3027,11 +3027,11 @@ async def test_404_before_session_initialization_preserves_method_not_found() -> def test_public_mcp_import_missing_dependency(missing_module: str) -> None: with patch.dict(sys.modules): for name in tuple(sys.modules): - if name.startswith(("litellm.experimental_mcp_client", "mcp.", "mcp_types.")) or name == "mcp": + if name.startswith(("litellm.mcp_client", "mcp.", "mcp_types.")) or name == "mcp": del sys.modules[name] with patch.dict(sys.modules, {missing_module: None}): with pytest.raises(ImportError) as caught: - importlib.import_module("litellm.experimental_mcp_client.client") + importlib.import_module("litellm.mcp_client.client") if missing_module in ("mcp", "httpx2"): assert "pip install 'litellm[mcp]'" in str(caught.value) @@ -3047,11 +3047,11 @@ def test_public_mcp_import_missing_dependency(missing_module: str) -> None: def test_public_mcp_import_preserves_incompatible_sdk_error() -> None: with patch.dict(sys.modules): for name in tuple(sys.modules): - if name.startswith("litellm.experimental_mcp_client"): + if name.startswith("litellm.mcp_client"): del sys.modules[name] with patch.dict(sys.modules, {"mcp": ModuleType("mcp")}): with pytest.raises(ImportError, match="cannot import name 'ClientSession'") as caught: - importlib.import_module("litellm.experimental_mcp_client.client") + importlib.import_module("litellm.mcp_client.client") assert not isinstance(caught.value, ModuleNotFoundError) assert caught.value.__cause__ is None @@ -3635,7 +3635,7 @@ async def test_modern_call_requires_schema_from_bounded_listing( from mcp.types import ToolsCapability - monkeypatch.setattr("litellm.experimental_mcp_client.tools.MCP_TOOL_LISTING_MAX_PAGES", 2) + monkeypatch.setattr("litellm.mcp_client.tools.MCP_TOOL_LISTING_MAX_PAGES", 2) methods: Final[SimpleQueue[str]] = SimpleQueue() def respond(request: httpx2.Request) -> httpx2.Response: @@ -5047,7 +5047,7 @@ async def test_persistent_session_reports_an_upstream_cancellation_as_a_runtime_ @pytest.mark.asyncio async def test_closing_persistent_session_fails_a_caller_blocked_on_a_full_queue_instead_of_hanging() -> None: - from litellm.experimental_mcp_client.client import _MAX_PENDING_OPERATIONS + from litellm.mcp_client.client import _MAX_PENDING_OPERATIONS app: Final = _stateful_upstream() async with app.router.lifespan_context(app): @@ -5089,8 +5089,9 @@ async def test_persistent_session_ends_after_a_broken_stream_so_the_next_call_ge @pytest.mark.parametrize("session_started_in_request", [True, False]) async def test_persistent_session_releases_the_current_requests_scope_response(session_started_in_request) -> None: from mcp.server.context import ServerRequestContext - from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var - from litellm.proxy._experimental.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse + + from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var + from litellm.proxy.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse app = _stateful_upstream() async with app.router.lifespan_context(app): @@ -5205,8 +5206,8 @@ async def test_cancelling_persistent_operation_stops_work_and_keeps_session_usab @pytest.mark.asyncio @pytest.mark.parametrize("header_name", ["Authorization", "x-upstream-token"]) async def test_subscription_fingerprint_preserves_context_across_managed_renewal(header_name): - from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import ClientCredentialsBearerAuth - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ClientCredentialsConfig + from litellm.proxy.mcp_server.outbound_credentials.client_credentials import ClientCredentialsBearerAuth + from litellm.proxy.mcp_server.outbound_credentials.types import ClientCredentialsConfig async def refetch(token: str) -> str | None: return None @@ -5234,7 +5235,7 @@ async def test_subscription_fingerprint_preserves_context_across_managed_renewal @pytest.mark.asyncio async def test_notification_callback_keeps_stream_errors_separate_from_protocol_errors() -> None: - from litellm.experimental_mcp_client.client import _receive_session_message + from litellm.mcp_client.client import _receive_session_message stream_error = asyncio.get_running_loop().create_future() receiver = AsyncMock() diff --git a/tests/unit/experimental_mcp_client/test_tools.py b/tests/unit/mcp_client/test_tools.py similarity index 96% rename from tests/unit/experimental_mcp_client/test_tools.py rename to tests/unit/mcp_client/test_tools.py index ac7afe4c74b..ca3cf29bd90 100644 --- a/tests/unit/experimental_mcp_client/test_tools.py +++ b/tests/unit/mcp_client/test_tools.py @@ -2,8 +2,6 @@ import json from unittest.mock import AsyncMock, MagicMock import pytest - - from mcp.types import ( CallToolRequestParams, CallToolResult, @@ -13,14 +11,14 @@ from mcp.types import ( ) from mcp.types import Tool as MCPTool -from litellm.experimental_mcp_client.tools import ( - list_tools_with_pagination, - transform_mcp_tool_to_anthropic_tool, +from litellm.mcp_client.tools import ( _get_function_arguments, _normalize_mcp_input_schema, call_mcp_tool, call_openai_tool, + list_tools_with_pagination, load_mcp_tools, + transform_mcp_tool_to_anthropic_tool, transform_mcp_tool_to_openai_responses_api_tool, transform_mcp_tool_to_openai_tool, transform_openai_tool_call_request_to_mcp_tool_call_request, @@ -131,7 +129,7 @@ async def test_load_mcp_tools_follows_pagination(mock_session): @pytest.mark.asyncio() @pytest.mark.parametrize("require_complete", [False, True]) async def test_pagination_walk_stops_at_page_cap(mock_session, monkeypatch, require_complete): - monkeypatch.setattr("litellm.experimental_mcp_client.tools.MCP_TOOL_LISTING_MAX_PAGES", 2) + monkeypatch.setattr("litellm.mcp_client.tools.MCP_TOOL_LISTING_MAX_PAGES", 2) mock_session.list_tools.side_effect = [ ListToolsResult( tools=[MCPTool(name="tool_0", description="0", inputSchema={})], @@ -189,10 +187,10 @@ async def test_pagination_walk_treats_empty_cursor_as_terminal(mock_session): async def test_pagination_walk_stops_at_whole_walk_deadline(mock_session, monkeypatch, require_complete): import anyio - from litellm.experimental_mcp_client.tools import list_tools_with_pagination + from litellm.mcp_client.tools import list_tools_with_pagination - monkeypatch.setattr("litellm.experimental_mcp_client.tools.MCP_CLIENT_TIMEOUT", 0.2) - monkeypatch.setattr("litellm.experimental_mcp_client.tools.MCP_TOOL_LISTING_TIMEOUT", 0.2) + monkeypatch.setattr("litellm.mcp_client.tools.MCP_CLIENT_TIMEOUT", 0.2) + monkeypatch.setattr("litellm.mcp_client.tools.MCP_TOOL_LISTING_TIMEOUT", 0.2) async def slow_page(params=None): await anyio.sleep(0.15) @@ -218,10 +216,10 @@ async def test_pagination_walk_stops_at_whole_walk_deadline(mock_session, monkey async def test_pagination_walk_honors_explicit_deadline_over_globals(mock_session, monkeypatch): import anyio - from litellm.experimental_mcp_client.tools import list_tools_with_pagination + from litellm.mcp_client.tools import list_tools_with_pagination - monkeypatch.setattr("litellm.experimental_mcp_client.tools.MCP_CLIENT_TIMEOUT", 0.1) - monkeypatch.setattr("litellm.experimental_mcp_client.tools.MCP_TOOL_LISTING_TIMEOUT", 0.1) + monkeypatch.setattr("litellm.mcp_client.tools.MCP_CLIENT_TIMEOUT", 0.1) + monkeypatch.setattr("litellm.mcp_client.tools.MCP_TOOL_LISTING_TIMEOUT", 0.1) async def slow_page(params=None): await anyio.sleep(0.15) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_init.py b/tests/unit/proxy/_experimental/mcp_server/test_init.py new file mode 100644 index 00000000000..2f933e11351 --- /dev/null +++ b/tests/unit/proxy/_experimental/mcp_server/test_init.py @@ -0,0 +1,30 @@ +import importlib +import sys +from collections.abc import Mapping + +import pytest + +import litellm.proxy.mcp_server.auth.user_api_key_auth_mcp +import litellm.proxy.mcp_server.server +import litellm.proxy.mcp_server.utils + +OLD_PACKAGE = "litellm.proxy._experimental.mcp_server" + + +def test_old_package_warns_and_deep_imports_resolve_to_proxy_mcp_server(monkeypatch: pytest.MonkeyPatch) -> None: + for name in tuple(sys.modules): + if name.startswith(OLD_PACKAGE): + monkeypatch.delitem(sys.modules, name) + + with pytest.warns(DeprecationWarning, match="litellm.proxy.mcp_server"): + old = importlib.import_module(OLD_PACKAGE) + + assert importlib.import_module(f"{OLD_PACKAGE}.server") is litellm.proxy.mcp_server.server + assert importlib.import_module(f"{OLD_PACKAGE}.utils") is litellm.proxy.mcp_server.utils + assert ( + importlib.import_module(f"{OLD_PACKAGE}.auth.user_api_key_auth_mcp") + is litellm.proxy.mcp_server.auth.user_api_key_auth_mcp + ) + attrs: Mapping[str, object] = vars(old) + assert attrs["server"] is litellm.proxy.mcp_server.server + assert attrs["auth"] is litellm.proxy.mcp_server.auth diff --git a/tests/unit/proxy/anthropic_endpoints/test_gateway_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_gateway_endpoints.py index 8c2d8e4f157..0f064622f2d 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_gateway_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_gateway_endpoints.py @@ -20,21 +20,6 @@ from fastapi import FastAPI from fastapi.testclient import TestClient from litellm.caching.dual_cache import DualCache -from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ( - MintedProxyCredential, - ProxyCredentialMintFailure, -) -from litellm.proxy._experimental.mcp_server.outbound_credentials.session_credentials import ( - SessionRefreshOpened, - open_session_refresh_bearer, - session_keys_from_master_key, -) -from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import ( - SESSION_REFRESH_PREFIX, - MintedSessionToken, - SessionPrincipal, - mint_session_refresh_token, -) from litellm.proxy._types import ProxyException from litellm.proxy.anthropic_endpoints import gateway_endpoints from litellm.proxy.management_endpoints.ui_sso import ( @@ -42,6 +27,21 @@ from litellm.proxy.management_endpoints.ui_sso import ( _hash_cli_sso_secret, _set_cli_sso_flow, ) +from litellm.proxy.mcp_server.gateway_dcr_flow import ( + MintedProxyCredential, + ProxyCredentialMintFailure, +) +from litellm.proxy.mcp_server.outbound_credentials.session_credentials import ( + SessionRefreshOpened, + open_session_refresh_bearer, + session_keys_from_master_key, +) +from litellm.proxy.mcp_server.outbound_credentials.session_token import ( + SESSION_REFRESH_PREFIX, + MintedSessionToken, + SessionPrincipal, + mint_session_refresh_token, +) from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMiddleware _DEVICE_CODE_GRANT: Final = "urn:ietf:params:oauth:grant-type:device_code" diff --git a/tests/unit/proxy/auth/test_login_utils.py b/tests/unit/proxy/auth/test_login_utils.py index ec9b9a04f72..37ea05f8ba9 100644 --- a/tests/unit/proxy/auth/test_login_utils.py +++ b/tests/unit/proxy/auth/test_login_utils.py @@ -9,8 +9,8 @@ import hashlib import os from collections.abc import Mapping from contextlib import ExitStack -from typing import TYPE_CHECKING, Final from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING, Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -638,10 +638,10 @@ class TestEncodeUiSessionJwt: this helper must now be accepted.""" from unittest.mock import MagicMock - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( + from litellm.proxy.auth.login_utils import encode_ui_session_jwt + from litellm.proxy.mcp_server.byok_oauth_endpoints import ( user_id_from_session_cookie, ) - from litellm.proxy.auth.login_utils import encode_ui_session_jwt token_object = {"user_id": "cornell-user", "key": "sk-abc", "login_method": "sso"} with patch("litellm.proxy.auth.login_utils.LITELLM_UI_SESSION_DURATION", "24h"): diff --git a/tests/unit/proxy/auth/test_mcp_ip_filtering.py b/tests/unit/proxy/auth/test_mcp_ip_filtering.py index c403daf733f..b28dd8bbab5 100644 --- a/tests/unit/proxy/auth/test_mcp_ip_filtering.py +++ b/tests/unit/proxy/auth/test_mcp_ip_filtering.py @@ -36,7 +36,7 @@ def _make_server(server_id, available_on_public_internet=False): def _make_manager(servers): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) diff --git a/tests/unit/proxy/db/mcp_server/test_db.py b/tests/unit/proxy/db/mcp_server/test_db.py index e2440e49f19..51c350dc1ce 100644 --- a/tests/unit/proxy/db/mcp_server/test_db.py +++ b/tests/unit/proxy/db/mcp_server/test_db.py @@ -3,8 +3,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest - -from litellm.proxy._experimental.mcp_server.db import ( +from litellm.proxy.mcp_server.db import ( approve_mcp_server, get_mcp_servers_by_team, reject_mcp_server, diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/akto/test_akto.py b/tests/unit/proxy/guardrails/guardrail_hooks/akto/test_akto.py index 8fa544819e9..e2f10225568 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/akto/test_akto.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/akto/test_akto.py @@ -3,9 +3,8 @@ import base64 import json import os from types import SimpleNamespace -from unittest.mock import AsyncMock, MagicMock, patch - from typing import Final, Literal +from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest @@ -1709,7 +1708,7 @@ async def test_one_text_masked_two_ways_blocks(akto_pre_call): @pytest.mark.asyncio async def test_masking_a_payload_too_deep_to_map_back_blocks(akto_pre_call): - from litellm.proxy._experimental.mcp_server.utils import MAX_STRUCTURED_CONTENT_SCAN_DEPTH + from litellm.proxy.mcp_server.utils import MAX_STRUCTURED_CONTENT_SCAN_DEPTH deep: object = CARD for _ in range(MAX_STRUCTURED_CONTENT_SCAN_DEPTH + 1): diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py index e4c7bcba951..0e6670c26e7 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_mcp_security.py @@ -63,7 +63,7 @@ class TestExtractMCPServerNames: class TestMCPSecurityGuardrailPreCall: @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" ) async def test_blocks_unregistered_server_chat_completions( self, mock_manager, guardrail @@ -93,7 +93,7 @@ class TestMCPSecurityGuardrailPreCall: @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" ) async def test_blocks_unregistered_server_responses_api( self, mock_manager, guardrail @@ -122,7 +122,7 @@ class TestMCPSecurityGuardrailPreCall: @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" ) async def test_allows_registered_servers(self, mock_manager, guardrail): """All MCP servers are registered - request passes through.""" @@ -211,8 +211,9 @@ class TestInitializeGuardrail: @pytest.mark.parametrize("call_type", ["acompletion", "aresponses"]) async def test_guardrail_observes_saved_server_creation_and_deletion_on_another_worker(guardrail, call_type): from unittest.mock import AsyncMock + from litellm.proxy._types import LiteLLM_MCPServerTable - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager manager = MCPServerManager() row = LiteLLM_MCPServerTable(server_id="peer-server", alias="peer_server", transport="http", @@ -226,7 +227,7 @@ async def test_guardrail_observes_saved_server_creation_and_deletion_on_another_ with ( patch("litellm.proxy.proxy_server.prisma_client", prisma), patch("litellm.proxy.proxy_server.general_settings", {}), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", manager), + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", manager), ): result = await guardrail.async_pre_call_hook(UserAPIKeyAuth(), MagicMock(), data, call_type) assert result == data diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py index a5c543528c2..83301db5d86 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py @@ -3243,7 +3243,7 @@ class TestPanwAirsMcpToolEventScan: with ( patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" ) as mock_manager, patch.object(handler, "_call_panw_api", new_callable=AsyncMock) as mock_api, ): diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py index bd1e9ae4068..e312378969a 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -15,27 +15,25 @@ from typing import Final, Literal from unittest.mock import MagicMock, patch import aiohttp +import pytest from aiohttp import web from aiohttp.client_proto import ResponseHandler from aiohttp.test_utils import TestServer -import pytest from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict - import litellm from litellm import mock_completion from litellm.caching.caching import DualCache +from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.presidio import ( - PresidioPerRequestConfig, OPTIONAL_PresidioPIIMasking, + PresidioPerRequestConfig, ) -from litellm.exceptions import GuardrailRaisedException from litellm.types.guardrails import LitellmParams, PiiAction, PiiEntityType from litellm.types.proxy.guardrails.guardrail_hooks.presidio import PresidioAnalyzeRequest, PresidioAnalyzeResponseItem from litellm.types.utils import Choices, Delta, Message, ModelResponse, StreamingChoices -from litellm.exceptions import BlockedPiiEntityError def _make_mock_session_iterator(json_response, status=200, content_type="application/json", text_response=""): @@ -4434,9 +4432,9 @@ async def test_pii_masking_replays_a_byte_identical_prefix_across_turns(mock_use async def test_initialized_presidio_scans_selected_surface(surface: str, action: PiiAction, has_tokens: bool) -> None: from mcp.types import CallToolResult, TextContent - from litellm.proxy._experimental.mcp_server.guardrail_translation.handler import MCPGuardrailTranslationHandler from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import UnifiedLLMGuardrails from litellm.proxy.guardrails.guardrail_initializers import initialize_presidio + from litellm.proxy.mcp_server.guardrail_translation.handler import MCPGuardrailTranslationHandler params: Final = LitellmParams( guardrail="presidio", diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index a132ce8eada..a6177ff9e3a 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -37,9 +37,6 @@ from litellm.llms.openai.responses.guardrail_translation.handler import ( OpenAIResponsesHandler, ) from litellm.llms.pass_through.guardrail_translation.handler import PassThroughEndpointHandler -from litellm.proxy._experimental.mcp_server.guardrail_translation.handler import ( - MCPGuardrailTranslationHandler, -) from litellm.proxy._types import LiteLLMRoutes, UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail import ( unified_guardrail as unified_module, @@ -47,6 +44,9 @@ from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail import ( from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( UnifiedLLMGuardrails, ) +from litellm.proxy.mcp_server.guardrail_translation.handler import ( + MCPGuardrailTranslationHandler, +) from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ( diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 345f37f1db4..b8dba5e1e68 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -4000,10 +4000,10 @@ async def test_billed_personal_key_auth_produces_team_rpm_descriptors(monkeypatc from litellm.models.budget import LiteLLM_BudgetTable from litellm.models.team_membership import LiteLLM_TeamMembership from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager - from litellm.proxy.auth import auth_checks from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager server: Final = MCPServer( server_id="math_a", diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index d6ee9b1d29e..4266c2ed6d1 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -1,30 +1,29 @@ import asyncio +import inspect +import json from collections.abc import Mapping from contextlib import ExitStack -from typing import Final -from types import SimpleNamespace -import json from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from typing import Final +from unittest.mock import AsyncMock, MagicMock, patch from uuid import UUID -import litellm import pytest import yaml -from fastapi.testclient import TestClient - -from unittest.mock import AsyncMock, MagicMock, patch - from fastapi import HTTPException +from fastapi.testclient import TestClient from pydantic import JsonValue -import inspect - +import litellm +from litellm.constants import PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS +from litellm.litellm_core_utils.duration_parser import duration_in_seconds +from litellm.models.object_permission import LiteLLM_ObjectPermissionTable from litellm.models.project import LiteLLM_ProjectTable from litellm.proxy._types import ( - GenerateKeyResponse, GenerateKeyRequest, + GenerateKeyResponse, KeyManagementRoutes, - NewUserRequest, LiteLLM_BudgetTable, LiteLLM_ObjectPermissionBase, LiteLLM_OrganizationTable, @@ -36,46 +35,42 @@ from litellm.proxy._types import ( LiteLLMKeyType, LitellmUserRoles, Member, + NewUserRequest, ProxyException, RegenerateKeyRequest, ResetSpendRequest, UpdateKeyRequest, ) -from litellm.models.object_permission import LiteLLM_ObjectPermissionTable from litellm.proxy.auth.auth_checks import ( delete_cache_key_object, jwt_key_mapping_cache_key, ) from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, project_cache_key -from litellm.constants import PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker -from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.proxy.management_endpoints.key_management_endpoints import ( + KeyProjectBindingError, _check_key_project_team, _check_key_project_team_on_mutation, _check_org_key_limits, _check_project_key_limits, _check_team_key_limits, - _is_team_key, - _key_generation_required_param_check, - _personal_key_generation_check, - _team_key_generation_check, _common_key_generation_helper, - _member_key_request, - _team_admin_requests_member_key, - KeyProjectBindingError, _effective_key_after_update, _effective_key_for_generate, _enforce_custom_key_policy, _enforce_upperbound_key_params, _execute_virtual_key_regeneration, _get_and_validate_existing_key, - list_key_helper, - persist_deleted_verification_tokens, + _is_team_key, + _key_generation_required_param_check, + _member_key_request, + _personal_key_generation_check, _process_single_key_update, _requested_end_user_budget_id, _save_deleted_verification_token_records, + _team_admin_requests_member_key, + _team_key_generation_check, _transform_verification_tokens_to_deleted_records, _validate_end_user_budget_id_change, _validate_max_budget, @@ -90,7 +85,9 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( generate_service_account_key_fn, key_aliases, key_generation_check, + list_key_helper, list_keys, + persist_deleted_verification_tokens, prepare_key_update_data, prepare_metadata_fields, reset_key_spend_fn, @@ -837,7 +834,7 @@ async def test_update_key_grandfathers_existing_mcp_servers(monkeypatch): } mock_mgr.get_allow_all_keys_server_ids.return_value = [] monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_mgr, ) @@ -10955,10 +10952,10 @@ class TestValidateKeyAliasFormat: _validate_key_alias_format("team/user@example.com") def test_validate_key_alias_format_invalid(self, monkeypatch): + from litellm.proxy._types import ProxyException from litellm.proxy.management_endpoints.key_management_endpoints import ( _validate_key_alias_format, ) - from litellm.proxy._types import ProxyException monkeypatch.setattr(litellm, "enable_key_alias_format_validation", True) invalid_aliases = [ @@ -12798,14 +12795,14 @@ class TestKeyAliasSkipValidationOnUnchanged: Updating a key without changing its key_alias should skip format validation — even if the alias wouldn't pass current rules. """ + # Temporarily make the regex reject '@' to simulate stricter rules + import re + + from litellm.proxy.management_endpoints import key_management_endpoints as mod from litellm.proxy.management_endpoints.key_management_endpoints import ( _validate_key_alias_format, ) - # Temporarily make the regex reject '@' to simulate stricter rules - import re - from litellm.proxy.management_endpoints import key_management_endpoints as mod - original_pattern = mod._KEY_ALIAS_PATTERN mod._KEY_ALIAS_PATTERN = re.compile( r"^[a-zA-Z0-9][a-zA-Z0-9_\-/\.]{0,253}[a-zA-Z0-9]$" @@ -20431,7 +20428,6 @@ async def test_regenerate_key_repoints_access_group_assigned_key_ids(monkeypatch from litellm.proxy.management_endpoints.key_management_endpoints import ( _execute_virtual_key_regeneration, ) - from litellm.proxy.utils import hash_token new_token_hash = hash_token("sk-newtoken1234ab12") diff --git a/tests/unit/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_mcp_management_endpoints.py index 65c4a8faeb7..4b1415c8879 100644 --- a/tests/unit/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -1,14 +1,11 @@ -from contextlib import nullcontext - - import asyncio +import json +import logging import os import sys import types -import json -import logging from collections.abc import Iterator, Mapping -from contextlib import ExitStack, contextmanager +from contextlib import ExitStack, contextmanager, nullcontext from dataclasses import dataclass, field from datetime import datetime, timedelta from types import SimpleNamespace @@ -17,28 +14,23 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -from pydantic import BaseModel, TypeAdapter, ValidationError -from respx import MockRouter from fastapi import FastAPI, HTTPException, Request from fastapi.testclient import TestClient +from pydantic import BaseModel, TypeAdapter, ValidationError +from respx import MockRouter import litellm from litellm._uuid import uuid from litellm.caching.caching import DualCache -from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.proxy.utils import ProxyLogging from litellm.constants import UI_SESSION_TOKEN_TEAM_ID +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.models.access_group import LiteLLM_AccessGroupTable from litellm.models.organization import LiteLLM_OrganizationTable from litellm.models.team import LiteLLM_TeamTable from litellm.models.user import LiteLLM_UserTable -from litellm.proxy.management_endpoints import ( - mcp_management_endpoints as mgmt_endpoints, -) - from litellm.proxy._types import ( - LiteLLM_ObjectPermissionTable, LiteLLM_MCPServerTable, + LiteLLM_ObjectPermissionTable, LitellmUserRoles, MakeMCPServersPublicRequest, MCPTransport, @@ -47,7 +39,11 @@ from litellm.proxy._types import ( UpdateMCPServerRequest, UserAPIKeyAuth, ) -from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerConfig, MCPServerManager +from litellm.proxy.management_endpoints import ( + mcp_management_endpoints as mgmt_endpoints, +) +from litellm.proxy.mcp_server.mcp_server_manager import MCPServerConfig, MCPServerManager +from litellm.proxy.utils import ProxyLogging from litellm.types.mcp import MCPAuth, MCPCredentials from litellm.types.mcp_server.mcp_server_manager import MCPServer, PinnedMCPTool @@ -173,7 +169,7 @@ def patch_proxy_general_settings(settings: dict): async def test_mcp_publication_list_and_detail_derive_current_status( from_db: bool, strict: bool, explicit: bool, expected_public: bool ) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager manager: Final = MCPServerManager() server: Final = MCPServer( @@ -227,7 +223,7 @@ async def test_mcp_publication_list_and_detail_derive_current_status( def test_mcp_publication_projection_excludes_unregistered_lifecycle_records( approval_status: str, strict: bool ) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager record: Final = LiteLLM_MCPServerTable( server_id="unregistered-server", @@ -275,7 +271,7 @@ async def test_mcp_publication_updates_runtime_only_after_successful_save( error_status: int | None, ) -> None: import litellm - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager manager: Final = MCPServerManager() server: Final = generate_mock_mcp_server_config_record(server_id="new-server") @@ -299,7 +295,7 @@ async def test_mcp_publication_updates_runtime_only_after_successful_save( patch("litellm.public_mcp_servers", previous_ids), patch("litellm.proxy.proxy_server.proxy_config", proxy_config), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", manager, ), ): @@ -1537,11 +1533,11 @@ class TestListMCPServers: mock_manager, ), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_servers_by_verificationtoken", + "litellm.proxy.mcp_server.db.get_mcp_servers_by_verificationtoken", AsyncMock(return_value=["env-server"]), ), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_servers", + "litellm.proxy.mcp_server.db.get_mcp_servers", AsyncMock(return_value=[generate_mock_mcp_server_db_record(server_id="env-server")]), ), patch( @@ -2343,7 +2339,7 @@ class TestTemporaryMCPSessionEndpoints: """The edit form authorizes against a saved server's own id, so a draft write would collide on the primary key. That row is already visible to every worker, so it is returned untouched and no draft is created.""" - from litellm.proxy._experimental.mcp_server.db import create_draft_mcp_server + from litellm.proxy.mcp_server.db import create_draft_mcp_server real_row = generate_mock_mcp_server_db_record(server_id="already-saved") real_row.approval_status = "active" @@ -2352,15 +2348,15 @@ class TestTemporaryMCPSessionEndpoints: with ( patch( - "litellm.proxy._experimental.mcp_server.db._db_find_mcp_server_row", + "litellm.proxy.mcp_server.db._db_find_mcp_server_row", AsyncMock(return_value=real_row), ), patch( - "litellm.proxy._experimental.mcp_server.db._db_find_mcp_server_rows", + "litellm.proxy.mcp_server.db._db_find_mcp_server_rows", AsyncMock(return_value=[]), ), - patch("litellm.proxy._experimental.mcp_server.db.create_mcp_server", create_call), - patch("litellm.proxy._experimental.mcp_server.db.delete_mcp_server", delete_call), + patch("litellm.proxy.mcp_server.db.create_mcp_server", create_call), + patch("litellm.proxy.mcp_server.db.delete_mcp_server", delete_call), ): result = await create_draft_mcp_server( MagicMock(), @@ -2381,7 +2377,7 @@ class TestTemporaryMCPSessionEndpoints: live two-worker proxy. The loser's session is in fact ready, because the winner wrote a draft for it, so it adopts that row instead of failing the caller. """ - from litellm.proxy._experimental.mcp_server.db import create_draft_mcp_server + from litellm.proxy.mcp_server.db import create_draft_mcp_server winner_draft = generate_mock_mcp_server_db_record(server_id="raced") winner_draft.approval_status = "draft" @@ -2389,13 +2385,13 @@ class TestTemporaryMCPSessionEndpoints: lookups = AsyncMock(side_effect=[None, winner_draft]) with ( - patch("litellm.proxy._experimental.mcp_server.db._db_find_mcp_server_row", lookups), + patch("litellm.proxy.mcp_server.db._db_find_mcp_server_row", lookups), patch( - "litellm.proxy._experimental.mcp_server.db._db_find_mcp_server_rows", + "litellm.proxy.mcp_server.db._db_find_mcp_server_rows", AsyncMock(return_value=[]), ), patch( - "litellm.proxy._experimental.mcp_server.db.create_mcp_server", + "litellm.proxy.mcp_server.db.create_mcp_server", AsyncMock(side_effect=Exception("duplicate key value violates unique constraint")), ), ): @@ -2412,19 +2408,19 @@ class TestTemporaryMCPSessionEndpoints: @pytest.mark.asyncio async def test_create_draft_mcp_server_reraises_when_the_create_failure_was_not_a_race(self): """A genuine database error must not be swallowed by the race-adoption path.""" - from litellm.proxy._experimental.mcp_server.db import create_draft_mcp_server + from litellm.proxy.mcp_server.db import create_draft_mcp_server with ( patch( - "litellm.proxy._experimental.mcp_server.db._db_find_mcp_server_row", + "litellm.proxy.mcp_server.db._db_find_mcp_server_row", AsyncMock(side_effect=[None, None]), ), patch( - "litellm.proxy._experimental.mcp_server.db._db_find_mcp_server_rows", + "litellm.proxy.mcp_server.db._db_find_mcp_server_rows", AsyncMock(return_value=[]), ), patch( - "litellm.proxy._experimental.mcp_server.db.create_mcp_server", + "litellm.proxy.mcp_server.db.create_mcp_server", AsyncMock(side_effect=Exception("connection refused")), ), pytest.raises(Exception, match="connection refused"), @@ -2440,10 +2436,10 @@ class TestTemporaryMCPSessionEndpoints: async def test_create_draft_mcp_server_prunes_drafts_past_their_lifetime(self): """Regression: abandoned OAuth sessions accumulated forever. Verified against a live proxy, where 12 drafts aged past the lifetime were still present and a 13th was added.""" - from litellm.proxy._experimental.mcp_server.db import create_draft_mcp_server - from datetime import timezone + from litellm.proxy.mcp_server.db import create_draft_mcp_server + now = datetime.now(timezone.utc) stale_one = generate_mock_mcp_server_db_record(server_id="stale-1") stale_one.updated_at = now - timedelta(hours=1) @@ -2456,14 +2452,14 @@ class TestTemporaryMCPSessionEndpoints: delete_call = AsyncMock() with ( - patch("litellm.proxy._experimental.mcp_server.db._db_find_mcp_server_rows", find_rows), + patch("litellm.proxy.mcp_server.db._db_find_mcp_server_rows", find_rows), patch( - "litellm.proxy._experimental.mcp_server.db._db_find_mcp_server_row", + "litellm.proxy.mcp_server.db._db_find_mcp_server_row", AsyncMock(return_value=None), ), - patch("litellm.proxy._experimental.mcp_server.db.delete_mcp_server", delete_call), + patch("litellm.proxy.mcp_server.db.delete_mcp_server", delete_call), patch( - "litellm.proxy._experimental.mcp_server.db.create_mcp_server", + "litellm.proxy.mcp_server.db.create_mcp_server", AsyncMock(return_value=generate_mock_mcp_server_db_record(server_id="fresh")), ), ): @@ -2483,11 +2479,11 @@ class TestTemporaryMCPSessionEndpoints: """Drafts are addressable only by their own id and must never appear in a listing, but a bare inequality would also drop pre-approval-workflow rows, since SQL evaluates NULL != 'draft' as NULL.""" - from litellm.proxy._experimental.mcp_server.db import get_all_mcp_servers + from litellm.proxy.mcp_server.db import get_all_mcp_servers find_rows = AsyncMock(return_value=[]) with patch( - "litellm.proxy._experimental.mcp_server.db._db_find_mcp_server_rows", + "litellm.proxy.mcp_server.db._db_find_mcp_server_rows", find_rows, ): await get_all_mcp_servers(MagicMock()) @@ -2499,11 +2495,11 @@ class TestTemporaryMCPSessionEndpoints: async def test_get_all_mcp_servers_propagates_read_failures(self): """Regression: a swallowed read failure returned [] and silently disabled the bulk-import dedupe, so a flaky DB read turned a re-import into duplicate servers.""" - from litellm.proxy._experimental.mcp_server.db import get_all_mcp_servers + from litellm.proxy.mcp_server.db import get_all_mcp_servers find_rows = AsyncMock(side_effect=RuntimeError("db down")) with patch( # test-quality-ok: the helper takes its row reader from module scope, matching the suite's pattern - "litellm.proxy._experimental.mcp_server.db._db_find_mcp_server_rows", + "litellm.proxy.mcp_server.db._db_find_mcp_server_rows", find_rows, ): with pytest.raises(RuntimeError, match="db down"): @@ -2681,7 +2677,7 @@ class TestTemporaryMCPSessionEndpoints: mock_manager, ), patch( - "litellm.proxy._experimental.mcp_server.ui_session_utils.build_effective_auth_contexts", + "litellm.proxy.mcp_server.ui_session_utils.build_effective_auth_contexts", AsyncMock(return_value=[non_admin]), ) as effective_contexts, patch.object(mgmt_endpoints, "build_effective_auth_contexts", effective_contexts), @@ -2769,7 +2765,7 @@ class TestTemporaryMCPSessionEndpoints: mock_manager, ), patch( - "litellm.proxy._experimental.mcp_server.ui_session_utils.build_effective_auth_contexts", + "litellm.proxy.mcp_server.ui_session_utils.build_effective_auth_contexts", AsyncMock(return_value=[ui_session_auth, team_context]), ) as effective_contexts, patch.object(mgmt_endpoints, "build_effective_auth_contexts", effective_contexts), @@ -3027,7 +3023,7 @@ class TestTemporaryMCPSessionEndpoints: with ( patch.dict(sys.modules, {"litellm.proxy.proxy_server": fake_proxy_server}), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), patch( @@ -3081,7 +3077,7 @@ class TestTemporaryMCPSessionEndpoints: with ( patch.dict(sys.modules, {"litellm.proxy.proxy_server": fake_proxy_server}), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), patch( @@ -3139,8 +3135,8 @@ class TestTemporaryMCPSessionEndpoints: from starlette.requests import Request - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.proxy.management_endpoints.mcp_management_endpoints import mcp_authorize + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager row: Final = LiteLLM_MCPServerTable( server_id="saved-server", @@ -3269,7 +3265,7 @@ class TestTemporaryMCPSessionEndpoints: if mint_mock is not None: patches.append( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.mint_ephemeral_dcr_client", + "litellm.proxy.mcp_server.discoverable_endpoints.mint_ephemeral_dcr_client", mint_mock, ) ) @@ -3324,7 +3320,7 @@ class TestTemporaryMCPSessionEndpoints: through to a gateway-side DCR mint and proceed with the minted client instead of dead-ending on a 400 missing_client_id. Both modes share the caller-held-client contract, so both get the fall-through.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( EphemeralDcrClient, ) @@ -3367,7 +3363,7 @@ class TestTemporaryMCPSessionEndpoints: return_value=nullcontext(server), ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.mint_ephemeral_dcr_client", + "litellm.proxy.mcp_server.discoverable_endpoints.mint_ephemeral_dcr_client", mint_mock, ), ): @@ -3448,7 +3444,7 @@ class TestTemporaryMCPSessionEndpoints: one (the minted flow runs the bridge short-circuit arm) instead of dead-ending on missing_client_id. The relay front door stays reserved for clients that present their own client_id.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( EphemeralDcrClient, ) @@ -3492,12 +3488,12 @@ class TestTemporaryMCPSessionEndpoints: callback forwarded, so the token endpoint recovers the ephemeral client and the real upstream code from it and authenticates the exchange with them, with no client_id supplied by the caller and none stored on the server.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - seal_passthrough_authorization_code, - ) from litellm.proxy.management_endpoints.mcp_management_endpoints import ( mcp_token, ) + from litellm.proxy.mcp_server.discoverable_endpoints import ( + seal_passthrough_authorization_code, + ) request = MagicMock() request.base_url = "https://litellm.example.com/" @@ -3549,13 +3545,13 @@ class TestTemporaryMCPSessionEndpoints: async def test_mcp_token_prefers_sealed_client_when_server_has_shared_client( self: "TestTemporaryMCPSessionEndpoints", ) -> None: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - seal_passthrough_authorization_code, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy.management_endpoints.mcp_management_endpoints import ( mcp_token, ) + from litellm.proxy.mcp_server.discoverable_endpoints import ( + seal_passthrough_authorization_code, + ) + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager request: Final = MagicMock() request.base_url = "https://litellm.example.com/" @@ -3590,7 +3586,7 @@ class TestTemporaryMCPSessionEndpoints: return_value=prisma_client, ), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + "litellm.proxy.mcp_server.db.get_mcp_server", new=mock_get_mcp_server, ), patch.object( @@ -3599,7 +3595,7 @@ class TestTemporaryMCPSessionEndpoints: new=mock_update_server, ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=fake_http_client, ), ): @@ -3640,12 +3636,12 @@ class TestTemporaryMCPSessionEndpoints: contract: a refresh_token grant that echoes a leftover sealed passthrough code (plus any verifier) must not recover the minted credentials, so a clientless server answers missing_client_id and the client re-runs authorize instead.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - seal_passthrough_authorization_code, - ) from litellm.proxy.management_endpoints.mcp_management_endpoints import ( mcp_token, ) + from litellm.proxy.mcp_server.discoverable_endpoints import ( + seal_passthrough_authorization_code, + ) request = MagicMock() request.base_url = "https://litellm.example.com/" @@ -3696,12 +3692,12 @@ class TestTemporaryMCPSessionEndpoints: """A sealed code is minted only for S256 PKCE flows, so redeeming one without the corresponding verifier is refused at the gateway rather than trusting the upstream to enforce the binding.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - seal_passthrough_authorization_code, - ) from litellm.proxy.management_endpoints.mcp_management_endpoints import ( mcp_token, ) + from litellm.proxy.mcp_server.discoverable_endpoints import ( + seal_passthrough_authorization_code, + ) request = MagicMock() server = generate_mock_mcp_server_config_record(server_id="server-1") @@ -3746,12 +3742,12 @@ class TestTemporaryMCPSessionEndpoints: """A sealed passthrough code is bound to the server it was minted for: presenting it at another server's token endpoint is a 400 before any upstream exchange, so a code cannot be replayed across a server boundary.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - seal_passthrough_authorization_code, - ) from litellm.proxy.management_endpoints.mcp_management_endpoints import ( mcp_token, ) + from litellm.proxy.mcp_server.discoverable_endpoints import ( + seal_passthrough_authorization_code, + ) request = MagicMock() server = generate_mock_mcp_server_config_record(server_id="server-1") @@ -4790,8 +4786,8 @@ async def test_health_discovery_respects_route_restricted_key_grants( ) -> None: from typing import Final - from litellm.proxy._experimental.mcp_server import mcp_server_manager from litellm.proxy._types import LiteLLM_ObjectPermissionTable + from litellm.proxy.mcp_server import mcp_server_manager monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") manager: Final = mcp_server_manager.MCPServerManager() @@ -4865,8 +4861,8 @@ async def test_health_checks_probe_shared_servers_once_across_auth_contexts( expected: tuple[str, ...], include_reachability: bool, ) -> None: - from litellm.proxy._experimental.mcp_server import mcp_server_manager from litellm.proxy._types import LiteLLM_ObjectPermissionTable + from litellm.proxy.mcp_server import mcp_server_manager monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") manager: Final = mcp_server_manager.MCPServerManager() @@ -4934,8 +4930,8 @@ async def test_health_reachability_requires_explicit_api_opt_in( detail: bool, flag: str | None, ) -> None: - from litellm.proxy._experimental.mcp_server import mcp_server_manager from litellm.proxy._types import LiteLLM_ObjectPermissionTable + from litellm.proxy.mcp_server import mcp_server_manager class HealthResponse(BaseModel): server_id: str @@ -5770,10 +5766,10 @@ async def test_store_mcp_oauth_user_credential_blocked_when_identity_binding_enf if not mgmt_endpoints.MCP_AVAILABLE: pytest.skip("MCP module not installed") - from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module from litellm.proxy.management_endpoints.mcp_management_endpoints import ( store_mcp_oauth_user_credential, ) + from litellm.proxy.mcp_server import mcp_server_manager as manager_module server_id = "srv-binding-1" bound_server = MCPServer( @@ -5875,10 +5871,10 @@ async def test_store_mcp_oauth_user_credential_invalidates_cached_token(): if not mgmt_endpoints.MCP_AVAILABLE: pytest.skip("MCP module not installed") - from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module from litellm.proxy.management_endpoints.mcp_management_endpoints import ( store_mcp_oauth_user_credential, ) + from litellm.proxy.mcp_server import mcp_server_manager as manager_module server_id = "srv-inv-1" user_id = "user-inv-1" @@ -5927,10 +5923,10 @@ async def test_delete_mcp_oauth_user_credential_invalidates_cached_token(): if not mgmt_endpoints.MCP_AVAILABLE: pytest.skip("MCP module not installed") - from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module from litellm.proxy.management_endpoints.mcp_management_endpoints import ( delete_mcp_oauth_user_credential, ) + from litellm.proxy.mcp_server import mcp_server_manager as manager_module server_id = "srv-inv-2" user_id = "user-inv-2" @@ -5971,10 +5967,10 @@ async def test_delete_mcp_oauth_user_credential_invalidates_when_record_already_ if not mgmt_endpoints.MCP_AVAILABLE: pytest.skip("MCP module not installed") - from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module from litellm.proxy.management_endpoints.mcp_management_endpoints import ( delete_mcp_oauth_user_credential, ) + from litellm.proxy.mcp_server import mcp_server_manager as manager_module server_id = "srv-inv-3" user_id = "user-inv-3" @@ -6018,10 +6014,10 @@ async def test_admin_revokes_another_users_byok_credential(): if not mgmt_endpoints.MCP_AVAILABLE: pytest.skip("MCP module not installed") - from litellm.proxy._experimental.mcp_server import server as mcp_server from litellm.proxy.management_endpoints.mcp_management_endpoints import ( delete_mcp_user_credential, ) + from litellm.proxy.mcp_server import server as mcp_server delete_mock = AsyncMock(return_value=None) invalidate_mock = AsyncMock() @@ -6087,10 +6083,10 @@ async def test_user_naming_themselves_still_deletes_own_byok_credential(): if not mgmt_endpoints.MCP_AVAILABLE: pytest.skip("MCP module not installed") - from litellm.proxy._experimental.mcp_server import server as mcp_server from litellm.proxy.management_endpoints.mcp_management_endpoints import ( delete_mcp_user_credential, ) + from litellm.proxy.mcp_server import server as mcp_server deleted_rows: list[tuple[str, str]] = [] # mutable-ok: test-local recorder for the fake delete boundary @@ -6126,10 +6122,10 @@ async def test_admin_revokes_another_users_oauth_credential(): if not mgmt_endpoints.MCP_AVAILABLE: pytest.skip("MCP module not installed") - from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module from litellm.proxy.management_endpoints.mcp_management_endpoints import ( delete_mcp_oauth_user_credential, ) + from litellm.proxy.mcp_server import mcp_server_manager as manager_module get_mock = AsyncMock(return_value={"type": "oauth2", "access_token": "mallory-tok"}) delete_mock = AsyncMock(return_value=None) @@ -7427,7 +7423,7 @@ class TestMCPUserEnvVarsAccessControl: def test_oauth2_flow_accepted_on_create_request(): """NewMCPServerRequest carries oauth2_flow through to the persisted dict.""" - from litellm.proxy._experimental.mcp_server.db import _prepare_mcp_server_data + from litellm.proxy.mcp_server.db import _prepare_mcp_server_data payload = NewMCPServerRequest( server_name="m2m-server", @@ -7506,8 +7502,8 @@ def test_dcr_bridge_rejected_on_create_when_auth_type_omitted(): @pytest.mark.parametrize("auth_type", ["true_passthrough", "oauth_delegate"]) def test_dcr_bridge_accepted_on_create_for_client_forwarded_modes(auth_type): - from litellm.proxy._experimental.mcp_server.db import _prepare_mcp_server_data from litellm.proxy._types import NewMCPServerRequest + from litellm.proxy.mcp_server.db import _prepare_mcp_server_data payload = NewMCPServerRequest( server_name="bridge-server", @@ -8037,7 +8033,7 @@ class TestConnectedAppViewAnnotation: AsyncMock(return_value=[caller_auth]), ), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", reload_mock, ), ): @@ -8069,7 +8065,7 @@ class TestConnectedAppViewAnnotation: mock_manager, ), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", AsyncMock(return_value=UserAPIKeyAuth(user_id="test_user_id")), ), ): @@ -8108,11 +8104,11 @@ class TestConnectedAppViewAnnotation: mock_manager, ), patch( - "litellm.proxy._experimental.mcp_server.ui_session_utils.resolve_ui_session_team_ids", + "litellm.proxy.mcp_server.ui_session_utils.resolve_ui_session_team_ids", AsyncMock(return_value=[]), ), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", AsyncMock(return_value=admitted_auth), ), ): @@ -8140,7 +8136,7 @@ class TestConnectedAppViewAnnotation: AsyncMock(return_value=[caller_auth]), ), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", AsyncMock(side_effect=HTTPException(status_code=401, detail="expired")), ), ): @@ -8168,7 +8164,7 @@ class TestConnectedAppViewAnnotation: AsyncMock(return_value=[caller_auth]), ), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", reload_mock, ), ): @@ -8202,7 +8198,7 @@ class TestConnectedAppViewAnnotation: AsyncMock(return_value=[caller_auth]), ), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", reload_mock, ), ): @@ -8233,7 +8229,7 @@ class TestConnectedAppViewAnnotation: AsyncMock(return_value=[caller_auth]), ), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", reload_mock, ), ): @@ -8433,10 +8429,10 @@ class TestGetMCPGatewaySessions: async def test_admin_roles_receive_live_session_report(self, role): from mcp.types import Implementation - from litellm.proxy._experimental.mcp_server import server as mcp_server from litellm.proxy.management_endpoints.mcp_management_endpoints import ( get_mcp_gateway_sessions, ) + from litellm.proxy.mcp_server import server as mcp_server from litellm.types.mcp import MCPGatewaySessionsResponse session_id = "gateway-sessions-endpoint-1" @@ -8470,7 +8466,7 @@ class TestGetMCPGatewaySessions: class TestDeleteMCPGatewaySessions: @pytest.fixture(autouse=True) def _forget_admin_terminated_ids(self): - from litellm.proxy._experimental.mcp_server import server as mcp_server + from litellm.proxy.mcp_server import server as mcp_server yield mcp_server._admin_terminated_session_ids.clear() @@ -8478,10 +8474,10 @@ class TestDeleteMCPGatewaySessions: @pytest.mark.asyncio @pytest.mark.parametrize("role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY]) async def test_non_full_admin_forbidden_before_any_session_is_touched(self, role): - from litellm.proxy._experimental.mcp_server import server as mcp_server from litellm.proxy.management_endpoints.mcp_management_endpoints import ( delete_mcp_gateway_sessions, ) + from litellm.proxy.mcp_server import server as mcp_server session_id = "gateway-terminate-forbidden-1" transport = MagicMock(terminate=AsyncMock()) @@ -8522,10 +8518,10 @@ class TestDeleteMCPGatewaySessions: @pytest.mark.asyncio async def test_admin_terminates_only_the_selected_session(self): - from litellm.proxy._experimental.mcp_server import server as mcp_server from litellm.proxy.management_endpoints.mcp_management_endpoints import ( delete_mcp_gateway_sessions, ) + from litellm.proxy.mcp_server import server as mcp_server from litellm.types.mcp import MCPGatewaySessionsTerminateResponse target_id = "11111111-target-session" @@ -8569,10 +8565,10 @@ class TestDeleteMCPGatewaySessions: @pytest.mark.asyncio async def test_admin_terminates_every_session_of_the_selected_user(self): - from litellm.proxy._experimental.mcp_server import server as mcp_server from litellm.proxy.management_endpoints.mcp_management_endpoints import ( delete_mcp_gateway_sessions, ) + from litellm.proxy.mcp_server import server as mcp_server def auth_user(user_id: str): return mcp_server.MCPAuthenticatedUser( @@ -8620,7 +8616,7 @@ class TestGetMcpToolsWireShape: tools=[Tool(name="add", description="Add", inputSchema=add_schema, outputSchema={"type": "integer"})] ) with patch( - "litellm.proxy._experimental.mcp_server.server._list_mcp_tools", + "litellm.proxy.mcp_server.server._list_mcp_tools", AsyncMock(return_value=listed), ): result = await mgmt_endpoints.get_mcp_tools(user_api_key_dict=generate_mock_user_api_key_auth()) @@ -8641,7 +8637,7 @@ class TestGetMcpToolsWireShape: ], ) async def test_config_server_edit_preserves_api_contract_without_creating_rows(role, expected_status): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager manager = MCPServerManager() server = generate_mock_mcp_server_config_record(server_id="read-only-config") @@ -8675,7 +8671,7 @@ async def test_config_server_edit_preserves_api_contract_without_creating_rows(r @pytest.mark.asyncio async def test_saved_server_authorize_denial_does_not_dispatch_upstream(): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager manager: Final = MCPServerManager() row: Final = LiteLLM_MCPServerTable(server_id="denied-peer", alias="denied_peer", transport=MCPTransport.http, @@ -8715,7 +8711,7 @@ class TestDuplicateIdentifierRejection: @staticmethod def _conflict(field: str, value: str, server_id: str = "existing-1"): - from litellm.proxy._experimental.mcp_server.db import McpIdentifierConflict + from litellm.proxy.mcp_server.db import McpIdentifierConflict return McpIdentifierConflict(field=field, value=value, server_id=server_id) @@ -8963,7 +8959,7 @@ class TestPinMCPServerTools: ), patch("litellm.proxy.management_endpoints.mcp_management_endpoints.set_mcp_server_pinned_tools", store_mock), patch("litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", manager), - patch("litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager", manager), + patch("litellm.proxy.mcp_server.rest_endpoints.global_mcp_server_manager", manager), patch.dict( sys.modules, { @@ -9171,9 +9167,9 @@ class _ResolutionEffects: patch.object(mgmt_endpoints, "merge_user_env_vars", self.env_merge), patch.object(mgmt_endpoints, "delete_user_env_vars", self.env_delete), patch.object(manager, "invalidate_user_oauth_token_cache", self.oauth_invalidate), - patch("litellm.proxy._experimental.mcp_server.server._invalidate_byok_cred_cache", self.byok_invalidate), + patch("litellm.proxy.mcp_server.server._invalidate_byok_cred_cache", self.byok_invalidate), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.invalidate_user_env_vars_cache", + "litellm.proxy.mcp_server.mcp_server_manager.invalidate_user_env_vars_cache", self.env_invalidate, ), ): @@ -10182,7 +10178,7 @@ class TestMCPServerResolutionCharacterization: with ( patch.object(mgmt_endpoints, "get_prisma_client_or_throw", return_value=prisma), patch.object(mgmt_endpoints, "global_mcp_server_manager", manager), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", manager), + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", manager), patch("litellm.proxy.proxy_server.prisma_client", prisma), patch("litellm.proxy.proxy_server.user_api_key_cache", _mock_mcp_resolution_cache()), patch("litellm.proxy.proxy_server.general_settings", {}), @@ -10293,7 +10289,7 @@ class TestMCPServerResolutionCharacterization: patch.object(mgmt_endpoints, "get_prisma_client_or_throw", return_value=prisma), patch.object(mgmt_endpoints, "global_mcp_server_manager", manager), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", manager, ), patch.object(manager, "add_server", add_server), @@ -10477,7 +10473,7 @@ class TestMCPServerResolutionCharacterization: patch.object(mgmt_endpoints, "get_prisma_client_or_throw", return_value=prisma), patch.object(mgmt_endpoints, "global_mcp_server_manager", manager), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", manager, ), patch.object(manager, "invalidate_user_oauth_token_cache", invalidate), @@ -11268,7 +11264,7 @@ class TestMCPServerResolutionCharacterization: MockRouter(assert_all_called=False) as httpx_mock, patch.object(mgmt_endpoints, "get_prisma_client_or_throw", return_value=prisma), patch.object(mgmt_endpoints, "global_mcp_server_manager", manager), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", manager), + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", manager), patch.object(manager, "add_server", add_server), patch.object(manager, "health_check_server", health_check), patch("litellm.proxy.proxy_server.prisma_client", prisma), diff --git a/tests/unit/proxy/management_endpoints/test_mcp_servers.py b/tests/unit/proxy/management_endpoints/test_mcp_servers.py index 77afe510267..4c0288f0e6f 100644 --- a/tests/unit/proxy/management_endpoints/test_mcp_servers.py +++ b/tests/unit/proxy/management_endpoints/test_mcp_servers.py @@ -9,14 +9,14 @@ from litellm.proxy._types import ( MCPTransport, NewMCPServerRequest, ) -from litellm.proxy._experimental.mcp_server.db import McpIdentifierConflict -from litellm.proxy._experimental.mcp_server.utils import validate_mcp_server_name from litellm.proxy.management_endpoints.mcp_management_endpoints import ( _redact_mcp_credentials, does_mcp_server_exist, raise_mcp_identifier_conflict, validate_and_normalize_mcp_server_payload, ) +from litellm.proxy.mcp_server.db import McpIdentifierConflict +from litellm.proxy.mcp_server.utils import validate_mcp_server_name def test_does_mcp_server_exist() -> None: diff --git a/tests/unit/proxy/management_endpoints/test_saml_sso.py b/tests/unit/proxy/management_endpoints/test_saml_sso.py index 181c5e65dae..ff2117ea533 100644 --- a/tests/unit/proxy/management_endpoints/test_saml_sso.py +++ b/tests/unit/proxy/management_endpoints/test_saml_sso.py @@ -31,7 +31,6 @@ from starlette.datastructures import URL from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import RedisCache -from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import SSOIdentityAssertion from litellm.proxy._types import LitellmUserRoles from litellm.proxy.management_endpoints.sso.saml_sso import ( _SAML_AUTHN_REQUEST_CACHE_PREFIX, @@ -42,6 +41,7 @@ from litellm.proxy.management_endpoints.sso.saml_sso import ( SAMLAuthHandler, ) from litellm.proxy.management_endpoints.types import CustomOpenID +from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store import SSOIdentityAssertion def _shared_cache(store: InMemoryCache | None = None) -> DualCache: diff --git a/tests/unit/proxy/management_endpoints/test_ui_sso.py b/tests/unit/proxy/management_endpoints/test_ui_sso.py index e1a939273ae..4135e9be6f6 100644 --- a/tests/unit/proxy/management_endpoints/test_ui_sso.py +++ b/tests/unit/proxy/management_endpoints/test_ui_sso.py @@ -10,9 +10,9 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -from pydantic import SecretStr import respx from fastapi import HTTPException, Request +from pydantic import SecretStr import litellm from litellm._uuid import uuid @@ -4119,7 +4119,7 @@ class TestCLIKeyRegenerationFlow: cli_state = f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:cli-new-session-key-456" # Mock the CLI callback and required proxy server components - from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( + from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store import ( assertion_from_sso_login, ) @@ -9625,7 +9625,7 @@ async def test_redirect_from_openid_persists_assertion_under_canonical_user_id() """The browser funnel persists the captured assertion AFTER canonical user resolution, keyed by the user_id admission will later resolve (the key-generation response user_id), not the raw IdP subject.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( + from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store import ( assertion_from_sso_login, ) @@ -9696,7 +9696,7 @@ async def test_redirect_from_openid_persists_the_login_exchanged_assertion(): """A server with ``id_jag_subject_source: refresh_token`` needs the refresh token the login assertion was exchanged for, so the funnel persists what the login-time exchange returns, not the raw capture.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( + from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store import ( assertion_from_sso_login, ) @@ -9921,12 +9921,12 @@ async def test_microsoft_login_asks_for_offline_access_only_while_an_id_jag_serv @pytest.mark.asyncio async def test_cli_completion_persists_assertion_under_db_user_id(): """The CLI funnel persists the captured assertion under the DB-resolved user_id.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( - assertion_from_sso_login, - ) from litellm.proxy.management_endpoints.ui_sso import ( _complete_cli_sso_callback_session, ) + from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store import ( + assertion_from_sso_login, + ) assertion = assertion_from_sso_login(_ema_id_token(), None) assert assertion is not None @@ -10041,12 +10041,12 @@ async def test_a_capturing_provider_that_returned_no_id_token_still_warns(monkey @pytest.mark.asyncio async def test_no_warning_when_the_assertion_was_captured(monkeypatch, caplog): - from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( - assertion_from_sso_login, - ) from litellm.proxy.management_endpoints.ui_sso import ( warn_if_id_jag_assertion_uncaptured, ) + from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store import ( + assertion_from_sso_login, + ) monkeypatch.setenv("GOOGLE_CLIENT_ID", "cid") assertion = assertion_from_sso_login(_ema_id_token(), None) diff --git a/tests/unit/proxy/management_helpers/test_object_permission_utils.py b/tests/unit/proxy/management_helpers/test_object_permission_utils.py index 33557713162..5814d8d2a97 100644 --- a/tests/unit/proxy/management_helpers/test_object_permission_utils.py +++ b/tests/unit/proxy/management_helpers/test_object_permission_utils.py @@ -19,9 +19,9 @@ from litellm.proxy.management_helpers.object_permission_utils import ( _extract_requested_mcp_access_groups, _extract_requested_mcp_server_ids, _resolve_team_allowed_mcp_servers, - set_object_permission, enforce_all_proxy_mcp_servers_grant_is_admin_only, prepare_object_permission_upsert, + set_object_permission, validate_key_mcp_servers_against_team, validate_key_search_tools_against_team, validate_key_vector_stores_against_team, @@ -277,7 +277,7 @@ def _make_mock_mcp_manager(*existing_ids: str, servers=None): return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -291,7 +291,7 @@ async def test_validate_no_object_permission(mock_access_groups, mock_allow_all) @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager("server-1", "server-2"), ) @patch( @@ -299,7 +299,7 @@ async def test_validate_no_object_permission(mock_access_groups, mock_allow_all) return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -316,7 +316,7 @@ async def test_validate_key_servers_within_team_scope( @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager("server-1", "server-outside"), ) @patch( @@ -324,7 +324,7 @@ async def test_validate_key_servers_within_team_scope( return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -344,7 +344,7 @@ async def test_validate_key_servers_outside_team_scope_raises( @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager("server-1", "global-server"), ) @patch( @@ -352,7 +352,7 @@ async def test_validate_key_servers_outside_team_scope_raises( return_value={"global-server"}, ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -369,7 +369,7 @@ async def test_validate_allow_all_keys_servers_always_allowed( @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager("global-server"), ) @patch( @@ -377,7 +377,7 @@ async def test_validate_allow_all_keys_servers_always_allowed( return_value={"global-server"}, ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -391,7 +391,7 @@ async def test_validate_no_team_only_allow_all_keys(mock_access_groups, mock_all @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager("private-server"), ) @patch( @@ -399,7 +399,7 @@ async def test_validate_no_team_only_allow_all_keys(mock_access_groups, mock_all return_value={"global-server"}, ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -418,7 +418,7 @@ async def test_validate_no_team_non_global_server_raises( @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager("private-server"), ) @patch( @@ -426,7 +426,7 @@ async def test_validate_no_team_non_global_server_raises( return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -444,7 +444,7 @@ async def test_validate_no_team_proxy_admin_can_assign_private_server( @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager("private-server"), ) @patch( @@ -452,7 +452,7 @@ async def test_validate_no_team_proxy_admin_can_assign_private_server( return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -475,7 +475,7 @@ async def test_validate_no_team_non_admin_private_server_still_raises( return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -493,7 +493,7 @@ async def test_validate_no_team_proxy_admin_can_assign_access_group( @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager("server-1", "server-outside"), ) @patch( @@ -501,7 +501,7 @@ async def test_validate_no_team_proxy_admin_can_assign_access_group( return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -522,7 +522,7 @@ async def test_validate_proxy_admin_still_bounded_by_team_scope( @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager("some-server"), ) @patch( @@ -530,7 +530,7 @@ async def test_validate_proxy_admin_still_bounded_by_team_scope( return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -569,7 +569,7 @@ def unified_mcp_prisma() -> Iterator[MagicMock]: manager.get_allow_all_keys_server_ids.return_value = [] with ( patch( # test-quality-ok: management helpers read this module singleton without a registry injection seam - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=manager, ), patch( # test-quality-ok: unified group resolver obtains its cache from the proxy singleton @@ -675,7 +675,7 @@ async def test_team_unified_access_group_without_servers_preserves_direct_grants @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager("server-outside"), ) @patch( @@ -683,7 +683,7 @@ async def test_team_unified_access_group_without_servers_preserves_direct_grants return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -703,7 +703,7 @@ async def test_validate_tool_permissions_validated_against_team( @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager(), # empty registry — all IDs are stale ) @patch( @@ -711,7 +711,7 @@ async def test_validate_tool_permissions_validated_against_team( return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -735,7 +735,7 @@ async def test_validate_stale_mcp_server_ids_are_silently_dropped( @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager(), # empty registry — all IDs are stale ) @patch( @@ -743,7 +743,7 @@ async def test_validate_stale_mcp_server_ids_are_silently_dropped( return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -765,7 +765,7 @@ async def test_validate_stale_ids_in_mcp_tool_permissions_silently_dropped( @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager(), # empty registry — all IDs are stale ) @patch( @@ -773,7 +773,7 @@ async def test_validate_stale_ids_in_mcp_tool_permissions_silently_dropped( return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -791,7 +791,7 @@ async def test_validate_stale_mcp_server_ids_are_removed_from_object_permission( @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager( "team-server", servers=[ @@ -808,7 +808,7 @@ async def test_validate_stale_mcp_server_ids_are_removed_from_object_permission( return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -827,7 +827,7 @@ async def test_validate_mcp_server_alias_outside_team_scope_raises( @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager( servers=[ _make_mock_mcp_server( @@ -843,7 +843,7 @@ async def test_validate_mcp_server_alias_outside_team_scope_raises( return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -878,7 +878,7 @@ def test_alias_grant_expands_on_other_region_after_save(): _drop_stale_object_permission_mcp_servers(object_permission, {"github-mcp": {"west-id"}}) assert object_permission["mcp_servers"] == ["github-mcp"] - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager expand = MCPServerManager.expand_permission_list assert expand(west_mgr, object_permission["mcp_servers"]) == ["west-id"] @@ -887,7 +887,7 @@ def test_alias_grant_expands_on_other_region_after_save(): @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager(), ) @patch( @@ -895,7 +895,7 @@ def test_alias_grant_expands_on_other_region_after_save(): return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -929,7 +929,7 @@ async def test_validate_db_mcp_server_alias_outside_team_scope_raises_when_regis return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -950,7 +950,7 @@ async def test_validate_access_groups_within_team_scope( return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -974,7 +974,7 @@ async def test_validate_access_groups_outside_team_scope_raises( return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -997,7 +997,7 @@ async def test_validate_access_groups_no_team_raises( return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=["server-from-group"], ) @@ -1018,7 +1018,7 @@ async def test_validate_team_access_groups_resolve_to_servers( @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -1037,7 +1037,7 @@ async def test_resolve_team_allowed_mcp_servers_string_tool_permissions( @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -1059,7 +1059,7 @@ async def test_resolve_team_allowed_mcp_servers_dict_tool_permissions( @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -1081,7 +1081,7 @@ async def test_resolve_team_all_proxy_sentinel_resolves_dynamically(mock_access_ team_perm.mcp_tool_permissions = {} with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_mgr, ): assert await _resolve_team_allowed_mcp_servers(team_perm) == {"srv-x", "srv-y"} @@ -1096,7 +1096,7 @@ async def test_resolve_team_all_proxy_sentinel_resolves_dynamically(mock_access_ @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager("srv-x", "srv-y", "srv-z"), ) @patch( @@ -1104,7 +1104,7 @@ async def test_resolve_team_all_proxy_sentinel_resolves_dynamically(mock_access_ return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -1127,7 +1127,7 @@ async def test_validate_key_scoped_to_server_added_after_team_all_proxy( @pytest.mark.asyncio @patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", new=_make_mock_mcp_manager("srv-x", "srv-z"), ) @patch( @@ -1135,7 +1135,7 @@ async def test_validate_key_scoped_to_server_added_after_team_all_proxy( return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -1278,7 +1278,7 @@ async def test_validate_search_tools_raises_when_not_subset(): return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -1301,7 +1301,7 @@ async def test_personal_non_admin_cannot_assign_mcp_toolsets( return_value=set(), ) @patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ) @@ -1403,7 +1403,7 @@ def _make_grandfather_fixtures(mcp_servers=None, mcp_tool_permissions=None): def _patch_grandfather_env(monkeypatch, mock_mgr): monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_mgr, ) monkeypatch.setattr( diff --git a/tests/unit/proxy/_experimental/mcp_server/faults/__init__.py b/tests/unit/proxy/mcp_server/__init__.py similarity index 100% rename from tests/unit/proxy/_experimental/mcp_server/faults/__init__.py rename to tests/unit/proxy/mcp_server/__init__.py diff --git a/tests/unit/proxy/_experimental/mcp_server/guardrail_translation/__init__.py b/tests/unit/proxy/mcp_server/auth/__init__.py similarity index 100% rename from tests/unit/proxy/_experimental/mcp_server/guardrail_translation/__init__.py rename to tests/unit/proxy/mcp_server/auth/__init__.py diff --git a/tests/unit/proxy/_experimental/mcp_server/auth/test_managed_agent_access.py b/tests/unit/proxy/mcp_server/auth/test_managed_agent_access.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/auth/test_managed_agent_access.py rename to tests/unit/proxy/mcp_server/auth/test_managed_agent_access.py index 1524aad5b66..fa4f80a2b96 100644 --- a/tests/unit/proxy/_experimental/mcp_server/auth/test_managed_agent_access.py +++ b/tests/unit/proxy/mcp_server/auth/test_managed_agent_access.py @@ -5,10 +5,10 @@ import pytest from fastapi import HTTPException from litellm.proxy import proxy_server -from litellm.proxy._experimental.mcp_server import mcp_server_manager -from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, LiteLLM_UserTable, UserAPIKeyAuth from litellm.proxy.auth import auth_checks +from litellm.proxy.mcp_server import mcp_server_manager +from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler from litellm.types.agents import AgentResponse from litellm.types.proxy.agent_identity import ManagedAgentContext @@ -219,7 +219,7 @@ async def test_autonomous_agent_toolset_and_access_group_revocations_bind_on_the writer revokes a tool or drops the server from the group, the next managed request must be denied even though the legacy cache still holds the warm grant and the replica still shows the old rows""" from litellm.caching.caching import DualCache - from litellm.proxy._experimental.mcp_server import toolset_db + from litellm.proxy.mcp_server import toolset_db warm_toolset: Final = _toolset_row("slack", "read") list_toolsets: Final = AsyncMock(return_value=[warm_toolset]) @@ -285,7 +285,7 @@ async def test_delegated_mcp_uses_explicit_team_grants_even_for_dashboard_admins ) for name in ("slack", "linear") } - from litellm.proxy._experimental.mcp_server import db + from litellm.proxy.mcp_server import db monkeypatch.setattr( db, @@ -322,7 +322,7 @@ async def test_delegated_mcp_uses_explicit_team_grants_even_for_dashboard_admins async def test_explicit_grants_never_fall_back_to_open_servers_on_resolution_failure( monkeypatch: pytest.MonkeyPatch, ) -> None: - from litellm.proxy._experimental.mcp_server import db + from litellm.proxy.mcp_server import db from litellm.types.mcp_server.mcp_server_manager import MCPServer manager: Final = mcp_server_manager.global_mcp_server_manager @@ -339,7 +339,7 @@ async def test_explicit_grants_never_fall_back_to_open_servers_on_resolution_fai @pytest.mark.asyncio async def test_absent_agent_policy_and_missing_delegated_subject_grant_no_servers() -> None: - from litellm.proxy._experimental.mcp_server.auth.managed_agent_access import managed_agent_servers + from litellm.proxy.mcp_server.auth.managed_agent_access import managed_agent_servers assert await managed_agent_servers(UserAPIKeyAuth()) == () auth: Final = actor(None, delegated=True) @@ -367,8 +367,8 @@ async def test_tool_policy_outage_after_server_admission_fails_closed(monkeypatc async def test_manager_preserves_managed_server_grants_across_open_channels( monkeypatch: pytest.MonkeyPatch, role: str | None, scoped: bool ) -> None: - from litellm.proxy._experimental.mcp_server import db - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPServerAccess + from litellm.proxy.mcp_server import db + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPServerAccess from litellm.types.mcp_server.mcp_server_manager import MCPServer manager: Final = mcp_server_manager.global_mcp_server_manager diff --git a/tests/unit/proxy/_experimental/mcp_server/auth/test_token_endpoint_auth.py b/tests/unit/proxy/mcp_server/auth/test_token_endpoint_auth.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/auth/test_token_endpoint_auth.py rename to tests/unit/proxy/mcp_server/auth/test_token_endpoint_auth.py index 47b5fac23eb..c65e7d4db26 100644 --- a/tests/unit/proxy/_experimental/mcp_server/auth/test_token_endpoint_auth.py +++ b/tests/unit/proxy/mcp_server/auth/test_token_endpoint_auth.py @@ -4,7 +4,7 @@ import base64 import pytest -from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( +from litellm.proxy.mcp_server.auth.token_endpoint_auth import ( TokenEndpointAuthConfigError, build_token_endpoint_client_auth, normalize_token_endpoint_auth_method, diff --git a/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py similarity index 95% rename from tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py rename to tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py index 16130364364..b7795b4d2f0 100644 --- a/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -16,12 +16,6 @@ from starlette.datastructures import Headers from litellm.caching.caching import DualCache from litellm.constants import MCP_ALL_TOOLS_WILDCARD from litellm.models.access_group import LiteLLM_AccessGroupTable -from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( - MCPRequestHandler, - UnloadableEntitlementError, - _agent_capped_servers, - is_mcp_admitted_user_subject, -) from litellm.proxy._types import ( UI_TEAM_ID, LiteLLM_ObjectPermissionTable, @@ -32,6 +26,12 @@ from litellm.proxy._types import ( SpecialMCPServerNames, UserAPIKeyAuth, ) +from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( + MCPRequestHandler, + UnloadableEntitlementError, + _agent_capped_servers, + is_mcp_admitted_user_subject, +) from litellm.types.agents import AgentCaller @@ -376,7 +376,7 @@ class TestMCPRequestHandler: def _real_manager_with_toolsets(self, toolset_perms): """A real MCPServerManager so the real expand_tool_permissions runs; only the DB-backed toolset lookup is stubbed""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager manager = MCPServerManager() manager.resolve_toolset_tool_permissions = AsyncMock(return_value=toolset_perms) @@ -392,7 +392,7 @@ class TestMCPRequestHandler: with ( patch.object(MCPRequestHandler, "_get_key_object_permission", return_value=key_object_permission), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), patch.object(MCPRequestHandler, "get_mcp_servers_from_access_groups", AsyncMock(return_value=[])), @@ -413,7 +413,7 @@ class TestMCPRequestHandler: with ( patch.object(MCPRequestHandler, "_get_key_object_permission", return_value=key_object_permission), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), patch.object(MCPRequestHandler, "get_mcp_servers_from_access_groups", AsyncMock(return_value=[])), @@ -433,7 +433,7 @@ class TestMCPRequestHandler: with ( patch.object(MCPRequestHandler, "_get_key_object_permission", return_value=key_object_permission), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), patch.object(MCPRequestHandler, "get_mcp_servers_from_access_groups", AsyncMock(return_value=[])), @@ -458,7 +458,7 @@ class TestMCPRequestHandler: with ( patch.object(MCPRequestHandler, "_get_key_object_permission", return_value=key_object_permission), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), patch.object(MCPRequestHandler, "get_mcp_servers_from_access_groups", AsyncMock(return_value=[])), @@ -483,7 +483,7 @@ class TestMCPRequestHandler: patch.object(MCPRequestHandler, "_get_key_object_permission", return_value=key_object_permission), patch.object(MCPRequestHandler, "_get_team_object_permission", AsyncMock(return_value=None)), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), ): @@ -507,7 +507,7 @@ class TestMCPRequestHandler: patch.object(MCPRequestHandler, "_get_key_object_permission", return_value=key_object_permission), patch.object(MCPRequestHandler, "_get_team_object_permission", AsyncMock(return_value=None)), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), ): @@ -539,7 +539,7 @@ class TestMCPRequestHandler: patch.object(MCPRequestHandler, "_get_key_object_permission", return_value=key_object_permission), patch.object(MCPRequestHandler, "_get_team_object_permission", AsyncMock(return_value=None)), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), ): @@ -620,7 +620,7 @@ class TestMCPRequestHandler: MCPRequestHandler, "_get_team_object_permission", AsyncMock(return_value=team_object_permission) ), patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", manager, ), ): @@ -654,7 +654,7 @@ class TestMCPRequestHandler: MCPRequestHandler, "_get_team_object_permission", AsyncMock(return_value=team_object_permission) ), patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", manager, ), ): @@ -706,7 +706,7 @@ class TestMCPRequestHandler: MCPRequestHandler, "_get_team_object_permission", AsyncMock(return_value=team_object_permission) ), patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), ): @@ -740,7 +740,7 @@ class TestMCPRequestHandler: MCPRequestHandler, "_get_team_object_permission", AsyncMock(return_value=team_object_permission) ), patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), ): @@ -760,7 +760,7 @@ class TestMCPRequestHandler: with ( patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), patch.object( # test-quality-ok: access-group lookup hits the DB, not under test here @@ -798,7 +798,7 @@ class TestMCPRequestHandler: AsyncMock(return_value=[]), ), patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), patch.object( # test-quality-ok: access-group lookup hits the DB, not under test here @@ -835,7 +835,7 @@ class TestMCPRequestHandler: MCPRequestHandler, "get_mcp_servers_from_access_groups", AsyncMock(return_value=[]) ), patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), patch.object( # test-quality-ok: stub the level's perm loader; the resolver reads module globals with no injection seam @@ -872,7 +872,7 @@ class TestMCPRequestHandler: MCPRequestHandler, "get_mcp_servers_from_access_groups", AsyncMock(return_value=[]) ), patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), patch.object( # test-quality-ok: stub the level's perm loader; the resolver reads module globals with no injection seam @@ -915,7 +915,7 @@ class TestMCPRequestHandler: AsyncMock(return_value=[]), ), patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), patch.object( # test-quality-ok: access-group lookup hits the DB, not under test here @@ -944,7 +944,7 @@ class TestMCPRequestHandler: MCPRequestHandler, "_get_org_object_permission", AsyncMock(return_value=org_object_permission) ), patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), ): @@ -974,7 +974,7 @@ class TestMCPRequestHandler: MCPRequestHandler, "get_mcp_servers_from_access_groups", AsyncMock(return_value=[]) ), patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), ): @@ -995,7 +995,7 @@ class TestMCPRequestHandler: MCPRequestHandler, "_get_user_object_permission", AsyncMock(return_value=user_object_permission) ), patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), ): @@ -1026,7 +1026,7 @@ class TestMCPRequestHandler: MCPRequestHandler, "get_mcp_servers_from_access_groups", AsyncMock(return_value=[]) ), patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), ): @@ -1054,7 +1054,7 @@ class TestMCPRequestHandler: MCPRequestHandler, "_get_team_object_permission", AsyncMock(return_value=team_object_permission) ), patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), ): @@ -1079,7 +1079,7 @@ class TestMCPRequestHandler: MCPRequestHandler, "get_mcp_servers_from_access_groups", AsyncMock(return_value=[]) ), patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), ): @@ -1101,7 +1101,7 @@ class TestMCPRequestHandler: MCPRequestHandler, "get_mcp_servers_from_access_groups", AsyncMock(return_value=[]) ), patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), ): @@ -1254,7 +1254,7 @@ class TestMCPRequestHandler: "litellm.proxy.auth.auth_checks.get_mcp_server_ids_from_access_groups", new=AsyncMock(return_value=[]), ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): result = await MCPRequestHandler._get_key_access_group_mcp_server_extras(auth) assert result == [] @@ -1269,7 +1269,7 @@ class TestMCPRequestHandler: "litellm.proxy.auth.auth_checks.get_mcp_server_ids_from_access_groups", new=AsyncMock(return_value=["alias-a", "srv-b"]), ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.expand_permission_list.return_value = ["srv-a", "srv-b"] result = await MCPRequestHandler._get_key_access_group_mcp_server_extras(auth) @@ -1426,7 +1426,7 @@ class TestMCPRequestHandler: ) with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth, ): # Call the method @@ -1600,7 +1600,7 @@ class TestMCPRequestHandler: ) with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth" + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth" ) as mock_user_api_key_auth: mock_user_api_key_auth.return_value = mock_auth_result @@ -1686,11 +1686,11 @@ class TestMCPOAuth2AuthFlow: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, return_value=UserAPIKeyAuth(user_id="admitted-user"), ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = oauth2_server ( @@ -1726,7 +1726,7 @@ class TestMCPOAuth2AuthFlow: return UserAPIKeyAuth(api_key=api_key, user_id="test-user") with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth, ) as mock_auth: ( @@ -1765,7 +1765,7 @@ class TestMCPOAuth2AuthFlow: with ( patch( # test-quality-ok: capturing the exact api_key handed to key validation is the regression under test - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth, ) as mock_auth ): @@ -1793,7 +1793,7 @@ class TestMCPOAuth2AuthFlow: return UserAPIKeyAuth(api_key=api_key, user_id="test-user") with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth, ) as mock_auth: ( @@ -1831,7 +1831,7 @@ class TestMCPOAuth2AuthFlow: raise HTTPException(status_code=500, detail="Internal server error") with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_server_error, ): with pytest.raises(HTTPException) as exc_info: @@ -1882,10 +1882,10 @@ class TestMCPOAuth2AuthFlow: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_proxy_exception, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = oauth2_server with pytest.raises(HTTPException) as exc_info: @@ -1921,7 +1921,7 @@ class TestMCPOAuth2AuthFlow: ) with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_500, ): with pytest.raises(ProxyException): @@ -1956,10 +1956,10 @@ class TestMCPPublicRouteGuard: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): # Explicit unresolvable target — proves auth still fails even # when the registry has no info to fall back to. @@ -1987,10 +1987,10 @@ class TestMCPPublicRouteGuard: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = None with pytest.raises(HTTPException) as exc_info: @@ -2011,7 +2011,7 @@ class TestMCPPublicRouteGuard: # No mock needed — public path should not call user_api_key_auth at all with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", ) as mock_auth: auth_result, *_rest = await MCPRequestHandler.process_mcp_request(scope) mock_auth.assert_not_called() @@ -2041,12 +2041,12 @@ class TestMCPPassthroughColdStartAdmission: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp._is_mcp_passthrough_cold_start" + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp._is_mcp_passthrough_cold_start" ) as mock_cold_start, ): mock_mgr.get_mcp_server_by_name.return_value = ( @@ -2081,10 +2081,10 @@ class TestMCPPassthroughColdStartAdmission: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = ( TestMCPPassthroughColdStartAdmission._make_passthrough_server() @@ -2109,10 +2109,10 @@ class TestMCPPassthroughColdStartAdmission: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = ( TestMCPPassthroughColdStartAdmission._make_passthrough_server() @@ -2137,14 +2137,14 @@ class TestMCPPassthroughColdStartAdmission: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.IPAddressUtils.get_mcp_client_ip", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.IPAddressUtils.get_mcp_client_ip", return_value="203.0.113.10", ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = None with pytest.raises(HTTPException) as exc_info: @@ -2168,10 +2168,10 @@ class TestMCPPassthroughColdStartAdmission: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_forbidden, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = ( TestMCPPassthroughColdStartAdmission._make_passthrough_server() @@ -2201,10 +2201,10 @@ class TestMCPPassthroughColdStartAdmission: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_server_error, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = ( TestMCPPassthroughColdStartAdmission._make_passthrough_server() @@ -2227,10 +2227,10 @@ class TestMCPPassthroughColdStartAdmission: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = ( TestMCPPassthroughColdStartAdmission._make_passthrough_server() @@ -2260,10 +2260,10 @@ class TestMCPPassthroughColdStartAdmission: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = ( TestMCPPassthroughColdStartAdmission._make_passthrough_server() @@ -2312,10 +2312,10 @@ class TestMCPOAuth2FallbackTargetGating: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = TestMCPOAuth2FallbackTargetGating._make_server( MCPAuth.api_key @@ -2343,10 +2343,10 @@ class TestMCPOAuth2FallbackTargetGating: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = None with pytest.raises(HTTPException) as exc_info: @@ -2381,10 +2381,10 @@ class TestMCPOAuth2FallbackTargetGating: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = TestMCPOAuth2FallbackTargetGating._make_server( MCPAuth.oauth2 @@ -2418,10 +2418,10 @@ class TestMCPOAuth2FallbackTargetGating: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = TestMCPOAuth2FallbackTargetGating._make_server( auth_type=MCPAuth.none, @@ -2446,14 +2446,14 @@ class TestMCPOAuth2FallbackTargetGating: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.IPAddressUtils.get_mcp_client_ip", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.IPAddressUtils.get_mcp_client_ip", return_value="203.0.113.10", ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = None with pytest.raises(HTTPException) as exc_info: @@ -2497,10 +2497,10 @@ class TestMCPOAuth2FallbackTargetGating: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.side_effect = mock_lookup with pytest.raises(HTTPException) as exc_info: @@ -2533,7 +2533,7 @@ class TestMCPOAuth2FallbackTargetGating: ) with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_no_code, ): with pytest.raises(ProxyException): @@ -2563,7 +2563,7 @@ class TestMCPDelegateAuthToUpstream: ) def test_legacy_delegate_cold_start_fails_closed_without_targets(self): - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( _is_legacy_delegate_cold_start, ) @@ -2571,7 +2571,7 @@ class TestMCPDelegateAuthToUpstream: def test_build_mcp_server_table_preserves_delegate_auth_to_upstream(self): """Registry → API list rows must expose delegate_auth_to_upstream for the UI.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp import MCPAuth @@ -2598,7 +2598,7 @@ class TestMCPDelegateAuthToUpstream: distinct from ``delegate_auth_to_upstream`` (oauth2-only). Both must round-trip independently so neither flag silently implies the other. """ - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp import MCPAuth @@ -2638,11 +2638,11 @@ class TestMCPDelegateAuthToUpstream: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, side_effect=HTTPException(status_code=401, detail="No key provided"), ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( auth_type=MCPAuth.oauth2, @@ -2668,11 +2668,11 @@ class TestMCPDelegateAuthToUpstream: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, side_effect=HTTPException(status_code=401, detail="Invalid API key"), ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( auth_type=MCPAuth.oauth2, @@ -2705,10 +2705,10 @@ class TestMCPDelegateAuthToUpstream: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( auth_type=MCPAuth.oauth2, @@ -2740,10 +2740,10 @@ class TestMCPDelegateAuthToUpstream: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( auth_type=MCPAuth.api_key, @@ -2788,10 +2788,10 @@ class TestMCPDelegateAuthToUpstream: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.side_effect = mock_lookup with pytest.raises(HTTPException) as exc_info: @@ -2818,10 +2818,10 @@ class TestMCPDelegateAuthToUpstream: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = None with pytest.raises(HTTPException) as exc_info: @@ -2846,11 +2846,11 @@ class TestMCPDelegateAuthToUpstream: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, return_value=UserAPIKeyAuth(user_id="real-user"), ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( auth_type=MCPAuth.oauth2, @@ -2878,11 +2878,11 @@ class TestMCPDelegateAuthToUpstream: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, return_value=UserAPIKeyAuth(user_id="real-user"), ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( auth_type=MCPAuth.oauth2, @@ -2936,10 +2936,10 @@ class TestMCPDelegateAuthToUpstream: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_auth_raises, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = m2m_server # No delegate bypass → normal auth is attempted → 401 raised @@ -2987,10 +2987,10 @@ class TestMCPDelegateAuthToUpstream: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_auth_raises, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = legacy_m2m_server with pytest.raises(HTTPException) as exc_info: @@ -3029,10 +3029,10 @@ class TestMCPDelegateAuthToUpstream: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_auth_raises, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = pkce_server auth, *_rest = await MCPRequestHandler.process_mcp_request(scope) @@ -3069,10 +3069,10 @@ class TestMCPDelegateAuthToUpstream: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_auth_raises, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = internal_server auth, *_rest = await MCPRequestHandler.process_mcp_request(scope) @@ -3081,7 +3081,7 @@ class TestMCPDelegateAuthToUpstream: async def test_get_allowed_servers_excludes_legacy_delegates(self): """Legacy delegated servers are never added to anonymous access.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp import MCPAuth @@ -3134,10 +3134,10 @@ class TestMCPDelegateAuthToUpstream: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( auth_type=MCPAuth.true_passthrough, @@ -3180,10 +3180,10 @@ class TestMCPDelegateAuthToUpstream: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.side_effect = mock_lookup with pytest.raises(HTTPException) as exc_info: @@ -3192,7 +3192,7 @@ class TestMCPDelegateAuthToUpstream: async def test_get_allowed_servers_includes_true_passthrough(self): """Anonymous callers can reach true_passthrough servers; admission is delegated upstream.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp import MCPAuth @@ -3230,7 +3230,7 @@ class TestMCPDelegateAuthToUpstream: allow-list which can include ``allow_all_keys`` servers that normally require a LiteLLM key. """ - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_mcp_servers_in_path, ) @@ -3307,10 +3307,10 @@ class TestMCPDelegateAuthToUpstream: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_auth_raises, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.side_effect = lookup_by_name with pytest.raises(HTTPException) as exc_info: @@ -3370,10 +3370,10 @@ class TestMCPDelegateAuthToUpstream: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_auth_raises, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.side_effect = lookup_by_name # Bypass MUST NOT fire — path-derived target is the non-delegate @@ -3568,7 +3568,7 @@ class TestMCPCustomHeaderName: ) with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth, ) as mock_auth: # Call the method @@ -3737,7 +3737,7 @@ class TestMCPAccessGroupsE2E: ) with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth, ) as mock_auth: # Call the method @@ -3786,7 +3786,7 @@ class TestMCPAccessGroupsE2E: ) with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth, ) as mock_auth: # Call the method @@ -3811,7 +3811,7 @@ class TestMCPAccessGroupsE2E: def test_mcp_path_based_server_segregation(monkeypatch): # Import the MCP server FastAPI app and context getter - from litellm.proxy._experimental.mcp_server.server import app, get_auth_context + from litellm.proxy.mcp_server.server import app, get_auth_context captured_mcp_servers = {} @@ -3848,21 +3848,21 @@ def test_mcp_path_based_server_segregation(monkeypatch): ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.server.session_manager_stateless", + "litellm.proxy.mcp_server.server.session_manager_stateless", MagicMock(handle_request=dummy_handle_request), ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.server.session_manager_stateful", + "litellm.proxy.mcp_server.server.session_manager_stateful", MagicMock(handle_request=dummy_handle_request), ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.server.initialize_session_managers", + "litellm.proxy.mcp_server.server.initialize_session_managers", AsyncMock(), ) # Patch user_api_key_auth to always return a dummy user monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", AsyncMock(return_value=UserAPIKeyAuth(api_key="test", user_id="user")), ) @@ -4121,10 +4121,10 @@ async def test_get_allowed_mcp_servers_for_team_uses_helper(): object_permission fields (mcp_servers, mcp_access_groups) and the unified team.access_group_ids → access_mcp_server_ids path. """ - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -4309,10 +4309,10 @@ async def test_get_allowed_mcp_servers_for_key_returns_empty_when_db_returns_non async def test_get_allowed_mcp_servers_for_key_prefers_in_memory_permission(): """Ensure in-memory object_permission is used without hitting the DB.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy._types import LiteLLM_ObjectPermissionTable + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import LiteLLM_ObjectPermissionTable from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -4597,8 +4597,8 @@ class TestAgentMCPPermissions: mock_agent.assert_called_once_with(user_api_key_auth) async def test_agent_access_group_server_ceiling_expands_group_servers(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy.agent_endpoints.auth.agent_access_groups import AgentAccessGroupCeiling + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -4733,7 +4733,7 @@ class TestAgentMCPPermissions: MCPRequestHandler, "_get_agent_object_permission", AsyncMock(return_value=agent_object_permission) ), patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling toolset tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), patch.object( # test-quality-ok: access-group lookup hits the DB, not under test here @@ -4945,7 +4945,7 @@ async def test_tool_permission_servers_included_in_allowed_servers(): Regression test for https://github.com/BerriAI/litellm/issues/21954 """ - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) from litellm.types.mcp import MCPTransport @@ -5101,7 +5101,7 @@ class TestOrgMCPPermissions: async def test_get_org_object_permission_no_prisma(self): auth = self._make_auth(org_id="org-123") with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_org_object_permission", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_org_object_permission", new_callable=AsyncMock, return_value=None, ): @@ -5326,7 +5326,7 @@ async def test_mcp_key_access_group_extras_when_team_authorized(): return_value=fake_ag, ), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_mgr, ), ] @@ -5363,7 +5363,7 @@ async def test_mcp_key_access_group_extras_when_key_directly_authorized(): return_value=fake_ag, ), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_mgr, ), ] @@ -5599,10 +5599,10 @@ async def test_team_access_group_ids_resolve_to_mcp_servers(): async def test_team_access_group_ids_union_with_object_permission(): """When both legacy object_permission and unified team.access_group_ids grant MCP servers, the final list is their union.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -5760,10 +5760,10 @@ async def test_allowed_mcp_servers_for_key_uses_object_permission_not_access_gro also carries access_group_ids that would resolve to other servers, those grants do NOT enter this (intersected) scope — only the object_permission server comes back. """ - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy._types import LiteLLM_ObjectPermissionTable + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import LiteLLM_ObjectPermissionTable from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -5841,10 +5841,10 @@ def test_expand_permission_list_does_not_honor_all_proxy_sentinel(): downstream). Concrete ids still resolve normally. If the sentinel were expanded here, any stored key/org/end_user permission holding it would silently gain every server.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy._types import SpecialMCPServerName + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import SpecialMCPServerName from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -5874,14 +5874,14 @@ async def test_get_allowed_mcp_servers_for_team_expands_all_proxy_sentinel_dynam registry without any change to its stored permission. Reverting the team-side expansion collapses this to the inert literal and the result no longer contains the real servers.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) from litellm.proxy._types import ( LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, SpecialMCPServerName, ) + from litellm.proxy.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -5952,13 +5952,13 @@ async def test_key_with_all_proxy_sentinel_does_not_grant_all_servers(): teamless key with the sentinel resolves to no real server — never srv-secret or the full registry. On the pre-hardening code the key path expanded the sentinel and this key would reach srv-secret.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) from litellm.proxy._types import ( LiteLLM_ObjectPermissionTable, SpecialMCPServerName, ) + from litellm.proxy.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -6000,14 +6000,14 @@ async def test_get_allowed_mcp_servers_team_all_proxy_key_scoped_to_one_end_to_e exactly that server (key ∩ all-servers == key). If the sentinel branch is reverted the team ceiling collapses to the literal marker, the intersection empties, and the result is [] instead of ["srv-x"].""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) from litellm.proxy._types import ( LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, SpecialMCPServerName, ) + from litellm.proxy.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -6121,10 +6121,10 @@ class TestMCPDcrBridgeDelegateAdmission: ): from pydantic import SecretStr - from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( envelope_keys_from_master_key, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( + from litellm.proxy.mcp_server.outbound_credentials.envelope import ( SealedEnvelope, UpstreamTokenGrant, key_hash_identity, @@ -6281,7 +6281,7 @@ class TestMCPDcrBridgeDelegateAdmission: server.alias = None with patch( # test-quality-ok: isolate the MCP registry when testing target selection - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" ) as mock_mgr: mock_mgr.get_mcp_server_by_name.return_value = server assert ( @@ -6303,11 +6303,11 @@ class TestMCPDcrBridgeDelegateAdmission: with ( patch( # test-quality-ok: observe the auth boundary while testing admission orchestration - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ) as mock_auth, patch( # test-quality-ok: isolate the MCP registry used by request admission - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" ) as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server() @@ -6342,12 +6342,12 @@ class TestMCPDcrBridgeDelegateAdmission: with ( patch( # test-quality-ok: force credential rejection through request admission - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, side_effect=HTTPException(status_code=401, detail="Invalid key"), ) as mock_auth, patch( # test-quality-ok: isolate the MCP registry used by request admission - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" ) as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server() @@ -6372,10 +6372,10 @@ class TestMCPDcrBridgeDelegateAdmission: } with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._patch_key_reload(return_value=self._reloaded_key()) as get_key_object, ): @@ -6418,10 +6418,10 @@ class TestMCPDcrBridgeDelegateAdmission: } with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._patch_user_reload( return_value=MagicMock( @@ -6461,10 +6461,10 @@ class TestMCPDcrBridgeDelegateAdmission: } with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._patch_user_reload( return_value=MagicMock( @@ -6497,7 +6497,7 @@ class TestMCPDcrBridgeDelegateAdmission: "headers": [(b"authorization", f"Bearer {envelope}".encode("latin-1"))], } with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._patch_user_reload(side_effect=self._wrapped_user_lookup_error(Exception())), ): @@ -6521,7 +6521,7 @@ class TestMCPDcrBridgeDelegateAdmission: "headers": [(b"authorization", f"Bearer {envelope}".encode("latin-1"))], } with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._patch_user_reload( side_effect=self._wrapped_user_lookup_error(ConnectionError("auth database unreachable")) @@ -6552,7 +6552,7 @@ class TestMCPDcrBridgeDelegateAdmission: } with ( patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling admission tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" ) as mock_mgr, patch( # test-quality-ok: the envelope opener reads master_key off the proxy module, no injection seam "litellm.proxy.proxy_server.master_key", self._MASTER_KEY @@ -6583,7 +6583,7 @@ class TestMCPDcrBridgeDelegateAdmission: "headers": [(b"authorization", f"Bearer {envelope}".encode("latin-1"))], } with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._patch_user_reload( return_value=MagicMock(user_id="offboarded-user", organization_id=None, metadata={"scim_active": False}) @@ -6617,10 +6617,10 @@ class TestMCPDcrBridgeDelegateAdmission: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), revoked, ): @@ -6642,7 +6642,7 @@ class TestMCPDcrBridgeDelegateAdmission: "headers": [(b"authorization", f"Bearer {envelope}".encode("latin-1"))], } with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._patch_key_reload(return_value=self._reloaded_key(blocked=True)), ): @@ -6665,7 +6665,7 @@ class TestMCPDcrBridgeDelegateAdmission: expired_at = datetime.now(timezone.utc) - timedelta(hours=1) with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._patch_key_reload(return_value=self._reloaded_key(expires=expired_at)), ): @@ -6688,7 +6688,7 @@ class TestMCPDcrBridgeDelegateAdmission: } with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._patch_key_reload(return_value=self._reloaded_key(), team_blocked=True), ): @@ -6712,7 +6712,7 @@ class TestMCPDcrBridgeDelegateAdmission: } with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._patch_key_reload( return_value=self._reloaded_key(), @@ -6738,7 +6738,7 @@ class TestMCPDcrBridgeDelegateAdmission: } with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._patch_key_reload( return_value=self._reloaded_key(), @@ -6765,7 +6765,7 @@ class TestMCPDcrBridgeDelegateAdmission: } with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._patch_key_reload( return_value=self._reloaded_key(project_id="project-restricted"), @@ -6778,7 +6778,7 @@ class TestMCPDcrBridgeDelegateAdmission: assert exc_info.value.status_code == 401 - _POLICY_GATE = "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.run_centralized_common_checks" + _POLICY_GATE = "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.run_centralized_common_checks" async def _enforce_with_gate_error(self, error): """Drive _enforce_admitted_live_policy with the centralized gate raising ``error`` and return @@ -6820,7 +6820,7 @@ class TestMCPDcrBridgeDelegateAdmission: "headers": [(b"authorization", f"Bearer {envelope}".encode("latin-1"))], } with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._patch_key_reload(side_effect=ConnectionError("could not reach database server")), ): @@ -6864,9 +6864,9 @@ class TestMCPDcrBridgeDelegateAdmission: [(True, "scoped", []), (False, "scoped", ["public"]), (True, "unscoped", ["public"])], ) async def test_master_envelope_respects_allow_all_scope(self, flag_enabled, scope, expected): - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPServerAccess - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.proxy._types import hash_token + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPServerAccess + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager manager = MCPServerManager() with patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY): @@ -6902,7 +6902,7 @@ class TestMCPDcrBridgeDelegateAdmission: "headers": [(b"authorization", f"Bearer {envelope}".encode("latin-1"))], } with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._patch_key_reload(return_value=self._reloaded_key(allowed_routes=["/chat/completions"])), ): @@ -6926,7 +6926,7 @@ class TestMCPDcrBridgeDelegateAdmission: "headers": [(b"authorization", f"Bearer {envelope}".encode("latin-1"))], } with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), patch("litellm.proxy.proxy_server.general_settings", {"allowed_routes": ["/chat/completions"]}), ): @@ -6961,10 +6961,10 @@ class TestMCPDcrBridgeDelegateAdmission: } with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._patch_key_reload(return_value=self._reloaded_key()), ): @@ -7015,7 +7015,7 @@ class TestMCPDcrBridgeDelegateAdmission: ``lookup_mcp_server_auth_in_headers`` called the way egress calls it (alias first, then server_name) must recover exactly that token. This pins the agreement between the two key hierarchies so neither side can drift and silently drop the forwarded token.""" - from litellm.proxy._experimental.mcp_server.utils import lookup_mcp_server_auth_in_headers + from litellm.proxy.mcp_server.utils import lookup_mcp_server_auth_in_headers envelope = self._mint_bridge_envelope() server = self._bridge_delegate_server(server_name=server_name, alias=alias) @@ -7054,11 +7054,11 @@ class TestMCPDcrBridgeDelegateAdmission: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, side_effect=HTTPException(status_code=401, detail="Invalid key"), ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), ): mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server(server_name=None, alias=None) @@ -7086,10 +7086,10 @@ class TestMCPDcrBridgeDelegateAdmission: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), ): mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server() @@ -7122,10 +7122,10 @@ class TestMCPDcrBridgeDelegateAdmission: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), ): mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server() @@ -7157,10 +7157,10 @@ class TestMCPDcrBridgeDelegateAdmission: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), ): mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server() @@ -7194,11 +7194,11 @@ class TestMCPDcrBridgeDelegateAdmission: with ( patch( # test-quality-ok: prove standard admission is never consulted for an envelope bearer - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ) as mock_auth, patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling challenge tests - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" ) as mock_mgr, patch( # test-quality-ok: envelope keys derive from the proxy master_key module global "litellm.proxy.proxy_server.master_key", self._MASTER_KEY @@ -7233,11 +7233,11 @@ class TestMCPDcrBridgeDelegateAdmission: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, side_effect=HTTPException(status_code=401, detail="Invalid key"), ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), ): mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server() @@ -7264,12 +7264,12 @@ class TestMCPDcrBridgeDelegateAdmission: with ( patch( # test-quality-ok: supply standard key admission through the auth boundary - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, return_value=admitted, ) as mock_auth, patch( # test-quality-ok: isolate the MCP registry used by request admission - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" ) as mock_mgr, patch( # test-quality-ok: configure key classification for the orchestration test "litellm.proxy.proxy_server.master_key", self._MASTER_KEY @@ -7299,12 +7299,12 @@ class TestMCPDcrBridgeDelegateAdmission: with ( patch( # test-quality-ok: force a non-401 auth result through request admission - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, side_effect=HTTPException(status_code=403, detail="Key blocked"), ), patch( # test-quality-ok: isolate the MCP registry used by request admission - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" ) as mock_mgr, patch( # test-quality-ok: configure key classification for the orchestration test "litellm.proxy.proxy_server.master_key", self._MASTER_KEY @@ -7339,10 +7339,10 @@ class TestMCPDcrBridgeDelegateAdmission: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), ): mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server() @@ -7379,10 +7379,10 @@ class TestMCPDcrBridgeDelegateAdmission: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), ): mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server( @@ -7421,10 +7421,10 @@ class TestMCPDcrBridgeDelegateAdmission: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), ): mock_mgr.get_mcp_server_by_name.side_effect = mock_lookup @@ -7554,7 +7554,7 @@ class TestMCPDcrBridgeDualCredential: ) with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new=AsyncMock(return_value=self._DELEGATE._reloaded_key()), ), patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), @@ -7583,11 +7583,11 @@ class TestMCPDcrBridgeDualCredential: presented_token = "aaa.bbb.ccc" if mapped_jwt else "sk-explicit-key" with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, return_value=explicit_auth, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._DELEGATE._patch_key_reload() as get_key_object, ): @@ -7618,11 +7618,11 @@ class TestMCPDcrBridgeDualCredential: envelope = self._DELEGATE._mint_bridge_envelope(key_hash=self._KEY_HASH) with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, return_value=self._DELEGATE._reloaded_key(api_key="a-different-key-hash"), ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._DELEGATE._patch_key_reload(), ): @@ -7640,12 +7640,12 @@ class TestMCPDcrBridgeDualCredential: envelope = self._DELEGATE._mint_bridge_envelope(user_id="sso-user-7") with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, return_value=UserAPIKeyAuth(user_id=presented_user, api_key="any-hash"), ), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" ) as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), ): @@ -7677,11 +7677,11 @@ class TestMCPDcrBridgeDualCredential: tampered = envelope[:-4] + "AAAA" with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, return_value=self._DELEGATE._reloaded_key(), ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), self._DELEGATE._patch_key_reload(), ): @@ -7703,8 +7703,8 @@ def test_gateway_challenge_metadata_contains_the_public_resource( from starlette.requests import Request from starlette.types import Scope - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import _gateway_dcr_challenge - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import _gateway_dcr_challenge + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_aggregate_protected_resource_response, oauth_protected_resource_root, ) @@ -7741,7 +7741,7 @@ class TestAggregateGatewayDcrChallenge: own protected-resource metadata, and must NOT fire for named-server targets, explicit litellm keys, or non-401 failures.""" - _AUTH_PATCH_TARGET = "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth" + _AUTH_PATCH_TARGET = "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth" _EXPECTED_RESOURCE_METADATA = 'resource_metadata="http://testserver/.well-known/oauth-protected-resource/mcp"' def _scope(self, path="/mcp", extra_headers=()): @@ -7880,7 +7880,7 @@ class TestAggregateGatewayDcrChallenge: ) with ( patch(self._AUTH_PATCH_TARGET, side_effect=self._auth_401()), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, ): manager.get_mcp_server_by_identifier.side_effect = lambda identifier, client_ip=None: { "alpha": alpha, @@ -7925,7 +7925,7 @@ class TestAggregateGatewayDcrChallenge: with ( patch(self._AUTH_PATCH_TARGET, side_effect=self._auth_401()), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" ) as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = server @@ -7973,7 +7973,7 @@ class TestAggregateGatewayDcrChallenge: patch.dict(os.environ, {"SERVER_ROOT_PATH": "/litellm"}), patch(self._AUTH_PATCH_TARGET, side_effect=self._auth_401()), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" ) as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = server @@ -8006,7 +8006,7 @@ class TestAggregateGatewayDcrChallenge: with ( patch(self._AUTH_PATCH_TARGET, side_effect=self._auth_401()), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" ) as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = resolved @@ -8018,7 +8018,7 @@ class TestAggregateGatewayDcrChallenge: def test_challenge_target_excludes_every_non_gateway_managed_mode(self): """Gateway challenges exclude unresolved, delegated, and client-forwarded targets.""" - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( _gateway_dcr_challenge_target, ) from litellm.types.mcp import MCPAuth @@ -8053,13 +8053,13 @@ class TestAggregateGatewayDcrChallenge: ] for resolved, expected in cases: with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" ) as mock_mgr: mock_mgr.get_mcp_server_by_identifier.return_value = resolved assert _gateway_dcr_challenge_target("/mcp/srv", None, None) == expected, resolved assert _gateway_dcr_challenge_target("/mcp/a,b", None, None) is None assert _gateway_dcr_challenge_target("/mcp", None, None) is None - with patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr: + with patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr: mock_mgr.get_mcp_server_by_identifier.return_value = _server(MCPAuth.oauth2) assert _gateway_dcr_challenge_target("/mcp/srv", ["other"], None) is None @@ -8108,10 +8108,10 @@ class TestGatewaySessionAdmission: _MASTER_KEY = "sk-gateway-session-admission-master-key" def _session_bearer(self, user_id="sso-user-42", client_id="llm_dcrc_abc"): - from litellm.proxy._experimental.mcp_server.outbound_credentials.session_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.session_credentials import ( session_keys_from_master_key, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import ( + from litellm.proxy.mcp_server.outbound_credentials.session_token import ( SessionPrincipal, mint_session_refresh_token, mint_session_token, @@ -8168,7 +8168,7 @@ class TestGatewaySessionAdmission: with ( patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ), self._patch_user_reload(user_id="org-user", organization_id="org-123"), @@ -8185,7 +8185,7 @@ class TestGatewaySessionAdmission: with ( patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ), self._patch_user_reload(user_id="rl-user", tpm_limit=1000, rpm_limit=50), @@ -8199,7 +8199,7 @@ class TestGatewaySessionAdmission: with ( patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ) as mock_auth, self._patch_user_reload(user_id="sso-user-42") as get_user_object, @@ -8250,7 +8250,7 @@ class TestGatewaySessionAdmission: with ( patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ), self._patch_user_reload(user_id="offboarded-user", active=False), @@ -8268,7 +8268,7 @@ class TestGatewaySessionAdmission: with ( patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ), self._patch_user_reload(user_id="sso-user-42"), @@ -8301,7 +8301,7 @@ class TestGatewaySessionAdmission: with ( patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ) as mock_auth, self._patch_user_reload(user_id="sso-user-42"), @@ -8333,7 +8333,7 @@ class TestGatewaySessionAdmission: ) with ( patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = server with pytest.raises(HTTPException) as exc_info: @@ -8353,7 +8353,7 @@ class TestGatewaySessionAdmission: with ( patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ), self._patch_user_reload(user_id="sso-user-42"), @@ -8441,7 +8441,7 @@ class TestUserSubjectTeamUnion: async def test_toolsets_of_a_team_that_dropped_the_user_from_its_roster_are_not_granted(self): """The user's cached team list still names team-revoked, but its live roster no longer lists the user, so its toolset is withheld exactly as its servers are on the aggregate /mcp.""" - from litellm.proxy._experimental.mcp_server.ui_session_utils import granted_toolset_ids + from litellm.proxy.mcp_server.ui_session_utils import granted_toolset_ids teams = { "team-kept": _make_team("team-kept", [], toolsets=["ts-kept"]), @@ -8455,7 +8455,7 @@ class TestUserSubjectTeamUnion: async def test_a_pinned_toolset_narrows_every_source_to_the_toolset_servers_and_tools(self): """On /toolset/{name}/mcp the admitted subject carries mcp_toolset_id; team-a's grant on srv1 and srv2 with every tool collapses to the toolset's srv1 and its one tool, and team-b's srv3 drops.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager teams = {"team-a": _make_team("team-a", ["srv1", "srv2"]), "team-b": _make_team("team-b", ["srv3"])} auth = _make_admitted_subject("sso-user") @@ -8478,7 +8478,7 @@ class TestUserSubjectTeamUnion: async def test_a_fresh_policy_pinned_toolset_bypasses_the_toolset_permission_cache(self): """A session admitted under requires_fresh_policy reads the pinned toolset from the writer, so a tool revoked from the toolset is gone on the very next request (Devin Review 4150024092).""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager teams = {"team-a": _make_team("team-a", ["srv1", "srv2"])} auth = _make_admitted_subject("sso-user") @@ -8646,7 +8646,7 @@ class TestUserSubjectTeamUnion: assert billed is None, "and billing agrees: the user is billed, not a team" def _manager_with(self, server_ids, allow_all=()): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -8688,7 +8688,7 @@ class TestUserSubjectTeamUnion: server = MagicMock(server_id="srv1") with self._patch(teams_by_id={"t-grant": t_grant}, user_teams=["t-grant"]): with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager.get_mcp_server_from_tool_name", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager.get_mcp_server_from_tool_name", MagicMock(return_value=server), ): billed = await MCPRequestHandler.billing_auth_for_tool_call( @@ -8744,7 +8744,7 @@ class TestUserSubjectTeamUnion: boom = AsyncMock(side_effect=RuntimeError("toolset resolution exploded")) with self._patch(teams_by_id=teams, user_teams=["t1"]): with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager.resolve_toolset_tool_permissions", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager.resolve_toolset_tool_permissions", boom, ): tools = await MCPRequestHandler.get_allowed_tools_for_server("srv1", auth) @@ -8867,7 +8867,7 @@ class TestUserSubjectTeamUnion: plain = _make_admitted_subject("plain-user") with self._patch(teams_by_id={}, user_teams=[]): with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager.operator_open_server_ids", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager.operator_open_server_ids", AsyncMock(return_value=set()), ): admin_tools = await MCPRequestHandler.get_allowed_tools_for_server("srv-any", admin) @@ -8912,7 +8912,7 @@ class TestUserSubjectTeamUnion: open_ids = AsyncMock(return_value={"srv-open"}) with self._patch(teams_by_id={}, user_teams=[]): with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager.operator_open_server_ids", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager.operator_open_server_ids", open_ids, ): tools = await MCPRequestHandler.get_allowed_tools_for_server("srv-open", auth) @@ -8927,7 +8927,7 @@ class TestUserSubjectTeamUnion: open_ids = AsyncMock(return_value={"srv-open"}) with self._patch(teams_by_id={}, user_teams=[]): with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager.operator_open_server_ids", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager.operator_open_server_ids", open_ids, ): open_tools = await MCPRequestHandler.get_allowed_tools_for_server("srv-open", auth) @@ -9290,7 +9290,7 @@ class TestAdmittedSubjectPerTeamOrgCap: patch("litellm.proxy.proxy_server.proxy_logging_obj", MagicMock()), ] if registry is not None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -9585,10 +9585,10 @@ class TestSessionBearerEgressScrub: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ) as mock_auth, - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = ttp_server (_auth, _mah, _srv, _sah, oauth2_headers, raw_headers) = await MCPRequestHandler.process_mcp_request(scope) @@ -9613,10 +9613,10 @@ class TestSessionBearerEgressScrub: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ), - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = ttp_server (_auth, _mah, _srv, _sah, oauth2_headers, _raw) = await MCPRequestHandler.process_mcp_request(scope) @@ -10296,7 +10296,7 @@ class TestUserMCPEntitlement: async def test_tool_call_is_rejected_at_call_time(self): """The end-to-end contract: a tool the human is not entitled to is refused when INVOKED, not merely hidden from the advertised list.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) from litellm.types.mcp import MCPTransport @@ -10348,7 +10348,7 @@ class TestUserMCPEntitlement: async def test_servers_named_only_under_tool_permissions_are_entitled(self): """Granting one tool on a server entitles the human to that server, so an admin never has to name it twice.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) from litellm.types.mcp import MCPTransport @@ -10778,10 +10778,10 @@ class TestScopedSessionAdmission: def _bearer(self, resource_server_ids): from datetime import datetime, timezone - from litellm.proxy._experimental.mcp_server.outbound_credentials.session_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.session_credentials import ( session_keys_from_master_key, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import ( + from litellm.proxy.mcp_server.outbound_credentials.session_token import ( SessionPrincipal, mint_session_token, ) @@ -10858,7 +10858,7 @@ async def test_fresh_mcp_user_permission_link_ignores_cached_and_replica_grants( @pytest.mark.asyncio @pytest.mark.parametrize("operation", ["servers", "tools"]) async def test_managed_agent_permission_resolution_outage_is_not_an_unrestricted_grant(monkeypatch, operation): - from litellm.proxy._experimental.mcp_server import mcp_server_manager + from litellm.proxy.mcp_server import mcp_server_manager from litellm.types.agents import AgentResponse auth = UserAPIKeyAuth(agent_id="managed") @@ -11006,7 +11006,7 @@ async def test_catalog_refresh_uses_current_virtual_key_policy_and_keeps_session @pytest.mark.asyncio async def test_admission_request_body_serves_stashed_peek_callable(): from litellm.constants import MCP_PEEKED_BODY_SCOPE_KEY - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import _admission_request + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import _admission_request admission_body: Final = b'{"method":"tools/list"}' @@ -11422,7 +11422,7 @@ class TestAccessGroupToolOwnershipAndFaults: assert await MCPRequestHandler.get_allowed_tools_for_server("srv", auth) == ["read"] async def test_legacy_tag_carried_by_the_server_keeps_the_key_unrestricted(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -11684,11 +11684,11 @@ async def test_token_minted_credential_lists_tools_for_every_scope_shape(monkeyp from mcp.types import ListToolsRequest, ListToolsResult, Tool from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import operations - from litellm.proxy._experimental.mcp_server.id_jag_grant import SubjectTokenRefusal, granted_scope - from litellm.proxy._experimental.mcp_server.operations import GatewayOperations, prepare_context from litellm.proxy._types import MCPTransport from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken + from litellm.proxy.mcp_server import operations + from litellm.proxy.mcp_server.id_jag_grant import SubjectTokenRefusal, granted_scope + from litellm.proxy.mcp_server.operations import GatewayOperations, prepare_context from litellm.types.mcp_server.mcp_server_manager import MCPServer monkeypatch.setenv("LITELLM_SALT_KEY", "sk-session-credential-tools-list") @@ -11768,9 +11768,9 @@ def personal_key_owner_teams(monkeypatch: pytest.MonkeyPatch, request: pytest.Fi srv-open (``allow_all_keys``, granted to no team).""" import litellm from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy._types import LiteLLM_UserTable from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -11821,8 +11821,8 @@ async def test_personal_key_mcp_servers_are_the_union_of_its_owner_team_grants( async def test_personal_key_mcp_listing_applies_agent_caller_ceiling( personal_key_owner_teams: UserAPIKeyAuth, ) -> None: - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPServerAccess from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_from_headers + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPServerAccess auth: Final = personal_key_owner_teams.model_copy(update={"agent_id": "agent-1"}) caller: Final = agent_caller_from_headers( @@ -11863,9 +11863,9 @@ async def test_personal_key_tool_call_bills_granting_team( ) -> None: from litellm.models.budget import LiteLLM_BudgetTable from litellm.models.team_membership import LiteLLM_TeamMembership - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager - from litellm.proxy._experimental.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR, get_server_prefix from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR, get_server_prefix team_membership: Final = LiteLLM_TeamMembership( user_id=personal_key_owner_teams.user_id, @@ -11923,9 +11923,9 @@ async def test_personal_key_mcp_billing_preserves_key_organization( team_org_id: str | None, expected_org_id: str, ) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager - from litellm.proxy._experimental.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR, get_server_prefix from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR, get_server_prefix async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: return LiteLLM_TeamTable( @@ -11958,8 +11958,8 @@ async def test_personal_key_mcp_billing_preserves_key_organization( async def test_personal_key_display_name_billing_uses_original_tool_grant( monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth ) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager server: Final = global_mcp_server_manager.registry["srv-a"].model_copy( update={ @@ -11986,8 +11986,8 @@ async def test_personal_key_display_name_billing_uses_allowed_servers( monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth ) -> None: from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -12035,9 +12035,9 @@ async def test_personal_key_display_name_billing_uses_allowed_servers( async def test_personal_key_tool_call_bills_lowest_team_when_both_allow( monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth ) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager - from litellm.proxy._experimental.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR, get_server_prefix from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR, get_server_prefix async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: return _personal_key_team_granting_tools(team_id, ("multiply",)) @@ -12056,8 +12056,8 @@ async def test_personal_key_tool_call_bills_lowest_team_when_both_allow( async def test_personal_key_attribution_preserves_hyphenated_native_tool_names( monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth ) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: return _personal_key_team_granting_tools( @@ -12087,8 +12087,8 @@ async def test_personal_key_guardrail_auth_preserves_native_tool_name_matching_s monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth, ) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: return _personal_key_team_granting_tools( @@ -12117,9 +12117,9 @@ async def test_personal_key_mcp_billing_inherits_adopted_organization_limits( monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth ) -> None: from litellm.models.budget import LiteLLM_BudgetTable - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager - from litellm.proxy.auth import auth_checks from litellm.proxy._types import LiteLLM_OrganizationTable + from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager organization: Final = LiteLLM_OrganizationTable( organization_id="org-1", @@ -12167,9 +12167,9 @@ async def test_personal_key_mcp_billing_keeps_key_organization_limits( monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth ) -> None: from litellm.models.budget import LiteLLM_BudgetTable - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager - from litellm.proxy.auth import auth_checks from litellm.proxy._types import LiteLLM_OrganizationTable + from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager organization: Final = LiteLLM_OrganizationTable( organization_id="org-1", @@ -12204,10 +12204,10 @@ async def test_personal_key_mcp_billing_keeps_key_organization_limits( async def test_personal_key_mcp_billing_keeps_team_controls_when_org_lookup_fails( monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth ) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy import proxy_server from litellm.proxy.auth import auth_checks from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: return _personal_key_team_granting_tools(team_id, ("divide",)).model_copy( @@ -12253,10 +12253,10 @@ async def test_personal_key_mcp_billing_keeps_team_id_when_membership_lookup_fai monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth, ) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy import proxy_server from litellm.proxy.auth import auth_checks from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: return _personal_key_team_granting_tools(team_id, ("divide",)) @@ -12307,9 +12307,9 @@ async def test_personal_key_mcp_billing_uses_fresh_team_and_membership_for_stric from litellm.models.budget import LiteLLM_BudgetTable from litellm.models.team_membership import LiteLLM_TeamMembership from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy.auth import auth_checks from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager async def _get_team_object(team_id: str, **kwargs: object) -> LiteLLM_TeamTable: database_value: Final = bool(kwargs.get("check_db_only")) @@ -12361,12 +12361,12 @@ async def test_personal_key_mcp_billing_uses_fresh_team_and_membership_for_stric async def test_personal_key_team_mcp_rpm_limit_charges_granting_team( monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth ) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy.auth import auth_checks from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PROXY_MaxParallelRequestsHandler_v3, RateLimitDescriptor, ) + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: team: Final = _personal_key_team_granting_tools(team_id, ("divide",) if team_id == "team-a" else ("multiply",)) @@ -12532,8 +12532,8 @@ async def test_personal_key_unreadable_team_permission_is_not_a_server_or_billin monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth, ) -> None: - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPServerAccess from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPServerAccess async def _unreadable_team(team_id: str, **_: object) -> LiteLLM_TeamTable: return LiteLLM_TeamTable( @@ -12558,8 +12558,8 @@ async def test_personal_key_team_server_access_group_read_fault_stays_scoped( monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth, ) -> None: - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPServerAccess from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPServerAccess async def _team_with_access_group(team_id: str, **_: object) -> LiteLLM_TeamTable: return LiteLLM_TeamTable(team_id=team_id, access_group_ids=["unreadable-group"]) @@ -12583,9 +12583,9 @@ async def test_personal_key_scoped_bare_tool_name_bills_granting_team( monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth, ) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy import proxy_server from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: return _personal_key_team(team_id).model_copy( @@ -12624,8 +12624,8 @@ async def test_personal_key_prefixed_tool_billing_resolves_server_without_loaded personal_key_owner_teams: UserAPIKeyAuth, ) -> None: from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -12674,8 +12674,8 @@ async def test_personal_key_scoped_tool_billing_prefers_scoped_server( personal_key_owner_teams: UserAPIKeyAuth, ) -> None: from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: server_id: Final = "srv-a" if team_id == "team-a" else "srv-b" diff --git a/tests/unit/proxy/_experimental/mcp_server/conftest.py b/tests/unit/proxy/mcp_server/conftest.py similarity index 93% rename from tests/unit/proxy/_experimental/mcp_server/conftest.py rename to tests/unit/proxy/mcp_server/conftest.py index de7ed4d45e8..da5b7d8d9a3 100644 --- a/tests/unit/proxy/_experimental/mcp_server/conftest.py +++ b/tests/unit/proxy/mcp_server/conftest.py @@ -70,7 +70,7 @@ def pytest_collection_modifyitems(config, items): @pytest.fixture def config_only_mcp_manager_factory(): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager class ConfigOnlyManager(MCPServerManager): def initialize_tool_name_to_mcp_server_name_mapping(self): @@ -81,13 +81,11 @@ def config_only_mcp_manager_factory(): @pytest.fixture(autouse=True) def _hermetic_mcp_server_registry(): - from litellm.proxy._experimental.mcp_server.catalog import CatalogSnapshots - - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.catalog import CatalogSnapshots + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry saved_tools = global_mcp_tool_registry.published_tools global_mcp_tool_registry.published_tools = {} diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/__init__.py b/tests/unit/proxy/mcp_server/faults/__init__.py similarity index 100% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/__init__.py rename to tests/unit/proxy/mcp_server/faults/__init__.py diff --git a/tests/unit/proxy/_experimental/mcp_server/faults/test_classify.py b/tests/unit/proxy/mcp_server/faults/test_classify.py similarity index 98% rename from tests/unit/proxy/_experimental/mcp_server/faults/test_classify.py rename to tests/unit/proxy/mcp_server/faults/test_classify.py index 30107db4055..33b5632cf23 100644 --- a/tests/unit/proxy/_experimental/mcp_server/faults/test_classify.py +++ b/tests/unit/proxy/mcp_server/faults/test_classify.py @@ -3,19 +3,19 @@ code and whose credentials the gateway presented, never on the upstream's HTTP s from typing import Final -import pytest import httpx +import pytest -from litellm.proxy._experimental.mcp_server.faults.classify import ( +from litellm.proxy.mcp_server.faults.classify import ( classify_upstream_dcr_rejection, classify_upstream_token_rejection, ) -from litellm.proxy._experimental.mcp_server.faults.types import ( +from litellm.proxy.mcp_server.faults.types import ( CallerRejected, GatewayRejected, UpstreamProtocolFault, - UpstreamReportedFault, UpstreamRegistrationRefused, + UpstreamReportedFault, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py b/tests/unit/proxy/mcp_server/faults/test_list_outcomes.py similarity index 93% rename from tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py rename to tests/unit/proxy/mcp_server/faults/test_list_outcomes.py index fdae575276d..df016d38ada 100644 --- a/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py +++ b/tests/unit/proxy/mcp_server/faults/test_list_outcomes.py @@ -14,18 +14,18 @@ import pytest from mcp import MCPError from mcp.types import CONNECTION_CLOSED, ErrorData -from litellm.proxy._experimental.mcp_server.exceptions import ( +from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError +from litellm.proxy.mcp_server.exceptions import ( MCPServerListError, MCPUpstreamAuthError, ) -from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( +from litellm.proxy.mcp_server.faults.list_outcomes import ( ServerListFault, ServerListOk, classify_list_exception, list_fault_http_status, outcome_wire_value, ) -from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError def test_carried_fault_passes_through(): @@ -202,7 +202,7 @@ async def test_cancelled_fetch_propagates_instead_of_masquerading_as_a_healthy_e import asyncio from unittest.mock import MagicMock - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager manager = MCPServerManager() client = MagicMock() @@ -230,8 +230,8 @@ async def test_client_side_cancelled_error_is_a_classified_fault_not_a_cancellat import asyncio from unittest.mock import AsyncMock, MagicMock - from litellm.experimental_mcp_client.client import SESSION_CLEANUP_TIMEOUT_MESSAGE - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.mcp_client.client import SESSION_CLEANUP_TIMEOUT_MESSAGE + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager manager = MCPServerManager() client = MagicMock() @@ -247,7 +247,7 @@ def test_auth_challenge_and_status_come_from_the_causal_response(): """An incidental 403 raised while handling the causal 401 (context chain) must not shadow it: the carrier channel and the challenge both derive from the response on the explicit causal chain, so the caller is challenged to authenticate rather than told it is forbidden.""" - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import upstream_auth_challenge + from litellm.proxy.mcp_server.faults.list_outcomes import upstream_auth_challenge causal = httpx.HTTPStatusError( "auth", @@ -277,7 +277,7 @@ def test_auth_challenge_and_status_come_from_the_causal_response(): def test_raise_classified_list_failure_routes_auth_to_upstream_auth_error(): """The single choice-point sends 401/403 through MCPUpstreamAuthError with the upstream's own challenge and everything else through MCPServerListError, so fetch sites cannot drift.""" - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import raise_classified_list_failure + from litellm.proxy.mcp_server.faults.list_outcomes import raise_classified_list_failure auth_exc = httpx.HTTPStatusError( "auth", @@ -302,7 +302,7 @@ def test_causal_auth_behind_unrelated_response_is_still_found(): """The auth scan must not end at the first response of any status: a causal 401 sitting deeper in the tree than an unrelated 5xx (retry attempts, multi-stream task groups) must still surface with its challenge, or the client is told upstream_error and never re-authenticates.""" - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import upstream_auth_challenge + from litellm.proxy.mcp_server.faults.list_outcomes import upstream_auth_challenge deep_auth = httpx.HTTPStatusError( "auth", @@ -381,7 +381,7 @@ def test_pure_non_auth_response_still_classifies_upstream_error(): def test_managed_scope_step_up_only_accepts_unambiguous_bearer_challenges(status, challenge, expected, client): import httpx2 - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import upstream_insufficient_scope + from litellm.proxy.mcp_server.faults.list_outcomes import upstream_insufficient_scope transport = httpx if client == "httpx" else httpx2 response = transport.Response(status, headers={"www-authenticate": challenge}, request=transport.Request("POST", "https://peer.test/mcp")) @@ -403,7 +403,7 @@ def _rate_limited_error(retry_after: str | None = None) -> httpx.HTTPStatusError @pytest.mark.asyncio async def test_retry_upstream_rate_limit_backs_off_then_reraises() -> None: - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + from litellm.proxy.mcp_server.faults.list_outcomes import retry_upstream_rate_limit sleeps: Final[list[float]] = [] calls: Final[list[httpx.HTTPStatusError]] = [] @@ -435,7 +435,7 @@ async def test_retry_upstream_rate_limit_backs_off_then_reraises() -> None: @pytest.mark.asyncio async def test_retry_upstream_rate_limit_honors_retry_after_up_to_cap() -> None: - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + from litellm.proxy.mcp_server.faults.list_outcomes import retry_upstream_rate_limit sleeps: Final[list[float]] = [] @@ -464,7 +464,7 @@ async def test_retry_upstream_rate_limit_parses_http_date_retry_after() -> None: import email.utils import time - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + from litellm.proxy.mcp_server.faults.list_outcomes import retry_upstream_rate_limit sleeps: Final[list[float]] = [] attempts: Final[list[None]] = [] @@ -495,7 +495,7 @@ async def test_retry_upstream_rate_limit_parses_http_date_retry_after() -> None: @pytest.mark.asyncio async def test_retry_upstream_rate_limit_passes_through_non_429() -> None: - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + from litellm.proxy.mcp_server.faults.list_outcomes import retry_upstream_rate_limit sleeps: Final[list[float]] = [] calls = 0 @@ -522,7 +522,7 @@ async def test_retry_upstream_rate_limit_passes_through_non_429() -> None: @pytest.mark.asyncio async def test_retry_upstream_rate_limit_negative_max_retries_is_a_single_attempt() -> None: - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + from litellm.proxy.mcp_server.faults.list_outcomes import retry_upstream_rate_limit sleeps: Final[list[float]] = [] calls = 0 @@ -555,7 +555,7 @@ async def test_retry_upstream_rate_limit_negative_max_retries_is_a_single_attemp @pytest.mark.asyncio async def test_retry_deadline_skips_sleep_that_would_pass_deadline() -> None: - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + from litellm.proxy.mcp_server.faults.list_outcomes import retry_upstream_rate_limit now: Final[list[float]] = [100.0] sleeps: Final[list[float]] = [] @@ -593,7 +593,7 @@ async def test_retry_deadline_skips_sleep_that_would_pass_deadline() -> None: async def test_retry_attempt_is_interrupted_at_deadline() -> None: import anyio - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + from litellm.proxy.mcp_server.faults.list_outcomes import retry_upstream_rate_limit calls = 0 failure: Final = _rate_limited_error("0") @@ -619,7 +619,7 @@ async def test_retry_attempt_is_interrupted_at_deadline() -> None: async def test_retry_deadline_does_not_bound_first_attempt() -> None: import anyio - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + from litellm.proxy.mcp_server.faults.list_outcomes import retry_upstream_rate_limit async def slow() -> str: await anyio.sleep(0.15) @@ -636,7 +636,7 @@ async def test_retry_deadline_does_not_bound_first_attempt() -> None: async def test_retry_upstream_rate_limit_never_retries_cancellation() -> None: import asyncio - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + from litellm.proxy.mcp_server.faults.list_outcomes import retry_upstream_rate_limit sleeps: Final[list[float]] = [] calls = 0 @@ -659,7 +659,7 @@ async def test_retry_upstream_rate_limit_never_retries_cancellation() -> None: def test_listing_failed_outright_only_when_every_server_faulted_non_auth() -> None: - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( + from litellm.proxy.mcp_server.faults.list_outcomes import ( ServerListOk, listing_failed_outright, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/faults/test_render_oauth.py b/tests/unit/proxy/mcp_server/faults/test_render_oauth.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/faults/test_render_oauth.py rename to tests/unit/proxy/mcp_server/faults/test_render_oauth.py index a6807ae1454..49d7c14c285 100644 --- a/tests/unit/proxy/_experimental/mcp_server/faults/test_render_oauth.py +++ b/tests/unit/proxy/mcp_server/faults/test_render_oauth.py @@ -6,16 +6,16 @@ from typing import Final, Literal import pytest -from litellm.proxy._experimental.mcp_server.faults.render_oauth import ( +from litellm.proxy.mcp_server.faults.render_oauth import ( dcr_fault_detail, render_token_fault, ) -from litellm.proxy._experimental.mcp_server.faults.types import ( +from litellm.proxy.mcp_server.faults.types import ( CallerRejected, GatewayRejected, UpstreamProtocolFault, - UpstreamReportedFault, UpstreamRegistrationRefused, + UpstreamReportedFault, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/faults/test_traversal.py b/tests/unit/proxy/mcp_server/faults/test_traversal.py similarity index 95% rename from tests/unit/proxy/_experimental/mcp_server/faults/test_traversal.py rename to tests/unit/proxy/mcp_server/faults/test_traversal.py index b8bf4da1dc4..a2ba75058f3 100644 --- a/tests/unit/proxy/_experimental/mcp_server/faults/test_traversal.py +++ b/tests/unit/proxy/mcp_server/faults/test_traversal.py @@ -7,7 +7,7 @@ import sys if sys.version_info < (3, 11): # BaseExceptionGroup is a builtin only from 3.11 from exceptiongroup import BaseExceptionGroup -from litellm.proxy._experimental.mcp_server.faults import iter_exception_tree +from litellm.proxy.mcp_server.faults import iter_exception_tree def test_yields_the_root_itself_first(): diff --git a/tests/unit/proxy/experimental/__init__.py b/tests/unit/proxy/mcp_server/guardrail_translation/__init__.py similarity index 100% rename from tests/unit/proxy/experimental/__init__.py rename to tests/unit/proxy/mcp_server/guardrail_translation/__init__.py diff --git a/tests/unit/proxy/_experimental/mcp_server/guardrail_translation/test_mcp_guardrail_handler.py b/tests/unit/proxy/mcp_server/guardrail_translation/test_mcp_guardrail_handler.py similarity index 99% rename from tests/unit/proxy/_experimental/mcp_server/guardrail_translation/test_mcp_guardrail_handler.py rename to tests/unit/proxy/mcp_server/guardrail_translation/test_mcp_guardrail_handler.py index f3e1bcf979f..a6f94cbd961 100644 --- a/tests/unit/proxy/_experimental/mcp_server/guardrail_translation/test_mcp_guardrail_handler.py +++ b/tests/unit/proxy/mcp_server/guardrail_translation/test_mcp_guardrail_handler.py @@ -11,11 +11,11 @@ import litellm.llms as litellm_llms from litellm.caching.caching import DualCache from litellm.exceptions import BlockedPiiEntityError from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.proxy._experimental.mcp_server.guardrail_translation.handler import ( +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.mcp_server.guardrail_translation.handler import ( MCPGuardrailTranslationHandler, ) -from litellm.proxy._experimental.mcp_server.utils import MAX_STRUCTURED_CONTENT_SCAN_DEPTH -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.mcp_server.utils import MAX_STRUCTURED_CONTENT_SCAN_DEPTH from litellm.proxy.utils import ProxyLogging from litellm.types.utils import GenericGuardrailAPIInputs @@ -698,7 +698,7 @@ async def test_structured_content_nested_too_deeply_is_blocked(): """Too deep to walk must block rather than pass the deeper values unscanned.""" from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server.utils import MAX_STRUCTURED_CONTENT_SCAN_DEPTH + from litellm.proxy.mcp_server.utils import MAX_STRUCTURED_CONTENT_SCAN_DEPTH handler = MCPGuardrailTranslationHandler() guardrail = SubstitutingGuardrail("jane@example.com", "") @@ -727,7 +727,7 @@ def test_too_deep_json_returns_a_sentinel_rather_than_raising(): the helper so this pins the contract without reloading the module and leaking that reload into other tests. """ - from litellm.proxy._experimental.mcp_server.utils import ( + from litellm.proxy.mcp_server.utils import ( MAX_STRUCTURED_CONTENT_SCAN_DEPTH, json_string_leaves, ) diff --git a/tests/unit/proxy/experimental/mcp_server/__init__.py b/tests/unit/proxy/mcp_server/outbound_credentials/__init__.py similarity index 100% rename from tests/unit/proxy/experimental/mcp_server/__init__.py rename to tests/unit/proxy/mcp_server/outbound_credentials/__init__.py diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_adapter.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_adapter.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_adapter.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_adapter.py index 9cd19e9a1c9..c39fb215ef3 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_adapter.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_adapter.py @@ -13,9 +13,9 @@ import pytest from fastapi import HTTPException from pydantic import ValidationError -from litellm.experimental_mcp_client.client import MCPClient -from litellm.proxy._experimental.mcp_server.exceptions import MCPServerURLCredentialsError -from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( +from litellm.mcp_client.client import MCPClient +from litellm.proxy.mcp_server.exceptions import MCPServerURLCredentialsError +from litellm.proxy.mcp_server.outbound_credentials.adapter import ( oauth_protected_resource_path, raise_public, raise_user_oauth_challenge, @@ -23,8 +23,8 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import to_subject, validate_static_credential, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Ok -from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( +from litellm.proxy.mcp_server.outbound_credentials.result import Error, Ok +from litellm.proxy.mcp_server.outbound_credentials.types import ( ApiKeyConfig, AuthorizationCodeConfig, ClientCredentialsConfig, @@ -601,7 +601,7 @@ def test_raise_user_oauth_challenge_per_request_prefix_is_routable(monkeypatch): def test_raise_token_exchange_challenge_is_rfc9728_invalid_token(): - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( + from litellm.proxy.mcp_server.outbound_credentials.adapter import ( raise_token_exchange_challenge, ) @@ -621,7 +621,7 @@ def test_raise_token_exchange_challenge_is_rfc9728_invalid_token(): def test_raise_token_exchange_challenge_includes_server_root_path(monkeypatch): - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( + from litellm.proxy.mcp_server.outbound_credentials.adapter import ( raise_token_exchange_challenge, ) @@ -633,7 +633,7 @@ def test_raise_token_exchange_challenge_includes_server_root_path(monkeypatch): def test_raise_token_exchange_challenge_per_request_prefix_is_routable(monkeypatch): - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( + from litellm.proxy.mcp_server.outbound_credentials.adapter import ( raise_token_exchange_challenge, ) @@ -645,7 +645,7 @@ def test_raise_token_exchange_challenge_per_request_prefix_is_routable(monkeypat def test_raise_token_exchange_challenge_static_form_is_unchanged_without_step_up(): - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( + from litellm.proxy.mcp_server.outbound_credentials.adapter import ( raise_token_exchange_challenge, ) @@ -661,7 +661,7 @@ def test_raise_token_exchange_challenge_static_form_is_unchanged_without_step_up def test_raise_token_exchange_challenge_uses_insufficient_claims_with_claims_present(): # Per the Microsoft claims-challenge format, a claims challenge MUST use error=insufficient_claims # (the value MSAL-family clients key on), and the claims ride base64-encoded, never raw. - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( + from litellm.proxy.mcp_server.outbound_credentials.adapter import ( raise_token_exchange_challenge, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_authz_code_refresher.py similarity index 98% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_authz_code_refresher.py index 3068e5a0116..3a1bb024464 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_authz_code_refresher.py @@ -2,10 +2,10 @@ import pytest -from litellm.proxy._experimental.mcp_server.outbound_credentials.authz_code_refresher import ( +from litellm.proxy.mcp_server.outbound_credentials.authz_code_refresher import ( AuthorizationCodeRefresher, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) @@ -374,7 +374,7 @@ async def test_cimd_refresh_on_fresh_replica_preserves_user_and_client_identity( @pytest.mark.asyncio async def test_saved_cimd_identity_refreshes_without_discovery(monkeypatch): - from litellm.proxy._experimental.mcp_server.outbound_credentials.v2_token_store import V2PerUserTokenStore + from litellm.proxy.mcp_server.outbound_credentials.v2_token_store import V2PerUserTokenStore from litellm.types.mcp_server.mcp_server_manager import MCPServer monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_bridge_credentials.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_bridge_credentials.py similarity index 98% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_bridge_credentials.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_bridge_credentials.py index f8fb22469f1..47b6c3c2fa9 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_bridge_credentials.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_bridge_credentials.py @@ -13,7 +13,7 @@ from datetime import datetime, timedelta, timezone import pytest from pydantic import SecretStr -from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( +from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( BridgeEnvelopeAdmitted, BridgeEnvelopeInvalid, BridgeRefreshInvalid, @@ -26,7 +26,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credenti open_bridge_refresh_envelope, resolve_bridge_envelope, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( +from litellm.proxy.mcp_server.outbound_credentials.envelope import ( ENVELOPE_PREFIX, EnvelopeIdentity, EnvelopeKeys, @@ -117,7 +117,7 @@ def test_refresh_jwt_wearing_the_access_prefix_is_rejected_at_the_edge(): """Belt-and-suspenders against a swapped wire prefix: a refresh JWT re-prefixed as an access envelope opens far enough to hit the signed kind claim, which rejects it, so admission fails closed rather than forwarding a refresh credential's contents upstream.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import REFRESH_ENVELOPE_PREFIX + from litellm.proxy.mcp_server.outbound_credentials.envelope import REFRESH_ENVELOPE_PREFIX keys = envelope_keys_from_master_key(_MASTER_KEY) swapped = ENVELOPE_PREFIX + _sealed_refresh(keys).removeprefix(REFRESH_ENVELOPE_PREFIX) diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_client_credentials.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_client_credentials.py similarity index 98% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_client_credentials.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_client_credentials.py index 77cbd8fa464..1086c374622 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_client_credentials.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_client_credentials.py @@ -10,7 +10,7 @@ import httpx2 import pytest from pydantic import SecretStr -from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import ( +from litellm.proxy.mcp_server.outbound_credentials.client_credentials import ( ClientCredentialsBearerAuth, ClientCredentialsTokenSource, TokenEndpointDenied, @@ -18,8 +18,8 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credenti TokenEndpointSuccess, TokenEndpointUnreachable, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Ok -from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( +from litellm.proxy.mcp_server.outbound_credentials.result import Error, Ok +from litellm.proxy.mcp_server.outbound_credentials.types import ( ClientCredentialsConfig, ) @@ -486,8 +486,9 @@ async def test_bearer_auth_advertises_the_header_it_will_occupy(): async def test_token_exchange_failure_diagnostics(mode, monkeypatch, caplog): import asyncio import logging + from litellm.llms.custom_httpx import http_handler - from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import post_client_credentials_grant + from litellm.proxy.mcp_server.outbound_credentials.client_credentials import post_client_credentials_grant class Poster: async def post(self, url, headers, data): @@ -557,7 +558,7 @@ def test_subscription_identity_tracks_grant_configuration(changed): def test_subscription_identity_requires_managed_grant_metadata(): - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import HeaderCarrier + from litellm.proxy.mcp_server.outbound_credentials.types import HeaderCarrier async def refetch(token: str) -> str | None: return None diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_dual_cache_token_backend.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_dual_cache_token_backend.py similarity index 95% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_dual_cache_token_backend.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_dual_cache_token_backend.py index f27bc987434..00db27a60b8 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_dual_cache_token_backend.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_dual_cache_token_backend.py @@ -2,13 +2,13 @@ import pytest -from litellm.proxy._experimental.mcp_server.outbound_credentials.dual_cache_token_backend import ( +from litellm.proxy.mcp_server.outbound_credentials.dual_cache_token_backend import ( DualCacheTokenCacheBackend, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.token_cache_codec import ( +from litellm.proxy.mcp_server.outbound_credentials.token_cache_codec import ( OAuthTokenCacheCodec, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_envelope.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_envelope.py similarity index 99% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_envelope.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_envelope.py index bd310339a1d..6da509a0d37 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_envelope.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_envelope.py @@ -19,7 +19,8 @@ import pytest from cryptography.hazmat.primitives.asymmetric import rsa from pydantic import SecretStr, ValidationError -from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( +from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value, encrypt_value +from litellm.proxy.mcp_server.outbound_credentials.envelope import ( ENVELOPE_ISSUER, ENVELOPE_PREFIX, MAX_ENVELOPE_BYTES, @@ -50,7 +51,6 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import open_refresh_envelope, user_identity, ) -from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value, encrypt_value _NOW = datetime(2026, 7, 9, 12, 0, 0, tzinfo=timezone.utc) _SIGNING_KEY = "unit-test-signing-key-0123456789abcdef0123456789abcdef" @@ -480,7 +480,7 @@ def test_open_size_guard_measures_bytes_not_characters(): _decode_claims to fail loudly proves the guard short-circuits before decode.""" from unittest.mock import patch - from litellm.proxy._experimental.mcp_server.outbound_credentials import envelope + from litellm.proxy.mcp_server.outbound_credentials import envelope multibyte_body = "é" * 7000 # 7000 chars, 14000 UTF-8 bytes candidate = ENVELOPE_PREFIX + multibyte_body @@ -500,7 +500,7 @@ def test_open_size_guard_rejects_oversize_character_count_before_decode(): character precheck makes this O(1) since UTF-8 byte length is never below character length.""" from unittest.mock import patch - from litellm.proxy._experimental.mcp_server.outbound_credentials import envelope + from litellm.proxy.mcp_server.outbound_credentials import envelope candidate = ENVELOPE_PREFIX + ("a" * (MAX_ENVELOPE_BYTES + 1)) assert len(candidate) > MAX_ENVELOPE_BYTES diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_httpx_auth.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_httpx_auth.py similarity index 95% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_httpx_auth.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_httpx_auth.py index 5a5eea60fce..aa75f52b540 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_httpx_auth.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_httpx_auth.py @@ -6,7 +6,7 @@ pin the header emission the api_key family and passthrough depend on. import httpx2 -from litellm.proxy._experimental.mcp_server.outbound_credentials import ( +from litellm.proxy.mcp_server.outbound_credentials import ( NoOpAuth, StaticHeaderAuth, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_login_assertion_exchange.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_login_assertion_exchange.py similarity index 96% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_login_assertion_exchange.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_login_assertion_exchange.py index ab2b5b3d966..754a56c2f00 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_login_assertion_exchange.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_login_assertion_exchange.py @@ -7,22 +7,23 @@ from datetime import datetime, timedelta, timezone import pytest from pydantic import SecretStr -from litellm.proxy._experimental.mcp_server.outbound_credentials.login_assertion_exchange import ( +from litellm.proxy._types import LiteLLM_MCPServerTable +from litellm.proxy.mcp_server.outbound_credentials.login_assertion_exchange import ( exchange_login_assertion_for_refresh_token, login_exchange_servers, refresh_token_exchange_targets, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Ok, Result -from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_refresher import SSOClientConfig -from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( +from litellm.proxy.mcp_server.outbound_credentials.result import Error, Ok, Result +from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_refresher import SSOClientConfig +from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store import ( SAML2_SUBJECT_TOKEN_TYPE, SSOIdentityAssertion, assertion_from_saml_login, login_refresh_token_expired, with_login_refresh_token, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.token_endpoint import ExchangedToken -from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( +from litellm.proxy.mcp_server.outbound_credentials.token_endpoint import ExchangedToken +from litellm.proxy.mcp_server.outbound_credentials.types import ( ClientAuth, ClientSecretAuth, CredError, @@ -32,7 +33,6 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( NoneConfig, ServerSpec, ) -from litellm.proxy._types import LiteLLM_MCPServerTable from litellm.types.mcp import MCPAuth, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_oauth_token_store.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_oauth_token_store.py similarity index 99% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_oauth_token_store.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_oauth_token_store.py index d72afaf46b6..88d095e53ab 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_oauth_token_store.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_oauth_token_store.py @@ -5,7 +5,7 @@ from typing import Dict, List, Optional, Tuple import pytest -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( CachedOAuthTokenStore, OAuthToken, RefreshingTokenStore, diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_per_user_oauth_store.py similarity index 92% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_per_user_oauth_store.py index 5e2bb0f3722..621e206b74c 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_per_user_oauth_store.py @@ -2,11 +2,11 @@ import asyncio import pytest -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( InvalidatableOAuthTokenStore, OAuthToken, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import ( +from litellm.proxy.mcp_server.outbound_credentials.per_user_oauth_store import ( LazyPerUserOAuthTokenStore, ServerLookup, ) @@ -43,8 +43,8 @@ def credential_database(monkeypatch): @pytest.mark.asyncio @pytest.mark.parametrize("scopes, expected", [(None, None), ((), []), (("read",), ["read"])]) async def test_credential_persistence_preserves_known_and_unknown_scopes(credential_database, scopes, expected): - from litellm.proxy._experimental.mcp_server.db import decode_oauth_payload - from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import _persist_credential + from litellm.proxy.mcp_server.db import decode_oauth_payload + from litellm.proxy.mcp_server.outbound_credentials.per_user_oauth_store import _persist_credential await _persist_credential("alice", "srv", "access", "refresh", 60, scopes) row = credential_database.upsert.call_args.kwargs["data"]["create"] @@ -61,9 +61,9 @@ async def test_credential_persistence_preserves_known_and_unknown_scopes(credent async def test_native_refresh_persists_actual_grant_through_storage_adapter( credential_database, prior_scopes, response_scope, expected, ): - from litellm.proxy._experimental.mcp_server.db import decode_oauth_payload - from litellm.proxy._experimental.mcp_server.outbound_credentials.authz_code_refresher import AuthorizationCodeRefresher - from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import _persist_credential + from litellm.proxy.mcp_server.db import decode_oauth_payload + from litellm.proxy.mcp_server.outbound_credentials.authz_code_refresher import AuthorizationCodeRefresher + from litellm.proxy.mcp_server.outbound_credentials.per_user_oauth_store import _persist_credential from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -337,8 +337,8 @@ async def test_enforcement_invalidates_cached_legacy_credentials_before_use(): async def test_enforced_cache_hit_avoids_credential_read_and_rejects_changed_policy(monkeypatch): from unittest.mock import AsyncMock - from litellm.proxy._experimental.mcp_server.oauth_identity_binding import current_binding_proof - from litellm.proxy._experimental.mcp_server.outbound_credentials import per_user_oauth_store as module + from litellm.proxy.mcp_server.oauth_identity_binding import current_binding_proof + from litellm.proxy.mcp_server.outbound_credentials import per_user_oauth_store as module from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -373,7 +373,7 @@ async def test_enforced_cache_hit_avoids_credential_read_and_rejects_changed_pol async def test_expired_unverified_credential_never_reaches_refresh(monkeypatch): from unittest.mock import AsyncMock - from litellm.proxy._experimental.mcp_server.outbound_credentials import per_user_oauth_store as module + from litellm.proxy.mcp_server.outbound_credentials import per_user_oauth_store as module from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_presented_token_store.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_presented_token_store.py similarity index 73% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_presented_token_store.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_presented_token_store.py index 8b715fa717b..cc7f2236e6f 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_presented_token_store.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_presented_token_store.py @@ -2,10 +2,10 @@ import pytest -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.presented_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.presented_token_store import ( PresentedOAuthTokenStore, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_redis_distributed_lock.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_redis_distributed_lock.py similarity index 96% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_redis_distributed_lock.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_redis_distributed_lock.py index e06f3bcd5cf..d50ef6d6ba6 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_redis_distributed_lock.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_redis_distributed_lock.py @@ -2,10 +2,10 @@ import pytest -from litellm.proxy._experimental.mcp_server.outbound_credentials.redis_distributed_lock import ( +from litellm.proxy.mcp_server.outbound_credentials.redis_distributed_lock import ( RedisDistributedLock, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.redis_refresh_coordinator import ( +from litellm.proxy.mcp_server.outbound_credentials.redis_refresh_coordinator import ( LockAcquisition, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_redis_refresh_coordinator.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_redis_refresh_coordinator.py similarity index 98% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_redis_refresh_coordinator.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_redis_refresh_coordinator.py index a6831562018..18bd4a66b65 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_redis_refresh_coordinator.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_redis_refresh_coordinator.py @@ -4,10 +4,10 @@ import asyncio import pytest -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.redis_refresh_coordinator import ( +from litellm.proxy.mcp_server.outbound_credentials.redis_refresh_coordinator import ( LockAcquisition, RedisRefreshCoordinator, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_resolver.py similarity index 98% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_resolver.py index 527475b86a9..f832f9560bd 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_resolver.py @@ -19,7 +19,7 @@ import jwt as pyjwt import pytest from pydantic import SecretStr -from litellm.proxy._experimental.mcp_server.outbound_credentials import ( +from litellm.proxy.mcp_server.outbound_credentials import ( ApiKeyConfig, AuthConfig, AuthorizationCodeConfig, @@ -46,23 +46,23 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials import ( TokenExchangeConfig, UpstreamCredentialProvider, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, TokenStoreUnavailable, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_refresher import ( +from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_refresher import ( RefreshingSSOAssertionStore, SSOAssertionRefresher, SSOClientConfig, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( +from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store import ( SAML2_SUBJECT_TOKEN_TYPE, AssertionStoreUnavailable, SSOIdentityAssertion, assertion_from_saml_login, with_login_refresh_token, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.token_endpoint import ( +from litellm.proxy.mcp_server.outbound_credentials.token_endpoint import ( ExchangedToken, ) @@ -1314,7 +1314,7 @@ async def test_passthrough_ignores_the_carrier_and_keeps_the_callers_slot(): async def test_resolved_source_matches_the_credential_sent_upstream( config: AuthConfig, subject: Subject, expected_source: str, expected_header: str | None ) -> None: - from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source + from litellm.proxy.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source store = _FakeTokenStore({("alice", "s"): OAuthToken(access_token="stored-alice")}) provider = UpstreamCredentialProvider(oauth_token_store=store) @@ -1328,7 +1328,7 @@ async def test_resolved_source_matches_the_credential_sent_upstream( @pytest.mark.asyncio async def test_resolved_source_preserves_missing_user_token_error() -> None: - from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source + from litellm.proxy.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source result = await resolve_credentials_with_source( UpstreamCredentialProvider(), _SUBJECT, _spec(AuthorizationCodeConfig()) @@ -1339,7 +1339,7 @@ async def test_resolved_source_preserves_missing_user_token_error() -> None: @pytest.mark.asyncio async def test_minted_token_sources_match_egress_and_do_not_fetch_twice() -> None: - from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source + from litellm.proxy.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source source = _FakeM2MSource(Ok(OAuthToken(access_token="m2m-at"))) m2m = await resolve_credentials_with_source( @@ -1363,7 +1363,7 @@ async def test_minted_token_sources_match_egress_and_do_not_fetch_twice() -> Non @pytest.mark.asyncio async def test_id_jag_source_describes_final_token_after_both_exchanges() -> None: - from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source + from litellm.proxy.mcp_server.outbound_credentials.resolver import resolve_credentials_with_source endpoint = _FakeTokenEndpoint(_two_leg_ok("resource-token")) provider = UpstreamCredentialProvider(token_endpoint=endpoint) @@ -1382,7 +1382,7 @@ async def test_client_credentials_subscription_identity_preserves_resolved_grant _SUBJECT, _spec(_M2M) ) assert isinstance(result, Ok) - from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.client_credentials import ( ClientCredentialsBearerAuth, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_result.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_result.py similarity index 90% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_result.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_result.py index 5d26a530214..96288a62c9a 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_result.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_result.py @@ -6,7 +6,7 @@ the runtime contract consumers rely on: each arm carries its payload and discrim type. The union is exercised for real where it is used (see PR2's parse_auth_spec_kind). """ -from litellm.proxy._experimental.mcp_server.outbound_credentials import ( +from litellm.proxy.mcp_server.outbound_credentials import ( Error, Ok, Result, diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_session_credentials.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_session_credentials.py similarity index 96% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_session_credentials.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_session_credentials.py index a49c43bedb5..f084d5c7d1d 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_session_credentials.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_session_credentials.py @@ -8,10 +8,10 @@ from cryptography.hazmat.primitives.asymmetric import rsa from jwt.utils import base64url_decode, base64url_encode from pydantic import SecretStr -from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( +from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( envelope_keys_from_master_key, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.session_credentials import ( +from litellm.proxy.mcp_server.outbound_credentials.session_credentials import ( NotSessionBearer, SessionBearerAdmitted, SessionBearerInvalid, @@ -24,7 +24,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.session_credent resolve_session_signing_keys, session_keys_from_master_key, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import ( +from litellm.proxy.mcp_server.outbound_credentials.session_token import ( SESSION_TTL_SECONDS, AsymmetricSessionKeys, MintedSessionToken, diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_session_token.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_session_token.py similarity index 99% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_session_token.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_session_token.py index 641aca2446e..2e648fb86ed 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_session_token.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_session_token.py @@ -9,7 +9,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa from jwt.utils import base64url_decode, base64url_encode from pydantic import SecretStr, ValidationError -from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import ( +from litellm.proxy.mcp_server.outbound_credentials.session_token import ( MAX_SESSION_TOKEN_BYTES, SESSION_ISSUER, SESSION_REFRESH_PREFIX, diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_sso_assertion_refresher.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_sso_assertion_refresher.py similarity index 99% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_sso_assertion_refresher.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_sso_assertion_refresher.py index 53d0d82a7f2..38d203bd77d 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_sso_assertion_refresher.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_sso_assertion_refresher.py @@ -20,12 +20,12 @@ import jwt as pyjwt import pytest from pydantic import SecretStr -from litellm.proxy._experimental.mcp_server.outbound_credentials.result import ( +from litellm.proxy.mcp_server.outbound_credentials.result import ( Error, Ok, Result, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_refresher import ( +from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_refresher import ( HttpxTokenEndpointTransport, RefreshFailure, RefreshingSSOAssertionStore, @@ -34,11 +34,11 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_r default_sso_assertion_store, sso_client_config, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( - with_login_refresh_token, +from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store import ( AssertionStoreUnavailable, SSOIdentityAssertion, assertion_from_saml_login, + with_login_refresh_token, ) SIGNING_KEY = "test-idp-signing-key-32-bytes-long-xxxx" diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_sso_assertion_store.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_sso_assertion_store.py similarity index 96% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_sso_assertion_store.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_sso_assertion_store.py index 3c8e7a9b448..8bcd802d497 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_sso_assertion_store.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_sso_assertion_store.py @@ -18,7 +18,8 @@ from unittest.mock import AsyncMock, MagicMock, patch import jwt as pyjwt import pytest -from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import ( +from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, encrypt_value_helper +from litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store import ( _ASSERTION_CACHE, SAML2_SUBJECT_TOKEN_TYPE, AssertionStoreUnavailable, @@ -35,7 +36,6 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_s rotate_sso_identity_assertions_master_key, with_login_refresh_token, ) -from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, encrypt_value_helper from litellm.types.mcp import MCPAuth SALT_KEY = "test-salt-key-for-sso-assertion-tests-1234" @@ -184,7 +184,7 @@ def test_assertion_without_exp_or_iss_still_retained(): @pytest.mark.asyncio async def test_retention_gate_requires_an_id_jag_server(): with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, patch("litellm.proxy.proxy_server.prisma_client", _make_prisma({}, db_has_id_jag_server=False)), ): manager.config_mcp_servers = { @@ -203,7 +203,7 @@ async def test_retention_gate_requires_an_id_jag_server(): async def test_retention_gate_reads_the_db_when_config_declares_no_id_jag_server(): """A DB-backed server added on another pod (or before this pod's DB load) must still enable retention off the authoritative DB row; False only when neither authority knows one.""" - with patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager: + with patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager: manager.config_mcp_servers = {"s1": _server_with_auth(MCPAuth.oauth2)} db_backed = _make_prisma({}, db_has_id_jag_server=True) with patch("litellm.proxy.proxy_server.prisma_client", db_backed): @@ -223,7 +223,7 @@ async def test_retention_gate_never_consults_the_registry_snapshot(): judge only the config declaration and the DB row, so a stale snapshot listing an id_jag server changes nothing.""" with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, patch("litellm.proxy.proxy_server.prisma_client", _make_prisma({}, db_has_id_jag_server=False)), ): manager.config_mcp_servers = {} @@ -237,7 +237,7 @@ async def test_retain_persists_when_only_the_db_knows_the_id_jag_server(): stored = {} prisma = _make_prisma(stored, db_has_id_jag_server=True) with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, patch("litellm.proxy.proxy_server.prisma_client", prisma), ): manager.config_mcp_servers = {} @@ -460,7 +460,7 @@ async def test_retain_noop_when_no_id_jag_server(): stored = {} prisma = _make_prisma(stored) with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, patch("litellm.proxy.proxy_server.prisma_client", prisma), ): manager.config_mcp_servers = {"s1": _server_with_auth(MCPAuth.oauth2)} @@ -476,7 +476,7 @@ async def test_retain_persists_when_id_jag_server_registered(): stored = {} prisma = _make_prisma(stored) with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, patch("litellm.proxy.proxy_server.prisma_client", prisma), ): manager.config_mcp_servers = {"s1": _server_with_auth(MCPAuth.oauth2_id_jag)} @@ -490,7 +490,7 @@ async def test_retain_persists_when_id_jag_server_registered(): async def test_retain_none_assertion_never_consults_gate_or_store(): gate = MagicMock() with patch( - "litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store.ema_assertion_retention_enabled", + "litellm.proxy.mcp_server.outbound_credentials.sso_assertion_store.ema_assertion_retention_enabled", gate, ): await retain_sso_identity_assertion_for_ema(user_id="user-a", assertion=None) @@ -502,7 +502,7 @@ async def test_retain_swallows_store_failure(): prisma = MagicMock() prisma.db.litellm_ssoidentityassertion.upsert = AsyncMock(side_effect=RuntimeError("db down")) with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as manager, patch("litellm.proxy.proxy_server.prisma_client", prisma), ): manager.config_mcp_servers = {"s1": _server_with_auth(MCPAuth.oauth2_id_jag)} diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_token_cache_codec.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_token_cache_codec.py similarity index 93% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_token_cache_codec.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_token_cache_codec.py index 12a04c4b4dc..d3cb701df33 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_token_cache_codec.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_token_cache_codec.py @@ -1,9 +1,9 @@ """Tests for the cache codec: encrypt on encode, drop the refresh_token, round-trip the bearer.""" -from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.token_cache_codec import ( +from litellm.proxy.mcp_server.outbound_credentials.token_cache_codec import ( OAuthTokenCacheCodec, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_token_endpoint.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_token_endpoint.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_token_endpoint.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_token_endpoint.py index db3a1a386a3..e9b2b8ba589 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_token_endpoint.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_token_endpoint.py @@ -13,31 +13,31 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import jwt -import litellm import pytest from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa +from pydantic import SecretStr -from litellm.proxy._experimental.mcp_server.outbound_credentials.result import ( +import litellm +from litellm.proxy.mcp_server.outbound_credentials.result import ( Error, Ok, Result, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.token_endpoint import ( +from litellm.proxy.mcp_server.outbound_credentials.token_endpoint import ( CLIENT_ASSERTION_TYPE, ExchangedToken, ExchangedTokenCache, TokenEndpointClient, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( +from litellm.proxy.mcp_server.outbound_credentials.types import ( ClientSecretAuth, CredError, PrivateKeyJwtAuth, ) -from pydantic import SecretStr _PATCH_TARGET = ( - "litellm.proxy._experimental.mcp_server.outbound_credentials." + "litellm.proxy.mcp_server.outbound_credentials." "token_endpoint.get_async_httpx_client" ) diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_token_exchange_provider.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_token_exchange_provider.py similarity index 96% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_token_exchange_provider.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_token_exchange_provider.py index cf15fdb3e26..40a10053dd2 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_token_exchange_provider.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_token_exchange_provider.py @@ -8,11 +8,11 @@ from unittest.mock import patch import pytest -from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchange_provider import ( +from litellm.proxy.mcp_server.outbound_credentials.token_exchange_provider import ( _post_exchange_endpoint, build_token_exchanger, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger import ( +from litellm.proxy.mcp_server.outbound_credentials.token_exchanger import ( OboTokenExchanger, SubjectTokenRejected, TokenExchangeClientError, diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_token_exchanger.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_token_exchanger.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_token_exchanger.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_token_exchanger.py index f2750cc3632..e1930931b03 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_token_exchanger.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_token_exchanger.py @@ -9,15 +9,15 @@ import asyncio import pytest from pydantic import SecretStr -from litellm.proxy._experimental.mcp_server.outbound_credentials import ( +from litellm.proxy.mcp_server.outbound_credentials import ( Error, Ok, ServerSpec, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger import ( +from litellm.proxy.mcp_server.outbound_credentials.token_exchanger import ( OboTokenExchanger, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( +from litellm.proxy.mcp_server.outbound_credentials.types import ( TokenExchangeConfig, ) @@ -127,7 +127,7 @@ async def test_client_secret_post_keeps_creds_in_body_with_no_auth_header(): async def test_exchange_maps_idp_rejection_to_unauthorized(): """An IdP 4xx (surfaced as SubjectTokenRejected by the post adapter) is non-retryable: it maps to ``unauthorized`` (the 401 OBO challenge), not the retryable ``upstream_unavailable`` (503).""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger import ( + from litellm.proxy.mcp_server.outbound_credentials.token_exchanger import ( SubjectTokenRejected, ) @@ -144,7 +144,7 @@ async def test_exchange_maps_gateway_fault_to_misconfigured(): """A gateway-fault RFC 6749 code (invalid_client / invalid_target / ..., surfaced as TokenExchangeClientError) is the gateway's problem, not the caller's, so it maps to misconfigured (500) rather than the retryable 503 or the 401 OBO challenge the caller can't act on.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger import ( + from litellm.proxy.mcp_server.outbound_credentials.token_exchanger import ( TokenExchangeClientError, ) @@ -298,7 +298,7 @@ async def test_non_bearer_token_type_is_logged(): post = _RecordingPost({"access_token": "x", "token_type": "N_A", "expires_in": 3600}) with patch( - "litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger.verbose_logger" + "litellm.proxy.mcp_server.outbound_credentials.token_exchanger.verbose_logger" ) as mock_logger: await OboTokenExchanger(post, clock=_Clock()).exchange("jwt", _SERVER, _CONFIG) assert mock_logger.warning.called @@ -542,7 +542,7 @@ async def test_distributed_coordinator_refresh_and_reread_use_the_cache(): @pytest.mark.asyncio async def test_step_up_rejection_threads_claims_into_unauthorized(): - from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger import ( + from litellm.proxy.mcp_server.outbound_credentials.token_exchanger import ( SubjectTokenRejected, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_types.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_types.py similarity index 99% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_types.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_types.py index 7f5c17de8b4..1f50444f09e 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_types.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_types.py @@ -10,7 +10,7 @@ corresponding guarantee is mutated away. import pytest from pydantic import SecretStr, TypeAdapter, ValidationError -from litellm.proxy._experimental.mcp_server.outbound_credentials import ( +from litellm.proxy.mcp_server.outbound_credentials import ( Ambient, ApiKeyConfig, AuthConfig, diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_v2_token_store.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_v2_token_store.py similarity index 95% rename from tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_v2_token_store.py rename to tests/unit/proxy/mcp_server/outbound_credentials/test_v2_token_store.py index 97d14b65f5a..f531491cfa3 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_v2_token_store.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_v2_token_store.py @@ -2,7 +2,7 @@ import pytest -from litellm.proxy._experimental.mcp_server.outbound_credentials.v2_token_store import ( +from litellm.proxy.mcp_server.outbound_credentials.v2_token_store import ( V2PerUserTokenStore, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_byok_credential_cache.py b/tests/unit/proxy/mcp_server/test_byok_credential_cache.py similarity index 96% rename from tests/unit/proxy/_experimental/mcp_server/test_byok_credential_cache.py rename to tests/unit/proxy/mcp_server/test_byok_credential_cache.py index 0ec4b431276..7ca169366a6 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_byok_credential_cache.py +++ b/tests/unit/proxy/mcp_server/test_byok_credential_cache.py @@ -2,15 +2,15 @@ import json import pytest -from litellm.proxy._experimental.mcp_server.byok_credential_cache import ( +from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import AuthCacheInvalidationSubscriber +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.mcp_server.byok_credential_cache import ( CachedByokCredential, byok_credential_cache, byok_credential_cache_key, cache_byok_credential, get_cached_byok_credential, ) -from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import AuthCacheInvalidationSubscriber -from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache class _FakeRedisCache: diff --git a/tests/unit/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py b/tests/unit/proxy/mcp_server/test_byok_oauth_endpoints.py similarity index 92% rename from tests/unit/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py rename to tests/unit/proxy/mcp_server/test_byok_oauth_endpoints.py index b2364647e53..37da00ab2ba 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py +++ b/tests/unit/proxy/mcp_server/test_byok_oauth_endpoints.py @@ -1,4 +1,5 @@ -from litellm.proxy._experimental.mcp_server import operations as mcp_operations +from litellm.proxy.mcp_server import operations as mcp_operations + """ Unit tests for the BYOK OAuth 2.1 authorization server endpoints. @@ -23,12 +24,12 @@ import pytest from fastapi import HTTPException from fastapi.testclient import TestClient -from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( +from litellm.proxy._types import MCPTransport +from litellm.proxy.mcp_server.byok_oauth_endpoints import ( _byok_auth_codes, _verify_pkce, router, ) -from litellm.proxy._types import MCPTransport # --------------------------------------------------------------------------- # _verify_pkce @@ -64,10 +65,10 @@ def test_verify_pkce_tampered_challenge(): from fastapi import FastAPI -from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.mcp_server.byok_oauth_endpoints import ( _byok_session_auth, ) -from litellm.proxy._types import UserAPIKeyAuth _test_app = FastAPI() _test_app.include_router(router) @@ -101,8 +102,8 @@ def unauthenticated_client(): @pytest.mark.parametrize("byok_first", [True, False]) def test_byok_challenge_discovers_api_key_flow(monkeypatch, byok_first): - from litellm.proxy._experimental.mcp_server import byok_oauth_endpoints, discoverable_endpoints - from litellm.proxy._experimental.mcp_server.oauth_utils import get_byok_www_authenticate + from litellm.proxy.mcp_server import byok_oauth_endpoints, discoverable_endpoints + from litellm.proxy.mcp_server.oauth_utils import get_byok_www_authenticate monkeypatch.delenv("PROXY_BASE_URL", raising=False) monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) @@ -137,7 +138,7 @@ def test_byok_challenge_discovers_api_key_flow(monkeypatch, byok_first): ], ) def test_byok_challenge_preserves_external_base(monkeypatch, base_url, root_path, expected): - from litellm.proxy._experimental.mcp_server.oauth_utils import get_byok_www_authenticate + from litellm.proxy.mcp_server.oauth_utils import get_byok_www_authenticate monkeypatch.setenv("PROXY_BASE_URL", base_url) monkeypatch.setenv("SERVER_ROOT_PATH", root_path) @@ -147,7 +148,7 @@ def test_byok_challenge_preserves_external_base(monkeypatch, base_url, root_path def test_byok_discovery_preserves_per_request_prefixes(monkeypatch): from fastapi import FastAPI - from litellm.proxy._experimental.mcp_server.server import _check_byok_credential + from litellm.proxy.mcp_server.server import _check_byok_credential from litellm.proxy.middleware.per_request_root_path_middleware import PerRequestRootPathMiddleware from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -385,25 +386,25 @@ async def test_token_endpoint_success(): with ( patch( - "litellm.proxy._experimental.mcp_server.byok_oauth_endpoints.store_user_credential", + "litellm.proxy.mcp_server.byok_oauth_endpoints.store_user_credential", mock_store, ), patch( - "litellm.proxy._experimental.mcp_server.byok_oauth_endpoints.router", + "litellm.proxy.mcp_server.byok_oauth_endpoints.router", ), ): # Import the actual handler function directly - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( + from litellm.proxy.mcp_server.byok_oauth_endpoints import ( byok_token, ) mock_request = MagicMock() # Patch module-level globals in the function's module with patch( - "litellm.proxy._experimental.mcp_server.byok_oauth_endpoints.store_user_credential", + "litellm.proxy.mcp_server.byok_oauth_endpoints.store_user_credential", mock_store, ): - import litellm.proxy._experimental.mcp_server.byok_oauth_endpoints as mod + import litellm.proxy.mcp_server.byok_oauth_endpoints as mod original_prisma = None original_master_key = None @@ -454,7 +455,7 @@ async def test_token_endpoint_success(): @pytest.mark.asyncio async def test_token_endpoint_invalid_code(): - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import byok_token + from litellm.proxy.mcp_server.byok_oauth_endpoints import byok_token mock_request = MagicMock() with ( @@ -486,7 +487,7 @@ async def test_token_endpoint_expired_code(): ttl=-10, # already expired ) - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import byok_token + from litellm.proxy.mcp_server.byok_oauth_endpoints import byok_token mock_request = MagicMock() with ( @@ -517,7 +518,7 @@ async def test_token_endpoint_wrong_verifier(): redirect_uri="http://127.0.0.1:3000/cb", ) - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import byok_token + from litellm.proxy.mcp_server.byok_oauth_endpoints import byok_token mock_request = MagicMock() with ( @@ -538,7 +539,7 @@ async def test_token_endpoint_wrong_verifier(): @pytest.mark.asyncio async def test_token_endpoint_unsupported_grant_type(): - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import byok_token + from litellm.proxy.mcp_server.byok_oauth_endpoints import byok_token mock_request = MagicMock() with ( @@ -565,7 +566,7 @@ async def test_token_endpoint_unsupported_grant_type(): @pytest.mark.asyncio async def test_check_byok_credential_not_byok(): """Non-BYOK servers should pass through without any DB check.""" - from litellm.proxy._experimental.mcp_server.server import _check_byok_credential + from litellm.proxy.mcp_server.server import _check_byok_credential from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -581,7 +582,7 @@ async def test_check_byok_credential_not_byok(): @pytest.mark.asyncio async def test_check_byok_credential_no_user_id(): """BYOK server with no user identity → 401.""" - from litellm.proxy._experimental.mcp_server.server import _check_byok_credential + from litellm.proxy.mcp_server.server import _check_byok_credential from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -601,8 +602,8 @@ async def test_check_byok_credential_no_user_id(): @pytest.mark.asyncio async def test_check_byok_credential_missing_credential(monkeypatch): """BYOK server with a known user but no stored credential → 401.""" - from litellm.proxy._experimental.mcp_server.server import _check_byok_credential from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import _check_byok_credential from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -613,7 +614,7 @@ async def test_check_byok_credential_missing_credential(monkeypatch): ) user_auth = UserAPIKeyAuth(user_id="user-99", api_key="sk-test") - from litellm.proxy._experimental.mcp_server import server as server_module + from litellm.proxy.mcp_server import server as server_module monkeypatch.delenv("PROXY_BASE_URL", raising=False) monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) @@ -622,7 +623,7 @@ async def test_check_byok_credential_missing_credential(monkeypatch): with ( patch( - "litellm.proxy._experimental.mcp_server.db.get_user_credential", + "litellm.proxy.mcp_server.db.get_user_credential", new=AsyncMock(return_value=None), ), patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), @@ -647,9 +648,9 @@ async def test_check_byok_credential_missing_credential(monkeypatch): async def test_execute_byok_tool_missing_credential_advertises_api_key_flow(monkeypatch): from datetime import datetime, timezone - from litellm.proxy._experimental.mcp_server import server as mcp_module from litellm.proxy import proxy_server from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server import server as mcp_module from litellm.types.mcp_server.mcp_server_manager import MCPServer monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com/proxy") @@ -680,8 +681,8 @@ async def test_execute_byok_tool_missing_credential_advertises_api_key_flow(monk @pytest.mark.asyncio async def test_check_byok_credential_has_credential(): """BYOK server with a valid stored credential → no error raised.""" - from litellm.proxy._experimental.mcp_server.server import _check_byok_credential from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import _check_byok_credential from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -696,7 +697,7 @@ async def test_check_byok_credential_has_credential(): with ( patch( - "litellm.proxy._experimental.mcp_server.db.get_user_credential", + "litellm.proxy.mcp_server.db.get_user_credential", new=AsyncMock(return_value="some-credential-value"), ), patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), @@ -708,9 +709,9 @@ async def test_check_byok_credential_has_credential(): @pytest.mark.asyncio async def test_invalidate_byok_cred_cache_evicts_locally_and_broadcasts_the_same_key(): """A revoked credential must stop being served here and on every peer worker within the TTL.""" - from litellm.proxy._experimental.mcp_server import server as server_module - from litellm.proxy._experimental.mcp_server.byok_credential_cache import byok_credential_cache_key from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server import server as server_module + from litellm.proxy.mcp_server.byok_credential_cache import byok_credential_cache_key from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer(server_id="byok-revoke", name="byok-server", transport=MCPTransport.http, is_byok=True) @@ -721,7 +722,7 @@ async def test_invalidate_byok_cred_cache_evicts_locally_and_broadcasts_the_same with ( patch( # test-quality-ok: the DB row lookup is the only seam below the credential resolver; no Prisma fake exists - "litellm.proxy._experimental.mcp_server.db.get_user_credential", new=db_lookup + "litellm.proxy.mcp_server.db.get_user_credential", new=db_lookup ), patch( # test-quality-ok: the resolver reads the module-level prisma_client singleton; the suite's only seam "litellm.proxy.proxy_server.prisma_client", MagicMock() @@ -746,8 +747,8 @@ async def test_check_byok_credential_db_unavailable_fails_closed(): Regression for GHSA-6762: previously returned silently, bypassing the ownership check during DB outage windows. """ - from litellm.proxy._experimental.mcp_server.server import _check_byok_credential from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import _check_byok_credential from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -859,7 +860,7 @@ async def test_token_endpoint_rejects_missing_user_id_in_code_record(): """Defense in depth: if a code record somehow lacks user_id (older format / manual DB write), /token must reject rather than fall back to the form's client_id.""" - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( + from litellm.proxy.mcp_server.byok_oauth_endpoints import ( byok_token, ) @@ -1050,7 +1051,7 @@ def test_authorize_post_rejects_non_loopback_redirect_uri(client): async def test_token_endpoint_rejects_redirect_uri_mismatch(): """RFC 6749 §4.1.3 / OAuth 2.1 §4.1.3: if redirect_uri was sent at /authorize, the /token redirect_uri MUST match exactly.""" - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( + from litellm.proxy.mcp_server.byok_oauth_endpoints import ( byok_token, ) @@ -1104,7 +1105,7 @@ async def test_token_endpoint_accepts_spec_compliant_client_id(): identifier, not the user. It must not be cross-checked against the record's user_id — that cross-check would break spec-compliant MCP clients that pass e.g. client_id="claude-desktop".""" - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( + from litellm.proxy.mcp_server.byok_oauth_endpoints import ( byok_token, ) @@ -1122,7 +1123,7 @@ async def test_token_endpoint_accepts_spec_compliant_client_id(): with ( patch( - "litellm.proxy._experimental.mcp_server.byok_oauth_endpoints.store_user_credential", + "litellm.proxy.mcp_server.byok_oauth_endpoints.store_user_credential", new=AsyncMock(), ), patch("litellm.proxy.proxy_server.master_key", "test-master"), @@ -1146,7 +1147,7 @@ async def test_token_endpoint_rejects_client_id_mismatch(): client_id, the token request must submit the same value. An attacker who steals a code from another client (different native app) can't redeem it.""" - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( + from litellm.proxy.mcp_server.byok_oauth_endpoints import ( byok_token, ) @@ -1220,7 +1221,7 @@ async def test_token_endpoint_missing_master_key_preserves_code_and_db(): code or writing the credential — otherwise a misconfigured deploy burns the code and persists the key with no way for the user to retrieve a session token without restarting the flow.""" - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( + from litellm.proxy.mcp_server.byok_oauth_endpoints import ( byok_token, ) @@ -1240,7 +1241,7 @@ async def test_token_endpoint_missing_master_key_preserves_code_and_db(): mock_store = AsyncMock() with ( patch( - "litellm.proxy._experimental.mcp_server.byok_oauth_endpoints.store_user_credential", + "litellm.proxy.mcp_server.byok_oauth_endpoints.store_user_credential", mock_store, ), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), @@ -1308,7 +1309,7 @@ async def test_token_endpoint_accepts_oauth21_client_omitting_redirect_uri(): Enforcement still fires when the client DOES submit a value that disagrees with the record (see test_token_endpoint_rejects_redirect_uri_mismatch). """ - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( + from litellm.proxy.mcp_server.byok_oauth_endpoints import ( byok_token, ) @@ -1327,7 +1328,7 @@ async def test_token_endpoint_accepts_oauth21_client_omitting_redirect_uri(): with ( patch( - "litellm.proxy._experimental.mcp_server.byok_oauth_endpoints.store_user_credential", + "litellm.proxy.mcp_server.byok_oauth_endpoints.store_user_credential", new=AsyncMock(), ), patch("litellm.proxy.proxy_server.master_key", "test-master"), @@ -1345,7 +1346,7 @@ async def test_token_endpoint_accepts_oauth21_client_omitting_redirect_uri(): def test_validate_loopback_redirect_uri_rejects_fragment(): - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_loopback_redirect_uri, ) @@ -1357,7 +1358,7 @@ def test_validate_loopback_redirect_uri_rejects_fragment(): def test_validate_loopback_redirect_uri_rejects_malformed_cleanly(): """Malformed / unparseable URIs should surface as 400 invalid_request, not a 500 from an unhandled exception inside ip_address().""" - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_loopback_redirect_uri, ) @@ -1388,7 +1389,7 @@ def _make_trusted_request(base_url: str = "https://llm.example.com/"): def test_validate_trusted_redirect_uri_accepts_same_origin(): - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1401,7 +1402,7 @@ def test_validate_trusted_redirect_uri_same_origin_normalizes_default_port(): otherwise produce a proxy_base of ``https://llm.example.com:443`` which wouldn't literally match the browser's port-less ``llm.example.com`` redirect_uri even though both represent the same origin.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1415,7 +1416,7 @@ def test_validate_trusted_redirect_uri_same_origin_normalizes_default_port(): def test_validate_trusted_redirect_uri_accepts_loopback(): - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1432,7 +1433,7 @@ def test_validate_trusted_redirect_uri_accepts_loopback(): def test_validate_trusted_redirect_uri_rejects_cross_origin_by_default( monkeypatch, ): - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1444,7 +1445,7 @@ def test_validate_trusted_redirect_uri_rejects_cross_origin_by_default( def test_validate_trusted_redirect_uri_rejects_fragment_and_bad_scheme(): - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1460,7 +1461,7 @@ def test_validate_trusted_redirect_uri_rejects_fragment_and_bad_scheme(): def test_validate_trusted_redirect_uri_accepts_cursor_native_callback(): - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1471,14 +1472,14 @@ def test_validate_trusted_redirect_uri_accepts_cursor_native_callback(): def test_validate_trusted_redirect_uri_rejects_unlisted_native_callback( monkeypatch, ): - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) monkeypatch.setenv("MCP_TRUSTED_NATIVE_REDIRECT_URIS", "") # Clear defaults by patching — env-only path for this test monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.oauth_utils._DEFAULT_NATIVE_REDIRECT_URIS", + "litellm.proxy.mcp_server.oauth_utils._DEFAULT_NATIVE_REDIRECT_URIS", [], ) req = _make_trusted_request("http://localhost:4000/") @@ -1492,12 +1493,12 @@ def test_validate_trusted_redirect_uri_rejects_unlisted_native_callback( def test_validate_trusted_redirect_uri_accepts_env_native_redirect_uri( monkeypatch, ): - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.oauth_utils._DEFAULT_NATIVE_REDIRECT_URIS", + "litellm.proxy.mcp_server.oauth_utils._DEFAULT_NATIVE_REDIRECT_URIS", [], ) monkeypatch.setenv( @@ -1509,7 +1510,7 @@ def test_validate_trusted_redirect_uri_accepts_env_native_redirect_uri( def test_validate_trusted_redirect_uri_rejects_native_callback_with_fragment(): - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1522,7 +1523,7 @@ def test_validate_trusted_redirect_uri_rejects_native_callback_with_fragment(): def test_validate_trusted_redirect_uri_rejects_native_callback_with_query(): - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1536,12 +1537,12 @@ def test_validate_trusted_redirect_uri_rejects_native_callback_with_query(): def test_validate_trusted_redirect_uri_native_path_case_insensitive(monkeypatch): - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.oauth_utils._DEFAULT_NATIVE_REDIRECT_URIS", + "litellm.proxy.mcp_server.oauth_utils._DEFAULT_NATIVE_REDIRECT_URIS", [], ) monkeypatch.setenv( @@ -1555,12 +1556,12 @@ def test_validate_trusted_redirect_uri_native_path_case_insensitive(monkeypatch) def test_validate_trusted_redirect_uri_native_wildcard_respects_path_boundary( monkeypatch, ): - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.oauth_utils._DEFAULT_NATIVE_REDIRECT_URIS", + "litellm.proxy.mcp_server.oauth_utils._DEFAULT_NATIVE_REDIRECT_URIS", [], ) monkeypatch.setenv( @@ -1580,12 +1581,12 @@ def test_validate_trusted_redirect_uri_native_wildcard_respects_path_boundary( def test_validate_trusted_redirect_uri_native_wildcard_directory_prefix( monkeypatch, ): - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.oauth_utils._DEFAULT_NATIVE_REDIRECT_URIS", + "litellm.proxy.mcp_server.oauth_utils._DEFAULT_NATIVE_REDIRECT_URIS", [], ) monkeypatch.setenv( @@ -1600,7 +1601,7 @@ def test_validate_trusted_redirect_uri_rejects_scheme_mismatch_on_same_host(): """Regression: an attacker who can serve http on the proxy's own host (e.g. by MITMing an unencrypted LAN hop) must not be able to pass same-origin validation — scheme must match as well as host.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1622,7 +1623,7 @@ def test_validate_trusted_redirect_uri_rejects_userinfo(monkeypatch): allowlist, and wildcard allowlist — so the bypass is closed on every path through the validator. """ - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1675,7 +1676,7 @@ def test_validate_trusted_redirect_uri_rejects_backslash_in_netloc(monkeypatch): exact-entry, and wildcard — by bouncing the netloc before any matching runs. """ - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1709,7 +1710,7 @@ def test_validate_trusted_redirect_uri_allowlist_entry_with_default_port(monkeyp away for the same-origin compare; the allowlist side has to apply the same normalization or the comparison is asymmetric and silently fails.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( _parse_trusted_redirect_origins, validate_trusted_redirect_uri, ) @@ -1729,7 +1730,7 @@ def test_validate_trusted_redirect_uri_allowlist_entry_with_default_port(monkeyp def test_validate_trusted_redirect_uri_accepts_exact_allowlisted_host(monkeypatch): - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1754,7 +1755,7 @@ def test_validate_trusted_redirect_uri_allowlist_rejects_http_even_on_listed_hos """An attacker must not be able to elevate to the allowlist by serving http:// on the listed host — only https is accepted for non-loopback allowlist entries.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1769,7 +1770,7 @@ def test_validate_trusted_redirect_uri_wildcard_allowlist(monkeypatch): """``*.suffix`` entries match any strictly-deeper subdomain of ``suffix`` but must not match the bare suffix, nor unrelated domains that happen to end with the same characters.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1794,7 +1795,7 @@ def test_validate_trusted_redirect_uri_wildcard_allowlist(monkeypatch): def test_validate_trusted_redirect_uri_wildcard_rejects_http(monkeypatch): """The https-only gate applies to wildcard entries too.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1810,7 +1811,7 @@ def test_validate_trusted_redirect_uri_wildcard_host_with_port_still_matches( ): """Wildcard entries don't express port constraints — an allowlisted subdomain should match regardless of explicit port on the URL.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1822,7 +1823,7 @@ def test_validate_trusted_redirect_uri_wildcard_host_with_port_still_matches( def test_validate_trusted_redirect_uri_accepts_ipv6_loopback_with_default_port(): """IPv6 loopback with explicit ``:443`` on an ``https`` URL should still match — exercises ``_strip_default_port``'s IPv6 branch.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1835,7 +1836,7 @@ def test_validate_trusted_redirect_uri_tolerates_malformed_env_entries(monkeypat ``*.``, non-numeric ports). None of those should raise; unmatched entries must simply fail to grant access while well-formed entries in the same list continue to work.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1863,7 +1864,7 @@ def test_validate_trusted_redirect_uri_rejects_wildcard_entry_with_dot_leading_s whose own leading ``.`` makes it look like a deeper subdomain. Operators who mistype an extra dot should get an ignored entry, not a broader match than they intended.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1885,7 +1886,7 @@ def test_validate_trusted_redirect_uri_falls_through_when_origin_lookup_fails(): """If ``get_request_base_url`` can't determine the proxy's origin, same-origin is skipped silently but loopback + allowlist paths are still reachable.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1905,7 +1906,7 @@ def test_strip_default_port_empty_netloc(): """``_strip_default_port("", "")`` should round-trip — validator rejects empty-netloc URLs upstream so this is purely a defensive contract on the helper itself.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import _strip_default_port + from litellm.proxy.mcp_server.oauth_utils import _strip_default_port assert _strip_default_port("https", "") == "" @@ -1914,7 +1915,7 @@ def test_strip_default_port_handles_non_numeric_port(): """Raw netloc with a non-numeric port is returned unchanged. Reached in practice when a malformed ``Host`` header survives upstream parsing — we stay out of its way rather than 500ing.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import _strip_default_port + from litellm.proxy.mcp_server.oauth_utils import _strip_default_port assert _strip_default_port("https", "foo.com:bar") == "foo.com:bar" assert _strip_default_port("https", "[::1]:bar") == "[::1]:bar" @@ -1923,7 +1924,7 @@ def test_strip_default_port_handles_non_numeric_port(): def test_validate_trusted_redirect_uri_rejects_public_ip_without_allowlist(): """A redirect_uri whose host is a public IP (parseable by ``ip_address`` but not loopback) must fail all three tiers and 400.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) @@ -1937,7 +1938,7 @@ def test_parse_trusted_redirect_origins_drops_bare_path_entries(monkeypatch): """``/foo`` has a scheme-less leading slash and would strip to the empty string — drop silently rather than allowlisting empty origins.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( _parse_trusted_redirect_origins, ) @@ -1965,12 +1966,14 @@ async def test_authenticated_browser_identity_checks_embedded_session( monkeypatch: pytest.MonkeyPatch, case: str, expected: str | None ) -> None: from datetime import datetime, timezone + import jwt from starlette.requests import Request + from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import get_authenticated_browser_user_id - from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_bearer_token from litellm.proxy.auth.auth_checks import LITELLM_SESSION_TOKEN_PREFIX + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_bearer_token + from litellm.proxy.mcp_server.byok_oauth_endpoints import get_authenticated_browser_user_id monkeypatch.setenv("LITELLM_SALT_KEY", "local-test-encryption-key") monkeypatch.setattr(proxy_server, "master_key", "local-test-cookie-signing-key-32-bytes") diff --git a/tests/unit/proxy/_experimental/mcp_server/test_callback_oauth_error_responses.py b/tests/unit/proxy/mcp_server/test_callback_oauth_error_responses.py similarity index 94% rename from tests/unit/proxy/_experimental/mcp_server/test_callback_oauth_error_responses.py rename to tests/unit/proxy/mcp_server/test_callback_oauth_error_responses.py index 11ef40b9961..5a66689a5bb 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_callback_oauth_error_responses.py +++ b/tests/unit/proxy/mcp_server/test_callback_oauth_error_responses.py @@ -27,7 +27,7 @@ def _mock_mcp_client_ip(): from unittest.mock import patch with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", + "litellm.proxy.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", return_value=None, ): yield @@ -45,7 +45,7 @@ def callback_test_client(monkeypatch): monkeypatch.setenv("LITELLM_SALT_KEY", "sk-test-salt-for-LIT-2750") - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( router, ) @@ -100,7 +100,7 @@ class TestCallbackOAuthErrorResponses: """When state decodes to a trusted (loopback) redirect_uri, propagate the error back so the MCP client's OAuth library can surface it instead of timing out waiting on the loopback.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( encode_state_with_base_url, ) @@ -136,7 +136,7 @@ class TestCallbackOAuthErrorResponses: """If the state minted earlier carries a redirect_uri that the proxy no longer trusts, we must surface the error inline rather than 302-ing to an attacker-controlled URL (open-redirect).""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( encode_state_with_base_url, ) @@ -188,7 +188,7 @@ class TestCallbackOAuthErrorResponses: """Regression: the successful (``code``+``state``) flow must still redirect back to the trusted client redirect_uri with the original state preserved.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( encode_state_with_base_url, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_capabilities.py b/tests/unit/proxy/mcp_server/test_capabilities.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/test_capabilities.py rename to tests/unit/proxy/mcp_server/test_capabilities.py index e5104e7941f..55d338e64aa 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_capabilities.py +++ b/tests/unit/proxy/mcp_server/test_capabilities.py @@ -15,7 +15,7 @@ from mcp.types import ( ) from mcp_types.version import HANDSHAKE_PROTOCOL_VERSIONS -from litellm.proxy._experimental.mcp_server.capabilities import ( +from litellm.proxy.mcp_server.capabilities import ( GATEWAY_OPERATIONS, REVISION_SUPPORT, TRANSLATION_PAIRS, @@ -130,6 +130,7 @@ async def test_version_policy_gates_the_actual_sdk_handshake(versions, accepted) def test_modern_discovery_requires_opt_in_and_keeps_unsupported_features_disabled() -> None: from pydantic import TypeAdapter + from litellm.types.mcp import MCPAdvertisedVersions configured: Final = TypeAdapter(MCPAdvertisedVersions).validate_python(["2026-07-28"]) @@ -154,7 +155,7 @@ def test_modern_discovery_requires_opt_in_and_keeps_unsupported_features_disable @pytest.mark.parametrize("path", ["/mcp/sse", "/mcp/example/sse/"]) def test_modern_protocol_is_rejected_on_legacy_sse_paths(path: str, monkeypatch: pytest.MonkeyPatch) -> None: from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.server import unsupported_protocol_version + from litellm.proxy.mcp_server.server import unsupported_protocol_version monkeypatch.setitem(proxy_server.general_settings, "mcp_advertised_versions", ["2025-11-25", "2026-07-28"]) assert unsupported_protocol_version({"path": "/mcp", "headers": [(b"mcp-protocol-version", b"2026-07-28")]}) is None @@ -194,8 +195,9 @@ def test_subscription_flags_require_a_supported_protocol_pair(revision, upstream async def test_subscription_discovery_never_probes_unsupported_targets(mode, monkeypatch): from types import SimpleNamespace from unittest.mock import AsyncMock - from litellm.proxy._experimental.mcp_server import subscriptions - from litellm.proxy._experimental.mcp_server.contracts import OperationContext + + from litellm.proxy.mcp_server import subscriptions + from litellm.proxy.mcp_server.contracts import OperationContext probe = AsyncMock() target = SimpleNamespace( diff --git a/tests/unit/proxy/_experimental/mcp_server/test_catalog.py b/tests/unit/proxy/mcp_server/test_catalog.py similarity index 95% rename from tests/unit/proxy/_experimental/mcp_server/test_catalog.py rename to tests/unit/proxy/mcp_server/test_catalog.py index 4f6530e5de1..45c648b81d2 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_catalog.py +++ b/tests/unit/proxy/mcp_server/test_catalog.py @@ -19,15 +19,15 @@ from mcp.types import ( Tool, ) -from litellm.proxy._experimental.mcp_server import catalog -from litellm.proxy._experimental.mcp_server.contracts import OperationContext -from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError +from litellm.proxy.mcp_server import catalog +from litellm.proxy.mcp_server.contracts import OperationContext +from litellm.proxy.mcp_server.faults.list_outcomes import ( SERVER_OUTCOMES_META_KEY, AggregateToolListing, ServerListOk, ) -from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -55,10 +55,10 @@ async def test_catalog_refresh_keeps_selected_toolset_with_current_grants_and_co from litellm.caching.dual_cache import DualCache from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import operations - from litellm.proxy._experimental.mcp_server.server import apply_toolset_scope - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry from litellm.proxy._types import LiteLLM_ObjectPermissionTable + from litellm.proxy.mcp_server import operations + from litellm.proxy.mcp_server.server import apply_toolset_scope + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry from litellm.types.mcp_server.mcp_toolset import MCPToolset, MCPToolsetTool grants: Final = LiteLLM_ObjectPermissionTable( @@ -145,7 +145,7 @@ def rate_limit_catalog_setup( monkeypatch: pytest.MonkeyPatch, rejected_server_ids: frozenset[str] ) -> tuple[tuple[MCPServer, MCPServer], OperationContext, AsyncMock]: from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import operations + from litellm.proxy.mcp_server import operations monkeypatch.setenv("LITELLM_SALT_KEY", "catalog-rate-limit-test-key") servers: Final = ( @@ -322,7 +322,7 @@ async def test_changed_upstream_revision_requires_restart(monkeypatch): @pytest.mark.asyncio async def test_failed_upstream_remains_visible_when_other_sources_continue(monkeypatch): - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import SERVER_OUTCOMES_META_KEY + from litellm.proxy.mcp_server.faults.list_outcomes import SERVER_OUTCOMES_META_KEY monkeypatch.setenv("LITELLM_SALT_KEY", "shared-test-key") @@ -366,8 +366,8 @@ async def test_cursor_cannot_be_reused_for_a_different_catalog_kind(monkeypatch) @pytest.mark.asyncio async def test_authenticated_state_with_invalid_catalog_schema_is_rejected(monkeypatch): - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok - from litellm.proxy._experimental.mcp_server.state_tokens import seal_state + from litellm.proxy.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.state_tokens import seal_state monkeypatch.setenv("LITELLM_SALT_KEY", "shared-test-key") token = seal_state({"not": "catalog state"}, purpose="mcp.catalog.list.v1:tools/list", expires_at=200, now=100) @@ -426,8 +426,8 @@ async def test_page_limit_rejects_an_unending_upstream(monkeypatch): def test_caller_binding_covers_identity_scope_and_forwarded_credentials(change): from dataclasses import replace - from litellm.proxy._experimental.mcp_server.contracts import OperationContext from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.contracts import OperationContext original = OperationContext(_caller=UserAPIKeyAuth(api_key="synthetic-key", user_id="owner")) assert catalog._caller_scope(original, ()) != catalog._caller_scope(replace(original, **change), ()) @@ -436,8 +436,8 @@ def test_caller_binding_covers_identity_scope_and_forwarded_credentials(change): def test_caller_binding_ignores_transport_headers_and_header_case(): from dataclasses import replace - from litellm.proxy._experimental.mcp_server.contracts import OperationContext from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.contracts import OperationContext original = OperationContext( _caller=UserAPIKeyAuth(user_id="owner"), raw_headers={"Authorization": "Bearer synthetic"} @@ -510,8 +510,8 @@ async def test_gateway_tools_continuation_rejects_an_upstream_failure(monkeypatc from mcp.types import PaginatedRequestParams from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import operations - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ServerListOk, classify_list_exception + from litellm.proxy.mcp_server import operations + from litellm.proxy.mcp_server.faults.list_outcomes import ServerListOk, classify_list_exception from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -546,8 +546,8 @@ async def test_optional_gateway_catalog_reports_initial_failure_and_rejects_fail from mcp import types from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import operations - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import SERVER_OUTCOMES_META_KEY + from litellm.proxy.mcp_server import operations + from litellm.proxy.mcp_server.faults.list_outcomes import SERVER_OUTCOMES_META_KEY from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -729,6 +729,7 @@ async def test_discovery_cache_uses_upstream_freshness_and_configured_cap( upstream_ttl: int, limit: float, expected_ttl: int ) -> None: from unittest.mock import AsyncMock + from mcp.types import ListPromptsResult, Prompt from pydantic import TypeAdapter @@ -759,7 +760,8 @@ async def test_discovery_cache_uses_upstream_freshness_and_configured_cap( def test_partial_optional_catalog_never_advertises_freshness() -> None: from mcp.types import ListPromptsRequest, ListPromptsResult - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import SERVER_OUTCOMES_META_KEY + + from litellm.proxy.mcp_server.faults.list_outcomes import SERVER_OUTCOMES_META_KEY result: Final = catalog.combine_optional_catalog( ListPromptsRequest(), diff --git a/tests/unit/proxy/_experimental/mcp_server/test_client_allowlist.py b/tests/unit/proxy/mcp_server/test_client_allowlist.py similarity index 99% rename from tests/unit/proxy/_experimental/mcp_server/test_client_allowlist.py rename to tests/unit/proxy/mcp_server/test_client_allowlist.py index d1f852e504e..e05f23cbf33 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_client_allowlist.py +++ b/tests/unit/proxy/mcp_server/test_client_allowlist.py @@ -3,7 +3,7 @@ from typing import Final import pytest -from litellm.proxy._experimental.mcp_server.client_allowlist import ( +from litellm.proxy.mcp_server.client_allowlist import ( MCP_ALLOWED_CLIENTS_SETTING, MCP_CLIENT_ID_HEADER_SETTING, MCP_CLIENT_ID_JWT_FIELD_SETTING, diff --git a/tests/unit/proxy/_experimental/mcp_server/test_contracts.py b/tests/unit/proxy/mcp_server/test_contracts.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/test_contracts.py rename to tests/unit/proxy/mcp_server/test_contracts.py index adb17590d74..db30f2a4af9 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_contracts.py +++ b/tests/unit/proxy/mcp_server/test_contracts.py @@ -4,8 +4,8 @@ from typing import Final import pytest from pydantic import SecretStr -from litellm.proxy._experimental.mcp_server.operations import prepare_context from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.mcp_server.operations import prepare_context def test_guardrail_bearer_stays_out_of_egress_and_context_representations() -> None: diff --git a/tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py b/tests/unit/proxy/mcp_server/test_db_credentials.py similarity index 96% rename from tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py rename to tests/unit/proxy/mcp_server/test_db_credentials.py index ce31a83075c..db2572ef954 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py +++ b/tests/unit/proxy/mcp_server/test_db_credentials.py @@ -18,7 +18,15 @@ from unittest.mock import AsyncMock, MagicMock import pytest from prisma.models import LiteLLM_MCPServerTable as PrismaMCPServer -from litellm.proxy._experimental.mcp_server.db import ( +from litellm.proxy._types import LiteLLM_MCPServerTable, NewMCPServerRequest, UpdateMCPServerRequest +from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + SecretMapDecodeError, + decode_secret_map, + decrypt_value_helper, + encrypt_secret_map, + encrypt_value_helper, +) +from litellm.proxy.mcp_server.db import ( _decode_user_credential, _prepare_mcp_server_data, create_mcp_server, @@ -39,14 +47,6 @@ from litellm.proxy._experimental.mcp_server.db import ( store_user_oauth_credential, update_mcp_server, ) -from litellm.proxy._types import LiteLLM_MCPServerTable, NewMCPServerRequest, UpdateMCPServerRequest -from litellm.proxy.common_utils.encrypt_decrypt_utils import ( - SecretMapDecodeError, - decode_secret_map, - decrypt_value_helper, - encrypt_secret_map, - encrypt_value_helper, -) from litellm.types.mcp import MCPAuth, MCPTransport SALT_KEY = "test-salt-key-for-byok-credential-tests-1234" @@ -121,7 +121,7 @@ def _identity_server(**overrides): ], ) def test_mcp_oauth_token_identity_changes_on_mint_relevant_fields(overrides): - from litellm.proxy._experimental.mcp_server.db import mcp_oauth_token_identity + from litellm.proxy.mcp_server.db import mcp_oauth_token_identity assert mcp_oauth_token_identity(_identity_server()) != mcp_oauth_token_identity(_identity_server(**overrides)) @@ -134,13 +134,13 @@ def test_mcp_oauth_token_identity_changes_on_mint_relevant_fields(overrides): ], ) def test_mcp_oauth_token_identity_stable_on_non_mint_fields(overrides): - from litellm.proxy._experimental.mcp_server.db import mcp_oauth_token_identity + from litellm.proxy.mcp_server.db import mcp_oauth_token_identity assert mcp_oauth_token_identity(_identity_server()) == mcp_oauth_token_identity(_identity_server(**overrides)) def _encrypted_creds_json(client_id: str = "cid", client_secret: str = "csec") -> str: - from litellm.proxy._experimental.mcp_server.db import encrypt_credentials + from litellm.proxy.mcp_server.db import encrypt_credentials encrypted = encrypt_credentials( credentials={"client_id": client_id, "client_secret": client_secret, "scopes": ["a"]}, @@ -154,7 +154,7 @@ def test_mcp_oauth_token_identity_stable_across_reencryption(): saves of the SAME plaintext produce different ciphertext. The identity must compare decrypted values; comparing ciphertext would flag every routine save as a mint-relevant change and purge per-user tokens that are still valid.""" - from litellm.proxy._experimental.mcp_server.db import mcp_oauth_token_identity + from litellm.proxy.mcp_server.db import mcp_oauth_token_identity first = _encrypted_creds_json() second = _encrypted_creds_json() @@ -166,7 +166,7 @@ def test_mcp_oauth_token_identity_stable_across_reencryption(): def test_mcp_oauth_token_identity_detects_change_under_encryption(): - from litellm.proxy._experimental.mcp_server.db import mcp_oauth_token_identity + from litellm.proxy.mcp_server.db import mcp_oauth_token_identity unchanged = _identity_server(credentials=_encrypted_creds_json()) changed = _identity_server(credentials=_encrypted_creds_json(client_id="other")) @@ -174,7 +174,7 @@ def test_mcp_oauth_token_identity_detects_change_under_encryption(): def test_mcp_oauth_token_identity_reads_client_and_scopes_from_json_string_credentials(): - from litellm.proxy._experimental.mcp_server.db import mcp_oauth_token_identity + from litellm.proxy.mcp_server.db import mcp_oauth_token_identity credentials: Final = json.dumps( {"client_id": "cid", "client_secret": "csec", "scopes": ["a"], "upstream_resource": "api://audience"} @@ -198,7 +198,7 @@ def test_mcp_oauth_token_identity_reads_client_and_scopes_from_json_string_crede @pytest.mark.parametrize("credentials", ["[]", '["client_id"]', '"cid"', "5", "null", "not json"]) def test_mcp_oauth_token_identity_treats_stored_credentials_that_are_not_a_json_object_as_empty(credentials): - from litellm.proxy._experimental.mcp_server.db import mcp_oauth_token_identity + from litellm.proxy.mcp_server.db import mcp_oauth_token_identity assert mcp_oauth_token_identity(_identity_server(credentials=credentials)) == ( "https://up.example.com/mcp", @@ -235,7 +235,7 @@ def _byok_row(user_id: str, server_id: str = "srv-1"): @pytest.mark.asyncio async def test_purge_user_oauth_credentials_for_server_invalidates_each_user(): """The purge must route each (user, server) row through the invalidator exactly once.""" - from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server + from litellm.proxy.mcp_server.db import purge_user_oauth_credentials_for_server prisma = MagicMock() prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[_oauth_row("alice"), _oauth_row("bob")]) @@ -259,7 +259,7 @@ async def test_purge_user_oauth_credentials_for_server_invalidates_each_user(): async def test_list_server_user_credentials_types_each_row_without_leaking_the_secret(): """The admin view of one server's stored credentials names the user and the kind of credential (OAuth2 vs BYOK) and echoes OAuth expiry, but never the token or key itself.""" - from litellm.proxy._experimental.mcp_server.db import list_server_user_credentials + from litellm.proxy.mcp_server.db import list_server_user_credentials oauth_row = _legacy_row( json.dumps( @@ -307,7 +307,7 @@ async def test_purge_user_oauth_credentials_for_server_spares_byok_rows(): """Regression: the purge used to delete_many on server_id alone, wiping BYOK API keys that share the LiteLLM_MCPUserCredentials table. Only rows holding an OAuth2 payload may be deleted (one batched query filtered to their user_ids), and only their users' token caches invalidated.""" - from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server + from litellm.proxy.mcp_server.db import purge_user_oauth_credentials_for_server prisma = MagicMock() prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[_byok_row("carol"), _oauth_row("alice")]) @@ -330,7 +330,7 @@ async def test_purge_user_oauth_credentials_for_server_spares_byok_rows(): @pytest.mark.asyncio async def test_purge_user_oauth_credentials_for_server_all_byok_is_noop(): """An api_key (BYOK-only) server whose identity tuple changes (e.g. its url) must purge nothing.""" - from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server + from litellm.proxy.mcp_server.db import purge_user_oauth_credentials_for_server prisma = MagicMock() prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[_byok_row("carol"), _byok_row("dave")]) @@ -348,8 +348,8 @@ async def test_purge_user_oauth_credentials_for_server_defaults_to_manager_inval invalidate_user_oauth_token_cache, the single point covering both the legacy per-user token cache and the v2 per-user OAuth token store; a wrong or no-op default silently leaves every cache serving tokens minted for the superseded config.""" - from litellm.proxy._experimental.mcp_server import mcp_server_manager - from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server + from litellm.proxy.mcp_server import mcp_server_manager + from litellm.proxy.mcp_server.db import purge_user_oauth_credentials_for_server prisma = MagicMock() prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[_oauth_row("alice")]) @@ -370,8 +370,8 @@ async def test_purge_user_oauth_credentials_for_server_defaults_to_manager_inval @pytest.mark.asyncio async def test_purge_user_oauth_credentials_for_server_logs_raced_rows(monkeypatch): - from litellm.proxy._experimental.mcp_server import db as db_module - from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server + from litellm.proxy.mcp_server import db as db_module + from litellm.proxy.mcp_server.db import purge_user_oauth_credentials_for_server prisma = MagicMock() prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[_oauth_row("alice")]) @@ -390,7 +390,7 @@ async def test_delete_mcp_server_invalidates_cached_tokens_for_enumerated_users( """Deleting a server must invalidate each enumerated user's cached per-user token: the caches are keyed by (user_id, server_id), so a re-created server reusing the same server_id would otherwise serve tokens minted for the deleted server until TTL.""" - from litellm.proxy._experimental.mcp_server.db import delete_mcp_server + from litellm.proxy.mcp_server.db import delete_mcp_server prisma = MagicMock() prisma.db.litellm_mcpservertable.delete = AsyncMock(return_value=MagicMock(server_id="srv-1")) @@ -411,7 +411,7 @@ async def test_delete_mcp_server_invalidates_cached_tokens_for_enumerated_users( @pytest.mark.asyncio async def test_delete_mcp_server_returns_none_without_cleanup_when_server_missing(): - from litellm.proxy._experimental.mcp_server.db import delete_mcp_server + from litellm.proxy.mcp_server.db import delete_mcp_server prisma = MagicMock() prisma.db.litellm_mcpservertable.delete = AsyncMock(return_value=None) @@ -425,7 +425,7 @@ async def test_delete_mcp_server_returns_none_without_cleanup_when_server_missin @pytest.mark.asyncio async def test_purge_user_oauth_credentials_for_server_noop_when_empty(): - from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server + from litellm.proxy.mcp_server.db import purge_user_oauth_credentials_for_server prisma = MagicMock() prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[]) @@ -1164,7 +1164,7 @@ def test_expiry_missing_expires_at_is_never_expired(): @pytest.mark.asyncio async def test_resolve_returns_valid_token_without_refreshing(monkeypatch): # A token good for 10 minutes must be returned as-is, with no refresh call. - import litellm.proxy._experimental.mcp_server.db as db_mod + import litellm.proxy.mcp_server.db as db_mod refresh = AsyncMock() monkeypatch.setattr(db_mod, "refresh_user_oauth_token", refresh) @@ -1183,7 +1183,7 @@ async def test_resolve_returns_valid_token_without_refreshing(monkeypatch): async def test_resolve_refreshes_expired_token_with_refresh_token(monkeypatch): # The core regression: an expired OBO cred with a refresh_token must mint a # new token rather than returning None (which left the UI tool list empty). - import litellm.proxy._experimental.mcp_server.db as db_mod + import litellm.proxy.mcp_server.db as db_mod refreshed = _oauth_cred(access_token="at-fresh", refresh_token="rt-2", expires_in_seconds=3600) refresh = AsyncMock(return_value=refreshed) @@ -1202,7 +1202,7 @@ async def test_resolve_refreshes_expired_token_with_refresh_token(monkeypatch): async def test_resolve_refreshes_token_expiring_within_buffer(monkeypatch): # A token still technically valid (30s left) but inside the 60s buffer must # be proactively refreshed, not handed back. - import litellm.proxy._experimental.mcp_server.db as db_mod + import litellm.proxy.mcp_server.db as db_mod refreshed = _oauth_cred(access_token="at-fresh", expires_in_seconds=3600) refresh = AsyncMock(return_value=refreshed) @@ -1220,7 +1220,7 @@ async def test_resolve_refreshes_token_expiring_within_buffer(monkeypatch): @pytest.mark.asyncio async def test_resolve_returns_none_when_expired_without_refresh_token(monkeypatch): # No refresh_token means nothing to refresh with — return None, never call refresh. - import litellm.proxy._experimental.mcp_server.db as db_mod + import litellm.proxy.mcp_server.db as db_mod refresh = AsyncMock() monkeypatch.setattr(db_mod, "refresh_user_oauth_token", refresh) @@ -1237,7 +1237,7 @@ async def test_resolve_returns_none_when_expired_without_refresh_token(monkeypat @pytest.mark.asyncio async def test_resolve_returns_none_when_refresh_fails(monkeypatch): # A failed refresh (provider returns nothing usable) must surface as None. - import litellm.proxy._experimental.mcp_server.db as db_mod + import litellm.proxy.mcp_server.db as db_mod refresh = AsyncMock(return_value=None) monkeypatch.setattr(db_mod, "refresh_user_oauth_token", refresh) @@ -1253,7 +1253,7 @@ async def test_resolve_returns_none_when_refresh_fails(monkeypatch): @pytest.mark.asyncio async def test_resolve_returns_none_for_missing_credential(monkeypatch): - import litellm.proxy._experimental.mcp_server.db as db_mod + import litellm.proxy.mcp_server.db as db_mod refresh = AsyncMock() monkeypatch.setattr(db_mod, "refresh_user_oauth_token", refresh) @@ -1301,7 +1301,7 @@ def _refresh_server(**overrides): async def _run_refresh(monkeypatch, server, response_body=None): - import litellm.proxy._experimental.mcp_server.db as db_mod + import litellm.proxy.mcp_server.db as db_mod captured: dict = {} @@ -1411,7 +1411,7 @@ async def test_rotate_user_env_vars_skips_undecryptable_rows(): async def test_refresh_user_oauth_token_uses_client_secret_basic(monkeypatch): """LIT-4091: a per-user refresh against a server with token_endpoint_auth_method=client_secret_basic sends HTTP Basic and keeps the secret out of the body.""" - import litellm.proxy._experimental.mcp_server.db as db_mod + import litellm.proxy.mcp_server.db as db_mod server = MagicMock(oauth_identity_binding=None) server.token_url = "https://idp.example.com/oauth2/token" @@ -1450,7 +1450,7 @@ async def test_refresh_user_oauth_token_uses_client_secret_basic(monkeypatch): async def test_refresh_user_oauth_token_defaults_to_client_secret_post(monkeypatch): """Backward compatibility: with no token_endpoint_auth_method the refresh keeps credentials in the body (client_secret_post) and sends no Authorization header.""" - import litellm.proxy._experimental.mcp_server.db as db_mod + import litellm.proxy.mcp_server.db as db_mod server = MagicMock(oauth_identity_binding=None) server.token_url = "https://idp.example.com/oauth2/token" @@ -1537,7 +1537,7 @@ async def test_master_key_rotation_reencrypts_oauth_client_store(monkeypatch): force a full re-authorization.""" import litellm.proxy.common_utils.encrypt_decrypt_utils as enc from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - from litellm.proxy._experimental.mcp_server.db import ( + from litellm.proxy.mcp_server.db import ( decrypt_credentials, encrypt_credentials, rotate_mcp_server_credentials_master_key, @@ -1580,7 +1580,7 @@ async def test_delete_mcp_server_cleans_oauth_client_store(): """Deleting a server must remove its server-scoped DCR client store entry alongside the per-user credential and env-var rows, or a re-created server reusing the same server_id would inherit the deleted server's OAuth client.""" - from litellm.proxy._experimental.mcp_server.db import delete_mcp_server + from litellm.proxy.mcp_server.db import delete_mcp_server prisma = MagicMock() prisma.db.litellm_mcpservertable.delete = AsyncMock(return_value=SimpleNamespace(server_id="s1")) @@ -1603,7 +1603,7 @@ def test_mcp_oauth_token_identity_changes_when_only_upstream_resource_is_edited( calling tools with the previous audience's token until it expires, which is the token-reuse RFC 8707 exists to stop. """ - from litellm.proxy._experimental.mcp_server.db import mcp_oauth_token_identity + from litellm.proxy.mcp_server.db import mcp_oauth_token_identity creds = {"client_id": "cid", "client_secret": "csec", "scopes": ["a"]} unset = _identity_server(credentials=dict(creds)) @@ -1736,7 +1736,8 @@ async def test_enforcement_rejects_preexisting_unverified_credential(): @pytest.mark.asyncio async def test_refresh_identity_rejection_returns_reauthentication_without_persisting(monkeypatch): from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server import db as module + + from litellm.proxy.mcp_server import db as module validator = AsyncMock(side_effect=HTTPException(status_code=403, detail="oauth_principal_mismatch")) monkeypatch.setattr(module, "enforce_oauth_identity_binding", validator) @@ -1752,9 +1753,9 @@ async def test_refresh_identity_rejection_returns_reauthentication_without_persi async def test_verified_legacy_cache_reads_avoid_database_and_reject_policy_changes(monkeypatch): from litellm.caching.caching import DualCache from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import db as module - from litellm.proxy._experimental.mcp_server.oauth2_token_cache import mcp_per_user_token_cache - from litellm.proxy._experimental.mcp_server.oauth_identity_binding import current_binding_proof + from litellm.proxy.mcp_server import db as module + from litellm.proxy.mcp_server.oauth2_token_cache import mcp_per_user_token_cache + from litellm.proxy.mcp_server.oauth_identity_binding import current_binding_proof from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -1789,8 +1790,8 @@ async def test_verified_legacy_cache_reads_avoid_database_and_reject_policy_chan async def test_unverified_legacy_cache_cannot_bypass_enforcement(monkeypatch): from litellm.caching.caching import DualCache from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import db as module - from litellm.proxy._experimental.mcp_server.oauth2_token_cache import mcp_per_user_token_cache + from litellm.proxy.mcp_server import db as module + from litellm.proxy.mcp_server.oauth2_token_cache import mcp_per_user_token_cache from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -1815,9 +1816,9 @@ async def test_saved_cimd_grant_refreshes_from_encrypted_storage_on_a_fresh_repl from urllib.parse import parse_qs from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import db as db_module - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import _store_per_user_token_server_side - from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import ( + from litellm.proxy.mcp_server import db as db_module + from litellm.proxy.mcp_server.discoverable_endpoints import _store_per_user_token_server_side + from litellm.proxy.mcp_server.outbound_credentials.per_user_oauth_store import ( LazyPerUserOAuthTokenStore, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -1869,7 +1870,7 @@ async def test_saved_cimd_grant_refreshes_from_encrypted_storage_on_a_fresh_repl @pytest.mark.asyncio @pytest.mark.parametrize("provider_scope, expected", [(None, ["a"]), ("b", ["b"]), ("", [])]) async def test_silent_refresh_preserves_omitted_scope_but_accepts_explicit_grant(monkeypatch, provider_scope, expected): - import litellm.proxy._experimental.mcp_server.db as db_mod + import litellm.proxy.mcp_server.db as db_mod await _run_refresh(monkeypatch, _refresh_server(), { "access_token": "new-access", diff --git a/tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/unit/proxy/mcp_server/test_discoverable_endpoints.py similarity index 90% rename from tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py rename to tests/unit/proxy/mcp_server/test_discoverable_endpoints.py index cedfa296515..c17f4b392cd 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/unit/proxy/mcp_server/test_discoverable_endpoints.py @@ -63,8 +63,8 @@ def _stored_grant(access_token="access-token", refresh_token=None, expires_in_se ], ) async def test_vendor_credential_state_agrees_with_egress_token_resolution(monkeypatch, fields, egress_has_token): - from litellm.proxy._experimental.mcp_server import db as mcp_db - from litellm.proxy._experimental.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server import db as mcp_db + from litellm.proxy.mcp_server import discoverable_endpoints monkeypatch.setattr(mcp_db, "MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS", 60) credential = _stored_grant(**fields) if fields is not None else None @@ -97,7 +97,7 @@ def mock_mcp_client_ip(): have proper client IP context. """ with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", + "litellm.proxy.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", return_value=None, ): yield @@ -111,8 +111,8 @@ def isolate_global_mcp_registry(monkeypatch): restore, entries leak into other test modules sharing the same worker and break assertions over the full registry contents. """ - from litellm.proxy._experimental.mcp_server.catalog import CatalogSnapshots - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.catalog import CatalogSnapshots + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager monkeypatch.setattr(global_mcp_server_manager, "catalog", CatalogSnapshots(global_mcp_server_manager)) snapshot = dict(global_mcp_server_manager.registry) @@ -169,8 +169,8 @@ def _resolved_oauth_metadata(): async def test_authorize_resolves_cold_oauth_metadata(monkeypatch): """The route hands the registered server to the flow, whose deferred-discovery join resolves the cold metadata; the redirect must land on the discovered authorization endpoint.""" - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager monkeypatch.setenv("LITELLM_SALT_KEY", "sk-test-salt-for-lit-6255") server = _unresolved_oauth_server() @@ -199,8 +199,8 @@ async def test_authorize_resolves_cold_oauth_metadata(monkeypatch): async def test_token_resolves_cold_oauth_metadata(): """The route hands the registered server to the exchange, whose deferred-discovery join resolves the cold metadata; the exchange must post to the discovered token endpoint.""" - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager server = _unresolved_oauth_server() global_mcp_server_manager.registry[server.server_id] = server @@ -241,8 +241,8 @@ async def test_token_resolves_cold_oauth_metadata(): async def test_register_resolves_cold_oauth_metadata(): """The route hands the registered server to the registration flow, whose deferred-discovery join resolves the cold metadata; DCR must post to the discovered registration endpoint.""" - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager server = _unresolved_oauth_server().model_copy(update={"client_id": None}) global_mcp_server_manager.registry[server.server_id] = server @@ -281,9 +281,9 @@ async def test_register_route_bridge_missing_registration_url_joins_discovery(): endpoint instead of short-circuiting to dummy credentials because authorization resolves.""" import json - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -340,9 +340,9 @@ async def test_token_route_bridge_missing_registration_url_joins_discovery(): registration endpoint must rejoin discovery at the exchange: the relay-vs-callback arm hinges on the registration url, so skipping discovery would swap the client's own redirect_uri for the gateway callback and the upstream would reject the code.""" - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -402,7 +402,7 @@ def trust_xff(): ``test_get_request_base_url_xff_trust_gate``. """ with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.IPAddressUtils.is_request_from_trusted_proxy", + "litellm.proxy.mcp_server.discoverable_endpoints.IPAddressUtils.is_request_from_trusted_proxy", return_value=True, ): yield @@ -414,13 +414,13 @@ async def test_authorize_endpoint_includes_response_type(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -451,7 +451,7 @@ async def test_authorize_endpoint_includes_response_type(): mock_request.headers = {} # Mock the encryption functions to avoid needing a signing key - with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: + with patch("litellm.proxy.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: mock_encrypt.return_value = "mocked_encrypted_state" # Call authorize endpoint @@ -481,13 +481,13 @@ async def test_authorize_endpoint_allows_client_forwarded_modes(auth_type_value) try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -512,7 +512,7 @@ async def test_authorize_endpoint_allows_client_forwarded_modes(auth_type_value) mock_request.base_url = "https://litellm.example.com/" mock_request.headers = {} - with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: + with patch("litellm.proxy.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: mock_encrypt.return_value = "mocked_encrypted_state" response = await authorize( @@ -534,13 +534,13 @@ async def test_authorize_endpoint_preserves_existing_query_params(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -568,7 +568,7 @@ async def test_authorize_endpoint_preserves_existing_query_params(): mock_request.base_url = "https://litellm.example.com/" mock_request.headers = {} - with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: + with patch("litellm.proxy.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: mock_encrypt.return_value = "mocked_encrypted_state" response = await authorize( @@ -610,7 +610,7 @@ async def test_authorize_preserves_contract_with_provider_defaults( ) -> None: from urllib.parse import parse_qs, urlparse - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import authorize_with_server + from litellm.proxy.mcp_server.discoverable_endpoints import authorize_with_server monkeypatch.setenv("LITELLM_SALT_KEY", "isolated-oauth-test-salt") is_bridge: Final = auth_mode != "direct" @@ -655,13 +655,13 @@ async def test_authorize_endpoint_forwards_pkce_parameters(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -692,7 +692,7 @@ async def test_authorize_endpoint_forwards_pkce_parameters(): mock_request.headers = {} # Mock the encryption function - with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: + with patch("litellm.proxy.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: mock_encrypt.return_value = "mocked_encrypted_state_with_pkce" # Call authorize endpoint with PKCE parameters @@ -724,13 +724,13 @@ async def test_token_endpoint_forwards_code_verifier(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( token_endpoint, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -774,7 +774,7 @@ async def test_token_endpoint_forwards_code_verifier(): from unittest.mock import AsyncMock with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client" + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client" ) as mock_get_client: mock_async_client = MagicMock() # Use AsyncMock for the async post method @@ -819,10 +819,10 @@ async def test_register_client_without_mcp_server_name_returns_dummy(server_name try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -835,7 +835,7 @@ async def test_register_client_without_mcp_server_name_returns_dummy(server_name mock_request.base_url = "https://proxy.litellm.example/" mock_request.headers = {} with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.read_request_body", + "litellm.proxy.mcp_server.discoverable_endpoints.read_request_body", new=AsyncMock(return_value={}), ): result = await register_client(request=mock_request, mcp_server_name=server_name) @@ -853,13 +853,13 @@ async def test_register_client_returns_existing_server_credentials(use_root): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -886,7 +886,7 @@ async def test_register_client_returns_existing_server_credentials(use_root): try: with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.read_request_body", + "litellm.proxy.mcp_server.discoverable_endpoints.read_request_body", new=AsyncMock(return_value={}), ): result = await register_client( @@ -907,13 +907,13 @@ async def test_register_client_remote_registration_success(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -958,11 +958,11 @@ async def test_register_client_remote_registration_success(): try: with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.read_request_body", + "litellm.proxy.mcp_server.discoverable_endpoints.read_request_body", new=AsyncMock(return_value=request_payload), ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), ): @@ -997,9 +997,9 @@ async def test_register_client_non_bridge_returns_client_redirect_not_gateway_ca verbatim makes a spec-compliant DCR client adopt /callback as its own redirect and loop.""" from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import register_client - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import register_client + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -1044,11 +1044,11 @@ async def test_register_client_non_bridge_returns_client_redirect_not_gateway_ca try: with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.read_request_body", + "litellm.proxy.mcp_server.discoverable_endpoints.read_request_body", new=AsyncMock(return_value=request_payload), ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), ): @@ -1071,9 +1071,9 @@ async def test_register_client_admin_client_id_echoes_client_redirect_uris(): the gateway /callback and self-redirect loop (#33699).""" from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import register_client - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import register_client + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -1100,7 +1100,7 @@ async def test_register_client_admin_client_id_echoes_client_redirect_uris(): try: with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.read_request_body", + "litellm.proxy.mcp_server.discoverable_endpoints.read_request_body", new=AsyncMock(return_value={"redirect_uris": [client_redirect]}), ): result = await register_client(request=mock_request, mcp_server_name=oauth2_server.server_name) @@ -1126,14 +1126,14 @@ async def test_dcr_full_loop_lands_on_client_redirect_not_gateway_callback(monke from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( _oauth_state_cookie_name, authorize_with_server, callback, register_client, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager - from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -1176,7 +1176,7 @@ async def test_dcr_full_loop_lands_on_client_redirect_not_gateway_callback(monke try: with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.read_request_body", + "litellm.proxy.mcp_server.discoverable_endpoints.read_request_body", new=AsyncMock( return_value={ "client_name": "Open WebUI", @@ -1187,7 +1187,7 @@ async def test_dcr_full_loop_lands_on_client_redirect_not_gateway_callback(monke ), ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), ): @@ -1249,9 +1249,9 @@ async def test_authorize_rejects_untrusted_cross_origin_redirect_with_allowlist_ the mechanism a legitimate separate-origin DCR client needs, not only to PROXY_BASE_URL.""" from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import authorize - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import authorize + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -1310,9 +1310,9 @@ async def test_register_client_malformed_redirect_uris_falls_back_to_gateway_cal callback so downstream never iterates a string as URIs or leaks non-string element types (#33699).""" from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import register_client - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import register_client + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -1337,7 +1337,7 @@ async def test_register_client_malformed_redirect_uris_falls_back_to_gateway_cal try: with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.read_request_body", + "litellm.proxy.mcp_server.discoverable_endpoints.read_request_body", new=AsyncMock(return_value={"redirect_uris": malformed_redirect_uris}), ): result = await register_client(request=mock_request, mcp_server_name=oauth2_server.server_name) @@ -1353,9 +1353,9 @@ async def test_register_client_valid_multi_redirect_uris_all_echoed(): unchanged, so the element-type guard does not narrow a legitimate multi-entry list (#33699).""" from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import register_client - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import register_client + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -1381,7 +1381,7 @@ async def test_register_client_valid_multi_redirect_uris_all_echoed(): client_redirects = ["https://app.example/cb", "http://127.0.0.1:6274/callback"] try: with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.read_request_body", + "litellm.proxy.mcp_server.discoverable_endpoints.read_request_body", new=AsyncMock(return_value={"redirect_uris": client_redirects}), ): result = await register_client(request=mock_request, mcp_server_name=oauth2_server.server_name) @@ -1405,13 +1405,13 @@ async def test_register_client_persists_dcr_client_identity(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client_with_server, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -1452,11 +1452,11 @@ async def test_register_client_persists_dcr_client_identity(): with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=prisma), - patch("litellm.proxy._experimental.mcp_server.db.update_mcp_server", new=mock_update), + patch("litellm.proxy.mcp_server.db.update_mcp_server", new=mock_update), patch.object(global_mcp_server_manager, "update_server", new=mock_update_server), ): response = await register_client_with_server( @@ -1502,13 +1502,13 @@ async def _register_persistence_attempted_for_auth_type(auth_type: MCPAuth) -> b to the browser either way, since the caller needs the minted client to run its own flow.""" from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client_with_server, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -1545,11 +1545,11 @@ async def _register_persistence_attempted_for_auth_type(auth_type: MCPAuth) -> b with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=prisma), - patch("litellm.proxy._experimental.mcp_server.db.update_mcp_server", new=mock_update), + patch("litellm.proxy.mcp_server.db.update_mcp_server", new=mock_update), patch.object(global_mcp_server_manager, "update_server", new=AsyncMock()), ): response = await register_client_with_server( @@ -1601,13 +1601,13 @@ async def test_register_client_persists_only_to_its_own_row_when_another_server_ upstream registration (adopting the sibling's client) or persist onto the wrong row.""" from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client_with_server, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer shared_url = "https://provider.example/mcp" @@ -1648,12 +1648,12 @@ async def test_register_client_persists_only_to_its_own_row_when_another_server_ with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), - patch("litellm.proxy._experimental.mcp_server.db.get_mcp_server", new=AsyncMock(side_effect=_get_row)), - patch("litellm.proxy._experimental.mcp_server.db.update_mcp_server", new=mock_update), + patch("litellm.proxy.mcp_server.db.get_mcp_server", new=AsyncMock(side_effect=_get_row)), + patch("litellm.proxy.mcp_server.db.update_mcp_server", new=mock_update), patch.object(global_mcp_server_manager, "update_server", new=AsyncMock()), ): response = await register_client_with_server( @@ -1684,13 +1684,13 @@ async def test_register_client_does_not_clobber_token_url_when_absent(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client_with_server, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -1727,11 +1727,11 @@ async def test_register_client_does_not_clobber_token_url_when_absent(): with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=prisma), - patch("litellm.proxy._experimental.mcp_server.db.update_mcp_server", new=mock_update), + patch("litellm.proxy.mcp_server.db.update_mcp_server", new=mock_update), patch.object(global_mcp_server_manager, "update_server", new=mock_update_server), ): await register_client_with_server( @@ -1755,13 +1755,13 @@ async def test_register_client_reuses_persisted_client_id_for_non_admin_when_reg try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client_with_server, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -1795,16 +1795,16 @@ async def test_register_client_reuses_persisted_client_id_for_non_admin_when_reg with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + "litellm.proxy.mcp_server.db.get_mcp_server", new=mock_get_mcp_server, ), patch( - "litellm.proxy._experimental.mcp_server.db.update_mcp_server", + "litellm.proxy.mcp_server.db.update_mcp_server", new=mock_update_mcp_server, ), patch.object(global_mcp_server_manager, "update_server", new=mock_update_server), @@ -1831,13 +1831,13 @@ async def test_register_client_reuse_refreshes_request_server_when_manager_updat try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client_with_server, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -1874,12 +1874,12 @@ async def test_register_client_reuse_refreshes_request_server_when_manager_updat with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + "litellm.proxy.mcp_server.db.get_mcp_server", new=mock_get_mcp_server, ), patch.object(global_mcp_server_manager, "update_server", new=mock_update_server), @@ -1908,13 +1908,13 @@ async def test_register_client_returns_reused_client_when_concurrent_persist_win try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client_with_server, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -1955,16 +1955,16 @@ async def test_register_client_returns_reused_client_when_concurrent_persist_win with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + "litellm.proxy.mcp_server.db.get_mcp_server", new=mock_get_mcp_server, ), patch( - "litellm.proxy._experimental.mcp_server.db.update_mcp_server", + "litellm.proxy.mcp_server.db.update_mcp_server", new=mock_update_mcp_server, ), patch.object(global_mcp_server_manager, "update_server", new=mock_update_server), @@ -2020,10 +2020,10 @@ async def test_register_client_re_registers_when_persisted_redirect_uri_no_longe try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client_with_server, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -2057,16 +2057,16 @@ async def test_register_client_re_registers_when_persisted_redirect_uri_no_longe with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + "litellm.proxy.mcp_server.db.get_mcp_server", new=mock_get_mcp_server, ), patch( - "litellm.proxy._experimental.mcp_server.db.update_mcp_server", + "litellm.proxy.mcp_server.db.update_mcp_server", new=mock_update_mcp_server, ), patch.object(global_mcp_server_manager, "update_server", new=mock_update_server), @@ -2106,7 +2106,7 @@ async def test_register_client_grandfathers_persisted_client_without_recorded_re try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client_with_server, ) except ImportError: @@ -2128,16 +2128,16 @@ async def test_register_client_grandfathers_persisted_client_without_recorded_re with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + "litellm.proxy.mcp_server.db.get_mcp_server", new=mock_get_mcp_server, ), patch( - "litellm.proxy._experimental.mcp_server.db.update_mcp_server", + "litellm.proxy.mcp_server.db.update_mcp_server", new=mock_update_mcp_server, ), ): @@ -2164,7 +2164,7 @@ async def test_register_client_keeps_persisted_client_when_recorded_redirect_uri try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client_with_server, ) except ImportError: @@ -2189,16 +2189,16 @@ async def test_register_client_keeps_persisted_client_when_recorded_redirect_uri with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + "litellm.proxy.mcp_server.db.get_mcp_server", new=mock_get_mcp_server, ), patch( - "litellm.proxy._experimental.mcp_server.db.update_mcp_server", + "litellm.proxy.mcp_server.db.update_mcp_server", new=mock_update_mcp_server, ), ): @@ -2226,10 +2226,10 @@ async def test_register_client_non_admin_reuses_persisted_client_despite_redirec try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client_with_server, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -2254,12 +2254,12 @@ async def test_register_client_non_admin_reuses_persisted_client_despite_redirec with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + "litellm.proxy.mcp_server.db.get_mcp_server", new=mock_get_mcp_server, ), patch.object(global_mcp_server_manager, "update_server", new=mock_update_server), @@ -2288,8 +2288,8 @@ async def test_register_client_without_replace_authority_adopts_concurrently_per try: from fastapi import Request - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -2310,23 +2310,23 @@ async def test_register_client_without_replace_authority_adopts_concurrently_per with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch.object(global_mcp_server_manager, "is_config_declared_server", return_value=True), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), - patch("litellm.proxy._experimental.mcp_server.db.get_mcp_server", new=AsyncMock(return_value=None)), - patch("litellm.proxy._experimental.mcp_server.db.update_mcp_server", new=AsyncMock(return_value=None)), + patch("litellm.proxy.mcp_server.db.get_mcp_server", new=AsyncMock(return_value=None)), + patch("litellm.proxy.mcp_server.db.update_mcp_server", new=AsyncMock(return_value=None)), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server_oauth_client_credentials", + "litellm.proxy.mcp_server.db.get_mcp_server_oauth_client_credentials", new=AsyncMock(side_effect=[None, winner]), ), patch( - "litellm.proxy._experimental.mcp_server.db.create_mcp_server_oauth_client_credentials_if_absent", + "litellm.proxy.mcp_server.db.create_mcp_server_oauth_client_credentials_if_absent", new=AsyncMock(return_value=False), ), patch( - "litellm.proxy._experimental.mcp_server.db.upsert_mcp_server_oauth_client_credentials", + "litellm.proxy.mcp_server.db.upsert_mcp_server_oauth_client_credentials", new=mock_upsert, ), ): @@ -2351,7 +2351,7 @@ async def test_persist_without_replace_authority_never_overwrites_a_row_client_i """A DB-declared server already holds a shared DCR client that this process cannot adopt (it was bound to another server URL). A caller without replace authority must get a failure, not fall through to update_mcp_server and overwrite the client every signed-in user depends on.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _persist_dcr_client_registration, ) @@ -2362,8 +2362,8 @@ async def test_persist_without_replace_authority_never_overwrites_a_row_client_i with ( patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), - patch("litellm.proxy._experimental.mcp_server.db.get_mcp_server", new=AsyncMock(return_value=row)), - patch("litellm.proxy._experimental.mcp_server.db.update_mcp_server", new=mock_update), + patch("litellm.proxy.mcp_server.db.get_mcp_server", new=AsyncMock(return_value=row)), + patch("litellm.proxy.mcp_server.db.update_mcp_server", new=mock_update), ): result = await _persist_dcr_client_registration( server, {"client_id": "my-client"}, "https://proxy.litellm.example/callback", replace_stale_client=False @@ -2383,13 +2383,13 @@ async def test_register_client_reuses_existing_client_id_without_re_dcr(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -2428,11 +2428,11 @@ async def test_register_client_reuses_existing_client_id_without_re_dcr(): try: with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.read_request_body", + "litellm.proxy.mcp_server.discoverable_endpoints.read_request_body", new=AsyncMock(return_value=request_payload), ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), ): @@ -2483,10 +2483,10 @@ async def test_public_register_route_persists_first_dcr_client_when_origin_is_co registers: the caller's grant types, response types, auth method and name are not sent upstream, so an anonymous caller cannot shape or phish the shared client.""" try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -2511,20 +2511,20 @@ async def test_public_register_route_persists_first_dcr_client_when_origin_is_co try: with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.read_request_body", + "litellm.proxy.mcp_server.discoverable_endpoints.read_request_body", new=AsyncMock(return_value=request_payload), ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + "litellm.proxy.mcp_server.db.get_mcp_server", new=AsyncMock(return_value=persisted_row), ), patch.object(global_mcp_server_manager, "update_server", new=mock_update_server), - patch("litellm.proxy._experimental.mcp_server.db.update_mcp_server", new=mock_update), + patch("litellm.proxy.mcp_server.db.update_mcp_server", new=mock_update), ): response = await register_client(request=mock_request, mcp_server_name=oauth2_server.server_name) finally: @@ -2548,10 +2548,10 @@ async def test_public_register_route_persists_first_dcr_client_for_config_server must reach the server-scoped OAuth client store, so a token refresh has a client_id and a second connect reuses the same client instead of minting another.""" try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -2578,27 +2578,27 @@ async def test_public_register_route_persists_first_dcr_client_for_config_server try: with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.read_request_body", + "litellm.proxy.mcp_server.discoverable_endpoints.read_request_body", new=AsyncMock(return_value=request_payload), ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch.object(global_mcp_server_manager, "is_config_declared_server", return_value=True), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), - patch("litellm.proxy._experimental.mcp_server.db.get_mcp_server", new=AsyncMock(return_value=None)), - patch("litellm.proxy._experimental.mcp_server.db.update_mcp_server", new=AsyncMock(return_value=None)), + patch("litellm.proxy.mcp_server.db.get_mcp_server", new=AsyncMock(return_value=None)), + patch("litellm.proxy.mcp_server.db.update_mcp_server", new=AsyncMock(return_value=None)), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server_oauth_client_credentials", + "litellm.proxy.mcp_server.db.get_mcp_server_oauth_client_credentials", new=AsyncMock(side_effect=lambda **kwargs: store.get("blob")), ), patch( - "litellm.proxy._experimental.mcp_server.db.create_mcp_server_oauth_client_credentials_if_absent", + "litellm.proxy.mcp_server.db.create_mcp_server_oauth_client_credentials_if_absent", new=mock_create, ), patch( - "litellm.proxy._experimental.mcp_server.db.upsert_mcp_server_oauth_client_credentials", + "litellm.proxy.mcp_server.db.upsert_mcp_server_oauth_client_credentials", new=mock_upsert, ), ): @@ -2623,10 +2623,10 @@ async def test_public_register_route_without_configured_origin_does_not_pin_the_ the body it sent, echo the upstream response, and persist nothing. An admin's Connect (or setting PROXY_BASE_URL) is what establishes the shared client.""" try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -2644,27 +2644,27 @@ async def test_public_register_route_without_configured_origin_does_not_pin_the_ try: with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.read_request_body", + "litellm.proxy.mcp_server.discoverable_endpoints.read_request_body", new=AsyncMock(return_value=request_payload), ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch.object(global_mcp_server_manager, "is_config_declared_server", return_value=True), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), - patch("litellm.proxy._experimental.mcp_server.db.get_mcp_server", new=AsyncMock(return_value=None)), - patch("litellm.proxy._experimental.mcp_server.db.update_mcp_server", new=mock_update), + patch("litellm.proxy.mcp_server.db.get_mcp_server", new=AsyncMock(return_value=None)), + patch("litellm.proxy.mcp_server.db.update_mcp_server", new=mock_update), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server_oauth_client_credentials", + "litellm.proxy.mcp_server.db.get_mcp_server_oauth_client_credentials", new=AsyncMock(return_value=None), ), patch( - "litellm.proxy._experimental.mcp_server.db.create_mcp_server_oauth_client_credentials_if_absent", + "litellm.proxy.mcp_server.db.create_mcp_server_oauth_client_credentials_if_absent", new=mock_create, ), patch( - "litellm.proxy._experimental.mcp_server.db.upsert_mcp_server_oauth_client_credentials", + "litellm.proxy.mcp_server.db.upsert_mcp_server_oauth_client_credentials", new=mock_upsert, ), ): @@ -2688,13 +2688,13 @@ async def test_authorize_endpoint_respects_x_forwarded_proto(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -2725,7 +2725,7 @@ async def test_authorize_endpoint_respects_x_forwarded_proto(): mock_request.headers = {"X-Forwarded-Proto": "https"} # Behind HTTPS proxy # Mock the encryption functions - with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: + with patch("litellm.proxy.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: mock_encrypt.return_value = "mocked_encrypted_state" # Call authorize endpoint @@ -2754,13 +2754,13 @@ async def test_token_endpoint_respects_x_forwarded_proto(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( token_endpoint, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -2804,7 +2804,7 @@ async def test_token_endpoint_respects_x_forwarded_proto(): mock_async_client.post = AsyncMock(return_value=mock_response) with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client" + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client" ) as mock_get_client: mock_get_client.return_value = mock_async_client @@ -2830,13 +2830,13 @@ async def test_oauth_protected_resource_respects_x_forwarded_proto(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( oauth_protected_resource_mcp, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -2883,13 +2883,13 @@ async def test_oauth_authorization_server_respects_x_forwarded_proto(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( oauth_authorization_server_mcp, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -2939,10 +2939,10 @@ async def test_register_client_respects_x_forwarded_proto(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -2957,7 +2957,7 @@ async def test_register_client_respects_x_forwarded_proto(): mock_request.headers = {"X-Forwarded-Proto": "https"} # Behind HTTPS proxy with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.read_request_body", + "litellm.proxy.mcp_server.discoverable_endpoints.read_request_body", new=AsyncMock(return_value={}), ): result = await register_client(request=mock_request) @@ -2977,13 +2977,13 @@ async def test_authorize_endpoint_respects_x_forwarded_host(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -3019,7 +3019,7 @@ async def test_authorize_endpoint_respects_x_forwarded_host(): } # Mock the encryption functions - with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: + with patch("litellm.proxy.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: mock_encrypt.return_value = "mocked_encrypted_state" # Call authorize endpoint @@ -3048,13 +3048,13 @@ async def test_token_endpoint_respects_x_forwarded_host(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( token_endpoint, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -3101,7 +3101,7 @@ async def test_token_endpoint_respects_x_forwarded_host(): mock_async_client.post = AsyncMock(return_value=mock_response) with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client" + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client" ) as mock_get_client: mock_get_client.return_value = mock_async_client @@ -3279,7 +3279,7 @@ def test_get_request_base_url_comprehensive( try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( get_request_base_url, ) except ImportError: @@ -3302,7 +3302,7 @@ def test_get_request_base_url_comprehensive( mock_request.headers.get = mock_get with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.IPAddressUtils.is_request_from_trusted_proxy", + "litellm.proxy.mcp_server.discoverable_endpoints.IPAddressUtils.is_request_from_trusted_proxy", return_value=True, ): result = get_request_base_url(mock_request) @@ -3363,7 +3363,7 @@ def test_get_request_base_url_xff_trust_gate(general_settings, direct_ip, expect try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( get_request_base_url, ) except ImportError: @@ -3403,10 +3403,10 @@ def test_xff_misconfig_warning_emitted_once(caplog): from fastapi import Request from litellm.proxy import auth as proxy_auth_pkg # noqa: F401 - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.auth import ip_address_utils + from litellm.proxy.mcp_server.discoverable_endpoints import ( get_request_base_url, ) - from litellm.proxy.auth import ip_address_utils except ImportError: pytest.skip("MCP discoverable endpoints not available") @@ -3441,7 +3441,7 @@ def test_get_request_base_url_honors_proxy_base_url_env(monkeypatch): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( get_request_base_url, ) except ImportError: @@ -3469,7 +3469,7 @@ def test_validate_trusted_redirect_uri_logs_diagnostic_on_rejection(caplog, monk try: from fastapi import HTTPException, Request - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) except ImportError: @@ -3531,7 +3531,7 @@ def test_validate_trusted_redirect_uri_follows_the_xff_trust_gate(direct_ip, exp try: from fastapi import HTTPException, Request - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( validate_trusted_redirect_uri, ) except ImportError: @@ -3584,8 +3584,8 @@ def test_get_request_base_url_rejects_malformed_proxy_base_url(bad_value, monkey try: from fastapi import Request - from litellm.proxy._experimental.mcp_server import oauth_utils - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server import oauth_utils + from litellm.proxy.mcp_server.oauth_utils import ( get_request_base_url, ) except ImportError: @@ -3623,8 +3623,8 @@ def test_get_request_base_url_malformed_proxy_base_url_warning_is_one_shot(monke try: from fastapi import Request - from litellm.proxy._experimental.mcp_server import oauth_utils - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server import oauth_utils + from litellm.proxy.mcp_server.oauth_utils import ( get_request_base_url, ) except ImportError: @@ -3667,13 +3667,13 @@ async def test_oauth_protected_resource_returns_empty_scopes_when_none(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_oauth_protected_resource_response, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -3724,13 +3724,13 @@ async def test_oauth_protected_resource_gateway_managed_oauth2_advertises_gatewa try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_oauth_protected_resource_response, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -3808,9 +3808,9 @@ def test_root_protected_resource_discovers_gateway(monkeypatch, server_count, by from fastapi import FastAPI from fastapi.testclient import TestClient - from litellm.proxy._experimental.mcp_server import byok_oauth_endpoints, discoverable_endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server import byok_oauth_endpoints, discoverable_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.types.mcp_server.mcp_server_manager import MCPServer monkeypatch.setenv("PROXY_BASE_URL", base_url) @@ -3858,7 +3858,7 @@ def test_root_protected_resource_discovers_gateway(monkeypatch, server_count, by async def test_unnamed_protected_resource_builder_uses_gateway_origin(monkeypatch): from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_oauth_protected_resource_response, ) @@ -3883,13 +3883,13 @@ async def test_oauth_authorization_server_returns_empty_scopes_when_none(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_oauth_authorization_server_response, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -3977,8 +3977,8 @@ def _create_id_lookup_oauth2_server(): async def test_authorize_resolves_server_by_id_when_name_lookup_fails(): from fastapi import Request - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager server = _create_id_lookup_oauth2_server() request = MagicMock(spec=Request) @@ -4014,8 +4014,8 @@ async def test_authorize_resolves_server_by_id_when_name_lookup_fails(): async def test_token_endpoint_resolves_server_by_id_when_name_lookup_fails(): from fastapi import Request - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager server = _create_id_lookup_oauth2_server() request = MagicMock(spec=Request) @@ -4057,8 +4057,8 @@ async def test_token_endpoint_resolves_server_by_id_when_name_lookup_fails(): async def test_register_client_resolves_server_by_id_when_name_lookup_fails(): from fastapi import Request - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager server = _create_id_lookup_oauth2_server().model_copy( update={"client_id": None, "client_secret": None, "registration_url": "https://provider.com/oauth/register"} @@ -4097,8 +4097,8 @@ async def test_register_client_resolves_server_by_id_when_name_lookup_fails(): async def test_protected_resource_metadata_resolves_server_by_id_when_name_lookup_fails(): from fastapi import Request - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager server = _create_id_lookup_oauth2_server() request = MagicMock(spec=Request) @@ -4128,8 +4128,8 @@ async def test_protected_resource_metadata_resolves_server_by_id_when_name_looku def test_authorization_server_metadata_resolves_server_by_id_when_name_lookup_fails(): from fastapi import Request - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager server = _create_id_lookup_oauth2_server() request = MagicMock(spec=Request) @@ -4161,10 +4161,10 @@ async def test_authorize_root_resolves_single_oauth2_server(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -4180,7 +4180,7 @@ async def test_authorize_root_resolves_single_oauth2_server(): try: with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper" + "litellm.proxy.mcp_server.discoverable_endpoints.encrypt_value_helper" ) as mock_encrypt: mock_encrypt.return_value = "mocked_encrypted_state" @@ -4208,10 +4208,10 @@ async def test_authorize_root_fails_with_multiple_oauth2_servers(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -4248,10 +4248,10 @@ async def test_authorize_root_does_not_resolve_private_server_for_external_clien try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -4267,7 +4267,7 @@ async def test_authorize_root_does_not_resolve_private_server_for_external_clien try: with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", + "litellm.proxy.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", return_value="198.51.100.10", ): with pytest.raises(HTTPException) as exc_info: @@ -4289,10 +4289,10 @@ async def test_token_root_resolves_single_oauth2_server(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( token_endpoint, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -4319,7 +4319,7 @@ async def test_token_root_resolves_single_oauth2_server(): try: with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client" + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client" ) as mock_get_client: mock_get_client.return_value = mock_async_client @@ -4354,10 +4354,10 @@ async def test_token_root_does_not_resolve_private_server_for_external_client(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( token_endpoint, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -4373,7 +4373,7 @@ async def test_token_root_does_not_resolve_private_server_for_external_client(): try: with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", + "litellm.proxy.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", return_value="198.51.100.10", ): with pytest.raises(HTTPException) as exc_info: @@ -4401,10 +4401,10 @@ async def test_register_root_does_aggregate_dcr_not_single_server_resolution(): from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -4419,7 +4419,7 @@ async def test_register_root_does_aggregate_dcr_not_single_server_resolution(): try: with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.read_request_body", + "litellm.proxy.mcp_server.discoverable_endpoints.read_request_body", new=AsyncMock(return_value={"redirect_uris": ["https://claude.ai/cb"]}), ), patch("litellm.proxy.proxy_server.master_key", "sk-test-salt-for-lit3637"), @@ -4442,10 +4442,10 @@ async def test_register_root_does_not_leak_a_private_server(): from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -4460,11 +4460,11 @@ async def test_register_root_does_not_leak_a_private_server(): try: with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.read_request_body", + "litellm.proxy.mcp_server.discoverable_endpoints.read_request_body", new=AsyncMock(return_value={"redirect_uris": ["https://claude.ai/cb"]}), ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", + "litellm.proxy.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", return_value="198.51.100.10", ), patch("litellm.proxy.proxy_server.master_key", "sk-test-salt-for-lit3637"), @@ -4484,10 +4484,10 @@ async def test_discovery_root_includes_server_name_prefix(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_oauth_authorization_server_response, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -4523,11 +4523,11 @@ async def test_discovery_root_does_not_expose_private_server_for_external_client try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_oauth_authorization_server_response, _build_oauth_protected_resource_response, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -4543,7 +4543,7 @@ async def test_discovery_root_does_not_expose_private_server_for_external_client try: with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", + "litellm.proxy.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", return_value="198.51.100.10", ): authorization_response = _build_oauth_authorization_server_response( @@ -4570,7 +4570,7 @@ async def test_discovery_root_does_not_expose_private_server_for_external_client async def test_oauth_callback_redirects_with_state(): """Test OAuth callback endpoint properly decodes state and redirects to client callback URL.""" try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( callback, ) except ImportError: @@ -4585,7 +4585,7 @@ async def test_oauth_callback_redirects_with_state(): "client_redirect_uri": "http://localhost:3000/ui/mcp/oauth/callback", } - with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.decode_state_hash") as mock_decode: + with patch("litellm.proxy.mcp_server.discoverable_endpoints.decode_state_hash") as mock_decode: mock_decode.return_value = mock_state_data # Call callback endpoint with code and state @@ -4611,13 +4611,13 @@ async def test_oauth_callback_preserves_client_redirect_uri_query(): try: from urllib.parse import parse_qs, urlparse - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( callback, ) except ImportError: pytest.skip("MCP discoverable endpoints not available") - with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.decode_state_hash") as mock_decode: + with patch("litellm.proxy.mcp_server.discoverable_endpoints.decode_state_hash") as mock_decode: mock_decode.return_value = { "base_url": "http://localhost:3000/ui/mcp/oauth/callback", "original_state": "test-uuid-state-123", @@ -4644,14 +4644,14 @@ async def test_oauth_callback_preserves_client_redirect_uri_query(): async def test_oauth_callback_handles_invalid_state(): """Test OAuth callback returns error page when state decryption fails.""" try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( callback, ) except ImportError: pytest.skip("MCP discoverable endpoints not available") # Mock state decoding to raise an exception - with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.decode_state_hash") as mock_decode: + with patch("litellm.proxy.mcp_server.discoverable_endpoints.decode_state_hash") as mock_decode: mock_decode.side_effect = Exception("Failed to decrypt state") # Call callback endpoint with invalid state @@ -4670,11 +4670,11 @@ async def test_oauth_callback_handles_invalid_state(): async def test_oauth_callback_accepts_same_origin_ui_redirect(): """UI OAuth flow: the callback should redirect to the proxy's own UI origin when the encrypted state carries a same-origin client_redirect_uri.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( callback, ) - with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.decode_state_hash") as mock_decode: + with patch("litellm.proxy.mcp_server.discoverable_endpoints.decode_state_hash") as mock_decode: mock_decode.return_value = { "base_url": "https://proxy.example.com/ui/mcp/oauth/callback", "original_state": "state-123", @@ -4707,13 +4707,13 @@ async def test_authorize_forwards_short_state_and_round_trips_via_cookie(monkeyp from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( _oauth_state_cookie_name, authorize_with_server, callback, decode_state_hash, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -4806,7 +4806,7 @@ async def test_callback_error_path_reads_cookie_and_clears_it(monkeypatch): from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _oauth_state_cookie_name, callback, encode_state_with_base_url, @@ -4879,7 +4879,7 @@ async def _authorize_then_callback(server, iss, monkeypatch, expected_issuer_ove from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _oauth_state_cookie_name, authorize_with_server, callback, @@ -5026,10 +5026,10 @@ async def test_oauth_authorize_includes_scopes_from_server_config(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize_with_server, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -5052,7 +5052,7 @@ async def test_oauth_authorize_includes_scopes_from_server_config(): mock_request.base_url = "https://litellm.example.com/" mock_request.headers = {} - with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: + with patch("litellm.proxy.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: mock_encrypt.return_value = "encrypted_state" # Call authorize without explicit scope parameter @@ -5082,10 +5082,10 @@ async def test_oauth_authorize_prefers_request_scope_over_server_config(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize_with_server, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -5107,7 +5107,7 @@ async def test_oauth_authorize_prefers_request_scope_over_server_config(): mock_request.base_url = "https://litellm.example.com/" mock_request.headers = {} - with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: + with patch("litellm.proxy.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: mock_encrypt.return_value = "encrypted_state" # Call authorize WITH explicit scope parameter @@ -5138,13 +5138,13 @@ async def test_token_endpoint_refresh_token_grant(): try: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( token_endpoint, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -5187,7 +5187,7 @@ async def test_token_endpoint_refresh_token_grant(): mock_async_client.post = AsyncMock(return_value=mock_response) with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client" + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client" ) as mock_get_client: mock_get_client.return_value = mock_async_client @@ -5225,13 +5225,13 @@ async def test_token_endpoint_refresh_token_grant(): async def test_token_endpoint_authorization_code_missing_code(): """Test that authorization_code grant rejects missing code param.""" try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( exchange_token_with_server, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -5278,13 +5278,13 @@ async def test_authorize_endpoint_rejects_non_loopback_redirect_uri(): primitive — reject with 400 before encoding anything into state.""" from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -5325,13 +5325,13 @@ async def test_authorize_endpoint_accepts_ipv4_loopback_range_and_ipv6_full_form would miss ``127.0.0.2`` and ``0:0:0:0:0:0:0:1``.""" from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -5360,7 +5360,7 @@ async def test_authorize_endpoint_accepts_ipv4_loopback_range_and_ipv6_full_form "http://localhost:3000/cb", ): with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper" + "litellm.proxy.mcp_server.discoverable_endpoints.encrypt_value_helper" ) as mock_encrypt: mock_encrypt.return_value = "mocked_encrypted_state" response = await authorize( @@ -5380,11 +5380,11 @@ async def test_callback_revalidates_loopback_on_decoded_base_url(): valid. /callback must re-validate the decoded base_url so those stale states can't be used as an open-redirect + code-theft primitive.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( callback, ) - with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.decode_state_hash") as mock_decode: + with patch("litellm.proxy.mcp_server.discoverable_endpoints.decode_state_hash") as mock_decode: mock_decode.return_value = { "base_url": "https://attacker.example.com/cb", "original_state": "s", @@ -5404,11 +5404,11 @@ async def test_callback_revalidates_loopback_on_decoded_base_url(): @pytest.mark.asyncio async def test_callback_revalidates_loopback_on_decoded_client_redirect_uri(): """If a state contains a full client_redirect_uri, validate that exact sink.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( callback, ) - with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.decode_state_hash") as mock_decode: + with patch("litellm.proxy.mcp_server.discoverable_endpoints.decode_state_hash") as mock_decode: mock_decode.return_value = { "base_url": "http://localhost:3000/cb", "original_state": "s", @@ -5428,11 +5428,11 @@ async def test_callback_revalidates_loopback_on_decoded_client_redirect_uri(): @pytest.mark.asyncio async def test_callback_rejects_state_missing_redirect_uri(): """Malformed state without a redirect target should fail with a structured 400.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( callback, ) - with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.decode_state_hash") as mock_decode: + with patch("litellm.proxy.mcp_server.discoverable_endpoints.decode_state_hash") as mock_decode: mock_decode.return_value = { "original_state": "s", "code_challenge": None, @@ -5454,10 +5454,10 @@ async def test_token_endpoint_sets_no_store_cache_control(): NOT be cached by intermediaries or the client.""" from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( exchange_token_with_server, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -5488,7 +5488,7 @@ async def test_token_endpoint_sets_no_store_cache_control(): fake_http_client.post = AsyncMock(return_value=fake_http_response) with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=fake_http_client, ): response = await exchange_token_with_server( @@ -5509,10 +5509,10 @@ async def test_token_endpoint_sets_no_store_cache_control(): async def _exchange_with_upstream_token_response(upstream_body): from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( exchange_token_with_server, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -5539,7 +5539,7 @@ async def _exchange_with_upstream_token_response(upstream_body): fake_http_client.post = AsyncMock(return_value=fake_http_response) with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=fake_http_client, ): response = await exchange_token_with_server( @@ -5618,7 +5618,7 @@ async def test_authorize_bridge_relay_passes_client_params_verbatim(auth_type_va and applies no gateway-side redirect trust (the upstream enforces its registered binding).""" from urllib.parse import parse_qs, urlparse - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize_with_server, ) from litellm.types.mcp import MCPAuth @@ -5655,7 +5655,7 @@ async def test_authorize_bridge_requires_s256_pkce(code_challenge, code_challeng """Bridge servers serve unauthenticated public clients, so the PKCE downgrade paths (missing challenge, or a method that is not S256; RFC 7636 defaults a missing method to plain) are rejected at the gateway on both bridge arms.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize_with_server, ) @@ -5682,7 +5682,7 @@ async def test_authorize_bridge_short_circuit_keeps_callback_and_redirect_trust( through the gateway callback with the state cookie.""" from urllib.parse import parse_qs, urlparse - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize_with_server, ) @@ -5701,7 +5701,7 @@ async def test_authorize_bridge_short_circuit_keeps_callback_and_redirect_trust( assert exc.value.status_code in (400, 403) with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper", + "litellm.proxy.mcp_server.discoverable_endpoints.encrypt_value_helper", return_value="mocked_encrypted_state", ): response = await authorize_with_server( @@ -5725,12 +5725,12 @@ async def test_authorize_non_bridge_client_forwarded_keeps_pre_bridge_contract() requirement and the gateway /callback relay (this is the browser-only Authorize path).""" from urllib.parse import parse_qs, urlparse - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize_with_server, ) with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper", + "litellm.proxy.mcp_server.discoverable_endpoints.encrypt_value_helper", return_value="mocked_encrypted_state", ): response = await authorize_with_server( @@ -5747,7 +5747,7 @@ async def test_authorize_non_bridge_client_forwarded_keeps_pre_bridge_contract() async def _bridge_token_post_data(server, redirect_uri): - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( exchange_token_with_server, ) @@ -5758,7 +5758,7 @@ async def _bridge_token_post_data(server, redirect_uri): fake_http_client.post = AsyncMock(return_value=fake_http_response) with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=fake_http_client, ): await exchange_token_with_server( @@ -5788,7 +5788,7 @@ async def test_token_bridge_relay_posts_client_redirect_uri(): @pytest.mark.asyncio async def test_token_bridge_relay_requires_redirect_uri(): - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( exchange_token_with_server, ) @@ -5818,10 +5818,10 @@ async def test_token_non_bridge_keeps_gateway_callback(): def _named_as_metadata_response(server): - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_oauth_authorization_server_response, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -5829,7 +5829,7 @@ def _named_as_metadata_response(server): global_mcp_server_manager.registry[server.server_id] = server try: with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", + "litellm.proxy.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", return_value=None, ): return _build_oauth_authorization_server_response( @@ -5865,7 +5865,7 @@ async def _bridge_register_response( replace_stale_client: bool = False, database_connected: bool = True, ) -> tuple[Response, MagicMock, AsyncMock]: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client_with_server, ) @@ -5882,15 +5882,15 @@ async def _bridge_register_response( with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._persist_dcr_client_registration", + "litellm.proxy.mcp_server.discoverable_endpoints._persist_dcr_client_registration", new_callable=AsyncMock, ) as mock_persist, patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._reuse_persisted_dcr_client_if_available", + "litellm.proxy.mcp_server.discoverable_endpoints._reuse_persisted_dcr_client_if_available", new_callable=AsyncMock, return_value=False, ), @@ -5949,7 +5949,7 @@ async def test_register_bridge_relay_surfaces_upstream_error_not_500(): RFC 7591 error body to the client, not a bare 500 that hides the real reason.""" import httpx - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client_with_server, ) @@ -5967,11 +5967,11 @@ async def test_register_bridge_relay_surfaces_upstream_error_not_500(): with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._reuse_persisted_dcr_client_if_available", + "litellm.proxy.mcp_server.discoverable_endpoints._reuse_persisted_dcr_client_if_available", new_callable=AsyncMock, return_value=False, ), @@ -5998,7 +5998,7 @@ async def test_register_non_bridge_upstream_error_relays_status_not_500(): opaque 500 that hides the real reason from the create-flow UI.""" import httpx - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client_with_server, ) @@ -6016,11 +6016,11 @@ async def test_register_non_bridge_upstream_error_relays_status_not_500(): with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._reuse_persisted_dcr_client_if_available", + "litellm.proxy.mcp_server.discoverable_endpoints._reuse_persisted_dcr_client_if_available", new_callable=AsyncMock, return_value=False, ), @@ -6151,8 +6151,8 @@ async def test_authorize_clientless_bridge_loads_shared_client_before_choosing_t kept one), the caller keeps its own upstream binding through the relay.""" from urllib.parse import parse_qs, urlparse - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import authorize_with_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.discoverable_endpoints import authorize_with_server + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager monkeypatch.setenv("PROXY_BASE_URL", "https://litellm.example.com") monkeypatch.setenv("LITELLM_SALT_KEY", "test-registration-binding") @@ -6167,7 +6167,7 @@ async def test_authorize_clientless_bridge_loads_shared_client_before_choosing_t return_value=prisma_client, ), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + "litellm.proxy.mcp_server.db.get_mcp_server", new=mock_get_mcp_server, ), patch.object( @@ -6176,7 +6176,7 @@ async def test_authorize_clientless_bridge_loads_shared_client_before_choosing_t new=mock_update_server, ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper", + "litellm.proxy.mcp_server.discoverable_endpoints.encrypt_value_helper", return_value="mocked_encrypted_state", ), ): @@ -6213,7 +6213,7 @@ async def test_token_clientless_bridge_loads_shared_client_before_choosing_the_a ) -> None: """The token leg mirrors authorize: with the shared client persisted it redeems the code with that client and the gateway /callback, and with none yet it relays the caller's own client and redirect.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager monkeypatch.setenv("PROXY_BASE_URL", "https://litellm.example.com") monkeypatch.setenv("LITELLM_SALT_KEY", "test-registration-binding") @@ -6225,7 +6225,7 @@ async def test_token_clientless_bridge_loads_shared_client_before_choosing_the_a with ( patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=prisma_client), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + "litellm.proxy.mcp_server.db.get_mcp_server", new=mock_get_mcp_server, ), patch.object( @@ -6251,10 +6251,10 @@ async def test_persist_dcr_client_for_client_forwarded_server_only_when_it_is_a_ ) -> None: """A DCR bridge keeps the shared gateway client on the server row (without stamping oauth2_flow on a client-forwarded row), while a client-forwarded server without the bridge still stores nothing.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _persist_dcr_client_registration, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) from litellm.types.mcp import MCPAuth @@ -6265,7 +6265,7 @@ async def test_persist_dcr_client_for_client_forwarded_server_only_when_it_is_a_ mock_update: Final = AsyncMock(return_value=MagicMock()) with ( patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=prisma), - patch("litellm.proxy._experimental.mcp_server.db.update_mcp_server", new=mock_update), + patch("litellm.proxy.mcp_server.db.update_mcp_server", new=mock_update), patch.object(global_mcp_server_manager, "update_server", new=AsyncMock()), patch.object(global_mcp_server_manager, "is_config_declared_server", return_value=False), ): @@ -6293,8 +6293,8 @@ async def test_token_sealed_ephemeral_client_is_not_replaced_by_persisted_gatewa server_has_shared_client: bool, ) -> None: """The token leg must use the sealed ephemeral client even if this worker's catalog has a shared client.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import exchange_token_with_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.discoverable_endpoints import exchange_token_with_server + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager monkeypatch.setenv("PROXY_BASE_URL", "https://litellm.example.com") monkeypatch.setenv("LITELLM_SALT_KEY", "test-registration-binding") @@ -6320,7 +6320,7 @@ async def test_token_sealed_ephemeral_client_is_not_replaced_by_persisted_gatewa return_value=prisma_client, ), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + "litellm.proxy.mcp_server.db.get_mcp_server", new=mock_get_mcp_server, ), patch.object( @@ -6329,7 +6329,7 @@ async def test_token_sealed_ephemeral_client_is_not_replaced_by_persisted_gatewa new=mock_update_server, ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=fake_http_client, ), ): @@ -6359,8 +6359,8 @@ _BRIDGE_MASTER_KEY = "sk-bridge-producer-master-key-0123456789abcdef" async def _exchange_for_bridge_server(server, upstream_body, key_hash, code="auth-code", fake_client_out=None): - from litellm.proxy._experimental.mcp_server.bridge_token_flow import _ResolvedKey - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import exchange_token_with_server + from litellm.proxy.mcp_server.bridge_token_flow import _ResolvedKey + from litellm.proxy.mcp_server.discoverable_endpoints import exchange_token_with_server fake_http_response = MagicMock() fake_http_response.json.return_value = upstream_body @@ -6376,11 +6376,11 @@ async def _exchange_for_bridge_server(server, upstream_body, key_hash, code="aut with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=fake_http_client, ), patch( - "litellm.proxy._experimental.mcp_server.bridge_token_flow._resolve_active_litellm_key", + "litellm.proxy.mcp_server.bridge_token_flow._resolve_active_litellm_key", new=key_resolver, ), patch("litellm.proxy.proxy_server.master_key", _BRIDGE_MASTER_KEY), @@ -6395,7 +6395,7 @@ async def _exchange_for_bridge_server(server, upstream_body, key_hash, code="aut client_secret=None, code_verifier="verifier", ) - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import is_bridge_authorization_code + from litellm.proxy.mcp_server.discoverable_endpoints import is_bridge_authorization_code # The key_hash path resolves the presented litellm key; the interactive SSO path recovers identity # from the gateway authorization code instead, so it never awaits the resolver. @@ -6414,7 +6414,7 @@ async def test_oauth_delegate_bridge_token_exchange_mints_envelope_not_raw_token never appears in the bearer the client receives.""" from datetime import datetime, timezone - from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( BridgeEnvelopeAdmitted, envelope_keys_from_master_key, resolve_bridge_envelope, @@ -6445,7 +6445,7 @@ def test_bridge_authorization_code_round_trips_and_rejects_hostile_input(): """The gateway authorization code seals and recovers the upstream code and the SSO user, and is total over hostile input: a raw upstream code (scripted path) opens to None, and a tampered or non-gateway value opens to None rather than raising.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( is_bridge_authorization_code, open_bridge_authorization_code, seal_bridge_authorization_code, @@ -6473,10 +6473,10 @@ async def test_interactive_bridge_token_exchange_mints_user_subject_envelope(): gateway code, not the sealed wrapper.""" from datetime import datetime, timezone - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( seal_bridge_authorization_code, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( BridgeEnvelopeAdmitted, envelope_keys_from_master_key, resolve_bridge_envelope, @@ -6508,7 +6508,7 @@ async def test_interactive_bridge_token_exchange_mints_user_subject_envelope(): async def test_interactive_bridge_gateway_code_for_another_server_is_rejected_400(): """A gateway authorization code is bound to the server it was minted for: presenting it at another server's token endpoint is a 400, so a code cannot be replayed across a server boundary.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( seal_bridge_authorization_code, ) from litellm.types.mcp import MCPAuth @@ -6530,11 +6530,11 @@ async def test_interactive_bridge_authorize_seals_sso_user_into_state(monkeypatc session cookie and seals it (and the target server) into the encrypted OAuth state, so the callback can later mint a user-bound gateway code; it still proceeds to the upstream redirect. The access gate runs for real against a granted resolver, so its interface stays exercised.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import authorize_with_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.discoverable_endpoints import authorize_with_server + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import UserAPIKeyAuth from litellm.types.mcp import MCPAuth server = _bridge_server(auth_type=MCPAuth.oauth_delegate, client_id="admin-client", registration_url=None) @@ -6549,7 +6549,7 @@ async def test_interactive_bridge_authorize_seals_sso_user_into_state(monkeypatc with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", new=AsyncMock(return_value=admitted), ), patch.object( @@ -6558,7 +6558,7 @@ async def test_interactive_bridge_authorize_seals_sso_user_into_state(monkeypatc new=AsyncMock(return_value=[server.server_id]), ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.encode_state_with_base_url", + "litellm.proxy.mcp_server.discoverable_endpoints.encode_state_with_base_url", side_effect=_capture, ), ): @@ -6590,11 +6590,11 @@ async def test_bridge_authorize_gates_on_the_egress_server_access_resolver( from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import authorize - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.discoverable_endpoints import authorize + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import UserAPIKeyAuth server = _bridge_server(auth_type=MCPAuth.oauth_delegate, client_id="upstream-app", registration_url=None) global_mcp_server_manager.registry.clear() @@ -6613,7 +6613,7 @@ async def test_bridge_authorize_gates_on_the_egress_server_access_resolver( try: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", new=AsyncMock(return_value=admitted), ) as mock_reload, patch.object( @@ -6622,7 +6622,7 @@ async def test_bridge_authorize_gates_on_the_egress_server_access_resolver( new=AsyncMock(return_value=allowed), ) as mock_allowed, patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper", + "litellm.proxy.mcp_server.discoverable_endpoints.encrypt_value_helper", return_value="mocked_encrypted_state", ), ): @@ -6666,8 +6666,8 @@ async def test_bridge_authorize_reload_failure_denies_or_stays_retryable( from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import authorize - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.discoverable_endpoints import authorize + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -6683,7 +6683,7 @@ async def test_bridge_authorize_reload_failure_denies_or_stays_retryable( try: with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", new=AsyncMock(side_effect=HTTPException(status_code=reload_status, detail="x")), ), ): @@ -6720,7 +6720,7 @@ async def test_bridge_authorize_reload_failure_denies_or_stays_retryable( async def test_interactive_bridge_authorize_without_session_redirects_to_login(): """Without a UI session there is no identity to bind, so the short-circuit bridge oauth_delegate authorize sends the browser through litellm login instead of proceeding to the upstream.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import authorize_with_server + from litellm.proxy.mcp_server.discoverable_endpoints import authorize_with_server from litellm.types.mcp import MCPAuth server = _bridge_server(auth_type=MCPAuth.oauth_delegate, client_id="admin-client", registration_url=None) @@ -6743,7 +6743,7 @@ async def test_interactive_bridge_callback_seals_user_into_gateway_code(): code, so the client's later token call can prove who signed in.""" from urllib.parse import parse_qs, urlparse - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( callback, is_bridge_authorization_code, ) @@ -6757,15 +6757,15 @@ async def test_interactive_bridge_callback_seals_user_into_gateway_code(): } with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._resolve_encoded_oauth_state", + "litellm.proxy.mcp_server.discoverable_endpoints._resolve_encoded_oauth_state", return_value="enc", ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.decode_state_hash", + "litellm.proxy.mcp_server.discoverable_endpoints.decode_state_hash", return_value=state_data, ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._get_validated_client_redirect_uri", + "litellm.proxy.mcp_server.discoverable_endpoints._get_validated_client_redirect_uri", return_value="http://127.0.0.1:60108/cb", ), patch("litellm.proxy.proxy_server.master_key", _BRIDGE_MASTER_KEY), @@ -6842,10 +6842,10 @@ async def test_bridge_access_envelope_never_carries_upstream_refresh_token(): refresh envelope returned as the response's refresh_token, encrypted, never in plaintext.""" from datetime import datetime, timezone - from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( envelope_keys_from_master_key, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( + from litellm.proxy.mcp_server.outbound_credentials.envelope import ( OpenedEnvelope, open_envelope, ) @@ -6877,7 +6877,7 @@ async def test_bridge_refresh_grant_with_non_envelope_is_invalid_grant_before_up token, so a refresh_token grant carrying a bare (non-envelope) value is invalid_grant, rejected in _prepare_bridge_refresh BEFORE any upstream exchange. Rejecting before the exchange means a bad refresh request can never consume or rotate an upstream refresh token.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import exchange_token_with_server + from litellm.proxy.mcp_server.discoverable_endpoints import exchange_token_with_server from litellm.types.mcp import MCPAuth server = _bridge_server(auth_type=MCPAuth.oauth_delegate) @@ -6885,7 +6885,7 @@ async def test_bridge_refresh_grant_with_non_envelope_is_invalid_grant_before_up fake_http_client.post = AsyncMock() with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=fake_http_client, ), patch("litellm.proxy.proxy_server.master_key", _BRIDGE_MASTER_KEY), @@ -6921,11 +6921,11 @@ def _mint_test_refresh_envelope( from pydantic import SecretStr - from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( build_bridge_refresh_token_response, envelope_keys_from_master_key, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( + from litellm.proxy.mcp_server.outbound_credentials.envelope import ( RefreshCredential, SealedEnvelope, key_hash_identity, @@ -6952,7 +6952,7 @@ async def _refresh_for_bridge_server( ``upstream_body``. Patching the single subject-revalidation dispatch covers both a key_hash and a user_id refresh envelope. Returns the response; the captured client exposes the POST call so a test can assert what refresh token was actually sent upstream.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import exchange_token_with_server + from litellm.proxy.mcp_server.discoverable_endpoints import exchange_token_with_server fake_http_response = MagicMock() fake_http_response.json.return_value = upstream_body @@ -6964,11 +6964,11 @@ async def _refresh_for_bridge_server( with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=fake_http_client, ), patch( - "litellm.proxy._experimental.mcp_server.bridge_token_flow._revalidate_active_subject", + "litellm.proxy.mcp_server.bridge_token_flow._revalidate_active_subject", new=AsyncMock(return_value=revalidate_result), ), patch("litellm.proxy.proxy_server.master_key", _BRIDGE_MASTER_KEY), @@ -6993,10 +6993,10 @@ async def test_bridge_mint_returns_refresh_envelope_that_opens_to_upstream_refre (under the same keys and server_id) to the upstream refresh token, so the client can renew later.""" from datetime import datetime, timezone - from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( envelope_keys_from_master_key, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( + from litellm.proxy.mcp_server.outbound_credentials.envelope import ( OpenedRefreshEnvelope, open_refresh_envelope, ) @@ -7060,10 +7060,10 @@ async def test_bridge_refresh_grant_rotates_refresh_envelope_wrapping_new_upstre that wraps the NEW upstream refresh token, so the rotation is carried through faithfully.""" from datetime import datetime, timezone - from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( envelope_keys_from_master_key, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( + from litellm.proxy.mcp_server.outbound_credentials.envelope import ( OpenedRefreshEnvelope, open_refresh_envelope, ) @@ -7132,12 +7132,12 @@ async def test_bridge_refresh_grant_renews_a_user_subject_envelope(): exchange received the real upstream refresh token, not the client's envelope.""" from datetime import datetime, timezone - from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( BridgeEnvelopeAdmitted, envelope_keys_from_master_key, resolve_bridge_envelope, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import user_identity + from litellm.proxy.mcp_server.outbound_credentials.envelope import user_identity from litellm.types.mcp import MCPAuth server = _bridge_server(auth_type=MCPAuth.oauth_delegate) @@ -7190,10 +7190,10 @@ async def test_bridge_refresh_re_seals_scope_when_upstream_omits_it_so_the_chain the upstream response had none, and a second refresh off it still re-requests the scope.""" from datetime import datetime, timezone - from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( envelope_keys_from_master_key, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( + from litellm.proxy.mcp_server.outbound_credentials.envelope import ( OpenedRefreshEnvelope, open_refresh_envelope, ) @@ -7229,7 +7229,7 @@ async def test_bridge_refresh_grant_with_deactivated_user_is_invalid_grant_befor the user no longer exists) cannot keep refreshing: subject re-validation reports no_active_key and the refresh is invalid_grant, returned BEFORE the upstream exchange. Revocation kills renewal for the user subject exactly as it does for the key subject.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import user_identity + from litellm.proxy.mcp_server.outbound_credentials.envelope import user_identity from litellm.types.mcp import MCPAuth server = _bridge_server(auth_type=MCPAuth.oauth_delegate) @@ -7248,19 +7248,19 @@ async def test_bridge_refresh_grant_with_deactivated_user_is_invalid_grant_befor async def test_revalidate_active_subject_dispatches_on_subject_type(): """Subject re-validation routes a key_hash envelope to the key reload and a user_id envelope to the user reload, so revocation gates renewal for either identity source through one dispatch point.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( + from litellm.proxy.mcp_server.bridge_token_flow import ( _ResolvedKey, _revalidate_active_subject, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import key_hash_identity, user_identity + from litellm.proxy.mcp_server.outbound_credentials.envelope import key_hash_identity, user_identity with ( patch( - "litellm.proxy._experimental.mcp_server.bridge_token_flow._reload_active_key_by_hash", + "litellm.proxy.mcp_server.bridge_token_flow._reload_active_key_by_hash", new=AsyncMock(return_value=_ResolvedKey(key_hash="kh", key=MagicMock())), ) as key_reload, patch( - "litellm.proxy._experimental.mcp_server.bridge_token_flow.reload_active_user_by_id", + "litellm.proxy.mcp_server.bridge_token_flow.reload_active_user_by_id", new=AsyncMock(return_value=None), ) as user_reload, ): @@ -7270,11 +7270,11 @@ async def test_revalidate_active_subject_dispatches_on_subject_type(): with ( patch( - "litellm.proxy._experimental.mcp_server.bridge_token_flow._reload_active_key_by_hash", + "litellm.proxy.mcp_server.bridge_token_flow._reload_active_key_by_hash", new=AsyncMock(), ) as key_reload2, patch( - "litellm.proxy._experimental.mcp_server.bridge_token_flow.reload_active_user_by_id", + "litellm.proxy.mcp_server.bridge_token_flow.reload_active_user_by_id", new=AsyncMock(return_value="no_active_key"), ) as user_reload2, ): @@ -7288,7 +7288,7 @@ def test_upstream_refresh_credential_expired_refresh_token_is_not_sealed(): not be sealed: _upstream_refresh_credential returns None so the exchange degrades to an access-only response, mirroring how the access grant refuses an already-elapsed access token rather than capping a dead token to the full refresh TTL. A live or unspecified lifetime still yields a credential.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import _upstream_refresh_credential + from litellm.proxy.mcp_server.bridge_token_flow import _upstream_refresh_credential assert _upstream_refresh_credential({"access_token": "A", "refresh_token": "R", "refresh_expires_in": 0}) is None assert _upstream_refresh_credential({"refresh_token": "R", "refresh_expires_in": -5}) is None @@ -7305,7 +7305,7 @@ async def test_bridge_refresh_upstream_invalid_grant_maps_to_invalid_grant(): client re-runs authorization_code, rather than surfacing the opaque upstream error it cannot act on.""" import httpx - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import exchange_token_with_server + from litellm.proxy.mcp_server.discoverable_endpoints import exchange_token_with_server from litellm.types.mcp import MCPAuth server = _bridge_server(auth_type=MCPAuth.oauth_delegate) @@ -7325,11 +7325,11 @@ async def test_bridge_refresh_upstream_invalid_grant_maps_to_invalid_grant(): with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=fake_http_client, ), patch( - "litellm.proxy._experimental.mcp_server.bridge_token_flow._revalidate_active_subject", + "litellm.proxy.mcp_server.bridge_token_flow._revalidate_active_subject", new=AsyncMock(return_value=None), ), patch("litellm.proxy.proxy_server.master_key", _BRIDGE_MASTER_KEY), @@ -7358,7 +7358,7 @@ async def test_bridge_refresh_upstream_error_detection_parses_json_not_substring as the classified upstream rejection rather than triggering a spurious authorization_code re-run.""" import httpx - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import exchange_token_with_server + from litellm.proxy.mcp_server.discoverable_endpoints import exchange_token_with_server from litellm.types.mcp import MCPAuth server = _bridge_server(auth_type=MCPAuth.oauth_delegate) @@ -7378,11 +7378,11 @@ async def test_bridge_refresh_upstream_error_detection_parses_json_not_substring with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=fake_http_client, ), patch( - "litellm.proxy._experimental.mcp_server.bridge_token_flow._revalidate_active_subject", + "litellm.proxy.mcp_server.bridge_token_flow._revalidate_active_subject", new=AsyncMock(return_value=None), ), patch("litellm.proxy.proxy_server.master_key", _BRIDGE_MASTER_KEY), @@ -7409,9 +7409,9 @@ async def test_revalidate_key_subject_revoked_when_owner_scim_deactivated(proxy_ """A key_hash refresh envelope whose key is still active but whose OWNING user was SCIM-deactivated must fail closed to no_active_key, mirroring how admission's _reject_if_admitted_owner_scim_deactivated revokes an offboarded owner's key. Without this, an offboarded user keeps renewing a live key.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import _ResolvedKey, _revalidate_active_subject - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import key_hash_identity from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import _ResolvedKey, _revalidate_active_subject + from litellm.proxy.mcp_server.outbound_credentials.envelope import key_hash_identity proxy_globals.user_api_key_cache = UserApiKeyCache() proxy_globals.prisma_client = object() @@ -7419,7 +7419,7 @@ async def test_revalidate_key_subject_revoked_when_owner_scim_deactivated(proxy_ resolved = _ResolvedKey(key_hash="kh", key=MagicMock(user_id="offboarded-owner")) with ( patch( - "litellm.proxy._experimental.mcp_server.bridge_token_flow._reload_active_key_by_hash", + "litellm.proxy.mcp_server.bridge_token_flow._reload_active_key_by_hash", new=AsyncMock(return_value=resolved), ), patch( @@ -7437,9 +7437,9 @@ async def test_revalidate_key_subject_active_owner_renews_and_missing_owner_fail """The key-owner SCIM gate blocks only an explicit scim_active False: an active owner renews (None), and a missing owner (get_user_object's wrapped ValueError) fails OPEN, since a key may outlive its owner record and a transient blip must not revoke a live key.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import _ResolvedKey, _revalidate_active_subject - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import key_hash_identity from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import _ResolvedKey, _revalidate_active_subject + from litellm.proxy.mcp_server.outbound_credentials.envelope import key_hash_identity proxy_globals.user_api_key_cache = UserApiKeyCache() proxy_globals.prisma_client = object() @@ -7448,7 +7448,7 @@ async def test_revalidate_key_subject_active_owner_renews_and_missing_owner_fail with ( patch( - "litellm.proxy._experimental.mcp_server.bridge_token_flow._reload_active_key_by_hash", + "litellm.proxy.mcp_server.bridge_token_flow._reload_active_key_by_hash", new=AsyncMock(return_value=resolved), ), patch( @@ -7460,7 +7460,7 @@ async def test_revalidate_key_subject_active_owner_renews_and_missing_owner_fail with ( patch( - "litellm.proxy._experimental.mcp_server.bridge_token_flow._reload_active_key_by_hash", + "litellm.proxy.mcp_server.bridge_token_flow._reload_active_key_by_hash", new=AsyncMock(return_value=resolved), ), patch( @@ -7477,7 +7477,7 @@ async def test_bridge_mint_fails_closed_before_upstream_when_master_key_unset(): misconfigured gateway returns a 500 server_error without consuming the single-use code, avoiding the burn-then-fail the pre-exchange phase exists to prevent. The failure is returned as an RFC 6749 error body, not raised.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import exchange_token_with_server + from litellm.proxy.mcp_server.discoverable_endpoints import exchange_token_with_server from litellm.types.mcp import MCPAuth server = _bridge_server(auth_type=MCPAuth.oauth_delegate) @@ -7485,11 +7485,11 @@ async def test_bridge_mint_fails_closed_before_upstream_when_master_key_unset(): fake_http_client.post = AsyncMock() with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=fake_http_client, ), patch( - "litellm.proxy._experimental.mcp_server.bridge_token_flow._resolve_active_litellm_key", + "litellm.proxy.mcp_server.bridge_token_flow._resolve_active_litellm_key", new=AsyncMock(return_value="no_active_key"), ), patch("litellm.proxy.proxy_server.master_key", None), @@ -7514,7 +7514,7 @@ async def _prepare_only_bridge_exchange(resolver_result): """Drive exchange_token_with_server for a bridge oauth_delegate authorization_code request with the identity resolver stubbed to a given tagged result, returning (response, post_mock) so a test can assert the mapped status and that the single-use code was never exchanged.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import exchange_token_with_server + from litellm.proxy.mcp_server.discoverable_endpoints import exchange_token_with_server from litellm.types.mcp import MCPAuth server = _bridge_server(auth_type=MCPAuth.oauth_delegate) @@ -7522,11 +7522,11 @@ async def _prepare_only_bridge_exchange(resolver_result): fake_http_client.post = AsyncMock() with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=fake_http_client, ), patch( - "litellm.proxy._experimental.mcp_server.bridge_token_flow._resolve_active_litellm_key", + "litellm.proxy.mcp_server.bridge_token_flow._resolve_active_litellm_key", new=AsyncMock(return_value=resolver_result), ), patch("litellm.proxy.proxy_server.master_key", _BRIDGE_MASTER_KEY), @@ -7589,7 +7589,7 @@ async def _exchange_for_bridge_server_with_jwt(jwt_auth_result, upstream_body=No presented credential is JWT-shaped, with _resolve_jwt_auth stubbed to a given result. Returns (response, post_mock) so a test can assert the minted envelope's sealed identity or the mapped error status.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import exchange_token_with_server + from litellm.proxy.mcp_server.discoverable_endpoints import exchange_token_with_server from litellm.types.mcp import MCPAuth request = _bridge_mock_request() @@ -7606,11 +7606,11 @@ async def _exchange_for_bridge_server_with_jwt(jwt_auth_result, upstream_body=No server = _bridge_server(auth_type=MCPAuth.oauth_delegate) with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=fake_http_client, ), patch( - "litellm.proxy._experimental.mcp_server.bridge_token_flow._resolve_jwt_auth", + "litellm.proxy.mcp_server.bridge_token_flow._resolve_jwt_auth", new=AsyncMock(return_value=jwt_auth_result), ), patch("litellm.proxy.proxy_server.master_key", _BRIDGE_MASTER_KEY), @@ -7644,12 +7644,12 @@ async def test_bridge_mint_unmapped_jwt_is_rejected_before_upstream(): async def test_bridge_mint_jwt_mapped_to_virtual_key_seals_key_hash_subject(): from datetime import datetime, timezone - from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( BridgeEnvelopeAdmitted, envelope_keys_from_master_key, resolve_bridge_envelope, ) - from litellm.proxy._types import UserAPIKeyAuth response, _post = await _exchange_for_bridge_server_with_jwt( UserAPIKeyAuth(token="mapped-key-hash-99", user_id="mapped-user") @@ -7748,7 +7748,7 @@ def _jwt_auth_patches(mapped_key): stack.enter_context(patch("litellm.proxy.proxy_server.user_api_key_cache", cache)) stack.enter_context( patch( - "litellm.proxy._experimental.mcp_server.bridge_token_flow._key_owner_scim_deactivated", + "litellm.proxy.mcp_server.bridge_token_flow._key_owner_scim_deactivated", new=AsyncMock(return_value=False), ) ) @@ -7760,8 +7760,8 @@ async def test_jwt_mapped_to_service_account_key_without_user_id_resolves(): """A JWT mapped to a team or service-account virtual key (no user_id) is still an active credential: _resolve_jwt_auth returns the mapped key, and the mint seals a key_hash-subject envelope rather than 400ing with no_identity.""" - from litellm.proxy._experimental.mcp_server import bridge_token_flow from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server import bridge_token_flow from litellm.types.mcp import MCPAuth mapped_key = UserAPIKeyAuth(user_id=None, token="svc-key-hash-1") @@ -7790,8 +7790,8 @@ async def test_jwt_mapped_to_service_account_key_without_user_id_resolves(): async def test_jwt_mapped_to_blocked_key_is_rejected(): """The relaxed gate is still active-state gated: a JWT mapped to a blocked virtual key resolves to None, so the mint cannot seal an envelope under it.""" - from litellm.proxy._experimental.mcp_server import bridge_token_flow from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server import bridge_token_flow mapped_key = UserAPIKeyAuth(user_id=None, token="blocked-key-hash-1", blocked=True) request = _bridge_mock_request() @@ -7816,8 +7816,8 @@ async def test_master_key_at_token_endpoint_mints_key_hash_envelope(): credential: presented at the bridge /token endpoint it must mint a key_hash-subject envelope (sealed under hash_token(master_key)) even with no database connection at all. A presented key that is NOT the master key still hits the unresolvable gate when prisma is down, unchanged.""" - from litellm.proxy._experimental.mcp_server import bridge_token_flow from litellm.proxy._types import hash_token + from litellm.proxy.mcp_server import bridge_token_flow from litellm.types.mcp import MCPAuth master = "sk-test-master-key-mint-0000" @@ -7920,14 +7920,14 @@ def test_bridge_reported_expires_in_can_be_zero_at_jwt_exp_boundary(): from fastapi.responses import JSONResponse - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _BridgeMintReady, finish_bridge_mint, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import ( envelope_keys_from_master_key, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import key_hash_identity + from litellm.proxy.mcp_server.outbound_credentials.envelope import key_hash_identity from litellm.types.mcp import MCPAuth ready = _BridgeMintReady( @@ -7951,7 +7951,7 @@ def test_classify_upstream_lifetime(): oversized) is "unspecified" so the envelope caps it, while a parseable non-positive value is "expired": the upstream reporting an already-dead token, which the mint must reject rather than silently give the 1h cap.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import _classify_upstream_lifetime + from litellm.proxy.mcp_server.bridge_token_flow import _classify_upstream_lifetime assert _classify_upstream_lifetime(300) == 300 assert _classify_upstream_lifetime(300.0) == 300 @@ -7984,7 +7984,7 @@ def test_bridge_grant_honors_and_rejects_upstream_lifetime(): """The grant validator honors a positive lifetime, leaves an unknown one None for the envelope to cap, and rejects an explicitly-expired one with "expired_lifetime" so a dead upstream token is never sealed into an hour-long envelope.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import _bridge_grant_from_token_response + from litellm.proxy.mcp_server.bridge_token_flow import _bridge_grant_from_token_response def grant(v): return _bridge_grant_from_token_response({"access_token": "x", "expires_in": v}) @@ -8065,10 +8065,10 @@ async def _exchange_persistence_attempted_for_auth_type(auth_type) -> bool: their contract is that the upstream token stays browser-held, minted/stored/refreshed nowhere.""" from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( exchange_token_with_server, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -8095,21 +8095,21 @@ async def _exchange_persistence_attempted_for_auth_type(auth_type) -> bool: with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=fake_http_client, ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.extract_user_id_from_request", + "litellm.proxy.mcp_server.discoverable_endpoints.extract_user_id_from_request", new_callable=AsyncMock, return_value="admin-user", ), patch( # test-quality-ok: this control tests persistence by auth mode; write-policy behavior is covered separately - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.authorize_oauth_credential_request", + "litellm.proxy.mcp_server.discoverable_endpoints.authorize_oauth_credential_request", new_callable=AsyncMock, return_value="admin-user", ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._store_per_user_token_server_side", + "litellm.proxy.mcp_server.discoverable_endpoints._store_per_user_token_server_side", new_callable=AsyncMock, ) as mock_store, ): @@ -8147,7 +8147,7 @@ async def test_token_exchange_persists_for_oauth2(): # ------------------------------------------------------------------- _OBO_RESOURCE = "https://litellm.example.com/mcp/obo_mcp" -_PATCH_ISSUERS = "litellm.proxy._experimental.mcp_server.discoverable_endpoints._jwt_auth_issuers" +_PATCH_ISSUERS = "litellm.proxy.mcp_server.discoverable_endpoints._jwt_auth_issuers" def _obo_server(scopes=None): @@ -8169,7 +8169,7 @@ def _obo_server(scopes=None): def test_obo_protected_resource_response_names_jwt_issuers(): """An OBO server's PRM points authorization_servers at the configured JWT issuers (the IdP that mints and validates the subject token), with the gateway resource echoed back.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _obo_protected_resource_response, ) @@ -8184,7 +8184,7 @@ def test_obo_protected_resource_response_names_jwt_issuers(): def test_obo_protected_resource_response_scopes_default_empty(): """A scopeless OBO server reports scopes_supported as [] rather than None.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _obo_protected_resource_response, ) @@ -8196,7 +8196,7 @@ def test_obo_protected_resource_response_scopes_default_empty(): def test_obo_protected_resource_response_falls_back_when_no_issuer(): """With no JWT issuer configured, the OBO branch returns None so the caller falls back to the gateway-default PRM (discovery still works, it just can't name the IdP).""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _obo_protected_resource_response, ) @@ -8206,10 +8206,10 @@ def test_obo_protected_resource_response_falls_back_when_no_issuer(): def test_obo_protected_resource_response_ignores_non_obo_server(): """Non-OBO servers are not handled by this branch (returns None -> gateway default).""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( _obo_protected_resource_response, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -8228,10 +8228,10 @@ async def test_build_oauth_protected_resource_response_obo_end_to_end(): authorization_servers, proving the extracted branch is wired into the public discovery path.""" from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_oauth_protected_resource_response, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -8282,11 +8282,11 @@ async def test_extract_user_id_reads_x_litellm_api_key_header(proxy_globals): """The LiteLLM key arrives on x-litellm-api-key (what Claude Desktop/Code send), not Authorization. Reading only Authorization dropped the identity, so the per-user token was never stored and the egress 401'd forever. Resolution must honor x-litellm-api-key.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( - extract_user_id_from_request, - ) from litellm.proxy._types import UserAPIKeyAuth, hash_token from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import ( + extract_user_id_from_request, + ) key = "sk-alice-key" cache = UserApiKeyCache() @@ -8307,11 +8307,11 @@ async def test_extract_user_id_rehydrates_cross_replica_dict_cache(proxy_globals """Cross-replica, async_get_cache hands back a serialized dict, not a UserAPIKeyAuth. Resolution must rehydrate it; the old getattr(cached, "user_id") returned None on a dict, which is exactly why a multi-replica gateway never found the stored token.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( - extract_user_id_from_request, - ) from litellm.proxy._types import hash_token from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import ( + extract_user_id_from_request, + ) key = "sk-alice-key" cache = UserApiKeyCache() @@ -8328,11 +8328,11 @@ async def test_extract_user_id_falls_back_to_db_on_cache_miss(proxy_globals): """A cache miss must read the key from the DB rather than returning None; the old code did a cache-only peek and skipped the DB, so any replica that hadn't just authenticated the key failed to store the token.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( - extract_user_id_from_request, - ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import ( + extract_user_id_from_request, + ) key = "sk-bob-key" @@ -8350,10 +8350,10 @@ async def test_extract_user_id_falls_back_to_db_on_cache_miss(proxy_globals): @pytest.mark.asyncio async def test_extract_user_id_none_without_litellm_key(proxy_globals): """No LiteLLM key on the request resolves to None without consulting the resolver.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import ( extract_user_id_from_request, ) - from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache proxy_globals.user_api_key_cache = UserApiKeyCache() proxy_globals.prisma_client = object() @@ -8367,11 +8367,11 @@ async def test_extract_user_id_rejects_blocked_key(proxy_globals): """A blocked LiteLLM key must not resolve an identity. get_key_object returns the DB row without checking blocked/expiry (the main auth pipeline does, and the public token endpoint bypasses it), so a revoked key could otherwise overwrite the stored per-user OAuth token for its user.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( - extract_user_id_from_request, - ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import ( + extract_user_id_from_request, + ) class _FakePrisma: async def get_data(self, token, table_name, parent_otel_span=None, proxy_logging_obj=None): @@ -8389,11 +8389,11 @@ async def test_extract_user_id_rejects_expired_key(proxy_globals): """An expired LiteLLM key must not resolve an identity, for the same reason as a blocked key.""" from datetime import datetime, timedelta, timezone - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( - extract_user_id_from_request, - ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import ( + extract_user_id_from_request, + ) expired = (datetime.now(timezone.utc) - timedelta(hours=1)).isoformat() @@ -8414,12 +8414,12 @@ async def test_resolve_active_litellm_key_returns_resolved_key_for_active_key(pr record. For an active key the resolver returns exactly hash_token(key), the same value get_key_object and the whole cache/DB layer key the record by, so the sealed reference resolves back to this key at admission.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( + from litellm.proxy._types import UserAPIKeyAuth, hash_token + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import ( _resolve_active_litellm_key, _ResolvedKey, ) - from litellm.proxy._types import UserAPIKeyAuth, hash_token - from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache key = "sk-alice-key" cache = UserApiKeyCache() @@ -8444,13 +8444,13 @@ async def test_resolve_active_litellm_key_resolves_key_without_user_id(proxy_glo presence wrongly rejected these keys with invalid_request; the active-state gate now checks only blocked and expiry, and the key hash (not the user) is what the mint seals. The per-user token store still gets no user for such a key, since there is none to key a stored credential by.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( + from litellm.proxy._types import UserAPIKeyAuth, hash_token + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import ( _resolve_active_litellm_key, _ResolvedKey, extract_user_id_from_request, ) - from litellm.proxy._types import UserAPIKeyAuth, hash_token - from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache key = "sk-team-scoped-key" cache = UserApiKeyCache() @@ -8473,11 +8473,11 @@ async def test_resolve_active_litellm_key_resolves_key_without_user_id(proxy_glo async def test_resolve_active_litellm_key_rejects_blocked_key(proxy_globals): """A blocked key must not yield a hash, so no gateway-bound envelope is minted for a revoked key; the mint fails closed with invalid_request instead.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( - _resolve_active_litellm_key, - ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import ( + _resolve_active_litellm_key, + ) class _FakePrisma: async def get_data(self, token, table_name, parent_otel_span=None, proxy_logging_obj=None): @@ -8496,11 +8496,11 @@ async def test_resolve_active_litellm_key_fails_closed_on_malformed_expiry(proxy returns invalid_request), not surface an unhandled 500. The active-state check runs outside the resolver's try, so it must be total over a bad expires rather than letting datetime.fromisoformat raise. Before the fix this raised a ValueError instead of returning None.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( - _resolve_active_litellm_key, - ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import ( + _resolve_active_litellm_key, + ) class _FakePrisma: async def get_data(self, token, table_name, parent_otel_span=None, proxy_logging_obj=None): @@ -8516,10 +8516,10 @@ async def test_resolve_active_litellm_key_fails_closed_on_malformed_expiry(proxy @pytest.mark.asyncio async def test_resolve_active_litellm_key_no_active_key_without_litellm_key(proxy_globals): """No LiteLLM key on the request yields no hash without consulting the resolver.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import ( _resolve_active_litellm_key, ) - from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache proxy_globals.user_api_key_cache = UserApiKeyCache() proxy_globals.prisma_client = object() @@ -8534,10 +8534,10 @@ async def test_resolve_active_litellm_key_db_outage_is_unavailable(proxy_globals the caller's fault, so the resolver reports "unavailable" (the mint statuses it 503) rather than collapsing it to the same value as a missing credential. is_database_service_unavailable_error classifies a connection error (an OSError) as an outage, matching admission's egress-side handling.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import ( _resolve_active_litellm_key, ) - from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache class _OutagePrisma: async def get_data(self, token, table_name, parent_otel_span=None, proxy_logging_obj=None): @@ -8557,10 +8557,10 @@ async def test_resolve_active_litellm_key_permanent_engine_fault_is_faulted(prox whose wording promises the outage is transient and asks the client to retry.""" from prisma.engine.errors import BinaryNotFoundError - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import ( _resolve_active_litellm_key, ) - from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache class _FaultedPrisma: async def get_data(self, token, table_name, parent_otel_span=None, proxy_logging_obj=None): @@ -8581,10 +8581,10 @@ async def test_resolve_active_litellm_key_transport_error_over_permanent_fault_i import httpx from prisma.engine.errors import BinaryNotFoundError - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import ( _resolve_active_litellm_key, ) - from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache class _ReconnectFailedPrisma: async def get_data(self, token, table_name, parent_otel_span=None, proxy_logging_obj=None): @@ -8605,10 +8605,10 @@ async def test_resolve_active_litellm_key_no_database_is_unresolvable(proxy_glob """With no database connection configured the gateway cannot verify the presented key at all, so the resolver reports "unresolvable" (the mint statuses it 500) instead of blaming the caller. Mirrors admission, which 500s a missing prisma_client on the egress side.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import ( _resolve_active_litellm_key, ) - from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache proxy_globals.user_api_key_cache = UserApiKeyCache() proxy_globals.prisma_client = None @@ -8639,8 +8639,8 @@ async def test_reload_active_user_by_id_missing_user_is_no_active_key(proxy_glob refresh path maps it to invalid_grant), not unresolvable/500. get_user_object catches the missing row and re-raises a bare ValueError, so a missing user must not be misclassified as a DB outage or an opaque gateway fault.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import reload_active_user_by_id from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import reload_active_user_by_id proxy_globals.user_api_key_cache = UserApiKeyCache() proxy_globals.prisma_client = object() @@ -8658,8 +8658,8 @@ async def test_reload_active_user_by_id_db_outage_is_unavailable(proxy_globals): a missing user, so the refresh path surfaces "unavailable" (a 503) rather than blaming the caller. get_user_object wraps the outage in a bare ValueError, so this exercises the chain-aware classifier; a raw ConnectionError would falsely pass even a chain-blind check because it is an OSError.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import reload_active_user_by_id from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import reload_active_user_by_id proxy_globals.user_api_key_cache = UserApiKeyCache() proxy_globals.prisma_client = object() @@ -8678,8 +8678,8 @@ async def test_reload_active_user_by_id_permanent_engine_fault_is_faulted(proxy_ wraps the fault in a bare ValueError, so the classification has to read the wrapped cause.""" from prisma.engine.errors import MismatchedVersionsError - from litellm.proxy._experimental.mcp_server.bridge_token_flow import reload_active_user_by_id from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import reload_active_user_by_id proxy_globals.user_api_key_cache = UserApiKeyCache() proxy_globals.prisma_client = object() @@ -8699,9 +8699,9 @@ async def test_load_active_user_by_id_reads_the_row_from_the_database_not_the_ca member never evicts the cached row, so a credential minted off the cached row refused the very first token exchange as not a member. The database source has to read the row from the database and leave the fresh row in the cache for the requests the credential makes next.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import load_active_user_by_id from litellm.proxy._types import LiteLLM_UserTable from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import load_active_user_by_id cache = UserApiKeyCache() await cache.async_set_cache( @@ -8728,12 +8728,12 @@ async def test_load_active_user_by_id_serves_a_cached_row_without_a_database_rea """Introspection and refresh revalidation run per call, so the loader's default source is the cache: a cached row answers without a database read, and only a caller that asks for the database row pays for one.""" - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( + from litellm.proxy._types import LiteLLM_UserTable + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.bridge_token_flow import ( load_active_user_by_id, reload_active_user_by_id, ) - from litellm.proxy._types import LiteLLM_UserTable - from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache cache = UserApiKeyCache() await cache.async_set_cache( @@ -8766,13 +8766,13 @@ async def test_token_endpoint_uses_client_secret_basic_when_configured(): from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( token_endpoint, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -8805,7 +8805,7 @@ async def test_token_endpoint_uses_client_secret_basic_when_configured(): mock_response.raise_for_status = MagicMock() with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client" + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client" ) as mock_get_client: mock_async_client = MagicMock() mock_async_client.post = AsyncMock(return_value=mock_response) @@ -8837,13 +8837,13 @@ async def test_token_endpoint_client_secret_basic_without_secret_returns_400(): inbound /token endpoint surfaces it as a 400 rather than silently posting a downgraded request.""" from fastapi import HTTPException, Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( token_endpoint, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -8903,10 +8903,10 @@ async def test_authorize_endpoint_rejects_non_oauth2_server(): """authorize() against a none-auth server returns an accurate 'does not use OAuth' 400, not the misleading 'client_id is required' that fired before the auth_type was checked.""" try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -8942,10 +8942,10 @@ async def test_token_endpoint_rejects_non_oauth2_server(): """token_endpoint() against a none-auth server returns 'does not use OAuth' 400 instead of the misleading 'token url is not set'.""" try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( token_endpoint, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -8984,10 +8984,10 @@ async def test_register_client_rejects_non_oauth2_server(): """register_client() against a named none-auth server returns 'does not use OAuth' 400 instead of the misleading 'authorization url is not set'.""" try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -9004,7 +9004,7 @@ async def test_register_client_rejects_non_oauth2_server(): try: with pytest.raises(HTTPException) as exc_info: with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.read_request_body", + "litellm.proxy.mcp_server.discoverable_endpoints.read_request_body", new=AsyncMock(return_value={}), ): await register_client(request=mock_request, mcp_server_name="access_group_server") @@ -9024,10 +9024,10 @@ async def test_register_client_rejects_non_oauth2_server(): async def test_oauth_protected_resource_for_gateway_owned_auth(auth_type, use_standard_pattern): from starlette.requests import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_oauth_protected_resource_response, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -9055,10 +9055,10 @@ async def test_oauth_protected_resource_for_gateway_owned_auth(auth_type, use_st async def test_oauth_authorization_server_404_for_non_oauth2_server(): """Discovery must not advertise a none-auth server as an OAuth authorization server.""" try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_oauth_authorization_server_response, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -9090,13 +9090,13 @@ async def test_oauth_protected_resource_passthrough_none_auth_not_404(): server that opted into OAuth pass-through must still proxy upstream metadata, it must NOT be 404'd. The gate has to sit after the pass-through branch.""" try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_oauth_protected_resource_response, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -9122,7 +9122,7 @@ async def test_oauth_protected_resource_passthrough_none_auth_not_404(): try: with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.fetch_upstream_oauth_protected_resource", + "litellm.proxy.mcp_server.discoverable_endpoints.fetch_upstream_oauth_protected_resource", new=AsyncMock(return_value={"authorization_servers": ["https://upstream-idp.example.com"]}), ): response = await _build_oauth_protected_resource_response( @@ -9140,10 +9140,10 @@ async def test_oauth_protected_resource_passthrough_none_auth_not_404(): async def test_oauth_protected_resource_404_for_unknown_server_name(): """Unknown server names must not produce metadata advertising nonexistent resources.""" try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_oauth_protected_resource_response, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -9168,10 +9168,10 @@ async def test_oauth_authorization_server_404_for_unknown_server_name(): """A named authorization-server discovery request for an unknown server returns 404, not a 200 metadata document pointing at non-existent /{name}/authorize and /{name}/token.""" try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_oauth_authorization_server_response, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) except ImportError: @@ -9195,8 +9195,8 @@ async def test_store_per_user_token_server_side_invalidates_v2_token_cache(): """A token stored by the OAuth callback (code exchange or refresh) drops the v2 per-user token cache entry, so egress stops serving the replaced token immediately instead of until its TTL.""" - from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server import mcp_server_manager as manager_module + from litellm.proxy.mcp_server.discoverable_endpoints import ( _store_per_user_token_server_side, ) from litellm.types.mcp import MCPAuth @@ -9218,11 +9218,11 @@ async def test_store_per_user_token_server_side_invalidates_v2_token_cache(): return_value=MagicMock(), ), patch( - "litellm.proxy._experimental.mcp_server.db.store_user_oauth_credential", + "litellm.proxy.mcp_server.db.store_user_oauth_credential", new=AsyncMock(return_value=None), ), patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.mcp_per_user_token_cache.set", + "litellm.proxy.mcp_server.oauth2_token_cache.mcp_per_user_token_cache.set", new=cache_set_mock, ), patch.object( @@ -9245,8 +9245,8 @@ async def test_store_per_user_token_server_side_invalidates_v2_token_cache(): async def test_store_per_user_token_server_side_skips_invalidate_when_db_write_fails(): """A failed DB write neither warms the v1 cache nor drops the v2 cache entry; the previously stored token is still the truth.""" - from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server import mcp_server_manager as manager_module + from litellm.proxy.mcp_server.discoverable_endpoints import ( _store_per_user_token_server_side, ) from litellm.types.mcp import MCPAuth @@ -9268,11 +9268,11 @@ async def test_store_per_user_token_server_side_skips_invalidate_when_db_write_f return_value=MagicMock(), ), patch( - "litellm.proxy._experimental.mcp_server.db.store_user_oauth_credential", + "litellm.proxy.mcp_server.db.store_user_oauth_credential", new=AsyncMock(side_effect=RuntimeError("db down")), ), patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.mcp_per_user_token_cache.set", + "litellm.proxy.mcp_server.oauth2_token_cache.mcp_per_user_token_cache.set", new=cache_set_mock, ), patch.object( @@ -9300,10 +9300,10 @@ async def test_token_exchange_pairs_client_secret_with_server_client_id(): placeholder makes the IdP reject the exchange with 401 on every re-auth.""" from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( exchange_token_with_server, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -9331,7 +9331,7 @@ async def test_token_exchange_pairs_client_secret_with_server_client_id(): mock_async_client.post = AsyncMock(return_value=mock_response) with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ): await exchange_token_with_server( @@ -9354,11 +9354,11 @@ async def test_token_exchange_pairs_client_secret_with_server_client_id(): async def test_token_exchange_refresh_passes_presented_refresh_ownership(): from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( exchange_token_with_server, ) - from litellm.proxy._experimental.mcp_server.oauth_identity_binding import RefreshTokenPresented - from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.oauth_identity_binding import RefreshTokenPresented from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer server = MCPServer( @@ -9386,15 +9386,15 @@ async def test_token_exchange_refresh_passes_presented_refresh_ownership(): with ( patch( # test-quality-ok: no injection seam exists for the exchange's HTTP and identity collaborators - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=client, ), patch( # test-quality-ok: no injection seam exists for request identity extraction - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.extract_user_id_from_request", + "litellm.proxy.mcp_server.discoverable_endpoints.extract_user_id_from_request", new=AsyncMock(return_value="user-a"), ), patch( # test-quality-ok: captures the ownership value at the exchange boundary - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.enforce_oauth_identity_binding", + "litellm.proxy.mcp_server.discoverable_endpoints.enforce_oauth_identity_binding", new=enforce, ), ): @@ -9419,11 +9419,11 @@ async def test_token_exchange_refresh_passes_presented_refresh_ownership(): async def test_token_exchange_authorization_code_passes_no_refresh_ownership(monkeypatch): from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( exchange_token_with_server, seal_bridge_authorization_code, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer monkeypatch.setenv("LITELLM_SALT_KEY", "identity-binding-test-salt") @@ -9452,15 +9452,15 @@ async def test_token_exchange_authorization_code_passes_no_refresh_ownership(mon with ( patch( # test-quality-ok: no injection seam exists for the exchange's HTTP and identity collaborators - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=client, ), patch( # test-quality-ok: no injection seam exists for request identity extraction - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.extract_user_id_from_request", + "litellm.proxy.mcp_server.discoverable_endpoints.extract_user_id_from_request", new=AsyncMock(return_value="user-a"), ), patch( # test-quality-ok: captures the ownership value at the exchange boundary - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.enforce_oauth_identity_binding", + "litellm.proxy.mcp_server.discoverable_endpoints.enforce_oauth_identity_binding", new=enforce, ), ): @@ -9498,10 +9498,10 @@ async def _exchange_with_upstream_response( caller-supplied-credentials flow (no stored client on the server).""" from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( exchange_token_with_server, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -9523,7 +9523,7 @@ async def _exchange_with_upstream_response( mock_async_client.post = AsyncMock(return_value=upstream_response) with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ): return await exchange_token_with_server( @@ -9671,10 +9671,10 @@ async def test_token_exchange_relays_rejection_when_http_client_raises(): from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( exchange_token_with_server, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -9694,7 +9694,7 @@ async def test_token_exchange_relays_rejection_when_http_client_raises(): mock_request.headers = {} with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=raising_client, ): response = await exchange_token_with_server( @@ -9720,7 +9720,7 @@ async def test_register_relays_rejection_when_http_client_raises(): so the register relay must catch it there, not only from the explicit raise_for_status.""" import httpx - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client_with_server, ) @@ -9740,11 +9740,11 @@ async def test_register_relays_rejection_when_http_client_raises(): with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=raising_client, ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._reuse_persisted_dcr_client_if_available", + "litellm.proxy.mcp_server.discoverable_endpoints._reuse_persisted_dcr_client_if_available", new_callable=AsyncMock, return_value=False, ), @@ -9783,7 +9783,7 @@ async def test_register_never_relays_out_of_contract_body_to_client(): client detail names only the status.""" import httpx - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client_with_server, ) @@ -9801,11 +9801,11 @@ async def test_register_never_relays_out_of_contract_body_to_client(): with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=raising_client, ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._reuse_persisted_dcr_client_if_available", + "litellm.proxy.mcp_server.discoverable_endpoints._reuse_persisted_dcr_client_if_available", new_callable=AsyncMock, return_value=False, ), @@ -9854,13 +9854,13 @@ async def test_persist_dcr_client_for_config_server_uses_side_store(): server so refresh can authenticate this process, and never call update_server(None) (which previously raised AttributeError on .approval_status, was swallowed, and reported a 200 that persisted nothing).""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( _persist_dcr_client_registration, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer config_server = MCPServer( @@ -9883,19 +9883,19 @@ async def test_persist_dcr_client_for_config_server_uses_side_store(): patch.object(global_mcp_server_manager, "is_config_declared_server", return_value=True), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), patch( - "litellm.proxy._experimental.mcp_server.db.update_mcp_server", + "litellm.proxy.mcp_server.db.update_mcp_server", new=AsyncMock(return_value=None), ), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + "litellm.proxy.mcp_server.db.get_mcp_server", new=AsyncMock(return_value=None), ), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server_oauth_client_credentials", + "litellm.proxy.mcp_server.db.get_mcp_server_oauth_client_credentials", new=AsyncMock(return_value=None), ), patch( - "litellm.proxy._experimental.mcp_server.db.upsert_mcp_server_oauth_client_credentials", + "litellm.proxy.mcp_server.db.upsert_mcp_server_oauth_client_credentials", new=mock_upsert, ), patch.object(global_mcp_server_manager, "update_server", new=mock_update_server), @@ -9935,11 +9935,11 @@ async def test_hydrate_config_server_applies_stored_dcr_client(monkeypatch, lega refresh_token grant can authenticate as the registered client instead of re-authenticating.""" import litellm.proxy.common_utils.encrypt_decrypt_utils as enc from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - from litellm.proxy._experimental.mcp_server.db import encrypt_credentials - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.db import encrypt_credentials + from litellm.proxy.mcp_server.discoverable_endpoints import ( hydrate_config_server_dcr_client, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -9975,7 +9975,7 @@ async def test_hydrate_config_server_applies_stored_dcr_client(monkeypatch, lega with ( patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server_oauth_client_credentials", + "litellm.proxy.mcp_server.db.get_mcp_server_oauth_client_credentials", new=AsyncMock(return_value=stored_blob), ), ): @@ -9994,14 +9994,14 @@ async def test_reuse_config_server_reads_store_with_real_crypto(monkeypatch): a re-authorize reuses the client instead of re-minting one.""" import litellm.proxy.common_utils.encrypt_decrypt_utils as enc from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - from litellm.proxy._experimental.mcp_server.db import encrypt_credentials - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.db import encrypt_credentials + from litellm.proxy.mcp_server.discoverable_endpoints import ( _reuse_persisted_dcr_client_if_available, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -10033,9 +10033,9 @@ async def test_reuse_config_server_reads_store_with_real_crypto(monkeypatch): with ( patch.object(global_mcp_server_manager, "is_config_declared_server", return_value=True), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), - patch("litellm.proxy._experimental.mcp_server.db.get_mcp_server", new=AsyncMock(return_value=None)), + patch("litellm.proxy.mcp_server.db.get_mcp_server", new=AsyncMock(return_value=None)), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server_oauth_client_credentials", + "litellm.proxy.mcp_server.db.get_mcp_server_oauth_client_credentials", new=store_lookup, ), ): @@ -10051,14 +10051,14 @@ async def test_temp_server_is_not_persisted_to_store(): """A rowless server that is NOT config-declared (a throwaway /server/oauth/session server) must not leave a permanent store row on persist, and the read must never consult the store for it. Its minted client is overlaid in memory for the session only.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( _persist_dcr_client_registration, _reuse_persisted_dcr_client_if_available, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer temp = MCPServer( @@ -10078,11 +10078,11 @@ async def test_temp_server_is_not_persisted_to_store(): with ( patch.object(global_mcp_server_manager, "is_config_declared_server", return_value=False), patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), - patch("litellm.proxy._experimental.mcp_server.db.update_mcp_server", new=AsyncMock(return_value=None)), - patch("litellm.proxy._experimental.mcp_server.db.get_mcp_server", new=AsyncMock(return_value=None)), - patch("litellm.proxy._experimental.mcp_server.db.upsert_mcp_server_oauth_client_credentials", new=upsert), + patch("litellm.proxy.mcp_server.db.update_mcp_server", new=AsyncMock(return_value=None)), + patch("litellm.proxy.mcp_server.db.get_mcp_server", new=AsyncMock(return_value=None)), + patch("litellm.proxy.mcp_server.db.upsert_mcp_server_oauth_client_credentials", new=upsert), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server_oauth_client_credentials", + "litellm.proxy.mcp_server.db.get_mcp_server_oauth_client_credentials", new=store_read, ), patch.object(global_mcp_server_manager, "update_server", new=AsyncMock()), @@ -10113,10 +10113,10 @@ async def test_temp_server_is_not_persisted_to_store(): async def test_hydrate_does_not_overwrite_explicit_config_client_id(): """An explicit client_id set in config.yaml wins: hydration must not overwrite it with a stale persisted store client, and must not even read the store when config already supplied a client.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( hydrate_config_server_dcr_client, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -10131,7 +10131,7 @@ async def test_hydrate_does_not_overwrite_explicit_config_client_id(): with ( patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server_oauth_client_credentials", + "litellm.proxy.mcp_server.db.get_mcp_server_oauth_client_credentials", new=store_read, ), ): @@ -10149,10 +10149,10 @@ async def test_reuse_does_not_inherit_store_client_when_a_row_exists(): server whose id collides with a config-declared server must not be able to load that config server's client from the store and send it to its own token endpoint. A row that exists but has no client_id yields no reusable client and must not fall back to the store.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.discoverable_endpoints import ( _reuse_persisted_dcr_client_if_available, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer submitted = MCPServer( @@ -10173,11 +10173,11 @@ async def test_reuse_does_not_inherit_store_client_when_a_row_exists(): with ( patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=MagicMock()), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server", + "litellm.proxy.mcp_server.db.get_mcp_server", new=AsyncMock(return_value=row_without_client), ), patch( - "litellm.proxy._experimental.mcp_server.db.get_mcp_server_oauth_client_credentials", + "litellm.proxy.mcp_server.db.get_mcp_server_oauth_client_credentials", new=store_lookup, ), ): @@ -10193,7 +10193,7 @@ async def test_load_servers_from_config_hydrates_dcr_clients(): """load_servers_from_config must invoke DCR-client hydration so config servers pick up their persisted client on startup; deleting the call site leaves a restarted server with no client_id and forces re-authentication on every token expiry.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -10210,7 +10210,7 @@ async def test_reload_servers_from_database_hydrates_dcr_clients(): reload_servers_from_database runs after the DB connects and must hydrate config servers' persisted DCR clients too, or a fresh pod has no client_id for a config server and forces re-authentication on the first token refresh.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -10240,8 +10240,8 @@ def test_aggregate_wellknown_routes_serve_gateway_metadata(): from fastapi import FastAPI from fastapi.testclient import TestClient - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import router - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.discoverable_endpoints import router + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -10273,8 +10273,8 @@ def test_as_aggregate_route_reserves_mcp_for_the_aggregate(): from fastapi import FastAPI from fastapi.testclient import TestClient - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import router - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.discoverable_endpoints import router + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -10311,7 +10311,7 @@ def test_well_known_root_suffix_reflects_server_root_path(): import os from unittest.mock import patch - from litellm.proxy._experimental.mcp_server.oauth_utils import well_known_root_suffix + from litellm.proxy.mcp_server.oauth_utils import well_known_root_suffix with patch.dict(os.environ, {"SERVER_ROOT_PATH": ""}): assert well_known_root_suffix() == "" @@ -10325,11 +10325,11 @@ def test_well_known_root_suffix_reflects_server_root_path(): async def test_root_resource_uses_gateway_without_changing_authorization_relay(): from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_oauth_authorization_server_response, _build_oauth_protected_resource_response, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -10360,11 +10360,11 @@ async def test_named_discovery_issuer_matches_protected_resource_authorization_s resolved the metadata from, which is the protected-resource authorization_servers entry.""" from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_oauth_authorization_server_response, _build_oauth_protected_resource_response, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -10395,10 +10395,10 @@ async def test_openid_configuration_issuer_stays_bare_origin_for_single_oauth2_s bare origin even when root discovery resolves the one configured OAuth2 server.""" from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( openid_configuration, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -10424,8 +10424,8 @@ def test_gateway_dcr_flow_routing_engages_only_for_llm_dcrc_clients(monkeypatch) from fastapi import FastAPI from fastapi.testclient import TestClient - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import router - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.discoverable_endpoints import router + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -10481,7 +10481,7 @@ async def test_authorize_wall_names_the_fix_for_urlless_servers(): """LIT-4629: the authorize wall previously said only "authorization url is not set" with no hint that spec-only servers never discover; the detail must now name both remedies (manual Authorization URL + Token URL, or an Issuer for RFC 8414 discovery).""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize_with_server, ) from litellm.types.mcp import MCPAuth, MCPTransport @@ -10518,7 +10518,7 @@ async def test_token_wall_names_the_fix_for_urlless_servers(): """The /token wall is the second stop on the same misconfiguration (LIT-4629): after an admin fills only the Authorization URL, the code exchange dies here; the detail must name the remedies like the authorize wall does.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( exchange_token_with_server, ) from litellm.types.mcp import MCPAuth, MCPTransport @@ -10559,7 +10559,7 @@ async def test_token_wall_names_the_fix_for_urlless_servers(): async def test_register_wall_names_the_fix_for_urlless_servers(): """The /register wall serves the same missing-authorization-url 400 as authorize; its detail must carry the same actionable remedies.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( register_client_with_server, ) from litellm.types.mcp import MCPAuth, MCPTransport @@ -10599,7 +10599,7 @@ async def test_authorize_wall_points_at_discovery_failure_for_url_servers(): discovery against that url failed (typically a misconfigured url); the old detail blamed "servers with no url", sending the operator down the wrong path. The detail must now name the discovery failure and point at the proxy logs where LIT-4658's warnings carry the reason.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize_with_server, ) from litellm.types.mcp import MCPAuth, MCPTransport @@ -10634,7 +10634,7 @@ async def test_authorize_wall_points_at_discovery_failure_for_url_servers(): @pytest.mark.asyncio async def test_token_wall_points_at_discovery_failure_for_url_servers(): - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( exchange_token_with_server, ) from litellm.types.mcp import MCPAuth, MCPTransport @@ -10673,7 +10673,7 @@ async def test_token_wall_points_at_discovery_failure_for_url_servers(): @pytest.mark.asyncio async def test_authorize_wall_names_the_issuer_for_anchored_servers(): - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize_with_server, ) from litellm.types.mcp import MCPAuth, MCPTransport @@ -10713,7 +10713,7 @@ async def test_authorize_uses_admin_entered_github_oauth_urls_after_issuer_yield the resolved authorization_url when a leftover issuer is treated as a pin (RFC 8414 yield). The UI authorize must still redirect to the admin-entered GitHub authorize URL instead of 400ing that discovery against api.githubcopilot.com failed.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize_with_server, ) from litellm.types.mcp import MCPAuth, MCPTransport @@ -10757,7 +10757,7 @@ async def test_authorize_uses_admin_entered_github_oauth_urls_after_issuer_yield def test_oauth_endpoints_count_admin_entered_urls_as_resolved(): """A leftover issuer empties the resolved authorize/token fields but must not keep the server on the deferred-discovery retry path when the admin already stored those URLs.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( oauth_endpoints_unresolved, ) from litellm.types.mcp import MCPAuth, MCPTransport @@ -10783,11 +10783,11 @@ async def test_token_exchange_with_configured_token_url_never_joins_discovery(mo """A server can hold an admin-entered Token URL while its Authorization URL is absent. The token exchange must post to that stored endpoint without awaiting deferred discovery, which can 503 against an unreachable issuer even though nothing it resolves is needed here.""" - from litellm.proxy._experimental.mcp_server import ( + from litellm.proxy.mcp_server import ( discoverable_endpoints, mcp_server_manager, ) - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( exchange_token_with_server, ) from litellm.types.mcp import MCPAuth, MCPTransport @@ -10852,7 +10852,7 @@ async def test_root_token_route_with_configured_token_url_never_joins_discovery( """A root POST /token that falls back to the sole OAuth2 server must reach the exchange's endpoint-gated discovery join instead of awaiting full discovery at the route: with the token url admin-entered, a failing or slow discovery must not turn the exchange into a 503.""" - from litellm.proxy._experimental.mcp_server import ( + from litellm.proxy.mcp_server import ( discoverable_endpoints, mcp_server_manager, ) @@ -10917,8 +10917,8 @@ async def test_bridge_authorize_relays_with_registration_url_resolved_by_deferre relay-vs-short-circuit call must read the resolved server: a client that registered itself through the front door keeps its own redirect binding instead of being routed through the gateway callback the upstream never granted it.""" - from litellm.proxy._experimental.mcp_server import mcp_server_manager - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server import mcp_server_manager + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize_with_server, ) from litellm.types.mcp import MCPAuth, MCPTransport @@ -10978,7 +10978,7 @@ def test_passthrough_authorization_code_round_trips_and_rejects_hostile_input(): and is total over hostile input: a raw upstream code opens to None, and a tampered or non-gateway value opens to None rather than raising, so every existing caller-supplied-client flow is untouched.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( open_passthrough_authorization_code, seal_passthrough_authorization_code, ) @@ -11001,7 +11001,7 @@ def test_passthrough_authorization_code_round_trips_and_rejects_hostile_input(): assert open_passthrough_authorization_code("raw-upstream-code") is None assert open_passthrough_authorization_code(sealed[:-4] + "aaaa") is None - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _BRIDGE_AUTH_CODE_PREFIX, _PASSTHROUGH_AUTH_CODE_PREFIX, open_bridge_authorization_code, @@ -11022,7 +11022,7 @@ async def test_authorize_with_ephemeral_dcr_client_seals_client_into_state(): """When mcp_authorize fell through to a gateway-side DCR mint, authorize_with_server seals the minted client and the target server into the encrypted OAuth state, so the callback can bind them into the forwarded authorization code while the gateway stores nothing.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( EphemeralDcrClient, authorize_with_server, ) @@ -11036,7 +11036,7 @@ async def test_authorize_with_ephemeral_dcr_client_seals_client_into_state(): return "mocked_encrypted_state" with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.encode_state_with_base_url", + "litellm.proxy.mcp_server.discoverable_endpoints.encode_state_with_base_url", side_effect=_capture, ): response = await authorize_with_server( @@ -11067,7 +11067,7 @@ async def test_callback_wraps_code_into_passthrough_code_for_ephemeral_dcr_state gateway never stored.""" from urllib.parse import parse_qs, urlparse - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( callback, open_passthrough_authorization_code, ) @@ -11083,15 +11083,15 @@ async def test_callback_wraps_code_into_passthrough_code_for_ephemeral_dcr_state } with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._resolve_encoded_oauth_state", + "litellm.proxy.mcp_server.discoverable_endpoints._resolve_encoded_oauth_state", return_value="enc", ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.decode_state_hash", + "litellm.proxy.mcp_server.discoverable_endpoints.decode_state_hash", return_value=state_data, ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._get_validated_client_redirect_uri", + "litellm.proxy.mcp_server.discoverable_endpoints._get_validated_client_redirect_uri", return_value="http://127.0.0.1:60108/cb", ), patch("litellm.proxy.proxy_server.master_key", _BRIDGE_MASTER_KEY), @@ -11117,7 +11117,7 @@ async def test_authorize_bridge_server_with_ephemeral_client_takes_short_circuit to an IdP that has the gateway callback registered, stranding the flow.""" from urllib.parse import parse_qs, urlparse - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( EphemeralDcrClient, authorize_with_server, ) @@ -11151,7 +11151,7 @@ async def test_callback_forwards_raw_code_when_dcr_state_lacks_server_binding(): one.""" from urllib.parse import parse_qs, urlparse - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import callback + from litellm.proxy.mcp_server.discoverable_endpoints import callback state_data = { "original_state": "client-state", @@ -11161,15 +11161,15 @@ async def test_callback_forwards_raw_code_when_dcr_state_lacks_server_binding(): } with ( patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._resolve_encoded_oauth_state", + "litellm.proxy.mcp_server.discoverable_endpoints._resolve_encoded_oauth_state", return_value="enc", ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.decode_state_hash", + "litellm.proxy.mcp_server.discoverable_endpoints.decode_state_hash", return_value=state_data, ), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._get_validated_client_redirect_uri", + "litellm.proxy.mcp_server.discoverable_endpoints._get_validated_client_redirect_uri", return_value="http://127.0.0.1:60108/cb", ), patch("litellm.proxy.proxy_server.master_key", _BRIDGE_MASTER_KEY), @@ -11206,7 +11206,7 @@ async def test_resolve_ephemeral_dcr_client_mint_set_is_exact(auth_type_value, d oauth_delegate dcr_bridge sign-in is left to its own browser-front-door flow. The UI gatewayMintsClientFor helper mirrors this exact set; ui/.../mcp_tools/types.test.tsx pins the frontend side against the same table, so a divergence fails on one side or the other.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( EphemeralDcrClient, resolve_ephemeral_dcr_client, ) @@ -11222,7 +11222,7 @@ async def test_resolve_ephemeral_dcr_client_mint_set_is_exact(auth_type_value, d mint_mock = AsyncMock(return_value=EphemeralDcrClient(client_id="minted", client_secret=None)) with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.mint_ephemeral_dcr_client", + "litellm.proxy.mcp_server.discoverable_endpoints.mint_ephemeral_dcr_client", mint_mock, ): result = await resolve_ephemeral_dcr_client( @@ -11245,7 +11245,7 @@ async def test_resolve_ephemeral_dcr_client_mint_set_is_exact(auth_type_value, d async def test_mint_ephemeral_dcr_client_returns_none_without_registration_endpoint(): """A server whose upstream exposes no RFC 7591 registration endpoint cannot mint, so the fall-through reports None and the caller keeps its existing missing_client_id failure.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( mint_ephemeral_dcr_client, ) from litellm.types.mcp import MCPAuth @@ -11258,7 +11258,7 @@ async def test_mint_ephemeral_dcr_client_returns_none_without_registration_endpo async def test_mint_ephemeral_dcr_client_posts_rfc7591_and_returns_client(): """The mint POSTs a public-client RFC 7591 registration bound to the gateway /callback and hands back the upstream's client without persisting it anywhere.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( mint_ephemeral_dcr_client, ) from litellm.types.mcp import MCPAuth @@ -11275,7 +11275,7 @@ async def test_mint_ephemeral_dcr_client_posts_rfc7591_and_returns_client(): mock_async_client.post = AsyncMock(return_value=mock_response) with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ): minted = await mint_ephemeral_dcr_client(_bridge_mock_request(), server) @@ -11295,7 +11295,7 @@ async def test_mint_ephemeral_dcr_client_reuses_minted_client_within_flow_ttl(): """Reloading the authorize page must not spam the upstream registration endpoint with orphan clients: within the OAuth state's lifetime a second mint for the same server and gateway origin reuses the cached client and performs no second upstream POST.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( mint_ephemeral_dcr_client, ) from litellm.types.mcp import MCPAuth @@ -11310,7 +11310,7 @@ async def test_mint_ephemeral_dcr_client_reuses_minted_client_within_flow_ttl(): mock_async_client.post = AsyncMock(return_value=mock_response) with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ): first = await mint_ephemeral_dcr_client(_bridge_mock_request(), server) @@ -11328,7 +11328,7 @@ async def test_mint_ephemeral_dcr_client_single_flights_concurrent_mints(): POST happens.""" import asyncio - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( mint_ephemeral_dcr_client, ) from litellm.types.mcp import MCPAuth @@ -11351,7 +11351,7 @@ async def test_mint_ephemeral_dcr_client_single_flights_concurrent_mints(): mock_async_client.post = AsyncMock(side_effect=_slow_post) with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ): first, second = await asyncio.gather( @@ -11376,7 +11376,7 @@ async def test_mint_ephemeral_dcr_client_unusable_registration_response_is_502(p """An upstream registration response without a usable client_id, whether the field is missing or an empty string, surfaces as a loud 502 instead of letting the authorize proceed with an empty client and fail opaquely at the IdP.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( mint_ephemeral_dcr_client, ) from litellm.types.mcp import MCPAuth @@ -11391,7 +11391,7 @@ async def test_mint_ephemeral_dcr_client_unusable_registration_response_is_502(p mock_async_client.post = AsyncMock(return_value=mock_response) with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ): with pytest.raises(HTTPException) as exc: @@ -11420,7 +11420,7 @@ async def test_token_exchange_authenticates_with_the_sealed_clients_own_auth_met import httpx - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( exchange_token_with_server, ) from litellm.types.mcp import MCPAuth @@ -11434,7 +11434,7 @@ async def test_token_exchange_authenticates_with_the_sealed_clients_own_auth_met mock_async_client.post = AsyncMock(return_value=upstream_response) with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ): await exchange_token_with_server( @@ -11529,7 +11529,7 @@ def test_resolve_upstream_resource_tristate_and_canonicalization(url, configured hygiene), and drop a trailing slash, while preserving a non-default port and the path case. An explicit value is never canonicalized, because it has to match what the authorization server expects byte for byte.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import resolve_upstream_resource + from litellm.proxy.mcp_server.oauth_utils import resolve_upstream_resource assert resolve_upstream_resource(_resource_server(url=url, upstream_resource=configured)) == expected @@ -11548,7 +11548,7 @@ def test_build_upstream_oauth2_token_request_bundles_resource_with_client_auth(c mandatory client authentication and no leg can authenticate without also naming the resource its sibling legs named. A leg that reverted to hand-building its body would drop the resource and diverge from the authorize leg, which a strict authorization server rejects as invalid_target.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import build_upstream_oauth2_token_request + from litellm.proxy.mcp_server.oauth_utils import build_upstream_oauth2_token_request req = build_upstream_oauth2_token_request( _resource_server(url=url, upstream_resource=configured), @@ -11566,7 +11566,7 @@ def test_build_upstream_oauth2_token_request_client_secret_basic_keeps_secret_ou also appear in the body, while the RFC 8707 resource still rides in the body.""" import base64 - from litellm.proxy._experimental.mcp_server.oauth_utils import build_upstream_oauth2_token_request + from litellm.proxy.mcp_server.oauth_utils import build_upstream_oauth2_token_request req = build_upstream_oauth2_token_request( _resource_server(upstream_resource="api://audience"), @@ -11585,7 +11585,7 @@ async def _authorize_query(server) -> dict: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( authorize_with_server, ) @@ -11593,7 +11593,7 @@ async def _authorize_query(server) -> dict: mock_request.base_url = "https://litellm.example.com/" mock_request.headers = {} - with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: + with patch("litellm.proxy.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: mock_encrypt.return_value = "encrypted_state" response = await authorize_with_server( request=mock_request, @@ -11612,7 +11612,7 @@ async def _authorize_query(server) -> dict: async def _token_body(server, grant_type: str) -> dict: from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( exchange_token_with_server, ) @@ -11627,7 +11627,7 @@ async def _token_body(server, grant_type: str) -> dict: mock_async_client.post = AsyncMock(return_value=mock_response) with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=mock_async_client, ): await exchange_token_with_server( @@ -11711,7 +11711,7 @@ def test_upstream_resource_auto_keeps_the_path_because_it_identifies_the_server( it would collide them onto one resource identifier and bind each token to the wrong audience, which is the exact confusion RFC 8707 exists to prevent. An operator whose path embeds a secret sets ``upstream_resource`` explicitly instead of using ``auto``.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import resolve_upstream_resource + from litellm.proxy.mcp_server.oauth_utils import resolve_upstream_resource first = resolve_upstream_resource( _resource_server(url="https://gw.example.com/team-a/mcp", upstream_resource="auto") @@ -11740,7 +11740,7 @@ async def test_upstream_resource_sent_on_dcr_bridge_relay_authorize(): """The DCR-bridge relay arm builds its own upstream authorize params, so it needs the resource too. Without it the relayed authorize would omit the resource while the gateway's token leg still sent one, which is itself an invalid_target.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _dcr_bridge_relays_client_registration, ) from litellm.types.mcp import MCPAuth @@ -11781,7 +11781,7 @@ def _no_proxy_base_url(monkeypatch): def _isolated_mcp_registry(): """Fixture-owned registry state: snapshot the shared registry, hand the test an empty one, restore afterwards so nothing leaks between cases.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -11798,7 +11798,7 @@ def _prefixed_discovery_client(prefixes): from fastapi import FastAPI from fastapi.testclient import TestClient - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import router + from litellm.proxy.mcp_server.discoverable_endpoints import router from litellm.proxy.middleware.per_request_root_path_middleware import ( PerRequestRootPathMiddleware, ) @@ -11817,7 +11817,7 @@ class TestPerRequestRootPathDiscovery: from fastapi import FastAPI from fastapi.testclient import TestClient - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import router + from litellm.proxy.mcp_server.discoverable_endpoints import router server = _create_oauth2_server(server_id="srv_a", name="server_a", server_name="server_a", alias="server_a") _isolated_mcp_registry[server.server_id] = server @@ -11891,7 +11891,7 @@ class TestPerRequestRootPathDiscovery: """The WWW-Authenticate resource_metadata URL a 401 advertises must land under the request's prefix, or the client is bounced to a document whose ``resource`` cannot match the URL it called.""" - from litellm.proxy._experimental.mcp_server.oauth_utils import ( + from litellm.proxy.mcp_server.oauth_utils import ( get_passthrough_resource_metadata_url, ) @@ -11935,8 +11935,8 @@ class TestPerRequestRootPathDiscovery: from fastapi import FastAPI, HTTPException, Request from fastapi.testclient import TestClient - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import router - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( + from litellm.proxy.mcp_server.discoverable_endpoints import router + from litellm.proxy.mcp_server.outbound_credentials.adapter import ( raise_user_oauth_challenge, ) from litellm.proxy.middleware.per_request_root_path_middleware import ( @@ -12083,9 +12083,9 @@ def _native_client_app(monkeypatch): from fastapi import FastAPI from fastapi.testclient import TestClient - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import router - from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ConsentTeam, MintedProxyCredential - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.discoverable_endpoints import router + from litellm.proxy.mcp_server.gateway_dcr_flow import ConsentTeam, MintedProxyCredential + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager monkeypatch.setenv("LITELLM_SALT_KEY", _NATIVE_CLIENT_MASTER_KEY) monkeypatch.setattr("litellm.proxy.proxy_server.master_key", _NATIVE_CLIENT_MASTER_KEY, raising=False) @@ -12099,10 +12099,10 @@ def _native_client_app(monkeypatch): return (ConsentTeam(team_id="team-a", team_alias="Team A"), ConsentTeam(team_id="team-b")) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.mint_proxy_credential", fake_mint + "litellm.proxy.mcp_server.discoverable_endpoints.mint_proxy_credential", fake_mint ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.lookup_consent_teams", fake_lookup + "litellm.proxy.mcp_server.discoverable_endpoints.lookup_consent_teams", fake_lookup ) global_mcp_server_manager.registry.clear() app = FastAPI() @@ -12215,9 +12215,9 @@ def test_gateway_authorize_routes_forward_server_scope(monkeypatch: pytest.Monke from http.cookies import SimpleCookie from urllib.parse import parse_qs, urlparse - from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import CONNECT_FLOW_COOKIE_PREFIX - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper + from litellm.proxy.mcp_server.gateway_dcr_flow import CONNECT_FLOW_COOKIE_PREFIX + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager monkeypatch.setattr( global_mcp_server_manager, @@ -12553,8 +12553,8 @@ def test_introspect_route_requires_virtual_key_auth_and_is_advertised(): from fastapi.routing import APIRoute from fastapi.testclient import TestClient - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import router from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.mcp_server.discoverable_endpoints import router route = next(r for r in router.routes if isinstance(r, APIRoute) and r.path == "/introspect") assert route.methods == {"POST"} @@ -12584,16 +12584,16 @@ def test_introspect_route_answers_for_authenticated_caller(monkeypatch): from fastapi import FastAPI from fastapi.testclient import TestClient - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import router - from litellm.proxy._experimental.mcp_server.outbound_credentials.session_credentials import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.mcp_server.discoverable_endpoints import router + from litellm.proxy.mcp_server.outbound_credentials.session_credentials import ( session_keys_from_master_key, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import ( + from litellm.proxy.mcp_server.outbound_credentials.session_token import ( SessionPrincipal, mint_session_token, ) - from litellm.proxy._types import UserAPIKeyAuth - from litellm.proxy.auth.user_api_key_auth import user_api_key_auth introspect_master_key = "sk-introspect-route-test" monkeypatch.setattr("litellm.proxy.proxy_server.master_key", introspect_master_key, raising=False) @@ -12602,7 +12602,7 @@ def test_introspect_route_answers_for_authenticated_caller(monkeypatch): return None monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.reload_active_user_by_id", fake_reload + "litellm.proxy.mcp_server.discoverable_endpoints.reload_active_user_by_id", fake_reload ) app = FastAPI() app.include_router(router) @@ -12631,7 +12631,7 @@ async def test_identity_bound_authorization_carries_nonce_and_caller_through_cal from fastapi import Request - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _oauth_state_cookie_name, authorize_with_server, callback, @@ -12667,11 +12667,11 @@ async def test_identity_bound_authorization_carries_nonce_and_caller_through_cal ) with ( patch( # test-quality-ok: isolate authenticated request resolution from the real encrypted OAuth round trip - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.authorize_oauth_credential_request", + "litellm.proxy.mcp_server.discoverable_endpoints.authorize_oauth_credential_request", new=AsyncMock(return_value="alice"), ), patch( # test-quality-ok: isolate user access lookup while testing nonce and caller preservation - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._user_can_reach_mcp_server", + "litellm.proxy.mcp_server.discoverable_endpoints._user_can_reach_mcp_server", new=AsyncMock(return_value=True), ), ): @@ -12708,18 +12708,18 @@ async def test_enforced_login_warms_verified_token_readable_without_database_loo from litellm.caching.caching import DualCache from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import db, mcp_server_manager - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import _store_per_user_token_server_side - from litellm.proxy._experimental.mcp_server.oauth2_token_cache import mcp_per_user_token_cache - from litellm.proxy._experimental.mcp_server.oauth_identity_binding import current_binding_proof - from litellm.proxy._experimental.mcp_server.outbound_credentials.dual_cache_token_backend import ( + from litellm.proxy.mcp_server import db, mcp_server_manager + from litellm.proxy.mcp_server.discoverable_endpoints import _store_per_user_token_server_side + from litellm.proxy.mcp_server.oauth2_token_cache import mcp_per_user_token_cache + from litellm.proxy.mcp_server.oauth_identity_binding import current_binding_proof + from litellm.proxy.mcp_server.outbound_credentials.dual_cache_token_backend import ( DualCacheTokenCacheBackend, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import CachedOAuthTokenStore - from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import ( + from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import CachedOAuthTokenStore + from litellm.proxy.mcp_server.outbound_credentials.per_user_oauth_store import ( LazyPerUserOAuthTokenStore, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.v2_token_store import V2PerUserTokenStore + from litellm.proxy.mcp_server.outbound_credentials.v2_token_store import V2PerUserTokenStore from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -12778,7 +12778,7 @@ async def test_dcr_refusal_is_actionable_without_upstream_body( import httpx - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( mint_ephemeral_dcr_client, register_client_with_server, ) @@ -12873,7 +12873,7 @@ def _gateway_root_path_discovery_router(monkeypatch: pytest.MonkeyPatch) -> "Ite (a second reload would hand earlier importers a different ``router`` object).""" import importlib - from litellm.proxy._experimental.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server import discoverable_endpoints snapshot: Final = dict(vars(discoverable_endpoints)) monkeypatch.setenv("SERVER_ROOT_PATH", "/gateway") @@ -12991,13 +12991,13 @@ async def test_oauth_exchange_stores_token_for_validated_jwt_user( ) -> None: import httpx - from litellm.proxy._experimental.mcp_server import discoverable_endpoints from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server import discoverable_endpoints from litellm.types.mcp_server.mcp_server_manager import MCPServer handler, signing_key = jwt_oauth_identity handler.litellm_jwtauth.custom_validate = lambda claims: policy_allowed - from litellm.proxy._experimental.mcp_server import mcp_server_manager + from litellm.proxy.mcp_server import mcp_server_manager manager: Final = MagicMock() manager.get_allowed_mcp_servers = AsyncMock(return_value=["jwt-oauth-server"] if server_allowed else []) @@ -13112,8 +13112,8 @@ async def test_oauth_jwt_identity_rejects_untrusted_or_inactive_owner( from litellm.models.user import LiteLLM_UserTable from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import mcp_server_manager - from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( + from litellm.proxy.mcp_server import mcp_server_manager + from litellm.proxy.mcp_server.bridge_token_flow import ( authorize_oauth_credential_request, extract_user_id_from_request, ) @@ -13159,8 +13159,8 @@ async def test_oauth_jwt_cannot_override_explicit_litellm_key( jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], blocked: bool, ) -> None: - from litellm.proxy._experimental.mcp_server.bridge_token_flow import extract_user_id_from_request from litellm.proxy._types import UserAPIKeyAuth, hash_token + from litellm.proxy.mcp_server.bridge_token_flow import extract_user_id_from_request handler, signing_key = jwt_oauth_identity key: Final = "sk-explicit-key" @@ -13183,9 +13183,9 @@ async def test_oauth_jwt_uses_configured_virtual_key_owner( mapping: str, ) -> None: from litellm.models.user import LiteLLM_UserTable - from litellm.proxy._experimental.mcp_server.bridge_token_flow import extract_user_id_from_request from litellm.proxy._types import UnregisteredJWTClientBehavior, UserAPIKeyAuth, hash_token from litellm.proxy.auth.auth_checks import jwt_key_mapping_cache_key + from litellm.proxy.mcp_server.bridge_token_flow import extract_user_id_from_request handler, signing_key = jwt_oauth_identity handler.litellm_jwtauth.virtual_key_claim_field = "sub" @@ -13220,7 +13220,7 @@ async def test_oauth_jwt_respects_custom_validation_and_email_policy( jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], allowed_domain: str | None, ) -> None: - from litellm.proxy._experimental.mcp_server.bridge_token_flow import extract_user_id_from_request + from litellm.proxy.mcp_server.bridge_token_flow import extract_user_id_from_request handler, signing_key = jwt_oauth_identity handler.litellm_jwtauth.custom_validate = lambda claims: True @@ -13237,9 +13237,9 @@ async def test_oauth_jwt_identity_preserves_separate_mcp_route_authorization( route_allowed: bool, ) -> None: from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.bridge_token_flow import extract_user_id_from_request from litellm.proxy._types import LitellmUserRoles, RoleBasedPermissions, RoleMapping from litellm.proxy.auth.handle_jwt import JWTAuthManager + from litellm.proxy.mcp_server.bridge_token_flow import extract_user_id_from_request handler, signing_key = jwt_oauth_identity handler.litellm_jwtauth.user_id_jwt_field = "sub" @@ -13298,8 +13298,8 @@ async def test_oauth_jwt_resolves_canonical_owner_without_cached_identity( ) -> None: from litellm.models.user import LiteLLM_UserTable from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.bridge_token_flow import extract_user_id_from_request from litellm.proxy.auth.handle_jwt import JWTAuthManager + from litellm.proxy.mcp_server.bridge_token_flow import extract_user_id_from_request handler, signing_key = jwt_oauth_identity external_id: Final = f"external-{identity}-{inactive}-{admin}" @@ -13345,7 +13345,7 @@ async def test_oauth_jwt_identity_does_not_provision_or_synchronize_teams( ) -> None: from litellm.models.user import LiteLLM_UserTable from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.bridge_token_flow import extract_user_id_from_request + from litellm.proxy.mcp_server.bridge_token_flow import extract_user_id_from_request handler, signing_key = jwt_oauth_identity handler.litellm_jwtauth.enforce_team_based_model_access = True @@ -13373,7 +13373,7 @@ async def test_oauth_refresh_revalidates_the_same_active_user_rule( ) -> None: from litellm.models.user import LiteLLM_UserTable from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.bridge_token_flow import reload_active_user_by_id + from litellm.proxy.mcp_server.bridge_token_flow import reload_active_user_by_id handler, _ = jwt_oauth_identity user_id: Final = f"jwt-owner-{state}" @@ -13402,10 +13402,10 @@ async def test_oauth_credential_write_keeps_virtual_key_permissions( ) -> None: import asyncio - from litellm.proxy._experimental.mcp_server import mcp_server_manager - from litellm.proxy._experimental.mcp_server.bridge_token_flow import authorize_oauth_credential_request from litellm.proxy._types import UserAPIKeyAuth, hash_token from litellm.proxy.auth.auth_checks import OrganizationNotFoundError, jwt_key_mapping_cache_key + from litellm.proxy.mcp_server import mcp_server_manager + from litellm.proxy.mcp_server.bridge_token_flow import authorize_oauth_credential_request handler, signing_key = jwt_oauth_identity monkeypatch.setattr( @@ -13471,9 +13471,9 @@ async def test_oauth_writer_preserves_claimed_team_instead_of_expanding_user_ros ) -> None: from litellm.models.user import LiteLLM_UserTable from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import mcp_server_manager - from litellm.proxy._experimental.mcp_server.bridge_token_flow import authorize_oauth_credential_request from litellm.proxy._types import LiteLLM_TeamTable, Member + from litellm.proxy.mcp_server import mcp_server_manager + from litellm.proxy.mcp_server.bridge_token_flow import authorize_oauth_credential_request handler, signing_key = jwt_oauth_identity handler.litellm_jwtauth.team_id_jwt_field = "team" @@ -13509,8 +13509,8 @@ async def test_oauth_write_denial_does_not_erase_identity_binding( jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], monkeypatch: pytest.MonkeyPatch, ) -> None: - from litellm.proxy._experimental.mcp_server import discoverable_endpoints, mcp_server_manager from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server import discoverable_endpoints, mcp_server_manager from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer _, signing_key = jwt_oauth_identity @@ -13570,8 +13570,8 @@ async def test_signed_oauth_callback_honors_credential_write_policy( from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import discoverable_endpoints, mcp_server_manager from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server import discoverable_endpoints, mcp_server_manager from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -13689,9 +13689,9 @@ async def test_identity_bound_authorize_preserves_presented_jwt_permissions( from litellm.models.user import LiteLLM_UserTable from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import discoverable_endpoints, mcp_server_manager from litellm.proxy._types import JWTIssuerConfig, MCPTransport, UserAPIKeyAuth, hash_token from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken + from litellm.proxy.mcp_server import discoverable_endpoints, mcp_server_manager from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer handler, signing_key = jwt_oauth_identity @@ -13825,8 +13825,8 @@ async def test_identity_bound_authorize_unrelated_bearer_uses_browser_session( from litellm.models.user import LiteLLM_UserTable from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import discoverable_endpoints, mcp_server_manager from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server import discoverable_endpoints, mcp_server_manager from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer handler, signing_key = jwt_oauth_identity @@ -13909,9 +13909,9 @@ async def test_authorize_observes_committed_peer_server_changes(monkeypatch, cha from fastapi import HTTPException, Request from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy._types import LiteLLM_MCPServerTable + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager monkeypatch.setenv("LITELLM_SALT_KEY", "catalog-consistency-test-key") stamp = datetime.now(timezone.utc) @@ -13966,9 +13966,9 @@ async def test_authorize_observes_committed_peer_server_changes(monkeypatch, cha @pytest.mark.asyncio async def test_update_server_drops_cached_upstream_oauth_metadata(): - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.proxy._types import LiteLLM_MCPServerTable, MCPTransport + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager from litellm.types.mcp_server.mcp_server_manager import MCPServer manager = MCPServerManager() @@ -14003,12 +14003,12 @@ async def test_update_server_drops_cached_upstream_oauth_metadata(): async def test_metadata_fetched_before_invalidation_does_not_repopulate_the_cache(): import asyncio - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.discoverable_endpoints import ( fetch_upstream_oauth_protected_resource, invalidate_oauth_metadata_cache, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -14028,7 +14028,7 @@ async def test_metadata_fetched_before_invalidation_does_not_repopulate_the_cach discoverable_endpoints._OAUTH_METADATA_CACHE.pop(cache_key, None) try: with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=client, ): in_flight: Final = asyncio.create_task(fetch_upstream_oauth_protected_resource(server)) @@ -14048,12 +14048,12 @@ async def test_metadata_fetched_before_invalidation_does_not_repopulate_the_cach async def test_fetch_waiting_on_a_lock_handoff_stays_tracked_through_invalidation(): import asyncio - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.discoverable_endpoints import ( fetch_upstream_oauth_protected_resource, invalidate_oauth_metadata_cache, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -14073,7 +14073,7 @@ async def test_fetch_waiting_on_a_lock_handoff_stays_tracked_through_invalidatio discoverable_endpoints._OAUTH_METADATA_CACHE.pop(cache_key, None) try: with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + "litellm.proxy.mcp_server.discoverable_endpoints.get_async_httpx_client", return_value=client, ): async with discoverable_endpoints._oauth_metadata_fetch_slot(cache_key): @@ -14099,8 +14099,8 @@ async def test_fetch_waiting_on_a_lock_handoff_stays_tracked_through_invalidatio def test_invalidating_an_idle_server_leaves_no_generation_behind(): - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import invalidate_oauth_metadata_cache + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.discoverable_endpoints import invalidate_oauth_metadata_cache server_ids: Final = tuple(f"churned-server-{i}" for i in range(50)) try: @@ -14129,7 +14129,7 @@ async def test_callback_rejects_missing_advertised_issuer(monkeypatch): ], ) def test_persisted_client_is_not_applied_to_another_upstream(issuer, url): - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _apply_persisted_dcr_credentials, _PersistedDcrCredentials, ) @@ -14154,7 +14154,7 @@ async def test_registration_does_not_write_client_after_upstream_edit(monkeypatc from prisma import models from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import _persist_dcr_client_registration + from litellm.proxy.mcp_server.discoverable_endpoints import _persist_dcr_client_registration server = _issuer_anchored_oauth_server().model_copy( update={"url": "https://resource.example/mcp", "client_id": None} @@ -14187,7 +14187,7 @@ async def test_registration_does_not_write_client_after_upstream_edit(monkeypatc "response_issuer", ["https://IDP.example.com", "https://idp.example.com/", "https://idp.example.com:443"] ) def test_issuer_validation_uses_exact_identifier(response_issuer): - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import _authorization_response_issuer_is_trusted + from litellm.proxy.mcp_server.discoverable_endpoints import _authorization_response_issuer_is_trusted assert ( _authorization_response_issuer_is_trusted(response_issuer, {"expected_issuer": "https://idp.example.com"}) @@ -14203,7 +14203,7 @@ def test_issuer_validation_uses_exact_identifier(response_issuer): ], ) def test_persisted_client_remains_usable_for_its_issuer(issuer, url, monkeypatch): - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _apply_persisted_dcr_credentials, _PersistedDcrCredentials, ) @@ -14240,7 +14240,7 @@ async def test_callback_does_not_forward_error_from_another_issuer(monkeypatch): async def test_bound_client_cannot_be_sent_to_a_different_issuer(monkeypatch, operation): from fastapi import Request - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -14287,7 +14287,7 @@ async def test_bound_client_cannot_be_sent_to_a_different_issuer(monkeypatch, op async def test_failed_registration_preserves_cached_credentials(monkeypatch): from fastapi import Request - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints server = _dcr_redirect_test_server("old-client").model_copy( update={ @@ -14333,7 +14333,7 @@ async def test_registration_without_database_keeps_client_in_temporary_server(mo from fastapi import Request from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints monkeypatch.setattr(proxy_server, "prisma_client", None) server = _dcr_redirect_test_server(None).model_copy( @@ -14367,9 +14367,9 @@ async def test_registration_without_database_keeps_client_in_temporary_server(mo @pytest.mark.parametrize("config_store", [False, True]) async def test_registration_does_not_overwrite_credentials_after_failed_identity_read(monkeypatch, config_store): from litellm.proxy import utils - from litellm.proxy._experimental.mcp_server import db - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server import db + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager server = _dcr_redirect_test_server(None) monkeypatch.setattr(utils, "get_prisma_client_or_throw", lambda _: MagicMock()) @@ -14399,9 +14399,9 @@ async def test_registration_does_not_overwrite_credentials_after_failed_identity @pytest.mark.parametrize("winner_available", [False, True]) async def test_registration_losing_conditional_write_reuses_only_a_matching_winner(monkeypatch, winner_available): from litellm.proxy import utils - from litellm.proxy._experimental.mcp_server import db - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints from litellm.proxy._types import LiteLLM_MCPServerTable + from litellm.proxy.mcp_server import db + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints server = _dcr_redirect_test_server(None).model_copy(update={"url": "https://upstream.example/mcp"}) row = LiteLLM_MCPServerTable.model_validate(server.model_dump(exclude_none=True)) @@ -14444,8 +14444,8 @@ async def test_register_preserves_client_application_type_only_for_bridge_relay( import respx from fastapi import FastAPI - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import router - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.discoverable_endpoints import router + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager server: Final = _bridge_server(auth_type=auth_type, server_id="application-client", alias="application-client") app: Final = FastAPI() @@ -14487,8 +14487,8 @@ async def test_register_rejects_invalid_application_type_before_upstream( import respx from fastapi import FastAPI - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import router - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.discoverable_endpoints import router + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager server: Final = _bridge_server(server_id="invalid-application-client", alias="invalid-application-client") app: Final = FastAPI() @@ -14520,8 +14520,8 @@ async def test_register_application_type_keeps_no_registration_endpoint_fallback import respx from fastapi import FastAPI - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import router - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.discoverable_endpoints import router + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager server: Final = _bridge_server( auth_type=MCPAuth.oauth2, @@ -14586,7 +14586,7 @@ def _cimd_request(): @pytest.mark.asyncio async def test_cimd_registration_returns_https_identity_without_dcr(monkeypatch, respx_mock): - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") response = await endpoints.register_client_with_server( @@ -14602,7 +14602,7 @@ async def test_cimd_registration_returns_https_identity_without_dcr(monkeypatch, async def test_cimd_authorization_uses_metadata_identity_and_s256(monkeypatch): from urllib.parse import parse_qs, urlparse - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") monkeypatch.setenv("LITELLM_SALT_KEY", "cimd-test-state-signing-key") @@ -14624,7 +14624,7 @@ async def test_cimd_authorization_uses_metadata_identity_and_s256(monkeypatch): async def test_cimd_authorization_rejects_missing_pkce(monkeypatch): from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") monkeypatch.setenv("LITELLM_SALT_KEY", "cimd-test-state-signing-key") @@ -14636,7 +14636,7 @@ async def test_cimd_authorization_rejects_missing_pkce(monkeypatch): def test_cimd_refresh_request_uses_same_identity_without_caller_secret(monkeypatch): - from litellm.proxy._experimental.mcp_server.oauth_utils import build_upstream_oauth2_token_request + from litellm.proxy.mcp_server.oauth_utils import build_upstream_oauth2_token_request monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") request = build_upstream_oauth2_token_request( @@ -14654,7 +14654,7 @@ def test_cimd_refresh_request_uses_same_identity_without_caller_secret(monkeypat async def test_cimd_without_stable_https_origin_reports_actionable_error(monkeypatch, base, respx_mock): from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints monkeypatch.delenv("PROXY_BASE_URL", raising=False) if base is not None: @@ -14669,7 +14669,7 @@ async def test_cimd_without_stable_https_origin_reports_actionable_error(monkeyp @pytest.mark.parametrize("base", [None, "http://gateway.example.com"]) @pytest.mark.asyncio async def test_cimd_without_https_origin_falls_back_to_available_dcr(monkeypatch, base, respx_mock): - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints monkeypatch.delenv("PROXY_BASE_URL", raising=False) if base is not None: @@ -14684,8 +14684,8 @@ async def test_cimd_without_https_origin_falls_back_to_available_dcr(monkeypatch @pytest.mark.asyncio async def test_cimd_yields_to_dynamic_registration_when_the_authorization_server_offers_both(monkeypatch, respx_mock): - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints - from litellm.proxy._experimental.mcp_server.oauth_utils import get_cimd_client_id + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server.oauth_utils import get_cimd_client_id monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") @@ -14700,7 +14700,7 @@ async def test_cimd_yields_to_dynamic_registration_when_the_authorization_server @pytest.mark.asyncio async def test_cimd_is_preferred_over_dynamic_registration_when_the_deployment_opts_in(monkeypatch, respx_mock): from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") @@ -14729,7 +14729,7 @@ async def test_cimd_is_preferred_over_dynamic_registration_when_the_deployment_o ], ) def test_cimd_preserves_existing_identity_and_other_auth_modes(monkeypatch, updates): - from litellm.proxy._experimental.mcp_server.oauth_utils import get_cimd_client_id + from litellm.proxy.mcp_server.oauth_utils import get_cimd_client_id monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") assert get_cimd_client_id(_cimd_oauth_server().model_copy(update=updates)) is None @@ -14740,7 +14740,7 @@ async def test_cimd_document_is_public_and_binds_configured_origin(monkeypatch): import httpx from fastapi import FastAPI - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com/proxy") app = FastAPI() @@ -14760,7 +14760,7 @@ async def test_cimd_document_is_public_and_binds_configured_origin(monkeypatch): async def test_cimd_document_is_unavailable_without_configured_https_origin(monkeypatch, base): from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints monkeypatch.delenv("PROXY_BASE_URL", raising=False) if base is not None: @@ -14772,7 +14772,7 @@ async def test_cimd_document_is_unavailable_without_configured_https_origin(monk @pytest.mark.asyncio async def test_cimd_upstream_client_rejection_is_gateway_fault(monkeypatch, respx_mock): - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") @@ -14803,8 +14803,8 @@ async def test_optional_cimd_discovery_preserves_the_callers_configured_endpoint ) -> None: from urllib.parse import parse_qs - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints - from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import mcp_server_manager as manager_module monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") monkeypatch.setenv("LITELLM_MCP_OAUTH_DISCOVERY_ON_STARTUP", "0") @@ -14874,8 +14874,8 @@ async def test_token_route_preserves_saved_cimd_grant_after_origin_change( from urllib.parse import parse_qs from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import db, mcp_server_manager - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import db, mcp_server_manager + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") monkeypatch.setenv("LITELLM_SALT_KEY", "saved-cimd-route-test-salt") @@ -14946,8 +14946,8 @@ async def test_saved_refresh_metadata_is_bound_to_the_callers_stored_grant( from types import SimpleNamespace from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import db - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import db + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints monkeypatch.setenv("LITELLM_SALT_KEY", "saved-cimd-owner-test-salt") server: Final = _cimd_oauth_server() @@ -15007,9 +15007,9 @@ async def test_managed_authorize_retains_the_callers_previous_grant( from urllib.parse import parse_qs, urlsplit from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import db - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server import db + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager _, signing_key = jwt_oauth_identity monkeypatch.setenv("LITELLM_SALT_KEY", "scope-grant-test-salt") @@ -15053,9 +15053,9 @@ async def test_managed_code_exchange_preserves_requested_scope_only_when_omitted from urllib.parse import parse_qs, urlsplit from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import db - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server import db + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager _, signing_key = jwt_oauth_identity monkeypatch.setenv("LITELLM_SALT_KEY", "requested-scope-test-salt") @@ -15117,7 +15117,7 @@ async def test_managed_authorize_requests_only_selected_scope_and_supported_refr from mcp.shared.auth import OAuthMetadata - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints monkeypatch.setenv("LITELLM_SALT_KEY", "metadata-scope-test-salt") metadata = OAuthMetadata( @@ -15143,7 +15143,7 @@ async def test_managed_authorize_requests_only_selected_scope_and_supported_refr @pytest.mark.asyncio @pytest.mark.parametrize("mismatch", ["server", "issuer", "user", "mode", "tampered", "invalid_payload"]) async def test_managed_scope_code_cannot_cross_oauth_boundaries(jwt_oauth_identity, monkeypatch, respx_mock, mismatch): - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints _, signing_key = jwt_oauth_identity monkeypatch.setenv("LITELLM_SALT_KEY", "scope-boundary-test-salt") @@ -15198,9 +15198,9 @@ async def test_managed_refresh_retains_only_the_actual_grant( from types import SimpleNamespace from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import db - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server import db + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager _, signing_key = jwt_oauth_identity monkeypatch.setenv("LITELLM_SALT_KEY", "refresh-scope-test-salt") @@ -15256,8 +15256,8 @@ async def test_managed_refresh_retains_only_the_actual_grant( @pytest.mark.asyncio @pytest.mark.parametrize("per_server_relay", [False, True]) async def test_managed_resource_metadata_excludes_refresh_permission(monkeypatch, per_server_relay): - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager server = _cimd_oauth_server().model_copy( update={ @@ -15281,8 +15281,8 @@ async def test_managed_resource_metadata_excludes_refresh_permission(monkeypatch ], ) async def test_connect_requires_the_requested_vendor_grant(monkeypatch, granted, required, expected): - from litellm.proxy._experimental.mcp_server import db - from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy.mcp_server import db + from litellm.proxy.mcp_server import discoverable_endpoints as endpoints credential = {**_stored_grant(), "scopes": granted} read = AsyncMock(return_value=credential) @@ -15308,8 +15308,8 @@ def test_aggregate_protected_resource_advertises_header_server_scopes( ) -> None: """A client that re-authenticates from cached discovery never sees the 401 challenge, so the aggregate PRM carries the x-mcp-servers selection as scopes_supported instead.""" - from litellm.proxy._experimental.mcp_server import mcp_server_manager - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server import mcp_server_manager + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_aggregate_protected_resource_response, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -15351,9 +15351,9 @@ async def test_token_endpoint_routes_the_jwt_bearer_grant_to_the_aggregate_gatew """A requesting app presents its identity provider's client_id, never a gateway DCR one, so the aggregate dispatch keys on the grant type for RFC 7523 and hands the assertion to the ID-JAG redeemer. A per-server token endpoint never serves that grant.""" - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import JWT_BEARER_GRANT_TYPE - from litellm.proxy._experimental.mcp_server.id_jag_grant import IdJagPrerequisites + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.gateway_dcr_flow import JWT_BEARER_GRANT_TYPE + from litellm.proxy.mcp_server.id_jag_grant import IdJagPrerequisites request = _mock_callback_request("https://litellm.example.com/") with patch("litellm.proxy.proxy_server.master_key", "sk-test-salt-for-lit9670"): @@ -15395,8 +15395,8 @@ async def test_token_endpoint_takes_the_client_id_from_http_basic_when_the_form_ header the request is a 400 invalid_request, not a 422.""" from base64 import b64encode - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import JWT_BEARER_GRANT_TYPE + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.gateway_dcr_flow import JWT_BEARER_GRANT_TYPE request = _mock_callback_request("https://litellm.example.com/") request.headers = {"authorization": "Basic " + b64encode(b"0oa-requesting-app:app-secret").decode()} @@ -15427,9 +15427,9 @@ def test_root_metadata_is_the_gateways_own_and_lists_the_jwt_bearer_grant_when_i exactly when the gateway serves it, under the same gates as the aggregate ``/mcp`` document.""" from fastapi import Request - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import JWT_BEARER_GRANT_TYPE - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.gateway_dcr_flow import JWT_BEARER_GRANT_TYPE + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager global_mcp_server_manager.registry.clear() request = MagicMock(spec=Request) @@ -15455,9 +15455,9 @@ def test_root_metadata_owned_by_a_resolved_oauth2_server_never_lists_the_gateway ``/.well-known/oauth-authorization-server/mcp`` document in that deployment.""" from fastapi import Request - from litellm.proxy._experimental.mcp_server import discoverable_endpoints - from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import JWT_BEARER_GRANT_TYPE - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server import discoverable_endpoints + from litellm.proxy.mcp_server.gateway_dcr_flow import JWT_BEARER_GRANT_TYPE + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager server = _create_oauth2_server() global_mcp_server_manager.registry.clear() diff --git a/tests/unit/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py b/tests/unit/proxy/mcp_server/test_gateway_dcr_flow.py similarity index 98% rename from tests/unit/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py rename to tests/unit/proxy/mcp_server/test_gateway_dcr_flow.py index 530667fdfdd..a426cea131a 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py +++ b/tests/unit/proxy/mcp_server/test_gateway_dcr_flow.py @@ -14,7 +14,8 @@ from starlette.requests import Request from starlette.responses import Response from litellm.caching.caching import DualCache -from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ( +from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper +from litellm.proxy.mcp_server.gateway_dcr_flow import ( _AUTH_CODE_DEBUG_KEY, ACCESS_TOKEN_TOKEN_TYPE, CONNECT_FLOW_COOKIE_PREFIX, @@ -48,21 +49,20 @@ from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ( revoke_refresh_token, supported_grant_types, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.session_credentials import ( +from litellm.proxy.mcp_server.outbound_credentials.session_credentials import ( SessionBearerAdmitted, SessionRefreshOpened, open_session_refresh_bearer, resolve_session_bearer, session_keys_from_master_key, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import ( +from litellm.proxy.mcp_server.outbound_credentials.session_token import ( SESSION_ISSUER, SESSION_REFRESH_PREFIX, SessionPrincipal, mint_session_refresh_token, mint_session_token, ) -from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper MASTER_KEY = "sk-gateway-dcr-flow-tests" REDIRECT_URI = "https://claude.ai/api/mcp/auth_callback" @@ -205,7 +205,7 @@ def test_pkce_mismatched_challenge_returns_false_never_raises(code_challenge): two str with non-ASCII content, but on bytes of unequal length it simply returns False. A review flagged this as an unhandled 500 on length mismatch; encoding both sides to bytes is exactly what makes that impossible, so the claim is pinned here rather than in a comment.""" - from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import _pkce_verifier_matches + from litellm.proxy.mcp_server.gateway_dcr_flow import _pkce_verifier_matches assert _pkce_verifier_matches("a" * 43, code_challenge) is False @@ -563,7 +563,7 @@ def test_permanent_db_fault_503_does_not_promise_a_retry_will_help(): """Both DB failures are 503 temporarily_unavailable (the only OAuth error a client reads as a server-side outage), so the description is the one place the two are told apart: a transient outage says retry, a fault that never heals must say retrying will not help and point at the deployment.""" - from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ( + from litellm.proxy.mcp_server.gateway_dcr_flow import ( _consent_lookup_failure_response, _mint_failure_response, _reload_failure_response, @@ -682,7 +682,7 @@ async def test_non_ascii_code_challenge_fails_grant_not_500(): async def test_single_use_guard_in_memory_is_single_use_within_process(): """No Redis configured (single-replica): the in-memory increment is authoritative — the first claim wins, a replay of the same id loses.""" - from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import _SingleUseGuard + from litellm.proxy.mcp_server.gateway_dcr_flow import _SingleUseGuard guard = _SingleUseGuard(DualCache()) # redis_cache is None assert await guard.claim("jti-inmem", 60) == "first" @@ -695,7 +695,7 @@ async def test_single_use_guard_uses_redis_as_sole_authority_when_configured(): first caller, >1 → replay), and the per-worker in-memory count is never consulted.""" from unittest.mock import AsyncMock, MagicMock - from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import _SingleUseGuard + from litellm.proxy.mcp_server.gateway_dcr_flow import _SingleUseGuard cache = DualCache() cache.redis_cache = MagicMock() @@ -716,7 +716,7 @@ async def test_single_use_guard_fails_closed_when_redis_errors(): Cursor/Veria replay-across-workers finding).""" from unittest.mock import AsyncMock, MagicMock - from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import _SingleUseGuard + from litellm.proxy.mcp_server.gateway_dcr_flow import _SingleUseGuard cache = DualCache() cache.redis_cache = MagicMock() @@ -734,7 +734,7 @@ async def test_single_use_guard_peek_fails_closed_when_redis_errors(): look unclaimed for exactly as long as Redis is down.""" from unittest.mock import AsyncMock, MagicMock - from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import _SingleUseGuard + from litellm.proxy.mcp_server.gateway_dcr_flow import _SingleUseGuard cache = DualCache() cache.redis_cache = MagicMock() @@ -756,7 +756,7 @@ async def test_single_use_guard_peek_reads_the_key_under_the_namespace_claim_wro peeked at the bare key would never be seen, and the revoked chain would keep renewing.""" from unittest.mock import AsyncMock, MagicMock - from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import _SingleUseGuard + from litellm.proxy.mcp_server.gateway_dcr_flow import _SingleUseGuard stored: dict[str, int] = {} # mutable-ok: the fake Redis store the test inspects @@ -1003,7 +1003,7 @@ def _scoped_mcp_server(name="github", auth_type="oauth2", **kw): SCOPED_RESOURCE = "https://llm.example.com/mcp/github" -_MANAGER_PATCH = "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" +_MANAGER_PATCH = "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" def _scoped_authorize( @@ -1475,7 +1475,7 @@ async def test_unknown_server_scope_redirects_with_invalid_scope_and_state() -> ], ) def test_gateway_server_scope_accepts_only_rfc6749_scope_tokens(server_name: str, expected: str | None) -> None: - from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import gateway_server_scope + from litellm.proxy.mcp_server.gateway_dcr_flow import gateway_server_scope assert gateway_server_scope(server_name) == expected @@ -1728,7 +1728,7 @@ async def test_resolve_scoped_resource_server_matrix(): modes all return None so nothing outside the served set can enter the scoped flow.""" from unittest.mock import patch - from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import resolve_scoped_resource_server + from litellm.proxy.mcp_server.gateway_dcr_flow import resolve_scoped_resource_server request = _request() github = _scoped_mcp_server() @@ -1757,7 +1757,7 @@ async def test_resource_resolution_is_identity_not_ip_filtered_access(): between authorize and token would turn a matching redemption into invalid_target.""" from unittest.mock import patch - from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import resolve_scoped_resource_server + from litellm.proxy.mcp_server.gateway_dcr_flow import resolve_scoped_resource_server with patch(_MANAGER_PATCH) as manager: manager.get_mcp_server_by_identifier.return_value = _scoped_mcp_server() @@ -2666,7 +2666,7 @@ async def test_introspect_accepts_rs256_signed_tokens_under_configured_signing(m from pydantic import SecretStr from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import AsymmetricSessionKeys + from litellm.proxy.mcp_server.outbound_credentials.session_token import AsymmetricSessionKeys private_pem = ( rsa.generate_private_key(public_exponent=65537, key_size=2048) @@ -3024,7 +3024,7 @@ async def test_aggregate_step_up_rejects_malformed_permissions(scope): async def test_step_up_validates_every_target_before_credential_reads_and_preserves_each_grant(): from unittest.mock import AsyncMock, patch - from litellm.proxy._experimental.mcp_server.oauth_utils import encode_upstream_scope + from litellm.proxy.mcp_server.oauth_utils import encode_upstream_scope client_id = (await _register([REDIRECT_URI]))["client_id"] first, second = _scoped_mcp_server("first"), _scoped_mcp_server("second") @@ -3075,7 +3075,7 @@ async def test_step_up_validates_every_target_before_credential_reads_and_preser async def test_named_resource_rejects_another_upstreams_permission_before_consent(): from unittest.mock import patch - from litellm.proxy._experimental.mcp_server.oauth_utils import encode_upstream_scope + from litellm.proxy.mcp_server.oauth_utils import encode_upstream_scope client_id = (await _register([REDIRECT_URI]))["client_id"] with patch(_MANAGER_PATCH) as manager: @@ -3094,7 +3094,7 @@ async def test_named_resource_rejects_another_upstreams_permission_before_consen ], ) def test_upstream_permission_scope_round_trip_preserves_identity_and_permission(identity, permission): - from litellm.proxy._experimental.mcp_server.oauth_utils import decode_upstream_scope, encode_upstream_scope + from litellm.proxy.mcp_server.oauth_utils import decode_upstream_scope, encode_upstream_scope encoded = encode_upstream_scope(identity, permission) decoded = decode_upstream_scope(encoded) @@ -3108,7 +3108,7 @@ def test_upstream_permission_scope_round_trip_preserves_identity_and_permission( async def test_upstream_step_up_cannot_finish_with_an_unproven_grant(state, expected): from unittest.mock import AsyncMock - from litellm.proxy._experimental.mcp_server.oauth_utils import encode_upstream_scope + from litellm.proxy.mcp_server.oauth_utils import encode_upstream_scope client_id = (await _register([REDIRECT_URI]))["client_id"] server = _scoped_mcp_server() @@ -3133,11 +3133,11 @@ async def test_sdk_completes_aggregate_step_up_discovery_consent_and_pkce(route) from mcp.client.auth import OAuthClientProvider from mcp.shared.auth import AuthorizationCodeResult, OAuthClientMetadata - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_aggregate_authorization_server_response, oauth_protected_resource_root, ) - from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError + from litellm.proxy.mcp_server.exceptions import MCPUpstreamAuthError server = _scoped_mcp_server() storage = AsyncMock() @@ -3209,7 +3209,7 @@ async def test_sdk_completes_aggregate_step_up_discovery_consent_and_pkce(route) ["tools.write", "litellm:mcp_scope::7772697465", "litellm:mcp_scope:6964:5A", "litellm:mcp_scope:6964:7772 697465"], ) def test_upstream_permission_rejects_noncanonical_tokens(scope): - from litellm.proxy._experimental.mcp_server.oauth_utils import decode_upstream_scope + from litellm.proxy.mcp_server.oauth_utils import decode_upstream_scope assert decode_upstream_scope(scope) is None @@ -3218,7 +3218,7 @@ def test_upstream_permission_rejects_noncanonical_tokens(scope): async def test_named_step_up_finish_describes_and_retains_its_single_server_ceiling(): from unittest.mock import AsyncMock, patch - from litellm.proxy._experimental.mcp_server.oauth_utils import encode_upstream_scope + from litellm.proxy.mcp_server.oauth_utils import encode_upstream_scope client_id = (await _register([REDIRECT_URI]))["client_id"] server = _scoped_mcp_server() @@ -3244,7 +3244,7 @@ async def test_named_step_up_finish_describes_and_retains_its_single_server_ceil async def test_upstream_step_up_inside_multi_server_scope_describes_every_sealed_server() -> None: from unittest.mock import AsyncMock, patch - from litellm.proxy._experimental.mcp_server.oauth_utils import encode_upstream_scope + from litellm.proxy.mcp_server.oauth_utils import encode_upstream_scope client_id: Final = (await _register([REDIRECT_URI]))["client_id"] alpha: Final = _scoped_mcp_server(name="alpha") @@ -3490,7 +3490,7 @@ def test_aggregate_metadata_advertises_basic_client_auth_only_with_the_grant_tha ``client_secret`` from the form, so ``client_secret_basic`` is advertised only when that grant is on. A client following the document otherwise sends Basic alone and authenticates with no secret.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _build_aggregate_authorization_server_response, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_id_jag_grant.py b/tests/unit/proxy/mcp_server/test_id_jag_grant.py similarity index 99% rename from tests/unit/proxy/_experimental/mcp_server/test_id_jag_grant.py rename to tests/unit/proxy/mcp_server/test_id_jag_grant.py index f4fa51d57ba..f5d7a07ddb4 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_id_jag_grant.py +++ b/tests/unit/proxy/mcp_server/test_id_jag_grant.py @@ -10,14 +10,22 @@ import pytest from fastapi import HTTPException from litellm.caching.caching import DualCache -from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ( +from litellm.proxy._types import IdJagRequestingApp, LiteLLM_JWTAuth, LiteLLMRoutes +from litellm.proxy.auth.handle_jwt import ( + ID_JAG_TYP, + AssertionRejection, + JWKSUnreachableError, + JWTHandler, + jwks_unavailable_exception, +) +from litellm.proxy.mcp_server.gateway_dcr_flow import ( AssertionIdentity, ConsentTeam, ReloadUserFailure, SubjectIdentity, SubjectTokenRefusal, ) -from litellm.proxy._experimental.mcp_server.id_jag_grant import ( +from litellm.proxy.mcp_server.id_jag_grant import ( ID_JAG_REQUIRED_CLAIMS, REJECTED_ASSERTION, TEAM_HEADER, @@ -37,14 +45,6 @@ from litellm.proxy._experimental.mcp_server.id_jag_grant import ( select_team, subject_resolver, ) -from litellm.proxy._types import IdJagRequestingApp, LiteLLM_JWTAuth, LiteLLMRoutes -from litellm.proxy.auth.handle_jwt import ( - ID_JAG_TYP, - AssertionRejection, - JWKSUnreachableError, - JWTHandler, - jwks_unavailable_exception, -) ISSUER = "https://idp.example.com" GATEWAY = "https://gateway.example.com" diff --git a/tests/unit/proxy/_experimental/mcp_server/test_idp_token_exchange.py b/tests/unit/proxy/mcp_server/test_idp_token_exchange.py similarity index 98% rename from tests/unit/proxy/_experimental/mcp_server/test_idp_token_exchange.py rename to tests/unit/proxy/mcp_server/test_idp_token_exchange.py index e94371a6056..036ee4a332b 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_idp_token_exchange.py +++ b/tests/unit/proxy/mcp_server/test_idp_token_exchange.py @@ -7,8 +7,10 @@ from prisma.engine.errors import BinaryNotFoundError from prisma.errors import DataError from litellm.caching.caching import DualCache -from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import SubjectIdentity, SubjectTokenRefusal -from litellm.proxy._experimental.mcp_server.idp_token_exchange import ( +from litellm.proxy._types import JWTIssuerConfig, LiteLLM_JWTAuth, ProxyException +from litellm.proxy.auth.handle_jwt import JWKSUnreachableError, JWTHandler, jwks_unavailable_exception +from litellm.proxy.mcp_server.gateway_dcr_flow import SubjectIdentity, SubjectTokenRefusal +from litellm.proxy.mcp_server.idp_token_exchange import ( REJECTED_SUBJECT_TOKEN, SUBJECT_TOKEN_CHECK_FAULTED, SUBJECT_TOKEN_CHECK_UNAVAILABLE, @@ -16,8 +18,6 @@ from litellm.proxy._experimental.mcp_server.idp_token_exchange import ( identity_from_subject_token, token_exchange_available, ) -from litellm.proxy._types import JWTIssuerConfig, LiteLLM_JWTAuth, ProxyException -from litellm.proxy.auth.handle_jwt import JWKSUnreachableError, JWTHandler, jwks_unavailable_exception IDP_JWT = "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ1MSJ9.idp-signature" REQUEST_HEADERS = {"x-litellm-team-id": "team-b", "user-agent": "lite/0.1"} diff --git a/tests/unit/proxy/_experimental/mcp_server/test_interactions.py b/tests/unit/proxy/mcp_server/test_interactions.py similarity index 94% rename from tests/unit/proxy/_experimental/mcp_server/test_interactions.py rename to tests/unit/proxy/mcp_server/test_interactions.py index 8286269525a..aa4c00b1f07 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_interactions.py +++ b/tests/unit/proxy/mcp_server/test_interactions.py @@ -4,9 +4,9 @@ import pytest from mcp import MCPError from mcp.types import CallToolRequest, CallToolRequestParams, InputRequiredResult -from litellm.proxy._experimental.mcp_server.contracts import OperationContext -from litellm.proxy._experimental.mcp_server.interactions import bind_target, open_continuation, seal_continuation from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.mcp_server.contracts import OperationContext +from litellm.proxy.mcp_server.interactions import bind_target, open_continuation, seal_continuation from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -130,7 +130,8 @@ def test_altered_continuations_are_rejected(change: str, monkeypatch: pytest.Mon def test_input_responses_without_state_and_unbound_results_are_rejected(monkeypatch: pytest.MonkeyPatch) -> None: from mcp.types import ElicitResult - from litellm.proxy._experimental.mcp_server.interactions import BoundInputRequiredResult + + from litellm.proxy.mcp_server.interactions import BoundInputRequiredResult monkeypatch.setenv("LITELLM_SALT_KEY", "test-salt") context: Final = OperationContext(_caller=UserAPIKeyAuth(user_id="alice")) @@ -144,8 +145,8 @@ def test_input_responses_without_state_and_unbound_results_are_rejected(monkeypa def test_authenticated_but_invalid_state_payload_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None: - from litellm.proxy._experimental.mcp_server.state_tokens import seal_state - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.state_tokens import seal_state monkeypatch.setenv("LITELLM_SALT_KEY", "test-salt") sealed: Final = seal_state( @@ -169,7 +170,8 @@ async def test_modern_sampling_errors_abort_without_relaying_form_input() -> Non ElicitRequestFormParams, ErrorData, ) - from litellm.proxy._experimental.mcp_server.interactions import ModernClientInteraction + + from litellm.proxy.mcp_server.interactions import ModernClientInteraction async def sampling(context: object, params: CreateMessageRequestParams) -> ErrorData: return ErrorData(code=-32603, message="Sampling refused by gateway policy") @@ -204,10 +206,12 @@ async def test_gateway_rejects_invalid_interaction_boundaries( invalid_response: Literal["legacy_state", "gateway_override", "unbound_result"], monkeypatch: pytest.MonkeyPatch ) -> None: from unittest.mock import AsyncMock, patch + from mcp.types import CreateMessageResult, ElicitResult, TextContent - from litellm.proxy._experimental.mcp_server import operations - from litellm.proxy._experimental.mcp_server.contracts import WireCompat - from litellm.proxy._experimental.mcp_server.interactions import BoundInputRequiredResult + + from litellm.proxy.mcp_server import operations + from litellm.proxy.mcp_server.contracts import WireCompat + from litellm.proxy.mcp_server.interactions import BoundInputRequiredResult monkeypatch.setenv("LITELLM_SALT_KEY", "test-salt") context: Final = OperationContext( diff --git a/tests/unit/proxy/_experimental/mcp_server/test_is_tool_name_prefixed.py b/tests/unit/proxy/mcp_server/test_is_tool_name_prefixed.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/test_is_tool_name_prefixed.py rename to tests/unit/proxy/mcp_server/test_is_tool_name_prefixed.py index 8f09e2410c4..f8dea399067 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_is_tool_name_prefixed.py +++ b/tests/unit/proxy/mcp_server/test_is_tool_name_prefixed.py @@ -6,8 +6,7 @@ Verifies fix for https://github.com/BerriAI/litellm/issues/25081 import pytest -from litellm.proxy._experimental.mcp_server.utils import is_tool_name_prefixed - +from litellm.proxy.mcp_server.utils import is_tool_name_prefixed # --------------------------------------------------------------------------- # Legacy behaviour (no known_server_prefixes passed) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_jwt_mcp_enforcement.py b/tests/unit/proxy/mcp_server/test_jwt_mcp_enforcement.py similarity index 99% rename from tests/unit/proxy/_experimental/mcp_server/test_jwt_mcp_enforcement.py rename to tests/unit/proxy/mcp_server/test_jwt_mcp_enforcement.py index eddfdc4f884..ea275373f2f 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_jwt_mcp_enforcement.py +++ b/tests/unit/proxy/mcp_server/test_jwt_mcp_enforcement.py @@ -8,20 +8,21 @@ Scenario: 4. EXPECTED: Team MCP permissions should be enforced """ -import pytest from unittest.mock import AsyncMock, MagicMock, patch +import pytest + from litellm.proxy._types import ( LiteLLM_JWTAuth, - LiteLLM_TeamTable, LiteLLM_ObjectPermissionTable, + LiteLLM_TeamTable, UserAPIKeyAuth, ) from litellm.proxy.auth.handle_jwt import JWTAuthManager, JWTHandler -from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( +from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) -from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager +from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager @pytest.mark.asyncio diff --git a/tests/unit/proxy/_experimental/mcp_server/test_jwt_mcp_simple.py b/tests/unit/proxy/mcp_server/test_jwt_mcp_simple.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/test_jwt_mcp_simple.py rename to tests/unit/proxy/mcp_server/test_jwt_mcp_simple.py index 7ee340a69f4..f258ba8b7e9 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_jwt_mcp_simple.py +++ b/tests/unit/proxy/mcp_server/test_jwt_mcp_simple.py @@ -2,13 +2,14 @@ Simple test to validate MCP permissions are enforced when calling MCP routes with JWT. """ -import pytest from unittest.mock import AsyncMock, MagicMock, patch +import pytest + from litellm.proxy._types import ( LiteLLM_JWTAuth, - LiteLLM_TeamTable, LiteLLM_ObjectPermissionTable, + LiteLLM_TeamTable, UserAPIKeyAuth, ) @@ -24,7 +25,7 @@ async def test_simple_jwt_mcp_permissions_enforced(): Expected: Only ["github-mcp", "slack-mcp"] should be allowed """ - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) @@ -86,7 +87,7 @@ async def test_simple_jwt_no_team_no_mcp_servers(): """ Simple test: JWT user with no team should get no MCP servers. """ - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) @@ -111,7 +112,7 @@ async def test_simple_jwt_team_id_required_for_mcp_permissions(): This is the key insight - if JWT auth doesn't set team_id, team MCP permissions won't be enforced. """ - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) @@ -176,8 +177,8 @@ async def test_jwt_auth_sets_team_id_for_mcp_route(): This is the critical test - when user calls /mcp/tools/list with JWT, the team_id from JWT groups must be set on UserAPIKeyAuth. """ - from litellm.proxy.auth.handle_jwt import JWTAuthManager, JWTHandler from litellm.caching import DualCache + from litellm.proxy.auth.handle_jwt import JWTAuthManager, JWTHandler from litellm.proxy.utils import ProxyLogging # Setup @@ -245,8 +246,8 @@ async def test_mcp_route_without_model_still_returns_team_id(): 4. Route check passes because "mcp_routes" is in team_allowed_routes 5. team_id is returned and set on UserAPIKeyAuth """ - from litellm.proxy.auth.handle_jwt import JWTAuthManager, JWTHandler from litellm.caching import DualCache + from litellm.proxy.auth.handle_jwt import JWTAuthManager, JWTHandler from litellm.proxy.utils import ProxyLogging # Setup diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_auth_header_extraction.py b/tests/unit/proxy/mcp_server/test_mcp_auth_header_extraction.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_auth_header_extraction.py rename to tests/unit/proxy/mcp_server/test_mcp_auth_header_extraction.py index b652a6d457e..bb47c1fdfe3 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_auth_header_extraction.py +++ b/tests/unit/proxy/mcp_server/test_mcp_auth_header_extraction.py @@ -6,14 +6,15 @@ Tests the fixes for: 2. Case-insensitive matching for server-specific auth headers in _call_regular_mcp_tool """ -import pytest from unittest.mock import AsyncMock, MagicMock + +import pytest from starlette.datastructures import Headers -from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( +from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) -from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager +from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager from litellm.types.mcp import MCPAuth, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_auth_priority.py b/tests/unit/proxy/mcp_server/test_mcp_auth_priority.py similarity index 91% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_auth_priority.py rename to tests/unit/proxy/mcp_server/test_mcp_auth_priority.py index 21a89d7ffcc..675848b0869 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_auth_priority.py +++ b/tests/unit/proxy/mcp_server/test_mcp_auth_priority.py @@ -7,8 +7,9 @@ Validates that: """ import pytest -from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager -from litellm.types.mcp import MCPAuth, MCPTransport, MCPSpecVersion + +from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager +from litellm.types.mcp import MCPAuth, MCPSpecVersion, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -53,7 +54,7 @@ async def test_mcp_server_config_auth_value_header_used(token_key, config_only_m """ import httpx2 - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_block_recording.py b/tests/unit/proxy/mcp_server/test_mcp_block_recording.py similarity index 96% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_block_recording.py rename to tests/unit/proxy/mcp_server/test_mcp_block_recording.py index 746130fb892..c59c37b2f37 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_block_recording.py +++ b/tests/unit/proxy/mcp_server/test_mcp_block_recording.py @@ -1,6 +1,7 @@ -from litellm.proxy._experimental.mcp_server import operations as mcp_operations +from litellm.proxy.mcp_server import operations as mcp_operations + """Tests for guardrail-block recording in -``litellm.proxy._experimental.mcp_server.operations.call_mcp_tool``. +``litellm.proxy.mcp_server.operations.call_mcp_tool``. A pre-call MCP guardrail block *raises* into ``call_mcp_tool``'s ``except Exception``. The failure spend-log row that the Guardrails Monitor's @@ -34,7 +35,7 @@ from unittest import mock import pytest from fastapi import HTTPException -from litellm.proxy._experimental.mcp_server import server +from litellm.proxy.mcp_server import server class _RecordingLoggingObj: diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_chat_completions.py b/tests/unit/proxy/mcp_server/test_mcp_chat_completions.py similarity index 99% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_chat_completions.py rename to tests/unit/proxy/mcp_server/test_mcp_chat_completions.py index 32cd883c1f7..e5fd50e832e 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_chat_completions.py +++ b/tests/unit/proxy/mcp_server/test_mcp_chat_completions.py @@ -206,7 +206,7 @@ async def test_completion_mcp_with_streaming_no_timeout_error(monkeypatch): ) # Create a mock streaming response - from unittest.mock import MagicMock, AsyncMock + from unittest.mock import AsyncMock, MagicMock logging_obj = MagicMock() logging_obj.model_call_details = {} @@ -281,18 +281,18 @@ async def test_completion_mcp_with_streaming_no_timeout_error(monkeypatch): acompletion_calls = [] # Create mock streaming response for initial call - from unittest.mock import MagicMock, AsyncMock + from unittest.mock import AsyncMock, MagicMock logging_obj = MagicMock() logging_obj.model_call_details = {} logging_obj.async_failure_handler = AsyncMock() from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + Delta, + Function, ModelResponseStream, StreamingChoices, - Delta, - ChatCompletionDeltaToolCall, - Function, ) # Create initial streaming chunks with tool_calls @@ -471,19 +471,19 @@ async def test_mcp_metadata_in_streaming_final_chunk(monkeypatch): from types import SimpleNamespace from unittest.mock import patch + from litellm.litellm_core_utils.litellm_logging import Logging from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) from litellm.responses.utils import ResponsesAPIRequestUtils - from litellm.utils import CustomStreamWrapper from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + Delta, + Function, ModelResponseStream, StreamingChoices, - Delta, - ChatCompletionDeltaToolCall, - Function, ) - from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.utils import CustomStreamWrapper dummy_tool = SimpleNamespace( name="local_search", @@ -568,7 +568,7 @@ async def test_mcp_metadata_in_streaming_final_chunk(monkeypatch): ] # Create a proper CustomStreamWrapper with logging_obj - from unittest.mock import MagicMock, AsyncMock + from unittest.mock import AsyncMock, MagicMock logging_obj = MagicMock() logging_obj.model_call_details = {} @@ -831,14 +831,14 @@ async def test_mcp_streaming_metadata_ordering(monkeypatch): LiteLLM_Proxy_MCP_Handler, ) from litellm.responses.utils import ResponsesAPIRequestUtils - from litellm.utils import CustomStreamWrapper from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + Delta, + Function, ModelResponseStream, StreamingChoices, - Delta, - ChatCompletionDeltaToolCall, - Function, ) + from litellm.utils import CustomStreamWrapper dummy_tool = SimpleNamespace( name="local_search", @@ -923,7 +923,7 @@ async def test_mcp_streaming_metadata_ordering(monkeypatch): ] # Create a proper CustomStreamWrapper with logging_obj - from unittest.mock import MagicMock, AsyncMock + from unittest.mock import AsyncMock, MagicMock logging_obj = MagicMock() logging_obj.model_call_details = {} diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_client_unit.py b/tests/unit/proxy/mcp_server/test_mcp_client_unit.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_client_unit.py rename to tests/unit/proxy/mcp_server/test_mcp_client_unit.py index a270ea16a39..1f804f3b425 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_client_unit.py +++ b/tests/unit/proxy/mcp_server/test_mcp_client_unit.py @@ -3,22 +3,22 @@ Unit tests for the MCPClient class - critical functionality only. """ import base64 +from unittest.mock import ANY, AsyncMock, MagicMock, patch + import pytest -from unittest.mock import AsyncMock, MagicMock, patch, ANY - -# Add the project root to the path - -import litellm.experimental_mcp_client.client as mcp_client_module -from litellm.experimental_mcp_client.client import MCPClient -from litellm.types.mcp import MCPAuth, MCPTransport from mcp.types import CallToolResult as MCPCallToolResult from mcp.types import Implementation, InitializeResult, ListToolsResult, PaginatedRequestParams, ServerCapabilities from mcp.types import Tool as MCPTool +# Add the project root to the path +import litellm.mcp_client.client as mcp_client_module +from litellm.mcp_client.client import MCPClient +from litellm.types.mcp import MCPAuth, MCPTransport + def test_mcp_client_uses_configurable_default_timeout(): """MCPClient should use MCP_CLIENT_TIMEOUT constant when no timeout is passed.""" - with patch("litellm.experimental_mcp_client.client.MCP_CLIENT_TIMEOUT", 120.0): + with patch("litellm.mcp_client.client.MCP_CLIENT_TIMEOUT", 120.0): # Client reads constant at runtime when timeout is None client = MCPClient( server_url="http://example.com", diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_cost_calculator.py b/tests/unit/proxy/mcp_server/test_mcp_cost_calculator.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_cost_calculator.py rename to tests/unit/proxy/mcp_server/test_mcp_cost_calculator.py index 5357e0dce9e..368080f55a5 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_cost_calculator.py +++ b/tests/unit/proxy/mcp_server/test_mcp_cost_calculator.py @@ -6,8 +6,7 @@ import pytest from fastapi import Request from fastapi.testclient import TestClient - -from litellm.proxy._experimental.mcp_server.cost_calculator import MCPCostCalculator +from litellm.proxy.mcp_server.cost_calculator import MCPCostCalculator class TestMCPCostCalculator: diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_custom_fields.py b/tests/unit/proxy/mcp_server/test_mcp_custom_fields.py similarity index 98% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_custom_fields.py rename to tests/unit/proxy/mcp_server/test_mcp_custom_fields.py index e3437bf16f6..78dde8f75df 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_custom_fields.py +++ b/tests/unit/proxy/mcp_server/test_mcp_custom_fields.py @@ -4,15 +4,16 @@ Test suite for MCP server custom fields functionality. Tests that mcp_info can accept arbitrary custom fields in addition to predefined ones. """ -import pytest +from typing import Any, Dict from unittest.mock import Mock, patch -from typing import Dict, Any + +import pytest + +from litellm.proxy._types import LiteLLM_MCPServerTable # Add the path to find the modules - -from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager +from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager from litellm.types.mcp import MCPAuth -from litellm.proxy._types import LiteLLM_MCPServerTable class TestMCPCustomFields: diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_debug.py b/tests/unit/proxy/mcp_server/test_mcp_debug.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_debug.py rename to tests/unit/proxy/mcp_server/test_mcp_debug.py index ed1ae0bb4fe..4f88aaa1d67 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_debug.py +++ b/tests/unit/proxy/mcp_server/test_mcp_debug.py @@ -9,13 +9,13 @@ import httpx import pytest from starlette.types import Message -from litellm.proxy._experimental.mcp_server.mcp_debug import ( +from litellm.proxy.mcp_server.mcp_debug import ( MCP_DEBUG_REQUEST_HEADER, MCPAuthDiagnostics, MCPDebug, describe_upstream_http_failure, ) -from litellm.proxy._experimental.mcp_server.outbound_credentials.types import AuthResolution +from litellm.proxy.mcp_server.outbound_credentials.types import AuthResolution class TestIsDebugEnabled: @@ -290,7 +290,7 @@ def test_failure_log_omits_unstructured_body(): @pytest.mark.asyncio @pytest.mark.parametrize("mode", ["error", "empty", "large", "timeout", "read_failure", "closed", "success", "cancel"]) async def test_error_capture_is_bounded_and_preserves_success_and_cancellation(mode): - from litellm.proxy._experimental.mcp_server.mcp_debug import capture_upstream_error_response + from litellm.proxy.mcp_server.mcp_debug import capture_upstream_error_response class Stream(httpx.AsyncByteStream): def __init__(self): @@ -349,7 +349,7 @@ def test_failure_preview_handles_empty_scalar_control_and_long_bodies(body): @pytest.mark.asyncio @pytest.mark.parametrize("slow_error", [False, True]) async def test_error_capture_preserves_httpx_auth_retry(slow_error): - from litellm.proxy._experimental.mcp_server.mcp_debug import capture_upstream_error_response + from litellm.proxy.mcp_server.mcp_debug import capture_upstream_error_response class RetryAuth(httpx.Auth): def auth_flow(self, request): @@ -467,8 +467,8 @@ async def test_concurrent_mcp_messages_record_on_their_own_http_scope(_mcp_reque from starlette.requests import Request - from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var - from litellm.proxy._experimental.mcp_server.mcp_debug import ( + from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var + from litellm.proxy.mcp_server.mcp_debug import ( MCP_AUTH_DIAGNOSTICS_SCOPE_KEY, record_auth_resolution, ) @@ -540,7 +540,8 @@ def test_oversized_request_omits_potentially_reflected_response_credentials(): @pytest.mark.asyncio async def test_streamed_error_redacts_reflected_credentials_before_capture(): import json - from litellm.proxy._experimental.mcp_server.mcp_debug import capture_upstream_error_response + + from litellm.proxy.mcp_server.mcp_debug import capture_upstream_error_response secret = "generic-credential-123" request = httpx.Request("POST", "https://upstream/token", data={"client_secret":secret}) @@ -566,7 +567,7 @@ def test_failure_diagnostics_omit_credential_bearing_url_paths(path): def test_deep_request_omits_response_when_credentials_cannot_be_inspected(): - from litellm.proxy._experimental.mcp_server.utils import MAX_STRUCTURED_CONTENT_SCAN_DEPTH + from litellm.proxy.mcp_server.utils import MAX_STRUCTURED_CONTENT_SCAN_DEPTH raw = "[" * (MAX_STRUCTURED_CONTENT_SCAN_DEPTH + 1) + '{"client_secret":"nested-credential"}' + "]" * (MAX_STRUCTURED_CONTENT_SCAN_DEPTH + 1) request = httpx.Request("POST", "https://upstream/token", content=raw, headers={"Content-Type": "application/json"}) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_discovery.py b/tests/unit/proxy/mcp_server/test_mcp_discovery.py similarity index 99% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_discovery.py rename to tests/unit/proxy/mcp_server/test_mcp_discovery.py index d35cb7234dc..35fc7440f99 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_discovery.py +++ b/tests/unit/proxy/mcp_server/test_mcp_discovery.py @@ -5,7 +5,6 @@ from typing import Final import pytest - class TestMCPRegistryFile: """Tests for the curated MCP registry JSON file.""" @@ -17,7 +16,6 @@ class TestMCPRegistryFile: "..", "..", "..", - "..", "litellm", "proxy", "mcp_registry.json", diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py b/tests/unit/proxy/mcp_server/test_mcp_elicitation_handler.py similarity index 98% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py rename to tests/unit/proxy/mcp_server/test_mcp_elicitation_handler.py index dd8dddf106e..08e3903ece4 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py +++ b/tests/unit/proxy/mcp_server/test_mcp_elicitation_handler.py @@ -12,21 +12,20 @@ from types import SimpleNamespace from unittest.mock import AsyncMock import pytest - from mcp.types import ( + REQUEST_TIMEOUT, ClientCapabilities, ElicitationCapability, - FormElicitationCapability, - UrlElicitationCapability, - REQUEST_TIMEOUT, ElicitRequestFormParams, ElicitRequestURLParams, ElicitResult, ErrorData, + FormElicitationCapability, + UrlElicitationCapability, ) -from litellm.proxy._experimental.mcp_server import elicitation_handler -from litellm.proxy._experimental.mcp_server.elicitation_handler import ( +from litellm.proxy.mcp_server import elicitation_handler +from litellm.proxy.mcp_server.elicitation_handler import ( _relay_elicitation_to_downstream, handle_elicitation_request, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_env_vars.py b/tests/unit/proxy/mcp_server/test_mcp_env_vars.py similarity index 93% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_env_vars.py rename to tests/unit/proxy/mcp_server/test_mcp_env_vars.py index 807061c946f..57ac6c0be2d 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_env_vars.py +++ b/tests/unit/proxy/mcp_server/test_mcp_env_vars.py @@ -1,7 +1,7 @@ """Tests for MCP env-var interpolation utilities. These cover the pure helpers in -``litellm.proxy._experimental.mcp_server.utils`` and do not require a DB +``litellm.proxy.mcp_server.utils`` and do not require a DB connection. The DB-backed per-user flow is exercised in higher-level tests in tests/integration/mcp. """ @@ -12,8 +12,6 @@ from unittest.mock import AsyncMock import pytest from respx import MockRouter -from litellm.types.mcp_server.mcp_server_manager import MCPServer - # Look up these names lazily on every access. Tests in this directory call # ``importlib.reload`` on the utils module to exercise registration logic, # which replaces ``MCPMissingUserEnvVarsError`` with a freshly-constructed @@ -21,7 +19,8 @@ from litellm.types.mcp_server.mcp_server_manager import MCPServer # old class object and ``pytest.raises(_u("MCPMissingUserEnvVarsError"))`` would # stop matching the new class. Accessing the attribute through the module # always picks up the current version. -import litellm.proxy._experimental.mcp_server.utils as _mcp_utils +import litellm.proxy.mcp_server.utils as _mcp_utils +from litellm.types.mcp_server.mcp_server_manager import MCPServer def _u(name: str): @@ -191,7 +190,7 @@ def mock_server(): async def test_resolve_static_headers_interpolates_globals_and_user( mock_server, monkeypatch ): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -216,7 +215,7 @@ async def test_resolve_static_headers_interpolates_globals_and_user( async def test_resolve_static_headers_raises_when_user_vars_missing( mock_server, monkeypatch ): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -250,7 +249,7 @@ async def test_resolve_static_headers_rechecks_db_before_raising_412( MCPMissingUserEnvVarsError the resolver must re-read with force_refresh and honor the fresh DB values. """ - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -288,7 +287,7 @@ async def test_resolve_static_headers_missing_is_non_blocking_for_listing( """With raise_on_missing=False (the tool-list path), missing per-user vars must NOT raise. Available vars interpolate; unfilled ${NAME} refs are left untouched so the server's tools still appear in the listing.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -317,7 +316,7 @@ async def test_resolve_static_headers_propagates_db_error_on_tool_call( ): """A DB failure on the tool-call path must surface as a real error, not be masked as a "missing credentials" MCPMissingUserEnvVarsError (412).""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -340,7 +339,7 @@ async def test_resolve_static_headers_swallows_db_error_on_listing( ): """On the listing path a DB failure is non-blocking: globals interpolate and unfilled per-user ${NAME} refs are left untouched.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -363,7 +362,7 @@ async def test_resolve_static_headers_swallows_db_error_on_listing( @pytest.mark.asyncio async def test_resolve_static_headers_passthrough_when_no_env_vars(): """Servers without env_vars should keep static_headers untouched.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -388,7 +387,7 @@ async def test_resolve_static_headers_unreferenced_user_var_is_not_blocking( """A per-user var declared by the admin but never referenced in static_headers must not block the request — only blocking-by-use is enforced.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -423,7 +422,7 @@ async def test_resolve_static_headers_stale_user_value_cannot_override_global( """A var that used to be user-scoped (so the user has a stored value) but is now global must resolve to the admin's global value, not the stale per-user row. Otherwise a user could override admin-configured headers indefinitely.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -460,7 +459,7 @@ async def test_resolve_static_headers_dual_scope_var_uses_global_without_412( global value (globals win in the merge), so the tool-call path must resolve it from the global instead of raising a 412 when the user hasn't filled it in. This happens during a global-to-user (or user-to-global) migration.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -504,7 +503,7 @@ async def test_resolve_static_headers_empty_global_does_not_cover_user_var( global carries no usable value, so the tool-call path still raises a 412 when the user hasn't supplied one, instead of silently interpolating an empty string into the header.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -543,7 +542,7 @@ async def test_resolve_static_headers_user_value_wins_over_empty_global( """When a global is empty, a value the user did supply must win the merge rather than being clobbered by the empty global. The header resolves to the user's value, not an empty string.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -603,7 +602,7 @@ def test_references_per_user_env_var(static_headers, env_vars, expected): """Only headers that actually reference a *per-user* var count: globals and declared-but-unreferenced user vars do not, since the userless probe can still resolve (or simply not need) them.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -624,7 +623,7 @@ def test_references_per_user_env_var(static_headers, env_vars, expected): async def test_health_check_reaches_servers_without_forwarding_per_user_env_vars( mock_server: MCPServer, monkeypatch: pytest.MonkeyPatch, respx_mock: MockRouter ) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -650,7 +649,7 @@ async def test_health_check_reaches_servers_without_forwarding_per_user_env_vars @pytest.mark.asyncio async def test_load_user_env_vars_returns_empty_without_user(): """No user auth → no per-user lookup is attempted.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -667,7 +666,7 @@ async def test_load_user_env_vars_returns_empty_without_user_id(): """User auth without a user_id (e.g. anonymous virtual key) → empty dict.""" from unittest.mock import MagicMock - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -689,7 +688,7 @@ async def test_load_user_env_vars_raises_when_db_unavailable(monkeypatch): satisfy (per-user env vars are unusable without a DB).""" from unittest.mock import MagicMock - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -715,7 +714,7 @@ async def test_resolve_static_headers_db_unavailable_is_not_missing_412( was None, making a DB outage look like "user has no credentials".""" from unittest.mock import MagicMock - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -736,8 +735,8 @@ async def test_load_user_env_vars_caches_within_ttl(env_vars_salt_key, monkeypat keeping the hot tool-call/tool-listing path off the DB.""" from unittest.mock import MagicMock - from litellm.proxy._experimental.mcp_server import mcp_server_manager as mgr_mod - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server import mcp_server_manager as mgr_mod + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -773,8 +772,8 @@ async def test_load_user_env_vars_force_refresh_bypasses_cache( process-local stale value cannot mask credentials stored on another worker.""" from unittest.mock import AsyncMock, MagicMock - from litellm.proxy._experimental.mcp_server import mcp_server_manager as mgr_mod - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server import mcp_server_manager as mgr_mod + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -818,8 +817,8 @@ async def test_load_user_env_vars_invalidation_forces_refetch( instead of serving the stale cached value.""" from unittest.mock import AsyncMock, MagicMock - from litellm.proxy._experimental.mcp_server import mcp_server_manager as mgr_mod - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server import mcp_server_manager as mgr_mod + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, invalidate_user_env_vars_cache, ) @@ -973,7 +972,7 @@ def _transactional_env_vars_prisma(read_delay: float = 0.0): async def test_merge_user_env_vars_does_not_persist_plaintext(env_vars_salt_key): """The per-user write path must encrypt values at rest; ``values_b64`` must never hold plaintext personal credentials, but must still round-trip.""" - from litellm.proxy._experimental.mcp_server.db import ( + from litellm.proxy.mcp_server.db import ( _decode_user_env_vars, merge_user_env_vars, ) @@ -997,7 +996,7 @@ async def test_merge_user_env_vars_does_not_persist_plaintext(env_vars_salt_key) async def test_get_user_env_vars_round_trip(env_vars_salt_key): from unittest.mock import MagicMock - from litellm.proxy._experimental.mcp_server.db import get_user_env_vars + from litellm.proxy.mcp_server.db import get_user_env_vars payload = {"CORP_USERNAME": "alice", "CORP_PASSWORD": "s3cret"} row = MagicMock() @@ -1010,7 +1009,7 @@ async def test_get_user_env_vars_round_trip(env_vars_salt_key): @pytest.mark.asyncio async def test_get_user_env_vars_returns_empty_for_missing_row(): - from litellm.proxy._experimental.mcp_server.db import get_user_env_vars + from litellm.proxy.mcp_server.db import get_user_env_vars prisma = _mock_env_vars_prisma(row=None) assert await get_user_env_vars(prisma, "alice", "srv-1") == {} @@ -1020,7 +1019,7 @@ async def test_get_user_env_vars_returns_empty_for_missing_row(): async def test_get_user_env_vars_stringifies_non_string_json_values(env_vars_salt_key): from types import SimpleNamespace - from litellm.proxy._experimental.mcp_server.db import get_user_env_vars + from litellm.proxy.mcp_server.db import get_user_env_vars row: Final = SimpleNamespace(values_b64=_encrypted_user_env_blob({"PORT": 8080, "DEBUG": True, "EMPTY": None})) @@ -1036,8 +1035,8 @@ async def test_get_user_env_vars_stringifies_non_string_json_values(env_vars_sal async def test_get_user_env_vars_treats_a_blob_that_is_not_a_json_object_as_unset(env_vars_salt_key, stored_json): from types import SimpleNamespace - from litellm.proxy._experimental.mcp_server.db import get_user_env_vars from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + from litellm.proxy.mcp_server.db import get_user_env_vars row: Final = SimpleNamespace(values_b64=encrypt_value_helper(stored_json)) @@ -1054,8 +1053,8 @@ async def test_decode_user_env_vars_warns_when_undecryptable( credentials" 412 for values they already stored.""" from unittest.mock import MagicMock - import litellm.proxy._experimental.mcp_server.db as mcp_db - from litellm.proxy._experimental.mcp_server.db import _decode_user_env_vars + import litellm.proxy.mcp_server.db as mcp_db + from litellm.proxy.mcp_server.db import _decode_user_env_vars blob = _encrypted_user_env_blob({"CORP_PASSWORD": "s3cret"}) @@ -1071,7 +1070,7 @@ async def test_decode_user_env_vars_warns_when_undecryptable( async def test_get_user_env_vars_bulk_distributes_results(env_vars_salt_key): from unittest.mock import AsyncMock, MagicMock - from litellm.proxy._experimental.mcp_server.db import get_user_env_vars_bulk + from litellm.proxy.mcp_server.db import get_user_env_vars_bulk blob1 = _encrypted_user_env_blob({"A": "1"}) blob2 = _encrypted_user_env_blob({"B": "2"}) @@ -1091,7 +1090,7 @@ async def test_get_user_env_vars_bulk_distributes_results(env_vars_salt_key): @pytest.mark.asyncio async def test_get_user_env_vars_bulk_empty_ids_short_circuits(): - from litellm.proxy._experimental.mcp_server.db import get_user_env_vars_bulk + from litellm.proxy.mcp_server.db import get_user_env_vars_bulk prisma = _mock_env_vars_prisma() assert await get_user_env_vars_bulk(prisma, "alice", []) == {} @@ -1103,7 +1102,7 @@ async def test_get_user_env_vars_bulk_empty_ids_short_circuits(): async def test_delete_user_env_vars_is_idempotent_delete_many(): """Delete must use ``delete_many`` so a missing row is a no-op rather than raising RecordNotFound; real DB errors are left to propagate.""" - from litellm.proxy._experimental.mcp_server.db import delete_user_env_vars + from litellm.proxy.mcp_server.db import delete_user_env_vars prisma = _mock_env_vars_prisma() await delete_user_env_vars(prisma, "alice", "srv-1") @@ -1118,7 +1117,7 @@ async def test_merge_user_env_vars_preserves_existing_and_prunes_disallowed( ): """Merging one update keeps the user's other stored values and drops any name the admin no longer declares as user-scoped.""" - from litellm.proxy._experimental.mcp_server.db import merge_user_env_vars + from litellm.proxy.mcp_server.db import merge_user_env_vars prisma = _transactional_env_vars_prisma() await merge_user_env_vars( @@ -1149,7 +1148,7 @@ async def test_merge_user_env_vars_serializes_concurrent_writes(env_vars_salt_ke both distinct values survive.""" import asyncio - from litellm.proxy._experimental.mcp_server.db import ( + from litellm.proxy.mcp_server.db import ( get_user_env_vars, merge_user_env_vars, ) @@ -1178,7 +1177,7 @@ async def test_merge_user_env_vars_acquires_lock_without_deserializing_void( from prisma.errors import RawQueryError - from litellm.proxy._experimental.mcp_server.db import merge_user_env_vars + from litellm.proxy.mcp_server.db import merge_user_env_vars class _Tx: def __init__(self): @@ -1231,7 +1230,7 @@ async def test_delete_mcp_server_removes_orphaned_user_env_vars(): it; there is no FK cascade, so skipping this leaves orphaned credentials.""" from unittest.mock import AsyncMock - from litellm.proxy._experimental.mcp_server.db import delete_mcp_server + from litellm.proxy.mcp_server.db import delete_mcp_server prisma = _mock_env_vars_prisma() prisma.db.litellm_mcpservertable.delete = AsyncMock(return_value=object()) @@ -1248,7 +1247,7 @@ async def test_delete_mcp_server_skips_env_var_cleanup_when_server_missing(): """A no-op delete (server not found) must not touch the env var table.""" from unittest.mock import AsyncMock - from litellm.proxy._experimental.mcp_server.db import delete_mcp_server + from litellm.proxy.mcp_server.db import delete_mcp_server prisma = _mock_env_vars_prisma() prisma.db.litellm_mcpservertable.delete = AsyncMock(return_value=None) @@ -1267,7 +1266,7 @@ async def test_delete_mcp_server_succeeds_when_orphan_cleanup_fails(): is already gone.""" from unittest.mock import AsyncMock - from litellm.proxy._experimental.mcp_server.db import delete_mcp_server + from litellm.proxy.mcp_server.db import delete_mcp_server deleted = object() prisma = _mock_env_vars_prisma() @@ -1289,7 +1288,7 @@ async def test_delete_mcp_server_removes_orphaned_user_credentials(): pointing at a now-missing server.""" from unittest.mock import AsyncMock - from litellm.proxy._experimental.mcp_server.db import delete_mcp_server + from litellm.proxy.mcp_server.db import delete_mcp_server prisma = _mock_env_vars_prisma() prisma.db.litellm_mcpservertable.delete = AsyncMock(return_value=object()) @@ -1306,7 +1305,7 @@ async def test_delete_mcp_server_skips_credential_cleanup_when_server_missing(): """A no-op delete (server not found) must not touch the credential table.""" from unittest.mock import AsyncMock - from litellm.proxy._experimental.mcp_server.db import delete_mcp_server + from litellm.proxy.mcp_server.db import delete_mcp_server prisma = _mock_env_vars_prisma() prisma.db.litellm_mcpservertable.delete = AsyncMock(return_value=None) @@ -1324,7 +1323,7 @@ async def test_delete_mcp_server_credential_cleanup_failure_still_cleans_env_var still succeed for the caller.""" from unittest.mock import AsyncMock - from litellm.proxy._experimental.mcp_server.db import delete_mcp_server + from litellm.proxy.mcp_server.db import delete_mcp_server deleted = object() prisma = _mock_env_vars_prisma() @@ -1361,11 +1360,11 @@ def test_prepare_mcp_server_data_encrypts_global_env_var_values(env_vars_salt_ke column, while ``scope="user"`` placeholders (not secrets) stay verbatim.""" import json - from litellm.proxy._experimental.mcp_server.db import ( + from litellm.proxy._types import MCPEnvVar + from litellm.proxy.mcp_server.db import ( _prepare_mcp_server_data, decrypt_global_env_var_values, ) - from litellm.proxy._types import MCPEnvVar req = _global_env_var_server_request( [ @@ -1400,8 +1399,8 @@ def test_prepare_mcp_server_data_skips_unset_env_vars_on_partial_update(): be written, even when the request object carries a non-None env_vars that was never marked as set. Otherwise a partial update could silently overwrite the stored values.""" - from litellm.proxy._experimental.mcp_server.db import _prepare_mcp_server_data from litellm.proxy._types import MCPEnvVar, UpdateMCPServerRequest + from litellm.proxy.mcp_server.db import _prepare_mcp_server_data data = UpdateMCPServerRequest.model_construct( _fields_set={"server_id"}, @@ -1420,8 +1419,8 @@ def test_prepare_mcp_server_data_writes_env_vars_when_set_on_partial_update( """A partial update that does set env_vars must serialize and encrypt them.""" import json - from litellm.proxy._experimental.mcp_server.db import _prepare_mcp_server_data from litellm.proxy._types import MCPEnvVar, UpdateMCPServerRequest + from litellm.proxy.mcp_server.db import _prepare_mcp_server_data data = UpdateMCPServerRequest( server_id="srv-1", @@ -1443,11 +1442,11 @@ async def test_build_mcp_server_from_table_decrypts_global_env_vars(env_vars_sal headers interpolate to the real secret instead of forwarding ciphertext.""" import json - from litellm.proxy._experimental.mcp_server.db import _prepare_mcp_server_data - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy._types import LiteLLM_MCPServerTable, MCPEnvVar + from litellm.proxy.mcp_server.db import _prepare_mcp_server_data + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) - from litellm.proxy._types import LiteLLM_MCPServerTable, MCPEnvVar req = _global_env_var_server_request( [MCPEnvVar(name="DB_PASSWORD", value="s3cr3t-p@ss", scope="global")] @@ -1481,14 +1480,14 @@ async def test_add_server_does_not_double_decrypt_global_env_vars(env_vars_salt_ upstream instead of the interpolated secret.""" import json - from litellm.proxy._experimental.mcp_server.db import ( + from litellm.proxy._types import LiteLLM_MCPServerTable, MCPEnvVar + from litellm.proxy.mcp_server.db import ( _prepare_mcp_server_data, decrypt_global_env_var_values, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) - from litellm.proxy._types import LiteLLM_MCPServerTable, MCPEnvVar req = _global_env_var_server_request( [MCPEnvVar(name="DB_PASSWORD", value="s3cr3t-p@ss", scope="global")] @@ -1529,16 +1528,16 @@ async def test_create_mcp_server_decrypts_env_vars_when_prisma_returns_json_stri (timestamps match), so headers forward ciphertext upstream.""" from unittest.mock import AsyncMock, MagicMock - from litellm.proxy._experimental.mcp_server.db import ( - _prepare_mcp_server_data, - create_mcp_server, - update_mcp_server, - ) from litellm.proxy._types import ( MCPEnvVar, NewMCPServerRequest, UpdateMCPServerRequest, ) + from litellm.proxy.mcp_server.db import ( + _prepare_mcp_server_data, + create_mcp_server, + update_mcp_server, + ) req = _global_env_var_server_request( [MCPEnvVar(name="DB_PASSWORD", value="s3cr3t-p@ss", scope="global")] @@ -1598,11 +1597,11 @@ def test_reencrypt_global_env_var_values_handles_json_string(env_vars_salt_key): helper must parse it instead of failing on ``dict(v)`` over a string.""" import json - from litellm.proxy._experimental.mcp_server.db import ( + from litellm.proxy._types import MCPEnvVar + from litellm.proxy.mcp_server.db import ( _prepare_mcp_server_data, _reencrypt_global_env_var_values, ) - from litellm.proxy._types import MCPEnvVar req = _global_env_var_server_request( [MCPEnvVar(name="DB_PASSWORD", value="s3cr3t-p@ss", scope="global")] @@ -1630,11 +1629,11 @@ async def test_rotate_mcp_user_env_vars_logs_rotated_and_skipped_counts( left untouched and counted as skipped.""" from unittest.mock import AsyncMock, MagicMock - import litellm.proxy._experimental.mcp_server.db as mcp_db - from litellm.proxy._experimental.mcp_server.db import ( + import litellm.proxy.mcp_server.db as mcp_db + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + from litellm.proxy.mcp_server.db import ( rotate_mcp_user_env_vars_master_key, ) - from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper def _row(user_id, server_id, blob): row = MagicMock() @@ -1693,12 +1692,12 @@ def test_decrypt_global_env_var_drops_undecryptable_value( import json from unittest.mock import MagicMock - import litellm.proxy._experimental.mcp_server.db as mcp_db - from litellm.proxy._experimental.mcp_server.db import ( + import litellm.proxy.mcp_server.db as mcp_db + from litellm.proxy._types import MCPEnvVar + from litellm.proxy.mcp_server.db import ( _prepare_mcp_server_data, decrypt_global_env_var_values, ) - from litellm.proxy._types import MCPEnvVar req = _global_env_var_server_request( [MCPEnvVar(name="DB_PASSWORD", value="s3cr3t-p@ss", scope="global")] diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_guardrail_usage_monitor.py b/tests/unit/proxy/mcp_server/test_mcp_guardrail_usage_monitor.py similarity index 99% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_guardrail_usage_monitor.py rename to tests/unit/proxy/mcp_server/test_mcp_guardrail_usage_monitor.py index 8d5e36e9795..765771de633 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_guardrail_usage_monitor.py +++ b/tests/unit/proxy/mcp_server/test_mcp_guardrail_usage_monitor.py @@ -21,7 +21,7 @@ from unittest import mock import pytest from litellm.exceptions import GuardrailRaisedException -from litellm.proxy._experimental.mcp_server import mcp_server_manager as MOD +from litellm.proxy.mcp_server import mcp_server_manager as MOD class _FakeLoggingObj: diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_header_alias_utils.py b/tests/unit/proxy/mcp_server/test_mcp_header_alias_utils.py similarity index 98% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_header_alias_utils.py rename to tests/unit/proxy/mcp_server/test_mcp_header_alias_utils.py index 6c24205c258..80c79505205 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_header_alias_utils.py +++ b/tests/unit/proxy/mcp_server/test_mcp_header_alias_utils.py @@ -1,6 +1,6 @@ """Tests for MCP header alias sanitization and auth header lookup.""" -from litellm.proxy._experimental.mcp_server.utils import ( +from litellm.proxy.mcp_server.utils import ( lookup_mcp_server_auth_in_headers, sanitize_mcp_alias_for_header, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py b/tests/unit/proxy/mcp_server/test_mcp_hook_extra_headers.py similarity index 96% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py rename to tests/unit/proxy/mcp_server/test_mcp_hook_extra_headers.py index c477a8d34cb..d0cf0199233 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py +++ b/tests/unit/proxy/mcp_server/test_mcp_hook_extra_headers.py @@ -19,8 +19,8 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from mcp.types import CallToolResult -from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager from litellm.proxy.utils import ProxyLogging from litellm.types.mcp import MCPAuth, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -412,7 +412,7 @@ class TestCallToolFlowsHookHeaders: new_callable=AsyncMock, return_value=CallToolResult(content=[], isError=False), ): - import litellm.proxy._experimental.mcp_server.mcp_server_manager as mgr_mod + import litellm.proxy.mcp_server.mcp_server_manager as mgr_mod proxy_logging = MagicMock(spec=ProxyLogging) @@ -1059,7 +1059,7 @@ class TestOpenApiByokCallTool: @pytest.mark.asyncio async def test_call_tool_openapi_byok_injects_request_auth_contextvar(self): """Playground/responses call call_tool directly; BYOK must reach OpenAPI handlers.""" - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( request_auth_header, ) @@ -1083,7 +1083,7 @@ class TestOpenApiByokCallTool: with patch.object(manager, "_resolve_mcp_server_for_tool_call", return_value=server): with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager._resolve_byok_mcp_auth_header", + "litellm.proxy.mcp_server.mcp_server_manager._resolve_byok_mcp_auth_header", new=AsyncMock(return_value="fc-test-key"), ): with patch.object( @@ -1113,21 +1113,21 @@ class TestFormatByokOpenapiAuthHeader: ) def test_api_key_auth_type(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _format_byok_openapi_auth_header, ) assert _format_byok_openapi_auth_header(self._server(MCPAuth.api_key), "secret") == "ApiKey secret" def test_basic_auth_type(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _format_byok_openapi_auth_header, ) assert _format_byok_openapi_auth_header(self._server(MCPAuth.basic), "secret") == "Basic secret" def test_defaults_to_bearer(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _format_byok_openapi_auth_header, ) @@ -1146,7 +1146,7 @@ class TestOpenapiForwardedExtraHeaders: ) def test_returns_none_without_extra_headers_config(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _openapi_forwarded_extra_headers, ) @@ -1154,7 +1154,7 @@ class TestOpenapiForwardedExtraHeaders: assert _openapi_forwarded_extra_headers(server, {"X-Custom": "v"}, None) is None def test_returns_none_without_raw_headers(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _openapi_forwarded_extra_headers, ) @@ -1162,7 +1162,7 @@ class TestOpenapiForwardedExtraHeaders: assert _openapi_forwarded_extra_headers(server, None, None) is None def test_forwards_configured_header_case_insensitively(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _openapi_forwarded_extra_headers, ) @@ -1171,7 +1171,7 @@ class TestOpenapiForwardedExtraHeaders: assert result == {"X-Custom": "v"} def test_returns_none_when_no_configured_header_is_present(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _openapi_forwarded_extra_headers, ) @@ -1179,7 +1179,7 @@ class TestOpenapiForwardedExtraHeaders: assert _openapi_forwarded_extra_headers(server, {"x-custom": "v"}, None) is None def test_skips_authorization_when_caller_header_must_be_stripped(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _openapi_forwarded_extra_headers, ) @@ -1189,7 +1189,7 @@ class TestOpenapiForwardedExtraHeaders: assert result is None def test_skips_non_string_header_entries(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _openapi_forwarded_extra_headers, ) @@ -1212,7 +1212,7 @@ class TestResolveByokMcpAuthHeader: @pytest.mark.asyncio async def test_non_byok_server_passes_header_through_unchanged(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _resolve_byok_mcp_auth_header, ) @@ -1222,7 +1222,7 @@ class TestResolveByokMcpAuthHeader: @pytest.mark.asyncio async def test_byok_server_uses_stored_credential_when_no_header_supplied(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _resolve_byok_mcp_auth_header, ) @@ -1230,7 +1230,7 @@ class TestResolveByokMcpAuthHeader: user_auth = UserAPIKeyAuth(user_id="user-1", api_key="sk-dashboard") with patch( - "litellm.proxy._experimental.mcp_server.operations._get_byok_credential", + "litellm.proxy.mcp_server.operations._get_byok_credential", new=AsyncMock(return_value="stored-cred"), ): result = await _resolve_byok_mcp_auth_header(server, user_auth, None) @@ -1241,7 +1241,7 @@ class TestResolveByokMcpAuthHeader: async def test_byok_server_raises_401_when_no_credential_stored(self, monkeypatch): from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _resolve_byok_mcp_auth_header, ) @@ -1250,7 +1250,7 @@ class TestResolveByokMcpAuthHeader: user_auth = UserAPIKeyAuth(user_id="user-1", api_key="sk-dashboard") with patch( - "litellm.proxy._experimental.mcp_server.operations._get_byok_credential", + "litellm.proxy.mcp_server.operations._get_byok_credential", new=AsyncMock(return_value=None), ): with pytest.raises(HTTPException) as exc_info: @@ -1264,7 +1264,7 @@ class TestResolveByokMcpAuthHeader: @pytest.mark.asyncio async def test_byok_server_checks_credential_and_keeps_caller_header_when_supplied(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _resolve_byok_mcp_auth_header, ) @@ -1273,7 +1273,7 @@ class TestResolveByokMcpAuthHeader: check_mock = AsyncMock(return_value=None) with patch( - "litellm.proxy._experimental.mcp_server.operations._check_byok_credential", + "litellm.proxy.mcp_server.operations._check_byok_credential", new=check_mock, ): result = await _resolve_byok_mcp_auth_header(server, user_auth, "caller-header") @@ -1304,13 +1304,13 @@ class TestOpenApiResolvedUpstreamAuth: async def test_call_tool_openapi_injects_v2_resolved_token_contextvar(self): """The managed spec_path arm resolves the v2 credential and sets the ContextVar; kills the mutant that drops the resolve_openapi_upstream_auth call in call_tool.""" - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( request_resolved_auth_headers, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.outbound_credentials.result import Ok manager = MCPServerManager() server = self._oauth_server() diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_logging.py b/tests/unit/proxy/mcp_server/test_mcp_logging.py similarity index 95% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_logging.py rename to tests/unit/proxy/mcp_server/test_mcp_logging.py index 44ba40afdd1..9548135b530 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_logging.py +++ b/tests/unit/proxy/mcp_server/test_mcp_logging.py @@ -11,14 +11,14 @@ from mcp.types import Tool as MCPTool import litellm from litellm.integrations.custom_logger import CustomLogger -from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( +from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth +from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) -from litellm.proxy._experimental.mcp_server.server import ( +from litellm.proxy.mcp_server.server import ( mcp_server_tool_call, set_auth_context, ) -from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth from litellm.types.mcp import MCPPostCallResponseObject @@ -111,7 +111,7 @@ async def test_mcp_cost_tracking(): local_mcp_server_manager = MCPServerManager() with patch( - "litellm.proxy._experimental.mcp_server.upstream.MCPClient", + "litellm.proxy.mcp_server.upstream.MCPClient", mock_client_constructor, ): # Load the server config @@ -138,11 +138,11 @@ async def test_mcp_cost_tracking(): # Patch the global manager in both modules where it's used with ( patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", local_mcp_server_manager, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", local_mcp_server_manager, ), ): @@ -244,7 +244,7 @@ async def test_mcp_cost_tracking_per_tool(): local_mcp_server_manager = MCPServerManager() with patch( - "litellm.proxy._experimental.mcp_server.upstream.MCPClient", + "litellm.proxy.mcp_server.upstream.MCPClient", mock_client_constructor, ): # Load the server config with per-tool costs @@ -289,11 +289,11 @@ async def test_mcp_cost_tracking_per_tool(): # Patch the global manager in both modules where it's used with ( patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", local_mcp_server_manager, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", local_mcp_server_manager, ), ): @@ -417,7 +417,7 @@ async def test_mcp_tool_call_hook(): local_mcp_server_manager = MCPServerManager() with patch( - "litellm.proxy._experimental.mcp_server.upstream.MCPClient", + "litellm.proxy.mcp_server.upstream.MCPClient", mock_client_constructor, ): # Load the server config @@ -447,11 +447,11 @@ async def test_mcp_tool_call_hook(): # Patch the global manager in both modules where it's used with ( patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", local_mcp_server_manager, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", local_mcp_server_manager, ), ): diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_max_concurrent_requests.py b/tests/unit/proxy/mcp_server/test_mcp_max_concurrent_requests.py similarity index 98% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_max_concurrent_requests.py rename to tests/unit/proxy/mcp_server/test_mcp_max_concurrent_requests.py index 8497d7bb7c4..f1896757874 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_max_concurrent_requests.py +++ b/tests/unit/proxy/mcp_server/test_mcp_max_concurrent_requests.py @@ -2,16 +2,16 @@ import asyncio from collections.abc import Awaitable, Callable from typing import Dict, Final, Optional, TypeVar, cast from unittest.mock import AsyncMock, patch + +import pytest from mcp import ClientSession from mcp.server.context import ServerRequestContext from mcp.types import CallToolRequestParams as MCPCallToolRequestParams -from litellm.experimental_mcp_client.client import MCPClient, PersistentMCPSession -from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var - -import pytest from mcp.types import CallToolResult, TextContent -from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager +from litellm.mcp_client.client import MCPClient, PersistentMCPSession +from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var +from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager from litellm.types.mcp import MCPAuth, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_metadata_preservation.py b/tests/unit/proxy/mcp_server/test_mcp_metadata_preservation.py similarity index 96% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_metadata_preservation.py rename to tests/unit/proxy/mcp_server/test_mcp_metadata_preservation.py index 86748d99063..118a4a9f17b 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_metadata_preservation.py +++ b/tests/unit/proxy/mcp_server/test_mcp_metadata_preservation.py @@ -9,11 +9,10 @@ which is critical for ChatGPT UI widget rendering. import pytest # Add the parent directory to the path so we can import litellm - from mcp.types import Tool as MCPTool -from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.proxy._types import MCPTransport +from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager from litellm.types.mcp_server.mcp_server_manager import MCPServer diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py b/tests/unit/proxy/mcp_server/test_mcp_oauth_passthrough.py similarity index 96% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py rename to tests/unit/proxy/mcp_server/test_mcp_oauth_passthrough.py index 1c59b7b87e0..80b4ea3c287 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py +++ b/tests/unit/proxy/mcp_server/test_mcp_oauth_passthrough.py @@ -15,15 +15,13 @@ import httpx import pytest from fastapi import HTTPException, Request - - -from litellm.proxy._experimental.mcp_server import discoverable_endpoints -from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( +from litellm.proxy._types import MCPTransport +from litellm.proxy.mcp_server import discoverable_endpoints +from litellm.proxy.mcp_server.discoverable_endpoints import ( _OAUTH_METADATA_CACHE, _OAUTH_METADATA_FETCH_LOCKS, _build_oauth_protected_resource_response, ) -from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -32,7 +30,7 @@ from litellm.types.mcp_server.mcp_server_manager import MCPServer def _mock_mcp_client_ip(): """Bypass IP-based access control in tests.""" with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", + "litellm.proxy.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", return_value=None, ): yield @@ -208,7 +206,7 @@ def test_is_oauth_passthrough_false_when_only_delegate_auth_to_upstream_set(): @pytest.mark.asyncio async def test_oauth_protected_resource_passthrough_proxies_upstream_metadata(): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -253,7 +251,7 @@ async def test_oauth_protected_resource_passthrough_proxies_upstream_metadata(): @pytest.mark.asyncio async def test_oauth_protected_resource_passthrough_cache_hit(): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -380,7 +378,7 @@ async def test_oauth_metadata_cache_expired_entry_is_refetched(): @pytest.mark.asyncio async def test_oauth_protected_resource_passthrough_network_error_returns_502(): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -444,7 +442,7 @@ async def test_oauth_protected_resource_gateway_managed_unchanged(): never fetch upstream metadata. Since LIT-4864 the advertised document is the gateway's own aggregate authorization server ({base}/mcp), which serves the keyless DCR flow for per-server URLs; the per-server relay endpoints remain for the keyed flow.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -505,7 +503,7 @@ async def test_oauth_protected_resource_oauth_delegate_returns_upstream_metadata gateway would make a strict IdP refuse to mint it or the upstream reject it. A regression that dropped oauth_delegate from the pass-through predicate would fall through to the gateway-AS branch and advertise LiteLLM as the AS.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -543,7 +541,7 @@ async def test_oauth_protected_resource_true_passthrough_returns_upstream_metada resource included, so the client treats the upstream as the resource and authorizes directly against it. A regression that rewrote resource (the gateway-proxied behavior) would break the transparent-proxy contract.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -582,7 +580,7 @@ async def test_oauth_protected_resource_dcr_bridge_returns_gateway_facade(auth_t facade: resource is the gateway URL the client dialed and authorization_servers names the gateway's per-server AS, so DCR-only clients (which enforce the RFC 9728 resource match) can register and sign in through the gateway. No upstream metadata fetch happens.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_cold_start.py b/tests/unit/proxy/mcp_server/test_mcp_oauth_passthrough_cold_start.py similarity index 90% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_cold_start.py rename to tests/unit/proxy/mcp_server/test_mcp_oauth_passthrough_cold_start.py index 67663448d65..3f6a44b0300 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_cold_start.py +++ b/tests/unit/proxy/mcp_server/test_mcp_oauth_passthrough_cold_start.py @@ -3,7 +3,6 @@ import pytest - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -40,11 +39,11 @@ def test_passthrough_cold_start_emits_401_with_matching_resource_metadata( route, expected_metadata_path ): """No auth headers on a passthrough server route emits matching metadata.""" - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( _is_mcp_passthrough_cold_start, _parse_mcp_server_names_from_path, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -93,10 +92,10 @@ def test_passthrough_cold_start_emits_401_with_matching_resource_metadata( def test_is_mcp_passthrough_cold_start_false_for_oauth2_server(): """Gateway-managed OAuth2 servers must not trigger the cold-start bypass.""" - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( _is_mcp_passthrough_cold_start, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -122,7 +121,7 @@ def test_is_mcp_passthrough_cold_start_false_for_oauth2_server(): def test_is_mcp_passthrough_cold_start_false_for_empty_servers(): """Aggregate /mcp route (no server list) must not trigger bypass.""" - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( _is_mcp_passthrough_cold_start, ) @@ -147,7 +146,7 @@ def test_is_mcp_passthrough_cold_start_false_for_empty_servers(): ], ) def test_parse_mcp_server_names_from_path(path, expected): - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( _parse_mcp_server_names_from_path, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py b/tests/unit/proxy/mcp_server/test_mcp_oauth_passthrough_tools.py similarity index 93% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py rename to tests/unit/proxy/mcp_server/test_mcp_oauth_passthrough_tools.py index 34b021df2b9..03a7373c908 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py +++ b/tests/unit/proxy/mcp_server/test_mcp_oauth_passthrough_tools.py @@ -1,4 +1,4 @@ -from litellm.proxy._experimental.mcp_server import operations as mcp_operations +from litellm.proxy.mcp_server import operations as mcp_operations """Unit tests for MCP OAuth passthrough tool-fetch behavior.""" @@ -11,21 +11,20 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest - if sys.version_info < (3, 11): from exceptiongroup import ExceptionGroup -from litellm.proxy._experimental.mcp_server.exceptions import ( +from litellm.proxy._types import MCPTransport +from litellm.proxy.mcp_server.exceptions import ( TOKEN_EXCHANGE_SUBJECT_ERROR, MCPServerListError, MCPUpstreamAuthError, ) -from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( +from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, _extract_upstream_auth_failure, ) -from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -341,8 +340,8 @@ async def test_aggregate_list_tools_absorbs_one_unauthenticated_server(): from mcp.types import Tool as MCPTool - from litellm.proxy._experimental.mcp_server import server as mcp_server from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server import server as mcp_server delegate = _http_server( "s1", "delegate_docs", auth_type=MCPAuth.oauth2, delegate_auth_to_upstream=True @@ -384,9 +383,9 @@ async def test_single_server_route_also_absorbs_upstream_auth_error(): re-auth surfacing is handled by a request-scope preemptive check, tracked separately.""" from unittest.mock import patch - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.mcp_context import mcp_gateway_server_name from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.mcp_context import mcp_gateway_server_name delegate = _http_server( "s1", "delegate_docs", auth_type=MCPAuth.oauth2, delegate_auth_to_upstream=True @@ -425,8 +424,8 @@ async def test_aggregate_with_single_accessible_server_still_absorbs(): permission sets.""" from unittest.mock import patch - from litellm.proxy._experimental.mcp_server import server as mcp_server from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server import server as mcp_server delegate = _http_server( "s1", "delegate_docs", auth_type=MCPAuth.oauth2, delegate_auth_to_upstream=True @@ -506,7 +505,7 @@ async def test_client_creation_failure_logs_sanitized_exchange(monkeypatch, capl ], ) def test_passthrough_admission_recognizes_only_matching_authorization(oauth_headers, server_headers, authorized): - from litellm.proxy._experimental.mcp_server.operations import _client_has_passthrough_authorization + from litellm.proxy.mcp_server.operations import _client_has_passthrough_authorization from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -528,7 +527,7 @@ def test_managed_step_up_challenge_uses_only_the_gateway_metadata(base_url, requ @pytest.mark.asyncio async def test_scope_challenge_replaces_pending_stream_headers_once(): - from litellm.proxy._experimental.mcp_server.scope_challenge import OAuthScopeResponse + from litellm.proxy.mcp_server.scope_challenge import OAuthScopeResponse send = AsyncMock() response = OAuthScopeResponse(send, "https://gateway.example", "/docs/mcp") @@ -554,14 +553,13 @@ async def test_scope_authorization_releases_progress_without_waiting_for_tool_re from mcp.server.context import ServerRequestContext from starlette.requests import Request - from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var - from litellm.proxy._experimental.mcp_server.scope_challenge import ( + from litellm.proxy.mcp_server.legacy_callbacks import get_scope_response, record_upstream_tool_authorization + from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var + from litellm.proxy.mcp_server.scope_challenge import ( SCOPE_RESPONSE_KEY, OAuthScopeResponse, ) - from litellm.proxy._experimental.mcp_server.legacy_callbacks import get_scope_response, record_upstream_tool_authorization - send = AsyncMock() gate = OAuthScopeResponse(send, "https://gateway.example", "/mcp/docs") request = Request({"type": "http", SCOPE_RESPONSE_KEY: gate}) @@ -578,7 +576,7 @@ async def test_scope_authorization_releases_progress_without_waiting_for_tool_re await record_upstream_tool_authorization(httpx2.Response(200, request=unread), pending=get_scope_response()) assert gate.pending # Use the actual HTTP response hook contract, before reading the tool response body. - from litellm.experimental_mcp_client.client import MCPClient + from litellm.mcp_client.client import MCPClient upstream = MCPClient(server_url="https://upstream.example/mcp") async with upstream._create_httpx_client_factory(transport=httpx2.MockTransport(lambda request: httpx2.Response(200, content=b"tool result")))() as client: @@ -602,7 +600,7 @@ async def test_scope_middleware_releases_waiting_response_on_validation_or_cance from mcp.server.context import ServerRequestContext from starlette.requests import Request - from litellm.proxy._experimental.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse, finish_scope_response + from litellm.proxy.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse, finish_scope_response send: Final = AsyncMock(side_effect=BrokenPipeError("peer disconnected") if fail_send else None) gate: Final = OAuthScopeResponse(send, "https://gateway.example", "/mcp/docs") @@ -627,8 +625,8 @@ async def test_scope_transport_owns_header_send_cancellation_and_errors(fail_sen import httpx2 from starlette.types import Message - from litellm.proxy._experimental.mcp_server.legacy_callbacks import record_upstream_tool_authorization - from litellm.proxy._experimental.mcp_server.scope_challenge import OAuthScopeResponse + from litellm.proxy.mcp_server.legacy_callbacks import record_upstream_tool_authorization + from litellm.proxy.mcp_server.scope_challenge import OAuthScopeResponse send_started: Final = asyncio.Event() send_stopped: Final = asyncio.Event() @@ -677,7 +675,7 @@ async def test_scope_transport_owns_header_send_cancellation_and_errors(fail_sen async def test_scope_authorization_sends_stream_headers_before_the_first_body_chunk(allowed_before_start: bool) -> None: from starlette.types import Message - from litellm.proxy._experimental.mcp_server.scope_challenge import OAuthScopeResponse + from litellm.proxy.mcp_server.scope_challenge import OAuthScopeResponse send: Final = AsyncMock() gate: Final = OAuthScopeResponse(send, "https://gateway.example", "/mcp/docs") @@ -698,7 +696,7 @@ async def test_scope_authorization_sends_stream_headers_before_the_first_body_ch @pytest.mark.asyncio async def test_scope_response_preserves_transport_validation_errors(): - from litellm.proxy._experimental.mcp_server.scope_challenge import OAuthScopeResponse + from litellm.proxy.mcp_server.scope_challenge import OAuthScopeResponse send = AsyncMock() gate = OAuthScopeResponse(send, "https://gateway.example", "/mcp/docs") @@ -718,9 +716,9 @@ async def test_streamable_sdk_relays_scope_challenge_before_committing_http_resp from mcp.server.streamable_http_manager import StreamableHTTPSessionManager from mcp.types import CallToolResult, TextContent - from litellm.proxy._experimental.mcp_server.legacy_callbacks import get_scope_response - from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var - from litellm.proxy._experimental.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse, finish_scope_response + from litellm.proxy.mcp_server.legacy_callbacks import get_scope_response + from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var + from litellm.proxy.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse, finish_scope_response async def call_tool(context, params): token = active_mcp_request_ctx_var.set(context) @@ -772,9 +770,9 @@ async def test_scope_challenges_are_isolated_between_concurrent_requests(): from mcp.server.context import ServerRequestContext from starlette.requests import Request - from litellm.proxy._experimental.mcp_server.legacy_callbacks import get_scope_response, record_upstream_tool_authorization - from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var - from litellm.proxy._experimental.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse + from litellm.proxy.mcp_server.legacy_callbacks import get_scope_response, record_upstream_tool_authorization + from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var + from litellm.proxy.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse async def request_scope(denied): send = AsyncMock() @@ -801,7 +799,7 @@ async def test_scope_challenges_are_isolated_between_concurrent_requests(): @pytest.mark.parametrize("body", [b"", b"{", b"[]", b'{"method":"initialize"}']) def test_non_tool_payloads_do_not_hold_response_headers(body): - from litellm.proxy._experimental.mcp_server.scope_challenge import is_tool_call_request, scope_response_from_request + from litellm.proxy.mcp_server.scope_challenge import is_tool_call_request, scope_response_from_request assert not is_tool_call_request(body) assert scope_response_from_request(None) is None @@ -812,7 +810,7 @@ async def test_scope_middleware_preserves_non_http_requests(): from mcp.server.context import ServerRequestContext from mcp.types import CallToolResult, TextContent - from litellm.proxy._experimental.mcp_server.scope_challenge import finish_scope_response + from litellm.proxy.mcp_server.scope_challenge import finish_scope_response context = ServerRequestContext(session=MagicMock(), lifespan_context={}, protocol_version="2025-11-25", method="tools/call") async def call_next(request): @@ -824,9 +822,10 @@ async def test_scope_middleware_preserves_non_http_requests(): @pytest.mark.parametrize("path", ["/mcp", "/toolset/demo/mcp"]) def test_managed_step_up_metadata_matches_the_sdk_configured_resource(path): - from mcp.shared.auth_utils import check_resource_allowed from urllib.parse import urlsplit + from mcp.shared.auth_utils import check_resource_allowed + error = MCPUpstreamAuthError(403, None, "docs", required_scope="tools.write") response = error.to_http_exception("https://gateway.test", path) challenge = response.headers["www-authenticate"] diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_partial_update.py b/tests/unit/proxy/mcp_server/test_mcp_partial_update.py similarity index 98% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_partial_update.py rename to tests/unit/proxy/mcp_server/test_mcp_partial_update.py index d6894979906..8e24e4fca80 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_partial_update.py +++ b/tests/unit/proxy/mcp_server/test_mcp_partial_update.py @@ -12,16 +12,16 @@ import json from unittest.mock import AsyncMock, MagicMock import pytest -from prisma import Json, models from fastapi import HTTPException +from prisma import Json, models -from litellm.proxy._experimental.mcp_server.db import ( +from litellm.proxy._types import NewMCPServerRequest, UpdateMCPServerRequest +from litellm.proxy.mcp_server.db import ( create_mcp_server, decrypt_credentials, set_mcp_server_pinned_tools, update_mcp_server, ) -from litellm.proxy._types import NewMCPServerRequest, UpdateMCPServerRequest from litellm.types.mcp_server.mcp_server_manager import PinnedMCPTool @@ -883,7 +883,7 @@ def _mock_toolset_prisma(): async def _run_toolset_update(payload: dict) -> dict: """The columns PUT /v1/mcp/toolset writes for this payload, minus the audit stamp every write carries. The prisma double is injected, so nothing is patched.""" - from litellm.proxy._experimental.mcp_server.toolset_db import update_mcp_toolset + from litellm.proxy.mcp_server.toolset_db import update_mcp_toolset from litellm.types.mcp_server.mcp_toolset import UpdateMCPToolsetRequest mock_prisma = _mock_toolset_prisma() @@ -943,7 +943,7 @@ async def test_find_identifier_conflict_reports_alias_hit(): Case-insensitive and cross-field matching is exercised end to end against real Postgres by test_duplicate_alias_is_rejected_so_tool_prefixes_cannot_collide. """ - from litellm.proxy._experimental.mcp_server.db import ( + from litellm.proxy.mcp_server.db import ( find_mcp_server_identifier_conflict, ) @@ -964,7 +964,7 @@ async def test_find_identifier_conflict_reports_alias_hit(): async def test_find_identifier_conflict_reports_server_name_when_alias_is_free(): """alias is checked first so the reported field is deterministic; a clean alias does not mask a colliding server_name.""" - from litellm.proxy._experimental.mcp_server.db import ( + from litellm.proxy.mcp_server.db import ( find_mcp_server_identifier_conflict, ) @@ -981,7 +981,7 @@ async def test_find_identifier_conflict_reports_server_name_when_alias_is_free() @pytest.mark.asyncio async def test_find_identifier_conflict_returns_none_when_free(): - from litellm.proxy._experimental.mcp_server.db import ( + from litellm.proxy.mcp_server.db import ( find_mcp_server_identifier_conflict, ) @@ -994,7 +994,7 @@ async def test_find_identifier_conflict_returns_none_when_free(): @pytest.mark.asyncio async def test_update_writing_alias_returns_conflict_instead_of_row(): - from litellm.proxy._experimental.mcp_server.db import McpIdentifierConflict + from litellm.proxy.mcp_server.db import McpIdentifierConflict mock_prisma = _mock_prisma() mock_prisma.db.litellm_mcpservertable.find_first = AsyncMock(return_value=_conflict_row()) @@ -1038,7 +1038,7 @@ async def test_update_writing_free_alias_returns_the_row(): async def test_clearing_alias_conflicts_on_the_fallback_server_name(): """alias: null drops the tool prefix to the stored server_name, which may already belong to another row, so that name goes through the conflict check.""" - from litellm.proxy._experimental.mcp_server.db import McpIdentifierConflict + from litellm.proxy.mcp_server.db import McpIdentifierConflict mock_prisma = _mock_prisma() existing = MagicMock() @@ -1061,7 +1061,7 @@ async def test_clearing_alias_conflicts_on_the_fallback_server_name(): async def test_clearing_alias_to_empty_string_conflicts_on_the_fallback_server_name(): """alias: "" publishes the stored server_name as the tool prefix, just like alias: null, so the fallback name must go through the conflict check too.""" - from litellm.proxy._experimental.mcp_server.db import McpIdentifierConflict + from litellm.proxy.mcp_server.db import McpIdentifierConflict mock_prisma = _mock_prisma() existing = MagicMock() @@ -1254,7 +1254,8 @@ async def test_url_edit_preserves_client_bound_to_previous_known_issuer_without_ @pytest.mark.parametrize("changed", [0, 1]) async def test_registration_write_requires_unchanged_server_revision(changed): from datetime import datetime, timezone - from litellm.proxy._experimental.mcp_server.db import _update_mcp_server_row + + from litellm.proxy.mcp_server.db import _update_mcp_server_row prisma = _mock_prisma() table = prisma.db.litellm_mcpservertable diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py b/tests/unit/proxy/mcp_server/test_mcp_proxy_mode.py similarity index 96% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py rename to tests/unit/proxy/mcp_server/test_mcp_proxy_mode.py index 7dadfc6493a..8d6a92a12a7 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py +++ b/tests/unit/proxy/mcp_server/test_mcp_proxy_mode.py @@ -1,4 +1,3 @@ -from litellm.proxy._experimental.mcp_server import operations as mcp_operations import asyncio import json import logging @@ -14,15 +13,16 @@ from pydantic import AnyUrl import litellm from litellm.integrations.custom_logger import CustomLogger -from litellm.proxy._experimental.mcp_server import server -from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_proxy_mode -from litellm.proxy._experimental.mcp_server.mcp_server_manager import ListedToolsCaller -from litellm.proxy._experimental.mcp_server.tool_search import ( +from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth +from litellm.proxy.mcp_server import operations as mcp_operations +from litellm.proxy.mcp_server import server +from litellm.proxy.mcp_server.mcp_context import _mcp_proxy_mode +from litellm.proxy.mcp_server.mcp_server_manager import ListedToolsCaller +from litellm.proxy.mcp_server.tool_search import ( handle_mcp_proxy_tool, mcp_proxy_tool_id, with_mcp_proxy_identity, ) -from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_completion_flow.py b/tests/unit/proxy/mcp_server/test_mcp_sampling_completion_flow.py similarity index 85% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_completion_flow.py rename to tests/unit/proxy/mcp_server/test_mcp_sampling_completion_flow.py index d17b407a1be..e838b8b088d 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_completion_flow.py +++ b/tests/unit/proxy/mcp_server/test_mcp_sampling_completion_flow.py @@ -11,10 +11,9 @@ from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import pytest - from mcp.types import CreateMessageResult, ErrorData -from litellm.proxy._experimental.mcp_server.sampling_handler import ( +from litellm.proxy.mcp_server.sampling_handler import ( _build_completion_kwargs, _run_guardrails_and_call_llm, handle_sampling_create_message, @@ -146,26 +145,26 @@ class TestHandleSamplingCreateMessagePipeline: ) with ( patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._resolve_model_from_preferences", + "litellm.proxy.mcp_server.sampling_handler._resolve_model_from_preferences", return_value="gpt-4o", ), patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._check_model_access", + "litellm.proxy.mcp_server.sampling_handler._check_model_access", new_callable=AsyncMock, return_value=None, ), patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._run_budget_checks", + "litellm.proxy.mcp_server.sampling_handler._run_budget_checks", new_callable=AsyncMock, return_value=None, ), patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._build_completion_kwargs", + "litellm.proxy.mcp_server.sampling_handler._build_completion_kwargs", new_callable=AsyncMock, return_value={"model": "gpt-4o", "messages": []}, ), patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._run_guardrails_and_call_llm", + "litellm.proxy.mcp_server.sampling_handler._run_guardrails_and_call_llm", new_callable=AsyncMock, return_value=response, ), @@ -187,21 +186,21 @@ class TestHandleSamplingCreateMessagePipeline: auth = SimpleNamespace(user_id="u1", api_key="sk-test", token="tok") with ( patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._resolve_model_from_preferences", + "litellm.proxy.mcp_server.sampling_handler._resolve_model_from_preferences", return_value="gpt-4o", ), patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._check_model_access", + "litellm.proxy.mcp_server.sampling_handler._check_model_access", new_callable=AsyncMock, return_value=None, ), patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._run_budget_checks", + "litellm.proxy.mcp_server.sampling_handler._run_budget_checks", new_callable=AsyncMock, return_value=None, ), patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._build_completion_kwargs", + "litellm.proxy.mcp_server.sampling_handler._build_completion_kwargs", new_callable=AsyncMock, side_effect=RateLimitError( "rate limited", llm_provider="openai", model="gpt-4o" @@ -220,21 +219,21 @@ class TestHandleSamplingCreateMessagePipeline: auth = SimpleNamespace(user_id="u1", api_key="sk-test", token="tok") with ( patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._resolve_model_from_preferences", + "litellm.proxy.mcp_server.sampling_handler._resolve_model_from_preferences", return_value="gpt-4o", ), patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._check_model_access", + "litellm.proxy.mcp_server.sampling_handler._check_model_access", new_callable=AsyncMock, return_value=None, ), patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._run_budget_checks", + "litellm.proxy.mcp_server.sampling_handler._run_budget_checks", new_callable=AsyncMock, return_value=None, ), patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._build_completion_kwargs", + "litellm.proxy.mcp_server.sampling_handler._build_completion_kwargs", new_callable=AsyncMock, side_effect=RuntimeError("kaboom"), ), diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_model_access.py b/tests/unit/proxy/mcp_server/test_mcp_sampling_model_access.py similarity index 92% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_model_access.py rename to tests/unit/proxy/mcp_server/test_mcp_sampling_model_access.py index 8975f42387b..7111638da29 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_model_access.py +++ b/tests/unit/proxy/mcp_server/test_mcp_sampling_model_access.py @@ -7,10 +7,11 @@ calls, preventing a malicious upstream MCP server from requesting inference on models the caller's API key is not authorized to use. """ -import pytest from unittest.mock import AsyncMock, MagicMock, patch -from litellm.proxy._experimental.mcp_server.sampling_handler import ( +import pytest + +from litellm.proxy.mcp_server.sampling_handler import ( _check_model_access, ) @@ -207,7 +208,7 @@ class TestSamplingAuthAndBudgetGating: @pytest.mark.asyncio async def test_should_deny_when_no_auth_context(self): """Sampling must reject calls with no user_api_key_auth.""" - from litellm.proxy._experimental.mcp_server.sampling_handler import ( + from litellm.proxy.mcp_server.sampling_handler import ( handle_sampling_create_message, ) @@ -236,7 +237,7 @@ class TestSamplingAuthAndBudgetGating: @pytest.mark.asyncio async def test_should_run_budget_checks(self): """Sampling must call _run_budget_checks after model access check.""" - from litellm.proxy._experimental.mcp_server.sampling_handler import ( + from litellm.proxy.mcp_server.sampling_handler import ( handle_sampling_create_message, ) @@ -254,17 +255,17 @@ class TestSamplingAuthAndBudgetGating: with ( patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._check_model_access", + "litellm.proxy.mcp_server.sampling_handler._check_model_access", new_callable=AsyncMock, return_value=None, ), patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._run_budget_checks", + "litellm.proxy.mcp_server.sampling_handler._run_budget_checks", new_callable=AsyncMock, return_value=None, ) as mock_budget, patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._resolve_model_from_preferences", + "litellm.proxy.mcp_server.sampling_handler._resolve_model_from_preferences", return_value="gpt-4o", ), patch( @@ -298,7 +299,8 @@ class TestSamplingAuthAndBudgetGating: async def test_should_deny_over_budget_caller(self): """When _run_budget_checks returns ErrorData, sampling must return it.""" from mcp.types import ErrorData - from litellm.proxy._experimental.mcp_server.sampling_handler import ( + + from litellm.proxy.mcp_server.sampling_handler import ( handle_sampling_create_message, ) @@ -318,17 +320,17 @@ class TestSamplingAuthAndBudgetGating: with ( patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._check_model_access", + "litellm.proxy.mcp_server.sampling_handler._check_model_access", new_callable=AsyncMock, return_value=None, ), patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._run_budget_checks", + "litellm.proxy.mcp_server.sampling_handler._run_budget_checks", new_callable=AsyncMock, return_value=budget_error, ), patch( - "litellm.proxy._experimental.mcp_server.sampling_handler._resolve_model_from_preferences", + "litellm.proxy.mcp_server.sampling_handler._resolve_model_from_preferences", return_value="gpt-4o", ), ): diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_model_resolution.py b/tests/unit/proxy/mcp_server/test_mcp_sampling_model_resolution.py similarity index 98% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_model_resolution.py rename to tests/unit/proxy/mcp_server/test_mcp_sampling_model_resolution.py index 0c8f7bd4814..a0280abbae7 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_model_resolution.py +++ b/tests/unit/proxy/mcp_server/test_mcp_sampling_model_resolution.py @@ -12,7 +12,7 @@ from unittest.mock import MagicMock, patch import pytest -from litellm.proxy._experimental.mcp_server.sampling_handler import ( +from litellm.proxy.mcp_server.sampling_handler import ( _resolve_model_from_preferences, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_priority_selection.py b/tests/unit/proxy/mcp_server/test_mcp_sampling_priority_selection.py similarity index 99% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_priority_selection.py rename to tests/unit/proxy/mcp_server/test_mcp_sampling_priority_selection.py index 24309ed0460..b6426ca88d6 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_priority_selection.py +++ b/tests/unit/proxy/mcp_server/test_mcp_sampling_priority_selection.py @@ -9,7 +9,7 @@ per the MCP spec. from types import SimpleNamespace from unittest.mock import patch -from litellm.proxy._experimental.mcp_server.sampling_handler import ( +from litellm.proxy.mcp_server.sampling_handler import ( _has_priorities, _resolve_model_from_preferences, _select_model_by_priority, diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_request_builder.py b/tests/unit/proxy/mcp_server/test_mcp_sampling_request_builder.py similarity index 98% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_request_builder.py rename to tests/unit/proxy/mcp_server/test_mcp_sampling_request_builder.py index d5c636baead..3d1b2072ae5 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_request_builder.py +++ b/tests/unit/proxy/mcp_server/test_mcp_sampling_request_builder.py @@ -7,7 +7,7 @@ so that header-dependent guardrails, routing hooks, and trace correlation function correctly. """ -from litellm.proxy._experimental.mcp_server.sampling_handler import ( +from litellm.proxy.mcp_server.sampling_handler import ( _build_sampling_request, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_response_conversion.py b/tests/unit/proxy/mcp_server/test_mcp_sampling_response_conversion.py similarity index 99% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_response_conversion.py rename to tests/unit/proxy/mcp_server/test_mcp_sampling_response_conversion.py index ba130f34964..de9c600b5c4 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_response_conversion.py +++ b/tests/unit/proxy/mcp_server/test_mcp_sampling_response_conversion.py @@ -18,7 +18,7 @@ from mcp.types import ( ToolUseContent, ) -from litellm.proxy._experimental.mcp_server.sampling_handler import ( +from litellm.proxy.mcp_server.sampling_handler import ( _convert_mcp_content_to_openai, _convert_mcp_tool_choice_to_openai, _convert_mcp_tools_to_openai, diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_tool_conversion.py b/tests/unit/proxy/mcp_server/test_mcp_sampling_tool_conversion.py similarity index 99% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_tool_conversion.py rename to tests/unit/proxy/mcp_server/test_mcp_sampling_tool_conversion.py index 167847afe1f..e537f03aeff 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_tool_conversion.py +++ b/tests/unit/proxy/mcp_server/test_mcp_sampling_tool_conversion.py @@ -12,12 +12,11 @@ from typing import Any, Dict from mcp.types import TextContent, ToolResultContent -from litellm.proxy._experimental.mcp_server.sampling_handler import ( +from litellm.proxy.mcp_server.sampling_handler import ( _convert_mcp_messages_to_openai, _convert_single_content, ) - # --------------------------------------------------------------------------- # Helpers — lightweight MCP type stand-ins # --------------------------------------------------------------------------- diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/unit/proxy/mcp_server/test_mcp_server.py similarity index 92% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py rename to tests/unit/proxy/mcp_server/test_mcp_server.py index de711d737d0..884c82760ea 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/unit/proxy/mcp_server/test_mcp_server.py @@ -2,24 +2,21 @@ import asyncio import json import os +from contextlib import asynccontextmanager, nullcontext from typing import Final +from unittest.mock import ANY, AsyncMock, MagicMock, patch import pytest -from unittest.mock import ANY, AsyncMock, MagicMock, patch -from contextlib import asynccontextmanager, nullcontext - +from mcp.types import CallToolResult, ListToolsResult, TextContent +from mcp.types import Tool as MCPTool from starlette.types import Receive, Scope, Send - -from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - MCPServerManager, +from litellm.proxy._types import LiteLLM_ObjectPermissionTable +from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServer, + MCPServerManager, MCPTransport, ) -from litellm.proxy._types import LiteLLM_ObjectPermissionTable -from mcp.types import Tool as MCPTool, CallToolResult, ListToolsResult -from mcp.types import TextContent - mcp_server_manager = MCPServerManager() @@ -77,7 +74,7 @@ async def test_mcp_server_manager_https_server(): return mock_client with patch( - "litellm.proxy._experimental.mcp_server.upstream.MCPClient", + "litellm.proxy.mcp_server.upstream.MCPClient", mock_client_constructor, ): await mcp_server_manager.load_servers_from_config( @@ -185,7 +182,7 @@ async def test_mcp_http_transport_list_tools_mock(): return mock_client with patch( - "litellm.proxy._experimental.mcp_server.upstream.MCPClient", + "litellm.proxy.mcp_server.upstream.MCPClient", mock_client_constructor, ): # Load server config with HTTP transport @@ -262,7 +259,7 @@ async def test_mcp_http_transport_call_tool_mock(): return mock_client with patch( - "litellm.proxy._experimental.mcp_server.upstream.MCPClient", + "litellm.proxy.mcp_server.upstream.MCPClient", mock_client_constructor, ): # Load server config with HTTP transport @@ -328,7 +325,7 @@ async def test_mcp_http_transport_call_tool_error_mock(): return mock_client with patch( - "litellm.proxy._experimental.mcp_server.upstream.MCPClient", + "litellm.proxy.mcp_server.upstream.MCPClient", mock_client_constructor, ): # Load server config with HTTP transport @@ -426,26 +423,26 @@ async def test_streamable_http_mcp_handler_mock(): with ( patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.server.session_manager_stateless", + "litellm.proxy.mcp_server.server.session_manager_stateless", mock_session_manager_stateless, ), patch( - "litellm.proxy._experimental.mcp_server.server.session_manager_stateful", + "litellm.proxy.mcp_server.server.session_manager_stateful", mock_session_manager_stateful, ), patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", AsyncMock(return_value=mock_auth_context), ), patch( - "litellm.proxy._experimental.mcp_server.server.set_auth_context", + "litellm.proxy.mcp_server.server.set_auth_context", ), ): - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, ) @@ -495,32 +492,32 @@ async def test_sse_mcp_handler_mock(): ) with ( - patch("litellm.proxy._experimental.mcp_server.server.serve_loop", serve), + patch("litellm.proxy.mcp_server.server.serve_loop", serve), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.server.sse", + "litellm.proxy.mcp_server.server.sse", mock_sse, ), patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new=AsyncMock(return_value=mock_auth_result), ), patch( - "litellm.proxy._experimental.mcp_server.server.set_auth_context", + "litellm.proxy.mcp_server.server.set_auth_context", ), patch( - "litellm.proxy._experimental.mcp_server.server._raise_preemptive_401_for_unauthenticated_servers", + "litellm.proxy.mcp_server.server._raise_preemptive_401_for_unauthenticated_servers", new=AsyncMock(), ), patch( - "litellm.proxy._experimental.mcp_server.server._check_passthrough_upstream_auth", + "litellm.proxy.mcp_server.server._check_passthrough_upstream_auth", new=AsyncMock(), ), ): - from litellm.proxy._experimental.mcp_server.server import handle_sse_mcp + from litellm.proxy.mcp_server.server import handle_sse_mcp # Call the handler await handle_sse_mcp(mock_scope, mock_receive, mock_send) @@ -560,33 +557,33 @@ async def test_sse_mcp_handler_propagates_passthrough_401(): with ( patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.server.sse", + "litellm.proxy.mcp_server.server.sse", ) as transport, patch( - "litellm.proxy._experimental.mcp_server.server.server.run", + "litellm.proxy.mcp_server.server.server.run", AsyncMock(), ), patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new=AsyncMock(return_value=mock_auth_result), ), patch( - "litellm.proxy._experimental.mcp_server.server.set_auth_context", + "litellm.proxy.mcp_server.server.set_auth_context", ), patch( - "litellm.proxy._experimental.mcp_server.server._raise_preemptive_401_for_unauthenticated_servers", + "litellm.proxy.mcp_server.server._raise_preemptive_401_for_unauthenticated_servers", new=AsyncMock(), ), patch( - "litellm.proxy._experimental.mcp_server.server._check_passthrough_upstream_auth", + "litellm.proxy.mcp_server.server._check_passthrough_upstream_auth", new=AsyncMock(side_effect=challenge), ), ): - from litellm.proxy._experimental.mcp_server.server import handle_sse_mcp + from litellm.proxy.mcp_server.server import handle_sse_mcp with pytest.raises(HTTPException) as excinfo: await handle_sse_mcp(mock_scope, mock_receive, mock_send) @@ -751,12 +748,13 @@ def test_generate_stable_server_id(): @pytest.mark.asyncio async def test_list_tools_rest_api_server_not_found(): """Test the list_tools REST API when server is not found""" - from litellm.proxy._experimental.mcp_server.rest_endpoints import ( - list_tool_rest_api, - global_mcp_server_manager, - ) from fastapi import Query + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.rest_endpoints import ( + global_mcp_server_manager, + list_tool_rest_api, + ) # Mock UserAPIKeyAuth with explicit permission to access the requested server id mock_user_auth = UserAPIKeyAuth( @@ -776,7 +774,7 @@ async def test_list_tools_rest_api_server_not_found(): # Mock the global_mcp_server_manager to allow the server ID but return None for the server with patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager" + "litellm.proxy.mcp_server.rest_endpoints.global_mcp_server_manager" ) as mock_manager: # Allow the server ID in permissions mock_manager.get_allowed_mcp_servers = AsyncMock( @@ -805,14 +803,15 @@ async def test_list_tools_rest_api_server_not_found(): @pytest.mark.asyncio async def test_list_tools_rest_api_success(): """Test the list_tools REST API successful case""" - from litellm.proxy._experimental.mcp_server.rest_endpoints import ( + from fastapi import Query + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.rest_endpoints import ( list_tool_rest_api, ) - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( ListMCPToolsRestAPIResponseObject, ) - from fastapi import Query - from litellm.proxy._types import UserAPIKeyAuth # Mock successful tools mock_tools = [ @@ -849,7 +848,7 @@ async def test_list_tools_rest_api_success(): # Mock the global_mcp_server_manager with patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager" + "litellm.proxy.mcp_server.rest_endpoints.global_mcp_server_manager" ) as mock_manager: mock_manager.get_allowed_mcp_servers = AsyncMock( return_value=["test-server-123"] @@ -862,7 +861,7 @@ async def test_list_tools_rest_api_success(): # Mock the _get_tools_for_single_server function with patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._get_tools_for_single_server" + "litellm.proxy.mcp_server.rest_endpoints._get_tools_for_single_server" ) as mock_get_tools: mock_get_tools.return_value = mock_tools @@ -881,14 +880,14 @@ async def test_list_tools_rest_api_success(): @pytest.mark.asyncio async def test_get_tools_from_mcp_servers(): """Test _get_tools_from_mcp_servers function with both specific and no server filters""" - from litellm.proxy._experimental.mcp_server.server import ( - _get_tools_from_mcp_servers, - ) from litellm.proxy._types import UserAPIKeyAuth - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServer, MCPTransport, ) + from litellm.proxy.mcp_server.server import ( + _get_tools_from_mcp_servers, + ) # Mock data mock_user_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user") @@ -946,7 +945,7 @@ async def test_get_tools_from_mcp_servers(): ) with patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", mock_manager, ): # Test with specific servers @@ -995,7 +994,7 @@ async def test_get_tools_from_mcp_servers(): ) with patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", mock_manager_2, ): result = await _get_tools_from_mcp_servers( @@ -1028,11 +1027,11 @@ async def test_get_tools_from_mcp_servers(): ) with patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", mock_manager, ): with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", AsyncMock(return_value=["server3_id"]), ): # Test with specific servers @@ -1112,7 +1111,7 @@ async def test_list_tools_only_returns_allowed_servers(monkeypatch): return mock_client with patch( - "litellm.proxy._experimental.mcp_server.upstream.MCPClient", + "litellm.proxy.mcp_server.upstream.MCPClient", mock_client_constructor, ): # Call list_tools @@ -1158,13 +1157,12 @@ async def test_mcp_server_manager_access_groups_from_config(config_only_mcp_mana assert config_server is not None assert set(config_server.access_groups) == {"group-a", "group-b"} # Check that the lookup logic finds the correct server ids - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( - MCPRequestHandler, - ) - # Patch global_mcp_server_manager for this test and restore afterwards to # avoid leaking state into other tests (e.g. the proxy MCP e2e suite). - import litellm.proxy._experimental.mcp_server.mcp_server_manager as mcp_server_manager_mod + import litellm.proxy.mcp_server.mcp_server_manager as mcp_server_manager_mod + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( + MCPRequestHandler, + ) original_manager = mcp_server_manager_mod.global_mcp_server_manager mcp_server_manager_mod.global_mcp_server_manager = test_manager @@ -1204,6 +1202,7 @@ async def test_mcp_server_manager_config_integration_with_database(config_only_m specifically testing access_groups and description fields. """ import datetime + from litellm.proxy._types import LiteLLM_MCPServerTable test_manager = config_only_mcp_manager_factory() @@ -1310,7 +1309,7 @@ def test_get_server_prefix_with_alias(): """ Test that get_server_prefix returns alias when present. """ - from litellm.proxy._experimental.mcp_server.utils import get_server_prefix + from litellm.proxy.mcp_server.utils import get_server_prefix # Create a mock server with alias mock_server = MagicMock() @@ -1326,7 +1325,7 @@ def test_get_server_prefix_without_alias(): """ Test that get_server_prefix falls back to server_name when alias is not present. """ - from litellm.proxy._experimental.mcp_server.utils import get_server_prefix + from litellm.proxy.mcp_server.utils import get_server_prefix # Create a mock server without alias mock_server = MagicMock() @@ -1342,7 +1341,7 @@ def test_get_server_prefix_fallback_to_server_id(): """ Test that get_server_prefix falls back to server_id when neither alias nor server_name are present. """ - from litellm.proxy._experimental.mcp_server.utils import get_server_prefix + from litellm.proxy.mcp_server.utils import get_server_prefix # Create a mock server without alias or server_name mock_server = MagicMock() @@ -1358,7 +1357,7 @@ def test_get_server_prefix_empty_strings(): """ Test that get_server_prefix handles empty strings correctly. """ - from litellm.proxy._experimental.mcp_server.utils import get_server_prefix + from litellm.proxy.mcp_server.utils import get_server_prefix # Create a mock server with empty strings mock_server = MagicMock() @@ -1409,7 +1408,7 @@ async def test_mcp_server_manager_alias_tool_prefixing(): return mock_client with patch( - "litellm.proxy._experimental.mcp_server.upstream.MCPClient", + "litellm.proxy.mcp_server.upstream.MCPClient", mock_client_constructor, ): # Get tools from server @@ -1469,7 +1468,7 @@ async def test_mcp_server_manager_server_name_tool_prefixing(): return mock_client with patch( - "litellm.proxy._experimental.mcp_server.upstream.MCPClient", + "litellm.proxy.mcp_server.upstream.MCPClient", mock_client_constructor, ): # Get tools from server @@ -1529,7 +1528,7 @@ async def test_mcp_server_manager_server_id_tool_prefixing(): return mock_client with patch( - "litellm.proxy._experimental.mcp_server.upstream.MCPClient", + "litellm.proxy.mcp_server.upstream.MCPClient", mock_client_constructor, ): # Get tools from server @@ -1748,7 +1747,7 @@ def test_normalize_server_name(): """ Test that normalize_server_name correctly replaces spaces with underscores. """ - from litellm.proxy._experimental.mcp_server.utils import normalize_server_name + from litellm.proxy.mcp_server.utils import normalize_server_name # Test basic space replacement assert normalize_server_name("My Server Name") == "My_Server_Name" @@ -1768,7 +1767,7 @@ def test_normalize_server_name(): def test_add_server_prefix_to_name(): """Ensure add_server_prefix_to_name correctly formats resource names.""" - from litellm.proxy._experimental.mcp_server.utils import add_server_prefix_to_name + from litellm.proxy.mcp_server.utils import add_server_prefix_to_name # Test basic prefixing result = add_server_prefix_to_name("send_email", "My Server") @@ -1789,7 +1788,7 @@ def test_add_server_prefix_to_name(): def test_get_server_auth_header_with_alias(): """Test _get_server_auth_header function with server alias.""" - from litellm.proxy._experimental.mcp_server.rest_endpoints import ( + from litellm.proxy.mcp_server.rest_endpoints import ( _get_server_auth_header, ) @@ -1824,7 +1823,7 @@ def test_get_server_auth_header_with_alias(): def test_get_server_auth_header_with_server_name(): """Test _get_server_auth_header function with server name (no alias).""" - from litellm.proxy._experimental.mcp_server.rest_endpoints import ( + from litellm.proxy.mcp_server.rest_endpoints import ( _get_server_auth_header, ) @@ -1859,7 +1858,7 @@ def test_get_server_auth_header_with_server_name(): def test_get_server_auth_header_fallback_to_default(): """Test _get_server_auth_header function fallback to default auth header.""" - from litellm.proxy._experimental.mcp_server.rest_endpoints import ( + from litellm.proxy.mcp_server.rest_endpoints import ( _get_server_auth_header, ) @@ -1887,7 +1886,7 @@ def test_get_server_auth_header_fallback_to_default(): def test_get_server_auth_header_hyphenated_alias_sanitized_header_key(): """Header keys use sanitized alias; lookup must match legacy hyphenated aliases.""" - from litellm.proxy._experimental.mcp_server.rest_endpoints import ( + from litellm.proxy.mcp_server.rest_endpoints import ( _get_server_auth_header, ) @@ -1907,7 +1906,7 @@ def test_get_server_auth_header_hyphenated_alias_sanitized_header_key(): def test_get_server_auth_header_no_auth_headers(): """Test _get_server_auth_header function with no auth headers.""" - from litellm.proxy._experimental.mcp_server.rest_endpoints import ( + from litellm.proxy.mcp_server.rest_endpoints import ( _get_server_auth_header, ) @@ -1926,11 +1925,12 @@ def test_get_server_auth_header_no_auth_headers(): def test_create_tool_response_objects(): """Test _create_tool_response_objects enriches mcp_info with server_id and alias.""" - from litellm.proxy._experimental.mcp_server.rest_endpoints import ( + from mcp.types import Tool as MCPTool + + from litellm.proxy.mcp_server.rest_endpoints import ( _create_tool_response_objects, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer - from mcp.types import Tool as MCPTool # Create mock tools mock_tools = [ @@ -1977,10 +1977,11 @@ def test_create_tool_response_objects(): @pytest.mark.asyncio async def test_get_tools_for_single_server(): """Test _get_tools_for_single_server function.""" - from litellm.proxy._experimental.mcp_server.rest_endpoints import ( + from mcp.types import Tool as MCPTool + + from litellm.proxy.mcp_server.rest_endpoints import ( _get_tools_for_single_server, ) - from mcp.types import Tool as MCPTool # Create a mock server (pin allowlist fields; MagicMock auto-attrs are truthy) mock_server = MagicMock() @@ -2001,7 +2002,7 @@ async def test_get_tools_for_single_server(): # Mock the global_mcp_server_manager with patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager" + "litellm.proxy.mcp_server.rest_endpoints.global_mcp_server_manager" ) as mock_manager: mock_manager.get_tools_from_server = AsyncMock(return_value=mock_tools) @@ -2033,10 +2034,11 @@ async def test_get_tools_for_single_server(): @pytest.mark.asyncio async def test_get_tools_for_single_server_applies_disallowed_tools_without_allowlist(): """REST listing must honor disallowed_tools even when no allowlist is set.""" - from litellm.proxy._experimental.mcp_server.rest_endpoints import ( + from mcp.types import Tool as MCPTool + + from litellm.proxy.mcp_server.rest_endpoints import ( _get_tools_for_single_server, ) - from mcp.types import Tool as MCPTool mock_server = MagicMock() mock_server.mcp_info = {"server_name": "zapier"} @@ -2062,7 +2064,7 @@ async def test_get_tools_for_single_server_applies_disallowed_tools_without_allo ] with patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager" + "litellm.proxy.mcp_server.rest_endpoints.global_mcp_server_manager" ) as mock_manager: mock_manager.get_tools_from_server = AsyncMock(return_value=mock_tools) @@ -2080,14 +2082,15 @@ async def test_rest_listing_hides_key_grants_dispatch_would_refuse(): therefore grants nothing, and REST listing that matched the prefixed spelling would advertise a tool the very next call refuses. """ - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from mcp.types import Tool as MCPTool + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) - from litellm.proxy._experimental.mcp_server.rest_endpoints import ( + from litellm.proxy.mcp_server.rest_endpoints import ( _get_tools_for_single_server, ) - from litellm.proxy._types import UserAPIKeyAuth - from mcp.types import Tool as MCPTool server_id = "3c6f6617-d23c-4f48-bfb0-f205e3b27bab" mock_server = MagicMock() @@ -2110,9 +2113,9 @@ async def test_rest_listing_hides_key_grants_dispatch_would_refuse(): ] with patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager" + "litellm.proxy.mcp_server.rest_endpoints.global_mcp_server_manager" ) as mock_manager, patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager" + "litellm.proxy.mcp_server.operations.global_mcp_server_manager" ) as mock_server_manager, patch.object( MCPRequestHandler, "get_allowed_tools_for_server", @@ -2133,11 +2136,11 @@ async def test_rest_listing_hides_key_grants_dispatch_would_refuse(): @pytest.mark.asyncio async def test_list_tool_rest_api_with_server_specific_auth(): """Test list_tool_rest_api with server-specific auth headers.""" - from litellm.proxy._experimental.mcp_server.rest_endpoints import list_tool_rest_api - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) - from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.rest_endpoints import list_tool_rest_api # Create mock request with server-specific auth headers mock_request = MagicMock() @@ -2162,7 +2165,7 @@ async def test_list_tool_rest_api_with_server_specific_auth(): # Mock the global_mcp_server_manager with patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager" + "litellm.proxy.mcp_server.rest_endpoints.global_mcp_server_manager" ) as mock_manager: mock_manager.get_allowed_mcp_servers = AsyncMock( return_value=["test-server-123"] @@ -2191,9 +2194,9 @@ async def test_list_tool_rest_api_with_server_specific_auth(): # Mock the _get_tools_for_single_server function with patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._get_tools_for_single_server" + "litellm.proxy.mcp_server.rest_endpoints._get_tools_for_single_server" ) as mock_get_tools: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( ListMCPToolsRestAPIResponseObject, ) @@ -2231,11 +2234,11 @@ async def test_list_tool_rest_api_with_server_specific_auth(): @pytest.mark.asyncio async def test_list_tool_rest_api_with_default_auth(): """Test list_tool_rest_api with default auth header when no server-specific header is found.""" - from litellm.proxy._experimental.mcp_server.rest_endpoints import list_tool_rest_api - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) - from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.rest_endpoints import list_tool_rest_api # Create mock request with default auth header only mock_request = MagicMock() @@ -2256,7 +2259,7 @@ async def test_list_tool_rest_api_with_default_auth(): # Mock the global_mcp_server_manager with patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager" + "litellm.proxy.mcp_server.rest_endpoints.global_mcp_server_manager" ) as mock_manager: mock_manager.get_allowed_mcp_servers = AsyncMock( return_value=["test-server-123"] @@ -2285,9 +2288,9 @@ async def test_list_tool_rest_api_with_default_auth(): # Mock the _get_tools_for_single_server function with patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._get_tools_for_single_server" + "litellm.proxy.mcp_server.rest_endpoints._get_tools_for_single_server" ) as mock_get_tools: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( ListMCPToolsRestAPIResponseObject, ) @@ -2325,11 +2328,11 @@ async def test_list_tool_rest_api_with_default_auth(): @pytest.mark.asyncio async def test_list_tool_rest_api_all_servers_with_auth(): """Test list_tool_rest_api for all servers with server-specific auth headers.""" - from litellm.proxy._experimental.mcp_server.rest_endpoints import list_tool_rest_api - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) - from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.rest_endpoints import list_tool_rest_api # Create mock request with server-specific auth headers mock_request = MagicMock() @@ -2354,7 +2357,7 @@ async def test_list_tool_rest_api_all_servers_with_auth(): # Mock the global_mcp_server_manager with patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager" + "litellm.proxy.mcp_server.rest_endpoints.global_mcp_server_manager" ) as mock_manager: # Create mock servers mock_zapier_server = MagicMock() @@ -2395,9 +2398,9 @@ async def test_list_tool_rest_api_all_servers_with_auth(): # Mock the _get_tools_for_single_server function with patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._get_tools_for_single_server" + "litellm.proxy.mcp_server.rest_endpoints._get_tools_for_single_server" ) as mock_get_tools: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( ListMCPToolsRestAPIResponseObject, ) @@ -2452,11 +2455,12 @@ async def test_list_tool_rest_api_all_servers_with_auth(): @pytest.mark.asyncio async def test_filter_tools_by_allowed_tools_integration(): """Test that filter_tools_by_allowed_tools works correctly via _get_tools_from_mcp_servers""" - from litellm.proxy._experimental.mcp_server.server import ( + from mcp.types import Tool as MCPTool + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _get_tools_from_mcp_servers, ) - from litellm.proxy._types import UserAPIKeyAuth - from mcp.types import Tool as MCPTool # Create a mock user auth mock_user_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user") @@ -2509,7 +2513,7 @@ async def test_filter_tools_by_allowed_tools_integration(): # Mock the global MCP server manager with patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager" + "litellm.proxy.mcp_server.operations.global_mcp_server_manager" ) as mock_manager: # Mock manager methods mock_manager.get_allowed_mcp_servers = AsyncMock( @@ -2526,7 +2530,7 @@ async def test_filter_tools_by_allowed_tools_integration(): # Mock the MCPClient constructor with patch( - "litellm.proxy._experimental.mcp_server.upstream.MCPClient", + "litellm.proxy.mcp_server.upstream.MCPClient", mock_client_constructor, ): # Call _get_tools_from_mcp_servers which should apply the filtering @@ -2567,11 +2571,12 @@ async def test_filter_tools_by_allowed_tools_integration(): @pytest.mark.asyncio async def test_filter_tools_by_disallowed_tools_integration(): """Test that filter_tools_by_allowed_tools works correctly with disallowed_tools via _get_tools_from_mcp_servers""" - from litellm.proxy._experimental.mcp_server.server import ( + from mcp.types import Tool as MCPTool + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _get_tools_from_mcp_servers, ) - from litellm.proxy._types import UserAPIKeyAuth - from mcp.types import Tool as MCPTool # Create a mock user auth mock_user_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user") @@ -2624,7 +2629,7 @@ async def test_filter_tools_by_disallowed_tools_integration(): # Mock the global MCP server manager with patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager" + "litellm.proxy.mcp_server.operations.global_mcp_server_manager" ) as mock_manager: # Mock manager methods mock_manager.get_allowed_mcp_servers = AsyncMock( @@ -2640,7 +2645,7 @@ async def test_filter_tools_by_disallowed_tools_integration(): # Mock the MCPClient constructor with patch( - "litellm.proxy._experimental.mcp_server.upstream.MCPClient", + "litellm.proxy.mcp_server.upstream.MCPClient", mock_client_constructor, ): # Call _get_tools_from_mcp_servers which should apply the filtering @@ -2681,11 +2686,12 @@ async def test_filter_tools_by_disallowed_tools_integration(): @pytest.mark.asyncio async def test_filter_tools_no_restrictions_integration(): """Test that filter_tools_by_allowed_tools returns all tools when no restrictions are set""" - from litellm.proxy._experimental.mcp_server.server import ( + from mcp.types import Tool as MCPTool + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _get_tools_from_mcp_servers, ) - from litellm.proxy._types import UserAPIKeyAuth - from mcp.types import Tool as MCPTool # Create a mock user auth mock_user_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user") @@ -2725,7 +2731,7 @@ async def test_filter_tools_no_restrictions_integration(): # Mock the global MCP server manager with patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager" + "litellm.proxy.mcp_server.operations.global_mcp_server_manager" ) as mock_manager: # Mock manager methods mock_manager.get_allowed_mcp_servers = AsyncMock( @@ -2742,7 +2748,7 @@ async def test_filter_tools_no_restrictions_integration(): # Mock the MCPClient constructor with patch( - "litellm.proxy._experimental.mcp_server.upstream.MCPClient", + "litellm.proxy.mcp_server.upstream.MCPClient", mock_client_constructor, ): # Call _get_tools_from_mcp_servers which should apply the filtering @@ -2766,10 +2772,10 @@ async def test_filter_tools_no_restrictions_integration(): @pytest.mark.asyncio async def test_mcp_access_group_permission_inheritance_integration(): """Integration test for MCP access group permission inheritance""" - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) - from litellm.proxy._types import UserAPIKeyAuth # Test scenario: team has access groups, key has no permissions -> should inherit # Use direct mocking of the helper functions instead of complex database mocking @@ -2806,10 +2812,10 @@ async def test_mcp_access_group_permission_inheritance_integration(): @pytest.mark.asyncio async def test_mcp_access_group_permission_intersection_integration(): """Integration test for MCP access group permission intersection""" - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) - from litellm.proxy._types import UserAPIKeyAuth # Test scenario: both team and key have access groups -> should intersect # Use direct mocking of the helper functions instead of complex database mocking @@ -2849,11 +2855,11 @@ async def test_mcp_access_group_permission_intersection_integration(): @pytest.mark.asyncio async def test_mcp_server_manager_with_access_groups_integration(config_only_mcp_manager_factory): """Integration test for MCPServerManager with access group filtering""" - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, MCPServerAccess, ) - from litellm.proxy._types import UserAPIKeyAuth # Create a test manager test_manager = config_only_mcp_manager_factory() @@ -2901,7 +2907,7 @@ async def test_mcp_server_manager_with_access_groups_integration(config_only_mcp @pytest.mark.asyncio async def test_get_allowed_mcp_servers_returns_registry_for_admin(config_only_mcp_manager_factory): from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) @@ -2936,7 +2942,7 @@ async def test_get_allowed_mcp_servers_returns_registry_for_admin(config_only_mc @pytest.mark.asyncio async def test_get_allowed_mcp_servers_returns_empty_for_non_admin_without_permissions(config_only_mcp_manager_factory): from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, MCPServerAccess, ) @@ -2972,7 +2978,7 @@ async def test_get_allowed_mcp_servers_returns_empty_for_non_admin_without_permi @pytest.mark.asyncio async def test_call_mcp_tool_uses_manager_permission_lookup(): - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( call_mcp_tool, global_mcp_server_manager, ) @@ -3002,14 +3008,14 @@ async def test_call_mcp_tool_uses_manager_permission_lookup(): ), patch.object(global_mcp_server_manager, "server_exposes_tool", return_value=True), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_tool_registry" + "litellm.proxy.mcp_server.operations.global_mcp_tool_registry" ) as mock_tool_registry, patch( - "litellm.proxy._experimental.mcp_server.operations._handle_managed_mcp_tool", + "litellm.proxy.mcp_server.operations._handle_managed_mcp_tool", new_callable=AsyncMock, ) as mock_handle_managed, patch( - "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed", + "litellm.proxy.mcp_server.server.MCPRequestHandler.is_tool_allowed", return_value=True, ), ): @@ -3038,7 +3044,7 @@ async def test_call_mcp_tool_resolves_unprefixed_tool_name_and_checks_permission Ensure `call_mcp_tool` resolves an unprefixed tool name on a request scoped to one server against that server and enforces server-level permissions with it. """ - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( call_mcp_tool, global_mcp_server_manager, ) @@ -3068,14 +3074,14 @@ async def test_call_mcp_tool_resolves_unprefixed_tool_name_and_checks_permission ), patch.object(global_mcp_server_manager, "server_exposes_tool", return_value=True), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_tool_registry" + "litellm.proxy.mcp_server.operations.global_mcp_tool_registry" ) as mock_tool_registry, patch( - "litellm.proxy._experimental.mcp_server.operations._handle_managed_mcp_tool", + "litellm.proxy.mcp_server.operations._handle_managed_mcp_tool", new_callable=AsyncMock, ) as mock_handle_managed, patch( - "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed", + "litellm.proxy.mcp_server.server.MCPRequestHandler.is_tool_allowed", return_value=True, ) as mock_is_allowed, ): @@ -3103,7 +3109,7 @@ async def test_call_mcp_tool_resolves_unprefixed_tool_name_and_checks_permission @pytest.mark.asyncio async def test_mcp_admission_body_peek_times_out_without_losing_the_body(monkeypatch: pytest.MonkeyPatch) -> None: try: - from litellm.proxy._experimental.mcp_server import server + from litellm.proxy.mcp_server import server except ImportError: pytest.skip("MCP server not available") @@ -3127,16 +3133,16 @@ async def test_mcp_admission_body_peek_times_out_without_losing_the_body(monkeyp @pytest.mark.parametrize("method", ["tools/list", "tools/call", ["tools/list"]]) async def test_mcp_routing_stashes_only_method_for_auth_and_replays_full_body(method: str | list[str]): try: - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.common_utils.http_parsing_utils import _read_request_body + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCP_PEEKED_BODY_SCOPE_KEY, _admission_request, ) - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, session_manager_stateless, ) - from litellm.proxy.common_utils.http_parsing_utils import _read_request_body except ImportError: pytest.skip("MCP server not available") @@ -3172,17 +3178,17 @@ async def test_mcp_routing_stashes_only_method_for_auth_and_replays_full_body(me with ( patch( # test-quality-ok: the ASGI handler reads auth from a module-level helper; the suite's only seam - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(MagicMock(), None, None, None, None, None), ), patch( # test-quality-ok: registry is empty in unit tests; key owns one server - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.server._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[MagicMock()], ), patch( # test-quality-ok: init flag is a module global; no injection seam - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object( # test-quality-ok: session managers are module-level singletons; the suite's only seam @@ -3206,15 +3212,15 @@ async def test_mcp_routing_stashes_only_method_for_auth_and_replays_full_body(me @pytest.mark.asyncio async def test_mcp_routing_batch_body_is_not_stashed_for_auth(): try: - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.common_utils.http_parsing_utils import _read_request_body + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCP_PEEKED_BODY_SCOPE_KEY, _admission_request, ) - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateless, ) - from litellm.proxy.common_utils.http_parsing_utils import _read_request_body except ImportError: pytest.skip("MCP server not available") @@ -3233,17 +3239,17 @@ async def test_mcp_routing_batch_body_is_not_stashed_for_auth(): with ( patch( # test-quality-ok: the ASGI handler reads auth from a module-level helper; the suite's only seam - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(MagicMock(), None, None, None, None, None), ), patch( # test-quality-ok: registry is empty in unit tests; key owns one server - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.server._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[MagicMock()], ), patch( # test-quality-ok: init flag is a module global; no injection seam - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object( # test-quality-ok: session managers are module-level singletons; the suite's only seam @@ -3262,15 +3268,15 @@ async def test_mcp_routing_batch_body_is_not_stashed_for_auth(): @pytest.mark.asyncio async def test_mcp_routing_truncated_json_body_is_not_stashed_for_auth(): try: - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.common_utils.http_parsing_utils import _read_request_body + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCP_PEEKED_BODY_SCOPE_KEY, _admission_request, ) - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateless, ) - from litellm.proxy.common_utils.http_parsing_utils import _read_request_body except ImportError: pytest.skip("MCP server not available") @@ -3289,17 +3295,17 @@ async def test_mcp_routing_truncated_json_body_is_not_stashed_for_auth(): with ( patch( # test-quality-ok: the ASGI handler reads auth from a module-level helper; the suite's only seam - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(MagicMock(), None, None, None, None, None), ), patch( # test-quality-ok: registry is empty in unit tests; key owns one server - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.server._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[MagicMock()], ), patch( # test-quality-ok: init flag is a module global; no injection seam - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object( # test-quality-ok: session managers are module-level singletons; the suite's only seam diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_identity_env.py b/tests/unit/proxy/mcp_server/test_mcp_server_identity_env.py similarity index 95% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_server_identity_env.py rename to tests/unit/proxy/mcp_server/test_mcp_server_identity_env.py index 1c65adac4c6..7b3472d32c2 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_identity_env.py +++ b/tests/unit/proxy/mcp_server/test_mcp_server_identity_env.py @@ -2,7 +2,7 @@ ``LITELLM_MCP_SERVER_NAME`` and ``LITELLM_MCP_SERVER_DESCRIPTION`` are read from the environment at import time in -``litellm.proxy._experimental.mcp_server.utils`` and must flow through to every +``litellm.proxy.mcp_server.utils`` and must flow through to every consumer, including the well-known registry entry built in ``mcp_management_endpoints``. The env values are reloaded into the modules and restored afterwards so the override does not leak into other tests. @@ -16,7 +16,7 @@ import pytest pytest.importorskip("mcp") -UTILS_MODULE = "litellm.proxy._experimental.mcp_server.utils" +UTILS_MODULE = "litellm.proxy.mcp_server.utils" MGMT_MODULE = "litellm.proxy.management_endpoints.mcp_management_endpoints" diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/unit/proxy/mcp_server/test_mcp_server_manager.py similarity index 96% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py rename to tests/unit/proxy/mcp_server/test_mcp_server_manager.py index ecd08e17cda..c76326c51f6 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/unit/proxy/mcp_server/test_mcp_server_manager.py @@ -1,13 +1,14 @@ -from litellm.proxy._experimental.mcp_server.upstream import resolve_upstream_auth -import importlib import asyncio + +# Add the parent directory to the path so we can import litellm +import contextlib import functools +import importlib import json import logging import os import sys import time -import anyio from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from datetime import datetime from pathlib import Path @@ -15,27 +16,17 @@ from types import SimpleNamespace from typing import Any, Dict, Final, Literal, Optional, TypeVar, cast from unittest.mock import AsyncMock, MagicMock, patch -import pytest -from fastapi import HTTPException -from respx import MockRouter - -from litellm.proxy._experimental.mcp_server.exceptions import ( - MCPServerListError, - MCPUpstreamAuthError, -) -from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ServerListFault - -# Add the parent directory to the path so we can import litellm - - -import contextlib - +import anyio import httpx import httpx2 +import pytest +from fastapi import HTTPException from mcp import ClientSession, ReadResourceResult, Resource from mcp.server.context import ServerRequestContext from mcp.types import ( CallToolRequestParams as MCPCallToolRequestParams, +) +from mcp.types import ( CallToolResult, GetPromptResult, ListToolsResult, @@ -46,30 +37,17 @@ from mcp.types import ( ) from mcp.types import Tool as MCPTool from pydantic import AnyUrl, TypeAdapter +from respx import MockRouter +import litellm +import litellm.llms as litellm_llms +from litellm.caching.caching import DualCache +from litellm.caching.llm_caching_handler import LLMClientCache from litellm.constants import MCP_METADATA_TIMEOUT, MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES -from litellm.proxy._experimental.mcp_server import discoverable_endpoints -from litellm.proxy._experimental.mcp_server.legacy_callbacks import ElicitationCallback -from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var -from litellm.proxy._experimental.mcp_server.tool_outcome import TextResult -from litellm.experimental_mcp_client.client import MCPClient, PersistentMCPSession -from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - ListedToolsCaller, - MCPServerManager, - global_mcp_server_manager, - _deserialize_json_dict, - _flow_endpoints_missing, - _mcp_oauth_discovery_on_startup_enabled, - oauth_endpoints_unresolved, - _deserialize_json_list, - _normalize_mcp_server_cost_info, - _obo_retry_applies, - resolve_openapi_tool_auth, - should_strip_caller_authorization, - listed_tools_caller_for, -) +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.mcp_client.client import MCPClient, PersistentMCPSession from litellm.proxy import proxy_server -from litellm.proxy.auth import auth_checks from litellm.proxy._types import ( LiteLLM_MCPServerTable, LiteLLM_ObjectPermissionTable, @@ -82,25 +60,45 @@ from litellm.proxy._types import ( MCPTransport, UserAPIKeyAuth, ) -from litellm.types.llms.custom_http import httpxSpecialProvider -from litellm.types.mcp import MCPAuth, MCPAuthType, MCPUpstreamProtocol -from litellm.types.mcp_server.mcp_server_manager import MCPOAuthMetadata, MCPServer, PinnedMCPTool -from litellm.caching.caching import DualCache -from litellm.caching.llm_caching_handler import LLMClientCache -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -import litellm -from litellm.integrations.custom_guardrail import CustomGuardrail -import litellm.llms as litellm_llms +from litellm.proxy.auth import auth_checks +from litellm.proxy.mcp_server import discoverable_endpoints +from litellm.proxy.mcp_server.exceptions import ( + MCPServerListError, + MCPUpstreamAuthError, +) +from litellm.proxy.mcp_server.faults.list_outcomes import ServerListFault +from litellm.proxy.mcp_server.legacy_callbacks import ElicitationCallback +from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var +from litellm.proxy.mcp_server.mcp_server_manager import ( + ListedToolsCaller, + MCPServerManager, + _deserialize_json_dict, + _deserialize_json_list, + _flow_endpoints_missing, + _mcp_oauth_discovery_on_startup_enabled, + _normalize_mcp_server_cost_info, + _obo_retry_applies, + global_mcp_server_manager, + listed_tools_caller_for, + oauth_endpoints_unresolved, + resolve_openapi_tool_auth, + should_strip_caller_authorization, +) +from litellm.proxy.mcp_server.tool_outcome import TextResult +from litellm.proxy.mcp_server.upstream import resolve_upstream_auth from litellm.proxy.utils import ProxyLogging from litellm.types.guardrails import GuardrailEventHooks from litellm.types.integrations.slack_alerting import AlertType +from litellm.types.llms.custom_http import httpxSpecialProvider +from litellm.types.mcp import MCPAuth, MCPAuthType, MCPUpstreamProtocol +from litellm.types.mcp_server.mcp_server_manager import MCPOAuthMetadata, MCPServer, PinnedMCPTool _TSessionResult = TypeVar("_TSessionResult") @pytest.mark.asyncio async def test_manager_sampling_preserves_explicit_headers_without_ambient_context(): - from litellm.proxy._experimental.mcp_server import server as legacy_server + from litellm.proxy.mcp_server import server as legacy_server caller = UserAPIKeyAuth(user_id="sampling-caller") upstream = MCPServer( @@ -115,8 +113,8 @@ async def test_manager_sampling_preserves_explicit_headers_without_ambient_conte client.call_tool = AsyncMock(return_value=CallToolResult(content=[])) assert legacy_server.get_active_auth_context() is None with ( - patch("litellm.proxy._experimental.mcp_server.upstream.MCPClient", return_value=client) as factory, - patch("litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", sampling), + patch("litellm.proxy.mcp_server.upstream.MCPClient", return_value=client) as factory, + patch("litellm.proxy.mcp_server.sampling_handler.handle_sampling_create_message", sampling), ): await MCPServerManager()._call_regular_mcp_tool( mcp_server=upstream, @@ -140,8 +138,8 @@ async def test_manager_sampling_preserves_explicit_headers_without_ambient_conte async def test_sampling_callback_keeps_creation_context_after_caller_switch(): from mcp.server.auth.middleware.auth_context import auth_context_var - from litellm.proxy._experimental.mcp_server import server as legacy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_sampling_callback + from litellm.proxy.mcp_server import server as legacy_server + from litellm.proxy.mcp_server.mcp_server_manager import _create_sampling_callback token = auth_context_var.set(None) recorder = AsyncMock() @@ -154,7 +152,7 @@ async def test_sampling_callback_keeps_creation_context_after_caller_switch(): original.models.append("bravo-model") headers["x-caller"] = "bravo" legacy_server.set_auth_context(UserAPIKeyAuth(user_id="bravo"), raw_headers={"x-caller": "bravo"}) - with patch("litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", recorder): + with patch("litellm.proxy.mcp_server.sampling_handler.handle_sampling_create_message", recorder): await callback(None, None) observed = recorder.await_args.kwargs assert observed["user_api_key_auth"].user_id == "alpha" @@ -168,18 +166,19 @@ async def test_sampling_callback_keeps_creation_context_after_caller_switch(): @pytest.mark.asyncio async def test_elicitation_callback_keeps_initiating_session(): - from litellm.proxy._experimental.mcp_server import server as legacy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_elicitation_callback - from types import SimpleNamespace + from mcp.types import ( ClientCapabilities, ElicitationCapability, - FormElicitationCapability, ElicitRequestFormParams, ElicitResult, + FormElicitationCapability, ) - from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var + + from litellm.proxy.mcp_server import server as legacy_server + from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var + from litellm.proxy.mcp_server.mcp_server_manager import _create_elicitation_callback capabilities = ClientCapabilities(elicitation=ElicitationCapability(form=FormElicitationCapability())) accepted = ElicitResult(action="accept") @@ -206,7 +205,7 @@ async def test_elicitation_callback_keeps_initiating_session(): async def test_sampling_callbacks_isolate_callers_and_cancellation(): from mcp.types import ErrorData - from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_sampling_callback + from litellm.proxy.mcp_server.mcp_server_manager import _create_sampling_callback started = asyncio.Event() cancelled = asyncio.Event() @@ -230,7 +229,7 @@ async def test_sampling_callbacks_isolate_callers_and_cancellation(): for label in ("alpha", "bravo", "cancelled") ) with patch( - "litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", record_sampling + "litellm.proxy.mcp_server.sampling_handler.handle_sampling_create_message", record_sampling ): tasks = tuple(asyncio.create_task(callback(None, None)) for callback in callbacks) await asyncio.wait_for(started.wait(), timeout=2) @@ -243,12 +242,12 @@ async def test_sampling_callbacks_isolate_callers_and_cancellation(): def _reload_mcp_manager_module(): - utils_module = sys.modules["litellm.proxy._experimental.mcp_server.utils"] - manager_module = sys.modules["litellm.proxy._experimental.mcp_server.mcp_server_manager"] + utils_module = sys.modules["litellm.proxy.mcp_server.utils"] + manager_module = sys.modules["litellm.proxy.mcp_server.mcp_server_manager"] importlib.reload(utils_module) reloaded = importlib.reload(manager_module) for name, module in tuple(sys.modules.items()): - if name.startswith("litellm.proxy._experimental.mcp_server.") and hasattr(module, "global_mcp_server_manager"): + if name.startswith("litellm.proxy.mcp_server.") and hasattr(module, "global_mcp_server_manager"): module.global_mcp_server_manager = reloaded.global_mcp_server_manager return reloaded @@ -265,7 +264,7 @@ def restore_mcp_manager_singleton(): bound: Final = tuple( (module, module.global_mcp_server_manager) for name, module in tuple(sys.modules.items()) - if name.startswith("litellm.proxy._experimental.mcp_server.") and hasattr(module, "global_mcp_server_manager") + if name.startswith("litellm.proxy.mcp_server.") and hasattr(module, "global_mcp_server_manager") ) yield for module, manager in bound: @@ -285,8 +284,8 @@ def restore_mcp_manager_singleton(): ) @pytest.mark.parametrize("explicit_key", [False, True]) def test_validated_admission_jwt_is_not_forwarded_as_an_extra_header(auth_type: MCPAuth, explicit_key: bool) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import _openapi_forwarded_extra_headers - from litellm.proxy._experimental.mcp_server.operations import _prepare_mcp_server_headers + from litellm.proxy.mcp_server.mcp_server_manager import _openapi_forwarded_extra_headers + from litellm.proxy.mcp_server.operations import _prepare_mcp_server_headers server: Final = MCPServer( server_id="header-boundary", @@ -325,8 +324,8 @@ def test_admission_authorization_is_not_forwarded_for_any_auth_type( auth_type: MCPAuth, monkeypatch: pytest.MonkeyPatch ) -> None: from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import _openapi_forwarded_extra_headers - from litellm.proxy._experimental.mcp_server.operations import _prepare_mcp_server_headers + from litellm.proxy.mcp_server.mcp_server_manager import _openapi_forwarded_extra_headers + from litellm.proxy.mcp_server.operations import _prepare_mcp_server_headers monkeypatch.setattr(proxy_server, "master_key", "test-master-key") server: Final = MCPServer( @@ -379,8 +378,8 @@ def test_no_master_key_preserves_caller_authorization_unless_auth_is_enabled( auth_type: MCPAuth, monkeypatch: pytest.MonkeyPatch ) -> None: from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import _openapi_forwarded_extra_headers - from litellm.proxy._experimental.mcp_server.operations import _prepare_mcp_server_headers + from litellm.proxy.mcp_server.mcp_server_manager import _openapi_forwarded_extra_headers + from litellm.proxy.mcp_server.operations import _prepare_mcp_server_headers monkeypatch.setattr(proxy_server, "master_key", None) monkeypatch.setattr(proxy_server, "general_settings", {}) @@ -423,8 +422,8 @@ def test_custom_auth_is_stripped_in_no_auth_dev_mode( auth_type: MCPAuth, monkeypatch: pytest.MonkeyPatch ) -> None: from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import _openapi_forwarded_extra_headers - from litellm.proxy._experimental.mcp_server.operations import _prepare_mcp_server_headers + from litellm.proxy.mcp_server.mcp_server_manager import _openapi_forwarded_extra_headers + from litellm.proxy.mcp_server.operations import _prepare_mcp_server_headers monkeypatch.setattr(proxy_server, "master_key", None) monkeypatch.setattr(proxy_server, "general_settings", {}) @@ -489,8 +488,8 @@ def test_custom_admission_header_controls_authorization_forwarding( monkeypatch: pytest.MonkeyPatch, ) -> None: from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import _openapi_forwarded_extra_headers - from litellm.proxy._experimental.mcp_server.operations import _prepare_mcp_server_headers + from litellm.proxy.mcp_server.mcp_server_manager import _openapi_forwarded_extra_headers + from litellm.proxy.mcp_server.operations import _prepare_mcp_server_headers monkeypatch.setattr(proxy_server, "master_key", "test-master-key") monkeypatch.setattr(proxy_server, "general_settings", {"litellm_key_header_name": "x-company-key"}) @@ -639,10 +638,10 @@ class TestMCPServerManager: """A caller-supplied per-request override must not substitute the stored authorization_code token: _create_mcp_client keeps the v2 spec and resolves through the injected provider rather than deferring to the v1 caller-override path.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import ( + from litellm.proxy.mcp_server.outbound_credentials.result import ( Ok, ) from litellm.types.mcp import MCPAuth @@ -674,10 +673,10 @@ class TestMCPServerManager: """A caller-supplied per-request override must not disable the ID-JAG exchange and forward an arbitrary bearer upstream: _create_mcp_client keeps the v2 spec and resolves through the injected provider rather than deferring to the v1 caller-override path.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import ( + from litellm.proxy.mcp_server.outbound_credentials.result import ( Ok, ) from litellm.types.mcp import MCPAuth @@ -1226,7 +1225,7 @@ class TestMCPServerManager: patch.object(manager, "_discover_oauth_metadata_for_server", new=discovery), patch.object(manager, "initialize_tool_name_to_mcp_server_name_mapping"), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager._oauth_discovery_now", + "litellm.proxy.mcp_server.mcp_server_manager._oauth_discovery_now", new=discovery_clock, ), ): @@ -1279,7 +1278,7 @@ class TestMCPServerManager: with ( patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.MCP_METADATA_TIMEOUT", + "litellm.proxy.mcp_server.mcp_server_manager.MCP_METADATA_TIMEOUT", 0.01, ), patch.object(manager, "_discover_oauth_metadata_for_server", side_effect=never_returns) as discovery, @@ -1446,7 +1445,7 @@ class TestMCPServerManager: with ( patch( - "litellm.proxy._experimental.mcp_server.db.MCPServerRepository", + "litellm.proxy.mcp_server.db.MCPServerRepository", return_value=repository, ), patch( @@ -1509,7 +1508,7 @@ class TestMCPServerManager: with ( patch( - "litellm.proxy._experimental.mcp_server.db.MCPServerRepository", + "litellm.proxy.mcp_server.db.MCPServerRepository", return_value=repository, ), patch( @@ -1651,7 +1650,7 @@ class TestMCPServerManager: "ensure_oauth_metadata_discovered", new=ensure_oauth_metadata_discovered, ), - patch("litellm.proxy._experimental.mcp_server.upstream.MCPClient"), + patch("litellm.proxy.mcp_server.upstream.MCPClient"), ): await manager._create_mcp_client(server) @@ -2568,7 +2567,7 @@ class TestMCPServerManager: won't help) and a genuine non-auth failure (e.g. 503) both keep the default isError degradation and stay a visible warning, mirroring the list path, rather than being relayed as a re-auth challenge.""" - from litellm.experimental_mcp_client.client import MCPClient + from litellm.mcp_client.client import MCPClient server = self._passthrough_call_server(MCPAuth.true_passthrough, server_id=f"pt-{status_code}") manager = MCPServerManager() @@ -2577,7 +2576,7 @@ class TestMCPServerManager: mock_client.error_tool_result = MCPClient.error_tool_result manager.create_mcp_client = AsyncMock(return_value=mock_client) - import litellm.proxy._experimental.mcp_server.mcp_server_manager as _mgr_mod + import litellm.proxy.mcp_server.mcp_server_manager as _mgr_mod with patch.object(_mgr_mod, "verbose_logger") as mock_log: result = await self._run_call_regular(manager, server) @@ -2637,8 +2636,8 @@ class TestMCPServerManager: # must reconstruct it onto the MCPServer, and the v2 adapter must map it onto the resolver # config. Without threading it through, an entra_obo server persisted via the API silently # falls back to rfc8693 and posts the wrong grant to the IdP. - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import to_server_spec - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import TokenExchangeConfig + from litellm.proxy.mcp_server.outbound_credentials.adapter import to_server_spec + from litellm.proxy.mcp_server.outbound_credentials.types import TokenExchangeConfig manager = MCPServerManager() row = LiteLLM_MCPServerTable( @@ -3239,10 +3238,10 @@ class TestMCPServerManager: """A caller-supplied per-server header (x-mcp-*) must NOT disable the OBO exchange: _create_mcp_client keeps the v2 spec and runs the resolver (which exchanges the subject), rather than deferring to v1 and forwarding the caller's header verbatim upstream.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.outbound_credentials.result import Ok exchanged_subjects = [] @@ -3269,10 +3268,10 @@ class TestMCPServerManager: """A guardrail/static Authorization (e.g. MCPJWTSigner) must NOT shadow the exchanged OBO token. The resolver-owned credential is authoritative: the conflicting header is dropped and the minted token is what reaches the upstream, not the injected JWT.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.outbound_credentials.result import Ok class _FakeProvider: async def resolve_credentials(self, subject, server): @@ -3297,10 +3296,10 @@ class TestMCPServerManager: the gateway-minted client_credentials bearer. Dropping the resolved auth here would also drop the one-shot 401 refetch that rides on it, so the resolver-owned credential is authoritative exactly as for token_exchange and authorization_code.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.outbound_credentials.result import Ok class _FakeProvider: async def resolve_credentials(self, subject, server): @@ -3348,10 +3347,10 @@ class TestMCPServerManager: """The dual-credential case: an ESB wants the gateway-minted token on its own header while a separate static Authorization passes through to the origin. Dropping Authorization here (the old name-blind behavior) deletes the second credential and the upstream 401s.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.outbound_credentials.result import Ok class _FakeProvider: async def resolve_credentials(self, subject, server): @@ -3371,10 +3370,10 @@ class TestMCPServerManager: """The negative class of the test above: when the two DO collide the resolver-owned credential is still authoritative, so the knob cannot be used to smuggle a second credential into the same slot.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.outbound_credentials.result import Ok class _FakeProvider: async def resolve_credentials(self, subject, server): @@ -3399,10 +3398,10 @@ class TestMCPServerManager: downstream consumer of that dict. httpx happens to collapse the two on the wire, which is exactly why this needs pinning rather than being left to luck. """ - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.outbound_credentials.result import Ok class _FakeProvider: async def resolve_credentials(self, subject, server): @@ -3430,8 +3429,8 @@ class TestMCPServerManager: """A subject the IdP rejects must raise the RFC 9728 401 challenge from the preflight, so a single-server route fails observably at the transport edge instead of the old behavior of the session opening and list_tools masking the failed exchange as an empty tool list.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import CredError + from litellm.proxy.mcp_server.outbound_credentials.result import Error + from litellm.proxy.mcp_server.outbound_credentials.types import CredError class _FakeProvider: async def resolve_credentials(self, subject, server): @@ -3455,8 +3454,8 @@ class TestMCPServerManager: async def test_preflight_token_exchange_maps_gateway_fault_to_public_status(self): """A gateway-fault CredError (e.g. invalid_client) must surface its public status (500) from the preflight, not the OBO 401 challenge and not an empty-success session.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import CredError + from litellm.proxy.mcp_server.outbound_credentials.result import Error + from litellm.proxy.mcp_server.outbound_credentials.types import CredError class _FakeProvider: async def resolve_credentials(self, subject, server): @@ -3477,10 +3476,10 @@ class TestMCPServerManager: async def test_preflight_token_exchange_noop_on_success_and_without_subject(self): """A successful exchange returns without raising, and a request with no bearer never reaches the resolver (the no-subject case is the existing preemptive challenge's job).""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.outbound_credentials.result import Ok resolved = [] @@ -3525,8 +3524,8 @@ class TestMCPServerManager: gets a 200 with an empty tool list and then 'tool not found', which is not what happened. 412 is a precondition, not an RFC 9728 discovery challenge, so it carries no WWW-Authenticate: there is nothing for the client to discover and retry against.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import CredError + from litellm.proxy.mcp_server.outbound_credentials.result import Error + from litellm.proxy.mcp_server.outbound_credentials.types import CredError summary = ( "ID-JAG requires an IdP identity assertion for this user and none is stored. " @@ -3553,8 +3552,8 @@ class TestMCPServerManager: async def test_preflight_id_jag_surfaces_assertion_store_outage_as_503(self): """A store outage is the other failure the session would swallow, and it is a different answer than 412: the user has nothing to fix by signing in again.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import CredError + from litellm.proxy.mcp_server.outbound_credentials.result import Error + from litellm.proxy.mcp_server.outbound_credentials.types import CredError class _FakeProvider: async def resolve_credentials(self, subject, server): @@ -3574,10 +3573,10 @@ class TestMCPServerManager: async def test_preflight_id_jag_preflights_litellm_key_and_skips_identity_bearer(self): """ID-JAG preflights when Authorization carries a LiteLLM key, but skips a caller identity bearer that the session passes through unchanged.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.outbound_credentials.result import Ok subjects = [] @@ -3635,8 +3634,8 @@ class TestMCPServerManager: other mode itself. The single-server call site no longer tests the mode before calling, so a mode that falls through here would start resolving its credential a second time, at connect, for flows that never had a connect-time resolution at all.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import CredError + from litellm.proxy.mcp_server.outbound_credentials.result import Error + from litellm.proxy.mcp_server.outbound_credentials.types import CredError calls = [] @@ -3677,11 +3676,11 @@ class TestMCPServerManager: ) async def test_preflight_token_exchange_strips_inbound_authorization_scheme(self, authorization): """The resolver posts inbound_token verbatim as subject_token.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import CredError, ServerSpec, Subject + from litellm.proxy.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.outbound_credentials.types import CredError, ServerSpec, Subject resolved: Final[list[str | None]] = [] @@ -3705,11 +3704,11 @@ class TestMCPServerManager: @pytest.mark.asyncio async def test_preflight_token_exchange_preserves_authorization_without_separator(self): - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import CredError, ServerSpec, Subject + from litellm.proxy.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.outbound_credentials.types import CredError, ServerSpec, Subject resolved: Final[list[str | None]] = [] @@ -4320,10 +4319,10 @@ class TestMCPServerManager: ) with ( patch( - "litellm.proxy._experimental.mcp_server.upstream.resolve_mcp_auth", + "litellm.proxy.mcp_server.upstream.resolve_mcp_auth", new_callable=AsyncMock, ) as mock_resolve, - patch("litellm.proxy._experimental.mcp_server.upstream.MCPClient") as mock_client_cls, + patch("litellm.proxy.mcp_server.upstream.MCPClient") as mock_client_cls, ): await manager._create_mcp_client(server=server, extra_headers={"Authorization": "Bearer upstream-token"}) mock_resolve.assert_not_awaited() @@ -4340,13 +4339,13 @@ class TestMCPServerManager: """A configured OBO server is owned end to end by the v2 token_exchange arm, even when the caller supplies an x-mcp-* override. This is what makes the v1 OBO handler unreachable, so if it ever defers to v1 again the deleted handler is silently needed back.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( + from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import ( + from litellm.proxy.mcp_server.outbound_credentials.resolver import ( UpstreamCredentialProvider, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.outbound_credentials.result import Ok class _StubExchanger: def __init__(self): @@ -4373,10 +4372,10 @@ class TestMCPServerManager: ) with ( patch( - "litellm.proxy._experimental.mcp_server.upstream.resolve_mcp_auth", + "litellm.proxy.mcp_server.upstream.resolve_mcp_auth", new_callable=AsyncMock, ) as mock_resolve, - patch("litellm.proxy._experimental.mcp_server.upstream.MCPClient") as mock_client_cls, + patch("litellm.proxy.mcp_server.upstream.MCPClient") as mock_client_cls, ): await manager._create_mcp_client( server=server, @@ -4416,10 +4415,10 @@ class TestMCPServerManager: ) with ( patch( - "litellm.proxy._experimental.mcp_server.upstream.resolve_mcp_auth", + "litellm.proxy.mcp_server.upstream.resolve_mcp_auth", new_callable=AsyncMock, ) as mock_resolve, - patch("litellm.proxy._experimental.mcp_server.upstream.MCPClient") as mock_client_cls, + patch("litellm.proxy.mcp_server.upstream.MCPClient") as mock_client_cls, ): await manager._create_mcp_client( server=server, @@ -4432,7 +4431,7 @@ class TestMCPServerManager: assert not kwargs["extra_headers"] or "authorization" not in {k.lower() for k in kwargs["extra_headers"]} def test_consumes_caller_authorization_per_mode(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _consumes_caller_authorization, ) @@ -4468,7 +4467,7 @@ class TestMCPServerManager: ) def test_caller_authorization_fans_out_only_with_second_consumer(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( caller_authorization_fans_out, ) @@ -4743,7 +4742,7 @@ class TestMCPServerManager: mock_client.get = AsyncMock(return_value=mock_response) with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ): servers, scopes = await manager._fetch_oauth_metadata_from_resource( @@ -4778,7 +4777,7 @@ class TestMCPServerManager: with ( patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ) as get_client, patch.object( @@ -4833,7 +4832,7 @@ class TestMCPServerManager: with ( patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ), patch.object( @@ -4880,7 +4879,7 @@ class TestMCPServerManager: mock_client.get = AsyncMock(side_effect=build_response) with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ): # The Azure issuer is cross-origin against the server_url — use @@ -4930,7 +4929,7 @@ class TestMCPServerManager: mock_client = MagicMock() mock_client.get = AsyncMock(side_effect=build_response) with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ): result = await manager._fetch_single_authorization_server_metadata(issuer, issuer, require_issuer=issuer) @@ -4967,7 +4966,7 @@ class TestMCPServerManager: mock_client = MagicMock() mock_client.get = AsyncMock(side_effect=build_response) with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ): result = await manager._fetch_single_authorization_server_metadata(issuer, issuer, require_issuer=issuer) @@ -4992,7 +4991,7 @@ class TestMCPServerManager: mock_client.get = AsyncMock(return_value=mock_response) with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ): result = await manager._fetch_single_authorization_server_metadata(issuer, issuer) @@ -5020,7 +5019,7 @@ class TestMCPServerManager: mock_client.get = AsyncMock(return_value=mock_response) with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ): azure = await manager._fetch_single_authorization_server_metadata(issuer, issuer, require_issuer=issuer) @@ -5061,7 +5060,7 @@ class TestMCPServerManager: with ( patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ), patch.object( @@ -5155,9 +5154,9 @@ class TestMCPServerManager: assert server.scopes is None # And the exchange precondition now fails closed before any IdP call. - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import to_server_spec - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error - from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger import ( + from litellm.proxy.mcp_server.outbound_credentials.adapter import to_server_spec + from litellm.proxy.mcp_server.outbound_credentials.result import Error + from litellm.proxy.mcp_server.outbound_credentials.token_exchanger import ( OboTokenExchanger, ) @@ -5220,7 +5219,7 @@ class TestMCPServerManager: assert minimal.subject_token_type == DEFAULT_SUBJECT_TOKEN_TYPE assert minimal.token_exchange_profile == "rfc8693" - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import to_server_spec + from litellm.proxy.mcp_server.outbound_credentials.adapter import to_server_spec spec = to_server_spec(full) assert spec is not None @@ -5390,7 +5389,7 @@ class TestMCPServerManager: @pytest.mark.asyncio @pytest.mark.parametrize("cancel", [False, True]) async def test_openapi_health_timeout_and_cancellation_cleanup(self, respx_mock, monkeypatch, cancel): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import _openapi_spec_health + from litellm.proxy.mcp_server.mcp_server_manager import _openapi_spec_health monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") started = asyncio.Event() @@ -5742,7 +5741,7 @@ class TestMCPServerManager: self, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch, cancel: bool ) -> None: monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") - monkeypatch.setattr("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCP_HEALTH_CHECK_TIMEOUT", 0.1) + monkeypatch.setattr("litellm.proxy.mcp_server.mcp_server_manager.MCP_HEALTH_CHECK_TIMEOUT", 0.1) manager: Final = MCPServerManager() server: Final = MCPServer( server_id="slow-health", @@ -6154,15 +6153,15 @@ class TestMCPServerManager: with ( patch( - "litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator.create_tool_function", + "litellm.proxy.mcp_server.openapi_to_mcp_generator.create_tool_function", side_effect=fake_create_tool_function, ), patch( - "litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator.build_input_schema", + "litellm.proxy.mcp_server.openapi_to_mcp_generator.build_input_schema", return_value={"type": "object", "properties": {}, "required": []}, ), patch( - "litellm.proxy._experimental.mcp_server.tool_registry.global_mcp_tool_registry.register_tool", + "litellm.proxy.mcp_server.tool_registry.global_mcp_tool_registry.register_tool", return_value=None, ), ): @@ -6244,15 +6243,15 @@ class TestMCPServerManager: return tool_func monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator.create_tool_function", + "litellm.proxy.mcp_server.openapi_to_mcp_generator.create_tool_function", fake_create_tool_function, ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator.build_input_schema", + "litellm.proxy.mcp_server.openapi_to_mcp_generator.build_input_schema", lambda *args, **kwargs: {"type": "object", "properties": {}, "required": []}, ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.tool_registry.global_mcp_tool_registry.register_tool", + "litellm.proxy.mcp_server.tool_registry.global_mcp_tool_registry.register_tool", lambda *args, **kwargs: None, ) await manager._register_openapi_tools( @@ -6851,7 +6850,7 @@ class TestMCPServerManager: stored = {"Authorization": "Bearer stored-user-token"} with patch( - "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db", + "litellm.proxy.mcp_server.operations._get_user_oauth_extra_headers_from_db", new=AsyncMock(return_value=stored), ) as mock_lookup: result = await manager._resolve_oauth2_headers_for_tool_call( @@ -6878,7 +6877,7 @@ class TestMCPServerManager: user_auth = UserAPIKeyAuth(api_key="sk-test", user_id="alice") with patch( - "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db", + "litellm.proxy.mcp_server.operations._get_user_oauth_extra_headers_from_db", new=AsyncMock(return_value={"Authorization": "Bearer should-not-be-used"}), ) as mock_lookup: result = await manager._resolve_oauth2_headers_for_tool_call( @@ -6904,7 +6903,7 @@ class TestMCPServerManager: user_auth = UserAPIKeyAuth(api_key="sk-test", user_id="alice") with patch( - "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db", + "litellm.proxy.mcp_server.operations._get_user_oauth_extra_headers_from_db", new=AsyncMock(side_effect=RuntimeError("redis down")), ): result = await manager._resolve_oauth2_headers_for_tool_call( @@ -7060,7 +7059,7 @@ class TestMCPServerManager: user_auth = UserAPIKeyAuth(api_key="sk-test") with patch( - "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db", + "litellm.proxy.mcp_server.operations._get_user_oauth_extra_headers_from_db", new=AsyncMock(return_value={"Authorization": "Bearer x"}), ) as mock_lookup: result = await manager._resolve_oauth2_headers_for_tool_call( @@ -7139,7 +7138,7 @@ class TestMCPServerManager: @pytest.mark.asyncio async def test_rest_endpoint_filters_by_allowed_tools(self): """Test that REST endpoint _get_tools_for_single_server respects allowed_tools configuration""" - from litellm.proxy._experimental.mcp_server.rest_endpoints import ( + from litellm.proxy.mcp_server.rest_endpoints import ( _get_tools_for_single_server, ) @@ -7160,7 +7159,7 @@ class TestMCPServerManager: tool3 = MCPTool(name="allowed_tool_2", description="This tool is also allowed", inputSchema={}) # Mock the global_mcp_server_manager._get_tools_from_server - from litellm.proxy._experimental.mcp_server import rest_endpoints + from litellm.proxy.mcp_server import rest_endpoints with patch.object( rest_endpoints.global_mcp_server_manager, @@ -7180,7 +7179,7 @@ class TestMCPServerManager: @pytest.mark.asyncio async def test_rest_endpoint_shows_all_when_allowed_tools_is_none(self): """Test that REST endpoint shows all tools when allowed_tools is None (backwards compatibility)""" - from litellm.proxy._experimental.mcp_server.rest_endpoints import ( + from litellm.proxy.mcp_server.rest_endpoints import ( _get_tools_for_single_server, ) @@ -7201,7 +7200,7 @@ class TestMCPServerManager: tool3 = MCPTool(name="tool_3", description="Tool 3", inputSchema={}) # Mock the global_mcp_server_manager._get_tools_from_server - from litellm.proxy._experimental.mcp_server import rest_endpoints + from litellm.proxy.mcp_server import rest_endpoints with patch.object( rest_endpoints.global_mcp_server_manager, @@ -7221,7 +7220,7 @@ class TestMCPServerManager: @pytest.mark.asyncio async def test_rest_endpoint_shows_all_when_allowed_tools_is_empty_list(self): """Test that REST endpoint shows all tools when allowed_tools is empty list (backwards compatibility)""" - from litellm.proxy._experimental.mcp_server.rest_endpoints import ( + from litellm.proxy.mcp_server.rest_endpoints import ( _get_tools_for_single_server, ) @@ -7240,7 +7239,7 @@ class TestMCPServerManager: tool2 = MCPTool(name="tool_2", description="Tool 2", inputSchema={}) # Mock the global_mcp_server_manager._get_tools_from_server - from litellm.proxy._experimental.mcp_server import rest_endpoints + from litellm.proxy.mcp_server import rest_endpoints with patch.object( rest_endpoints.global_mcp_server_manager, @@ -7656,8 +7655,8 @@ class TestMCPServerManager: from mcp.server.context import ServerRequestContext from starlette.requests import Request - from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var - from litellm.proxy._experimental.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse + from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var + from litellm.proxy.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse auth = self._unrestricted_auth() manager, logging = self._manager_ready_for_call_tool( @@ -8130,7 +8129,7 @@ class TestMCPServerManager: with ( patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), patch( - "litellm.proxy._experimental.mcp_server.db.get_user_credential", + "litellm.proxy.mcp_server.db.get_user_credential", AsyncMock(side_effect=RuntimeError("DB DOWN")), ), ): @@ -8156,11 +8155,11 @@ class TestMCPServerManager: """A REST listing records under the header the client sent (none here). tools/call then swaps the stored credential in, either before reaching ``call_tool`` (``execute_mcp_tool``) or inside it (the Responses API), and must still read that slot rather than one keyed by the credential.""" - from litellm.proxy._experimental.mcp_server.byok_credential_cache import ( + from litellm.proxy.mcp_server.byok_credential_cache import ( byok_credential_cache_key, cache_byok_credential, ) - from litellm.proxy._experimental.mcp_server.operations import byok_credential_cache + from litellm.proxy.mcp_server.operations import byok_credential_cache manager = MCPServerManager() server = MCPServer( @@ -8264,11 +8263,11 @@ class TestMCPServerManager: reads, even with the stored BYOK secret at hand in the cache, and tools/list sends upstream exactly what the caller supplied, so the static token, the M2M mint and MCPJWTSigner all behave as they did before the catalog existed, whatever the auth_type.""" - from litellm.proxy._experimental.mcp_server.byok_credential_cache import ( + from litellm.proxy.mcp_server.byok_credential_cache import ( byok_credential_cache_key, cache_byok_credential, ) - from litellm.proxy._experimental.mcp_server.operations import byok_credential_cache + from litellm.proxy.mcp_server.operations import byok_credential_cache manager = MCPServerManager() server = MCPServer( @@ -8531,7 +8530,7 @@ class TestMCPServerManager: ) def test_per_caller_listed_tools_evict_oldest_caller_and_keep_shared(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import _LISTED_TOOLS_CALLERS_PER_SERVER + from litellm.proxy.mcp_server.mcp_server_manager import _LISTED_TOOLS_CALLERS_PER_SERVER manager = MCPServerManager() server = MCPServer( @@ -8566,7 +8565,7 @@ class TestMCPServerManager: @pytest.mark.asyncio @pytest.mark.parametrize("add_prefix", [True, False]) async def test_openapi_listing_records_listed_tools(self, add_prefix): - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry server = MCPServer( server_id="petstore-id", @@ -8601,7 +8600,7 @@ class TestMCPServerManager: @pytest.mark.asyncio async def test_openapi_listing_ignores_overlapping_server_prefix(self): - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry server = MCPServer( server_id="pet-id", @@ -8647,7 +8646,7 @@ class TestMCPServerManager: async def test_get_tools_from_server_records_the_catalog_only_when_asked_to(self, openapi): """The startup fill, the implicit pre-call listing and the pin snapshot reuse this fetch without serving its result, so only a listing that asks to be recorded sets what tools/call hooks see.""" - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry if openapi: server = MCPServer( @@ -8720,11 +8719,11 @@ class TestMCPServerManager: when called. This verifies the fix where user_api_key_auth is passed through litellm_metadata from responses API. """ - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, MCPServerAccess, ) - from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth manager = MCPServerManager() @@ -8777,10 +8776,10 @@ class TestMCPServerManager: async def test_no_mcp_servers_sentinel_blocks_allow_all_keys(self): """A key scoped to no-mcp-servers gets zero servers even when allow_all_keys servers exist, and the inner resolver is never consulted.""" - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) - from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth manager = MCPServerManager() object_permission = LiteLLM_ObjectPermissionTable( @@ -8812,10 +8811,10 @@ class TestMCPServerManager: @pytest.mark.asyncio async def test_no_mcp_servers_sentinel_excludes_submitted_byom_servers(self): from litellm.proxy import proxy_server as proxy_server_module - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) - from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth class _Cache: async def async_get_cache(self, key: str): @@ -8860,10 +8859,10 @@ class TestMCPServerManager: @pytest.mark.asyncio async def test_explicitly_scoped_key_excludes_submitted_byom_servers(self): from litellm.proxy import proxy_server as proxy_server_module - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) - from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth cache = MagicMock() cache.async_get_cache = AsyncMock(return_value=["submitted-server"]) @@ -8912,14 +8911,14 @@ class TestMCPServerManager: @pytest.mark.asyncio async def test_toolset_scope_excludes_submitted_byom_servers(self): from litellm.proxy import proxy_server as proxy_server_module - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, MCPServerAccess, ) - from litellm.proxy._experimental.mcp_server.mcp_context import ( + from litellm.proxy.mcp_server.mcp_context import ( mcp_active_toolset_id, ) - from litellm.proxy._types import UserAPIKeyAuth cache = MagicMock() cache.async_get_cache = AsyncMock(return_value=["submitted-server"]) @@ -8998,7 +8997,7 @@ class TestMCPServerManager: patch.object(proxy_server_module, "user_api_key_cache", cache), patch.object(proxy_server_module, "prisma_client", MagicMock()), patch( - "litellm.proxy._experimental.mcp_server.db.get_active_submitted_mcp_server_ids_for_user", + "litellm.proxy.mcp_server.db.get_active_submitted_mcp_server_ids_for_user", AsyncMock(return_value=["submitted-server", "unknown-server"]), ), ): @@ -9014,10 +9013,10 @@ class TestMCPServerManager: @pytest.mark.asyncio async def test_get_allowed_mcp_servers_fallback_keeps_submitted_byom_servers(self): from litellm.proxy import proxy_server as proxy_server_module - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) - from litellm.proxy._types import UserAPIKeyAuth class _Cache: async def async_get_cache(self, key: str): @@ -9099,7 +9098,7 @@ class TestMCPServerManager: submitted_server_ids, expected_server_ids, ): # test-quality-ok: parameterized matrix covers the scope state machine - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPServerAccess + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPServerAccess manager = MCPServerManager() auth = UserAPIKeyAuth(api_key="sk-test", user_id="user-123") @@ -9131,7 +9130,7 @@ class TestMCPServerManager: when extracting server name from prefixed tool name (second case). This ensures the fix for using server_name instead of name works correctly. """ - from litellm.proxy._experimental.mcp_server.utils import ( + from litellm.proxy.mcp_server.utils import ( add_server_prefix_to_name, ) @@ -9317,9 +9316,9 @@ class TestMCPServerTimestamps: repo_instance = MagicMock() repo_instance.table.update = AsyncMock() with ( - patch("litellm.proxy._experimental.mcp_server.db.update_mcp_server", new=update_mcp_server_mock), + patch("litellm.proxy.mcp_server.db.update_mcp_server", new=update_mcp_server_mock), patch( - "litellm.proxy._experimental.mcp_server.db.MCPServerRepository", + "litellm.proxy.mcp_server.db.MCPServerRepository", return_value=repo_instance, ), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), @@ -9493,7 +9492,7 @@ class TestMCPServerTimestamps: build_mock = AsyncMock(return_value=previous_entry) with ( patch( - "litellm.proxy._experimental.mcp_server.db.MCPServerRepository", + "litellm.proxy.mcp_server.db.MCPServerRepository", return_value=repo_instance, ), patch( @@ -9585,7 +9584,7 @@ class TestMCPServerTimestamps: def test_carry_forward_skips_when_url_or_auth_type_changed(self): """Stale endpoints from a different upstream or auth mode must not carry forward.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( carry_forward_resolved_oauth_endpoints, ) @@ -9624,7 +9623,7 @@ class TestMCPServerTimestamps: this, re-pointing authorize to server B while the same MCP url keeps serving A's token endpoint recreates the RFC 9700 mix-up, durably, and the discovery gate alone cannot catch it because the stale endpoint comes from the registry, not from discovery.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( carry_forward_resolved_oauth_endpoints, ) @@ -9658,7 +9657,7 @@ class TestMCPServerTimestamps: endpoint) adopts the previous authorize endpoint AND its token endpoint together as a consistent group, and a rebuild that re-pins the same authorize endpoint (formatting aside) keeps carrying the corroborated token endpoint.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( carry_forward_resolved_oauth_endpoints, ) @@ -9720,7 +9719,7 @@ class TestMCPServerTimestamps: endpoint the issuer anchor distrusts would keep being served across rebuilds. Resource-driven scopes still carry as last-known-good. Anchoring is keyed on the explicit issuer_is_anchored flag, not on issuer truthiness, so a discovered issuer does not trip this fail-closed branch.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( carry_forward_resolved_oauth_endpoints, ) @@ -9760,7 +9759,7 @@ class TestMCPServerTimestamps: must still carry forward as last-known-good, the same as any resource-rooted server. This is the regression the explicit issuer_is_anchored flag prevents: keying fail-closed on issuer truthiness alone would drop the working endpoints the moment the server learned its issuer.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( carry_forward_resolved_oauth_endpoints, ) @@ -9799,7 +9798,7 @@ class TestMCPServerTimestamps: """The corroboration check must not fail on formatting-only differences an IdP legitimately emits: default port, trailing slash, host case, and query string are not identity, but a non-default port is.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _normalized_authorize_endpoint, ) @@ -9813,7 +9812,7 @@ class TestMCPServerTimestamps: """Issuer equality tolerates only URL-insignificant differences (scheme/host case, default port, a trailing slash). A different host, a non-string, an empty string, or a None issuer never matches, so a document that omits issuer fails closed under issuer-anchored discovery.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import _issuer_matches + from litellm.proxy.mcp_server.mcp_server_manager import _issuer_matches assert _issuer_matches("https://mcp.slack.com", "https://mcp.slack.com") assert not _issuer_matches("https://MCP.slack.com/", "https://mcp.slack.com") @@ -10340,7 +10339,7 @@ class TestLegacyDelegateAuthWarningLog: def test_warn_skipped_for_client_credentials(self, caplog): caplog.set_level(logging.WARNING, logger="LiteLLM") - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _warn_legacy_delegate_auth_if_applicable, ) @@ -10795,7 +10794,7 @@ class TestOAuthDiscoverySSRFGuard: mock_client.get = AsyncMock() with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ): result = await manager._fetch_single_authorization_server_metadata( @@ -10823,7 +10822,7 @@ class TestOAuthDiscoverySSRFGuard: mock_client.get = AsyncMock(return_value=mock_response) with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ): servers, scopes = await manager._fetch_oauth_metadata_from_resource( @@ -10846,7 +10845,7 @@ class TestOAuthDiscoverySSRFGuard: mock_client.get = AsyncMock() with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ): servers, scopes = await manager._fetch_oauth_metadata_from_resource( @@ -10865,7 +10864,7 @@ class TestOAuthDiscoverySSRFGuard: mock_client.get = AsyncMock() with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ): servers, scopes = await manager._fetch_oauth_metadata_from_resource( @@ -10893,7 +10892,7 @@ class TestOAuthDiscoverySSRFGuard: mock_client.get = AsyncMock() with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ): servers, scopes = await manager._fetch_oauth_metadata_from_resource( @@ -10916,7 +10915,7 @@ class TestOAuthDiscoverySSRFGuard: mock_client.get = AsyncMock() with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ): servers, scopes = await manager._fetch_oauth_metadata_from_resource( @@ -10942,7 +10941,7 @@ class TestOAuthDiscoverySSRFGuard: mock_client.get = AsyncMock() with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ): servers, scopes = await manager._fetch_oauth_metadata_from_resource( @@ -10973,7 +10972,7 @@ class TestOAuthDiscoverySSRFGuard: mock_client.get = AsyncMock(return_value=redirect_response) with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ): result = await manager._fetch_single_authorization_server_metadata( @@ -11007,7 +11006,7 @@ class TestOAuthDiscoverySSRFGuard: mock_client.get = AsyncMock(side_effect=fake_get) with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ): await manager._fetch_oauth_metadata_from_resource( @@ -11038,7 +11037,7 @@ class TestOAuthDiscoverySSRFGuard: mock_client.get = AsyncMock(side_effect=fake_get) with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ): await manager._fetch_single_authorization_server_metadata( @@ -11057,7 +11056,7 @@ class TestOAuthDiscoverySSRFGuard: mock_client.get = AsyncMock() with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ): result = await manager._fetch_single_authorization_server_metadata( @@ -11130,10 +11129,10 @@ class TestApprovalStatusGate: async def test_update_server_eviction_clears_openapi_routing_artifacts(self, tmp_path): """Rejecting a server must remove its OpenAPI tools and name mappings.""" - from litellm.proxy._experimental.mcp_server.tool_registry import ( + from litellm.proxy.mcp_server.tool_registry import ( global_mcp_tool_registry, ) - from litellm.proxy._experimental.mcp_server.utils import ( + from litellm.proxy.mcp_server.utils import ( add_server_prefix_to_name, get_server_prefix, ) @@ -11315,7 +11314,7 @@ class TestUserEnvVarsCacheEviction: @staticmethod def _patch_cache(monkeypatch, max_size): - from litellm.proxy._experimental.mcp_server import mcp_server_manager as m + from litellm.proxy.mcp_server import mcp_server_manager as m cache: Dict[Any, Any] = {} monkeypatch.setattr(m, "_user_env_vars_cache", cache) @@ -11553,7 +11552,7 @@ class TestCreateMcpClientV2Graft: return MCPServer(**base) async def test_none_mode_resolves_to_noop_auth(self): - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( NoOpAuth, ) @@ -11590,7 +11589,7 @@ class TestCreateMcpClientV2Graft: ], ) async def test_static_family_emits_expected_header(self, auth_type, token, expected_name, expected_value): - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) @@ -11606,7 +11605,7 @@ class TestCreateMcpClientV2Graft: async def test_basic_mode_base64_encodes(self): import base64 - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) @@ -11622,16 +11621,16 @@ class TestCreateMcpClientV2Graft: async def test_m2m_client_credentials_resolves_via_v2(self): # M2M (oauth2 client_credentials) is migrated: to_server_spec owns the server and the # v2 arm mints the token through the injected source; nothing flows to v1's auth_value. - from litellm.proxy._experimental.mcp_server.outbound_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials import ( UpstreamCredentialProvider, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.client_credentials import ( ClientCredentialsBearerAuth, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( + from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.outbound_credentials.result import Ok class _FakeM2MSource: async def get(self, server_id, config): @@ -11699,8 +11698,8 @@ class TestCreateMcpClientV2Graft: assert client._mcp_auth_value == "k-stdio" async def test_resolver_error_maps_to_http_exception(self): - from litellm.proxy._experimental.mcp_server.outbound_credentials import Error - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( + from litellm.proxy.mcp_server.outbound_credentials import Error + from litellm.proxy.mcp_server.outbound_credentials.types import ( CredError, ) @@ -11740,7 +11739,7 @@ class TestCreateMcpClientV2Graft: assert client._get_auth_headers()["Authorization"] == "Bearer hook-jwt" async def test_none_with_extra_header_stays_v2_without_clobbering(self): - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( NoOpAuth, ) @@ -11780,7 +11779,7 @@ class TestMCPToolsListAuthSurfacing: @pytest.mark.asyncio async def test_fetch_tools_with_timeout_surfaces_upstream_401(self): - from litellm.proxy._experimental.mcp_server.exceptions import ( + from litellm.proxy.mcp_server.exceptions import ( MCPUpstreamAuthError, ) @@ -11832,7 +11831,7 @@ class TestMCPToolsListAuthSurfacing: @pytest.mark.asyncio async def test_get_tools_from_server_surfaces_unusable_user_token(self): - from litellm.proxy._experimental.mcp_server.exceptions import ( + from litellm.proxy.mcp_server.exceptions import ( MCPUpstreamAuthError, ) @@ -11884,7 +11883,7 @@ class TestMCPToolsListAuthSurfacing: answered 401 without the WWW-Authenticate the client needs to start the OAuth flow.""" import httpx - from litellm.proxy._experimental.mcp_server.exceptions import ( + from litellm.proxy.mcp_server.exceptions import ( MCPUpstreamAuthError, ) @@ -11916,7 +11915,7 @@ class TestMCPToolsListAuthSurfacing: would fail the RFC 9728 resource match against the gateway URL it dialed.""" import httpx - from litellm.proxy._experimental.mcp_server.exceptions import ( + from litellm.proxy.mcp_server.exceptions import ( MCPUpstreamAuthError, ) from litellm.types.mcp import MCPAuth @@ -11954,7 +11953,7 @@ class TestMCPToolsListAuthSurfacing: clients at the upstream protected-resource metadata, which fails the RFC 9728 resource match against the gateway URL they dialed. Stripping it makes the single-server route fabricate the gateway well-known challenge, whose content is the bridge facade.""" - from litellm.proxy._experimental.mcp_server.exceptions import ( + from litellm.proxy.mcp_server.exceptions import ( MCPUpstreamAuthError, ) from litellm.types.mcp import MCPAuth @@ -11982,7 +11981,7 @@ class TestMCPToolsListAuthSurfacing: @pytest.mark.asyncio async def test_get_tools_from_server_suppresses_resolver_challenge_for_dcr_bridge(self): """The client-build-time HTTPException conversion path applies the same suppression.""" - from litellm.proxy._experimental.mcp_server.exceptions import ( + from litellm.proxy.mcp_server.exceptions import ( MCPUpstreamAuthError, ) from litellm.types.mcp import MCPAuth @@ -12011,7 +12010,7 @@ class TestMCPToolsListAuthSurfacing: @pytest.mark.asyncio async def test_aggregate_list_tools_absorbs_unauthenticated_server(self): - from litellm.proxy._experimental.mcp_server.exceptions import ( + from litellm.proxy.mcp_server.exceptions import ( MCPUpstreamAuthError, ) @@ -12096,7 +12095,7 @@ class _RetryFakeClient: """A fake MCPClient whose call_tool fails on the first attempt and (optionally) succeeds after.""" def __init__(self, *, raises=None, result=None) -> None: - from litellm.experimental_mcp_client.client import MCPClient + from litellm.mcp_client.client import MCPClient self._raises = raises self._result = result @@ -12175,7 +12174,7 @@ class TestOBOCallToolRetry: async def test_upstream_401_on_id_jag_evicts_the_cached_bearer_and_retries(self): """The retry path must invalidate the ID-JAG leg-2 bearer too: without eviction the rebuilt client resolves the same rejected token from the cache and the retry 401s identically.""" - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( + from litellm.proxy.mcp_server.outbound_credentials.types import ( IdJagConfig, ) @@ -12300,7 +12299,7 @@ class TestOBOCallToolRetry: @pytest.mark.asyncio async def test_session_closed_by_a_peers_refresh_retries_on_a_fresh_session_without_invalidating(self) -> None: - from litellm.experimental_mcp_client.client import UpstreamSessionClosedError + from litellm.mcp_client.client import UpstreamSessionClosedError manager: Final = self._manager() success: Final = CallToolResult(content=[], isError=False) @@ -12608,7 +12607,7 @@ class TestOBOEndpointDiscovery: with ( patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=mock_client, ), patch.object(manager, "_fetch_oauth_metadata_from_resource", AsyncMock(return_value=([], None))), @@ -12633,8 +12632,8 @@ async def test_preflight_challenge_carries_step_up_error_and_claims(): claims in the single-server 401 challenge, so an MSAL-family client can drive the step-up and retry.""" import base64 - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import CredError + from litellm.proxy.mcp_server.outbound_credentials.result import Error + from litellm.proxy.mcp_server.outbound_credentials.types import CredError claims = '{"access_token":{"acrs":{"essential":true,"value":"c1"}}}' @@ -12671,7 +12670,7 @@ async def test_preflight_challenge_carries_step_up_error_and_claims(): async def test_aggregate_list_still_absorbs_step_up_challenged_server(): """A step-up (claims-bearing) 401 from one server must not change the aggregate contract: the multi-server listing still absorbs it and returns the healthy servers' tools.""" - from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError + from litellm.proxy.mcp_server.exceptions import MCPUpstreamAuthError manager = MCPServerManager() good = MCPServer(server_id="good", name="good", transport=MCPTransport.http) @@ -13093,7 +13092,7 @@ async def test_resolve_toolset_tool_permissions_single_db_fetch_across_checks(): key's toolsets during one request; the shared cache must dedupe the DB fetch so the request costs a single toolset query however many checks run""" from litellm.caching.caching import DualCache - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -13104,7 +13103,7 @@ async def test_resolve_toolset_tool_permissions_single_db_fetch_across_checks(): with ( patch( - "litellm.proxy._experimental.mcp_server.toolset_db.list_mcp_toolsets", + "litellm.proxy.mcp_server.toolset_db.list_mcp_toolsets", list_toolsets_mock, ), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), @@ -13124,7 +13123,7 @@ async def test_resolve_toolset_tool_permissions_fresh_policy_sees_writer_revocat request even though the legacy cache still holds the old grant, and the fresh read must go to the writer, not the replica""" from litellm.caching.caching import DualCache - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -13137,7 +13136,7 @@ async def test_resolve_toolset_tool_permissions_fresh_policy_sees_writer_revocat with ( patch( - "litellm.proxy._experimental.mcp_server.toolset_db.list_mcp_toolsets", + "litellm.proxy.mcp_server.toolset_db.list_mcp_toolsets", list_toolsets_mock, ), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), @@ -13162,7 +13161,7 @@ async def test_resolve_toolset_tool_permissions_fresh_policy_propagates_db_fault """A fresh read that fails must raise so the managed-agent boundary fails closed; the legacy path keeps its swallow-to-empty behaviour""" from litellm.caching.caching import DualCache - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -13171,7 +13170,7 @@ async def test_resolve_toolset_tool_permissions_fresh_policy_propagates_db_fault with ( patch( - "litellm.proxy._experimental.mcp_server.toolset_db.list_mcp_toolsets", + "litellm.proxy.mcp_server.toolset_db.list_mcp_toolsets", list_toolsets_mock, ), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), @@ -13191,10 +13190,10 @@ class TestMaterializeAuthHeaders: @pytest.mark.asyncio async def test_static_header_auth_materializes_its_header(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _materialize_auth_headers, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) @@ -13203,13 +13202,13 @@ class TestMaterializeAuthHeaders: @pytest.mark.asyncio async def test_client_credentials_bearer_auth_materializes_bearer(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _materialize_auth_headers, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.client_credentials import ( ClientCredentialsBearerAuth, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( + from litellm.proxy.mcp_server.outbound_credentials.types import ( ClientCredentialsConfig, ) @@ -13224,13 +13223,13 @@ class TestMaterializeAuthHeaders: async def test_materialize_follows_the_minted_token_to_a_custom_header(self): # The OpenAPI arm reads header_name off the auth object rather than assuming Authorization, # so it carries the knob with no per-arm change. This pins that it stays that way. - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _materialize_auth_headers, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import ( + from litellm.proxy.mcp_server.outbound_credentials.client_credentials import ( ClientCredentialsBearerAuth, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( + from litellm.proxy.mcp_server.outbound_credentials.types import ( ClientCredentialsConfig, ) @@ -13243,10 +13242,10 @@ class TestMaterializeAuthHeaders: @pytest.mark.asyncio async def test_noop_and_none_materialize_to_none(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _materialize_auth_headers, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( NoOpAuth, ) @@ -13367,7 +13366,7 @@ class TestDiscoveryFailureLogging: secret_url = "https://typo-host.example.com/mcp/s/PATHSECRET/mcp" with ( patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=self._connect_error_client(secret_url), ), caplog.at_level(logging.WARNING, logger="LiteLLM"), @@ -13387,7 +13386,7 @@ class TestDiscoveryFailureLogging: url = "https://typo-host.example.com/mcp" with ( patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=self._connect_error_client(url), ), caplog.at_level(logging.WARNING, logger="LiteLLM"), @@ -13404,7 +13403,7 @@ class TestDiscoveryFailureLogging: client.get = AsyncMock(return_value=httpx.Response(404, request=httpx.Request("GET", url))) with ( patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.get_async_httpx_client", + "litellm.proxy.mcp_server.mcp_server_manager.get_async_httpx_client", return_value=client, ), caplog.at_level(logging.WARNING, logger="LiteLLM"), @@ -13731,7 +13730,7 @@ class TestServerToolListsHonorThePrefixBoundary: @pytest.mark.asyncio async def test_wire_form_allowlist_entry_follows_a_non_default_separator(self): - from litellm.proxy._experimental.mcp_server import utils as mcp_utils + from litellm.proxy.mcp_server import utils as mcp_utils server = self._aliased_server(allowed_tools=["petstore__getpetbyid"]) @@ -13885,7 +13884,7 @@ class TestOpenAPIRegistryKeyMatchesRegistration: @staticmethod def _register(server: MCPServer, base_tool_name: str) -> str: - from litellm.proxy._experimental.mcp_server.utils import ( + from litellm.proxy.mcp_server.utils import ( add_server_prefix_to_name, get_server_prefix, ) @@ -13893,7 +13892,7 @@ class TestOpenAPIRegistryKeyMatchesRegistration: return add_server_prefix_to_name(base_tool_name, get_server_prefix(server)) async def _call(self, server: MCPServer, registered_key: str, bare_tool_name: str) -> CallToolResult: - from litellm.proxy._experimental.mcp_server.tool_registry import ( + from litellm.proxy.mcp_server.tool_registry import ( global_mcp_tool_registry, ) @@ -13967,7 +13966,7 @@ class TestOpenAPIRegistryKeyMatchesRegistration: @pytest.mark.asyncio async def test_dispatch_follows_a_non_default_prefix_separator(self): - from litellm.proxy._experimental.mcp_server import utils as mcp_utils + from litellm.proxy.mcp_server import utils as mcp_utils server = MCPServer( server_id="dd7f2b9e-2c4a-4f1b-9e0a-8d3c6b5a4f21", @@ -14092,20 +14091,20 @@ class TestSessionResourceScopeIntersect: return auth def test_scope_reader_is_none_for_keys_and_unscoped_subjects(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager assert MCPServerManager._admitted_session_resource_scope(None) is None assert MCPServerManager._admitted_session_resource_scope(UserAPIKeyAuth(user_id="u")) is None assert MCPServerManager._admitted_session_resource_scope(self._admitted_auth(None)) is None def test_scope_reader_returns_sealed_scope_for_admitted_subjects(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager assert MCPServerManager._admitted_session_resource_scope(self._admitted_auth("b")) == frozenset({"b"}) def test_scope_reader_returns_all_sealed_server_ids(self): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager auth = self._admitted_auth("first") auth.mcp_session_resource_server_ids = ("first", "second") @@ -14119,8 +14118,8 @@ class TestSessionResourceScopeIntersect: single-exit shape, the old early return would hand a per-server bearer the registry.""" from unittest.mock import AsyncMock, patch - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -14148,8 +14147,8 @@ class TestSessionResourceScopeIntersect: async def test_admin_registry_seed_keeps_both_servers_in_a_multi_server_session(self): from unittest.mock import AsyncMock, patch - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -14184,14 +14183,14 @@ class TestSessionResourceScopeIntersect: than the whole set.""" from unittest.mock import AsyncMock, patch - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager manager = MCPServerManager() auth = self._admitted_auth("granted-id") with ( patch.object(MCPServerManager, "get_allow_all_keys_server_ids", return_value=["open-id", "granted-id"]), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_allowed_mcp_servers", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_allowed_mcp_servers", new_callable=AsyncMock, return_value=["granted-id", "other-id"], ), @@ -14208,7 +14207,7 @@ class TestSessionResourceScopeIntersect: with ( patch.object(MCPServerManager, "get_allow_all_keys_server_ids", return_value=["open-id", "granted-id"]), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_allowed_mcp_servers", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_allowed_mcp_servers", new_callable=AsyncMock, side_effect=RuntimeError("resolver down"), ), @@ -14281,7 +14280,7 @@ class TestClientForwardedDiscoveryFailureIsNotFatal: {"new": AsyncMock(return_value=None)} if failure == "incomplete" else {"side_effect": never_returns} ) with ( - patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCP_METADATA_TIMEOUT", 0.01), + patch("litellm.proxy.mcp_server.mcp_server_manager.MCP_METADATA_TIMEOUT", 0.01), patch.object(manager, "_discover_oauth_metadata_for_server", **discovery_patch), ): if not serves_without_endpoints: @@ -14506,8 +14505,8 @@ class TestOpenApiHandlerRelaysUpstreamAuth: @pytest.mark.asyncio async def test_upstream_auth_error_keeps_its_type(self): - from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.exceptions import MCPUpstreamAuthError + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry manager = MCPServerManager() server = self._server() @@ -14526,7 +14525,7 @@ class TestOpenApiHandlerRelaysUpstreamAuth: @pytest.mark.asyncio async def test_other_failures_still_become_an_error_result(self): - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry manager = MCPServerManager() @@ -14556,8 +14555,8 @@ class TestOpenApiHandlerRelaysUpstreamAuth: async def test_json_bodies_keep_verbatim_text_and_gain_structured_content(self, body, compat, expected_structured): """The OpenAPI arm used to stringify the response; now the text block is the upstream body byte for byte, exactly once, and JSON bodies carry structuredContent when the caller's revision admits it.""" - from litellm.proxy._experimental.mcp_server.tool_outcome import WireCompat, parse_http_body - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.tool_outcome import WireCompat, parse_http_body + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry manager = MCPServerManager() @@ -14747,7 +14746,7 @@ class TestConfigServerIdPinning: ) with ( patch( # test-quality-ok: the db reload path has no seam but its own repository - "litellm.proxy._experimental.mcp_server.db.MCPServerRepository", + "litellm.proxy.mcp_server.db.MCPServerRepository", return_value=repository, ), patch( # test-quality-ok: same, the prisma client is fetched inside the reload @@ -15203,8 +15202,8 @@ class TestLitellmAdmissionKeyIsNeverTheSubjectToken: @staticmethod def _recording_provider() -> MagicMock: - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import StaticHeaderAuth - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import StaticHeaderAuth + from litellm.proxy.mcp_server.outbound_credentials.result import Ok provider: Final = MagicMock() provider.resolve_credentials = AsyncMock( @@ -15504,8 +15503,7 @@ async def test_pre_call_tool_check_honors_guardrail_attached_to_key(monkeypatch, @pytest.mark.asyncio async def test_pre_call_tool_check_stamps_personal_key_team_rpm_limits(monkeypatch: pytest.MonkeyPatch) -> None: import litellm - - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler manager: Final = MCPServerManager() server: Final = MCPServer( @@ -15544,7 +15542,7 @@ async def test_pre_call_tool_check_stamps_personal_key_team_rpm_limits(monkeypat @pytest.mark.asyncio async def test_pre_call_tool_check_uses_inherited_team_auth_for_guardrails(monkeypatch: pytest.MonkeyPatch) -> None: - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler server: Final = MCPServer( server_id="srv-guardrails", @@ -15610,7 +15608,7 @@ async def test_pre_call_tool_check_uses_inherited_team_auth_for_guardrails(monke @pytest.mark.asyncio async def test_pre_call_tool_check_keeps_native_name_for_team_attribution(monkeypatch: pytest.MonkeyPatch) -> None: - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler server: Final = MCPServer( server_id="math-a", @@ -15711,10 +15709,10 @@ async def test_pre_call_tool_check_keeps_original_auth_for_team_key(monkeypatch: @pytest.mark.asyncio async def test_call_tool_passes_attributed_auth_to_guardrail_hooks(monkeypatch: pytest.MonkeyPatch) -> None: from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager - from litellm.proxy._experimental.mcp_server.operations import _run_post_mcp_call_guardrails - from litellm.proxy.auth import auth_checks from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, LiteLLM_UserTable + from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.operations import _run_post_mcp_call_guardrails original_auth: Final = UserAPIKeyAuth(api_key="sk-personal", user_id="alice") team: Final = LiteLLM_TeamTable( @@ -15826,13 +15824,12 @@ async def test_call_tool_passes_attributed_auth_to_guardrail_hooks(monkeypatch: async def test_virtual_mcp_calls_use_attributed_auth_for_logging_and_hooks(monkeypatch: pytest.MonkeyPatch) -> None: from mcp.types import CallToolResult, Tool - from litellm.proxy._experimental.mcp_server import operations, tool_search - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry from litellm.proxy import proxy_server + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, LiteLLM_UserTable from litellm.proxy.auth import auth_checks - from litellm.proxy._types import LiteLLM_ObjectPermissionTable - from litellm.proxy._types import LiteLLM_TeamTable, LiteLLM_UserTable + from litellm.proxy.mcp_server import operations, tool_search + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry from litellm.types.mcp_server.tool_registry import MCPTool original_auth: Final = UserAPIKeyAuth( @@ -15971,13 +15968,13 @@ async def test_debug_resolution_matches_final_header_conflict_winner( expected_source: str, expected_authorization: str | None, ) -> None: - from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var - from starlette.requests import Request from pydantic import SecretStr + from starlette.requests import Request - from litellm.proxy._experimental.mcp_server.auth.litellm_auth_handler import MCPAuthenticatedUser - from litellm.proxy._experimental.mcp_server.mcp_debug import MCP_AUTH_DIAGNOSTICS_SCOPE_KEY, MCPAuthDiagnostics - from litellm.proxy._experimental.mcp_server.outbound_credentials import ( + from litellm.proxy.mcp_server.auth.litellm_auth_handler import MCPAuthenticatedUser + from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var + from litellm.proxy.mcp_server.mcp_debug import MCP_AUTH_DIAGNOSTICS_SCOPE_KEY, MCPAuthDiagnostics + from litellm.proxy.mcp_server.outbound_credentials import ( ApiKeyConfig, AuthorizationCodeConfig, NoneConfig, @@ -15985,7 +15982,7 @@ async def test_debug_resolution_matches_final_header_conflict_winner( SharedKey, UpstreamCredentialProvider, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import OAuthToken + from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import OAuthToken from litellm.types.mcp_server.mcp_server_manager import MCPServer class Store: @@ -16041,10 +16038,10 @@ async def test_debug_reports_legacy_signing_and_non_http_transport( _mcp_request_ctx, monkeypatch, transport: Literal["http", "stdio"] ) -> None: monkeypatch.setenv("LITELLM_ENABLE_MCP_STDIO", "true") - from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var from starlette.requests import Request - from litellm.proxy._experimental.mcp_server.mcp_debug import MCP_AUTH_DIAGNOSTICS_SCOPE_KEY, MCPAuthDiagnostics + from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var + from litellm.proxy.mcp_server.mcp_debug import MCP_AUTH_DIAGNOSTICS_SCOPE_KEY, MCPAuthDiagnostics from litellm.types.mcp_server.mcp_server_manager import MCPServer diagnostics = MCPAuthDiagnostics() @@ -16255,7 +16252,7 @@ async def test_openapi_health_coalesces_concurrent_checks_and_reuses_results(res @pytest.mark.asyncio async def test_openapi_health_cache_expires_at_thirty_seconds(respx_mock, monkeypatch): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import _OpenAPIHealthProbe + from litellm.proxy.mcp_server.mcp_server_manager import _OpenAPIHealthProbe monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") clock = iter([0.0, 29.0, 30.0, 30.0]) @@ -16350,8 +16347,8 @@ class _DiscoveryClock: return self.now -from pydantic import TypeAdapter from mcp.types import JSONRPCMessage, ListPromptsResult, ListResourcesResult, ListResourceTemplatesResult +from pydantic import TypeAdapter _JSONRPC_ADAPTER = TypeAdapter(JSONRPCMessage) @@ -16359,7 +16356,7 @@ _JSONRPC_ADAPTER = TypeAdapter(JSONRPCMessage) @contextlib.contextmanager def _mcp_upstream(respond): """Drive the SDK's streamable-HTTP transport off an httpx2 MockTransport; respx only sees httpx.""" - from litellm.experimental_mcp_client.client import MCPClient + from litellm.mcp_client.client import MCPClient def make_client(self, *args, **kwargs): return httpx2.AsyncClient( @@ -16792,7 +16789,7 @@ async def test_discovery_cache_can_be_disabled(monkeypatch: pytest.MonkeyPatch) "value,expected", (("invalid", 60.0), ("nan", 60.0), ("inf", 60.0), ("-1", 60.0), ("12.5", 12.5)) ) def test_discovery_cache_ttl_validation(value: str, expected: float, monkeypatch: pytest.MonkeyPatch) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import _mcp_discovery_cache_ttl + from litellm.proxy.mcp_server.mcp_server_manager import _mcp_discovery_cache_ttl monkeypatch.setenv("LITELLM_MCP_DISCOVERY_CACHE_TTL", value) assert _mcp_discovery_cache_ttl() == expected @@ -16811,7 +16808,7 @@ def test_discovery_cache_keys_isolate_user_dependent_auth(auth_type: MCPAuth) -> def _register_local_tool(name: str, description: str) -> None: - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry async def _handler(**kwargs): return None @@ -16829,7 +16826,7 @@ def _openapi_server(name: str) -> MCPServer: @pytest.mark.asyncio async def test_openapi_listing_ignores_overlapping_server_prefix() -> None: - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry manager: Final = MCPServerManager() manager.create_mcp_client = AsyncMock(return_value=AsyncMock()) @@ -16850,7 +16847,7 @@ async def test_openapi_listing_ignores_overlapping_server_prefix() -> None: @pytest.mark.asyncio async def test_openapi_listing_finds_tools_registered_under_the_normalized_prefix() -> None: - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry manager: Final = MCPServerManager() manager.create_mcp_client = AsyncMock(return_value=AsyncMock()) @@ -16866,7 +16863,7 @@ async def test_openapi_listing_finds_tools_registered_under_the_normalized_prefi @pytest.mark.asyncio async def test_discovery_cache_retries_cancelled_fetches() -> None: - from litellm.proxy._experimental.mcp_server.catalog import _DiscoveryCache + from litellm.proxy.mcp_server.catalog import _DiscoveryCache cache: Final = _DiscoveryCache[ListPromptsResult](60, _DiscoveryClock(), TypeAdapter(ListPromptsResult)) @@ -16883,7 +16880,7 @@ async def test_discovery_cache_retries_cancelled_fetches() -> None: @pytest.mark.asyncio async def test_discovery_cache_cancels_fetch_when_last_waiter_leaves() -> None: - from litellm.proxy._experimental.mcp_server.catalog import _DiscoveryCache + from litellm.proxy.mcp_server.catalog import _DiscoveryCache cache: Final = _DiscoveryCache[ListPromptsResult](60, _DiscoveryClock(), TypeAdapter(ListPromptsResult)) entered: Final = asyncio.Event() @@ -16912,7 +16909,7 @@ async def test_discovery_cache_cancels_fetch_when_last_waiter_leaves() -> None: @pytest.mark.asyncio async def test_discovery_cache_bounds_detached_fetches_without_dropping_results() -> None: - from litellm.proxy._experimental.mcp_server.catalog import _DiscoveryCache + from litellm.proxy.mcp_server.catalog import _DiscoveryCache cache: Final = _DiscoveryCache[ListPromptsResult](60, _DiscoveryClock(), TypeAdapter(ListPromptsResult)) entered: Final[asyncio.Queue[None]] = asyncio.Queue() @@ -16944,10 +16941,11 @@ async def test_discovery_cache_bounds_detached_fetches_without_dropping_results( @pytest.mark.asyncio async def test_discovery_cache_tracks_resolved_credentials_across_workers() -> None: import respx - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import StaticHeaderAuth - from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import UpstreamCredentialProvider - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Ok, Result - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import CredError, ServerSpec, Subject + + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import StaticHeaderAuth + from litellm.proxy.mcp_server.outbound_credentials.resolver import UpstreamCredentialProvider + from litellm.proxy.mcp_server.outbound_credentials.result import Error, Ok, Result + from litellm.proxy.mcp_server.outbound_credentials.types import CredError, ServerSpec, Subject class CredentialSource(UpstreamCredentialProvider): def __init__(self) -> None: @@ -17020,7 +17018,8 @@ async def test_discovery_cache_tracks_resolved_credentials_across_workers() -> N @pytest.mark.asyncio async def test_discovery_resolves_stored_oauth_for_the_requesting_user() -> None: import respx - from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import OAuthToken + + from litellm.proxy.mcp_server.outbound_credentials.oauth_token_store import OAuthToken class TokenStore: def __init__(self) -> None: @@ -17058,7 +17057,7 @@ async def test_discovery_resolves_stored_oauth_for_the_requesting_user() -> None @pytest.mark.asyncio async def test_discovery_cache_evicts_results_at_capacity() -> None: - from litellm.proxy._experimental.mcp_server.catalog import _DiscoveryCache + from litellm.proxy.mcp_server.catalog import _DiscoveryCache cache: Final = _DiscoveryCache[ListPromptsResult](60, _DiscoveryClock(), TypeAdapter(ListPromptsResult)) @@ -17076,7 +17075,7 @@ async def test_discovery_cache_evicts_results_at_capacity() -> None: @pytest.mark.asyncio async def test_discovery_cache_invalidation_preserves_other_servers_and_pending_fetches() -> None: - from litellm.proxy._experimental.mcp_server.catalog import _DiscoveryCache + from litellm.proxy.mcp_server.catalog import _DiscoveryCache cache: Final = _DiscoveryCache[ListPromptsResult](60, _DiscoveryClock(), TypeAdapter(ListPromptsResult)) entered: Final = asyncio.Event() @@ -17108,7 +17107,7 @@ async def test_discovery_cache_invalidation_preserves_other_servers_and_pending_ @pytest.mark.asyncio @pytest.mark.parametrize("description", ("x" * 96_000, "é" * 40_000), ids=("ascii", "unicode")) async def test_discovery_cache_returns_oversized_results_without_retaining_them(description: str) -> None: - from litellm.proxy._experimental.mcp_server.catalog import _DiscoveryCache + from litellm.proxy.mcp_server.catalog import _DiscoveryCache cache: Final = _DiscoveryCache[ListPromptsResult](60, _DiscoveryClock(), TypeAdapter(ListPromptsResult)) fetch: Final = AsyncMock( @@ -17141,9 +17140,9 @@ class TestProtectedCredentialPreparation: credential: str | None, dispatch: str, ) -> None: - from litellm.proxy._experimental.mcp_server import operations - from litellm.proxy._experimental.mcp_server.server import _handle_local_mcp_tool - from litellm.proxy._experimental.mcp_server.utils import add_server_prefix_to_name, get_server_prefix + from litellm.proxy.mcp_server import operations + from litellm.proxy.mcp_server.server import _handle_local_mcp_tool + from litellm.proxy.mcp_server.utils import add_server_prefix_to_name, get_server_prefix spec_path: Final = tmp_path / "openapi.json" spec_path.write_text( @@ -17433,7 +17432,7 @@ class TestProtectedCredentialPreparation: forwarded: dict[str, str] | None, caller: str | None, ) -> None: - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( create_tool_function, request_auth_header, request_extra_headers, @@ -17465,8 +17464,9 @@ class TestProtectedCredentialPreparation: @pytest.mark.asyncio async def test_static_resolution_cancellation_closes_flow(self) -> None: from collections.abc import AsyncGenerator - from litellm.experimental_mcp_client.client import MCPClient - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import prepare_mcp_client + + from litellm.mcp_client.client import MCPClient + from litellm.proxy.mcp_server.outbound_credentials.adapter import prepare_mcp_client class CancelledAuth(httpx.Auth): closed = False @@ -17730,8 +17730,8 @@ class TestProtectedCredentialPreparation: @pytest.mark.asyncio @pytest.mark.parametrize("selected", [False, True]) async def test_request_selected_during_guardrail_runs_concurrently_with_tool(monkeypatch, selected): + from litellm.proxy.mcp_server import tool_registry from litellm.responses.mcp.request_context import MCPRequestContext - from litellm.proxy._experimental.mcp_server import tool_registry tool_started = asyncio.Event() guardrail_started = asyncio.Event() @@ -17809,8 +17809,9 @@ async def test_server_response_identifies_read_only_config(in_config, in_db, exp @pytest.mark.parametrize("with_caller,legacy_factory", [(True, False), (False, False), (True, True)]) async def test_client_sampling_does_not_fill_explicit_context_from_another_ambient_caller(with_caller, legacy_factory): from mcp.server.auth.middleware.auth_context import auth_context_var - from litellm.proxy._experimental.mcp_server import server as legacy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_sampling_callback + + from litellm.proxy.mcp_server import server as legacy_server + from litellm.proxy.mcp_server.mcp_server_manager import _create_sampling_callback upstream = MCPServer( server_id="explicit-empty", @@ -17828,8 +17829,8 @@ async def test_client_sampling_does_not_fill_explicit_context_from_another_ambie client_ip="192.0.2.99", ) with ( - patch("litellm.proxy._experimental.mcp_server.upstream.MCPClient") as factory, - patch("litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", sampling), + patch("litellm.proxy.mcp_server.upstream.MCPClient") as factory, + patch("litellm.proxy.mcp_server.sampling_handler.handle_sampling_create_message", sampling), ): if legacy_factory: callback = _create_sampling_callback(user_api_key_auth=UserAPIKeyAuth(user_id="explicit")) @@ -18106,6 +18107,7 @@ async def test_catalog_delete_drops_derived_tool_mapping(monkeypatch): @pytest.mark.parametrize("rediscover", [False, True]) async def test_catalog_publishes_rediscovered_routes_despite_concurrent_owner_change(monkeypatch, rediscover): from mcp.types import Tool + from litellm.proxy import proxy_server monkeypatch.setattr(proxy_server, "prisma_client", None) @@ -18165,7 +18167,7 @@ async def test_catalog_reload_preserves_concurrent_config_discovery_and_routes(a with ( patch("litellm.proxy.proxy_server.prisma_client", prisma), patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.hydrate_config_server_dcr_client", + "litellm.proxy.mcp_server.discoverable_endpoints.hydrate_config_server_dcr_client", side_effect=hydrate, ), ): @@ -18247,7 +18249,7 @@ async def test_catalog_operation_retains_routes_only_for_same_configured_target( @pytest.mark.parametrize("change", ["concurrent", "staged", "deleted", "staged_after_delete", "staged_with_concurrent"]) @pytest.mark.parametrize("mapped", [False, True]) async def test_catalog_reload_keeps_new_route_owner_over_earlier_route(change, mapped, monkeypatch): - from litellm.proxy._experimental.mcp_server import tool_registry + from litellm.proxy.mcp_server import tool_registry manager: Final = MCPServerManager() registry: Final = tool_registry.MCPToolRegistry() @@ -18377,7 +18379,8 @@ async def test_catalog_reload_retains_routes_discovered_for_a_late_server(change from datetime import timedelta from mcp.types import Tool - from litellm.proxy._experimental.mcp_server import tool_registry + + from litellm.proxy.mcp_server import tool_registry row: Final = LiteLLM_MCPServerTable( server_id="late-routes", @@ -18453,7 +18456,7 @@ async def test_catalog_reload_retains_routes_discovered_for_a_late_server(change async def test_catalog_openapi_refresh_does_not_restore_removed_operations( tmp_path, monkeypatch, respx_mock, retain_operation ): - from litellm.proxy._experimental.mcp_server import tool_registry + from litellm.proxy.mcp_server import tool_registry monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") manager: Final = MCPServerManager() @@ -18493,7 +18496,7 @@ async def test_catalog_openapi_refresh_does_not_restore_removed_operations( if retain_operation: assert manager.server_exposes_tool(manager.registry[row.server_id], "spec_refresh-retained") assert retained is not None - from litellm.proxy._experimental.mcp_server.tool_outcome import JsonResult + from litellm.proxy.mcp_server.tool_outcome import JsonResult retained_result = await retained.handler() assert isinstance(retained_result, JsonResult) @@ -18609,7 +18612,8 @@ async def test_catalog_background_lookup_after_operation_exit_observes_deletion( @pytest.mark.asyncio async def test_catalog_cancelled_openapi_refresh_retains_tools_and_discovery(monkeypatch): from datetime import timedelta - from litellm.proxy._experimental.mcp_server import tool_registry + + from litellm.proxy.mcp_server import tool_registry manager: Final = MCPServerManager() stamp: Final = datetime.now() @@ -18697,7 +18701,7 @@ async def test_catalog_snapshot_identity_is_independent_of_worker_oauth_discover @pytest.mark.asyncio async def test_catalog_failed_openapi_row_does_not_publish_partial_handlers(monkeypatch): - from litellm.proxy._experimental.mcp_server import tool_registry + from litellm.proxy.mcp_server import tool_registry manager: Final = MCPServerManager() registry: Final = tool_registry.MCPToolRegistry() @@ -18744,7 +18748,7 @@ async def test_catalog_unchanged_read_preserves_derived_initialize_instructions( @pytest.mark.asyncio async def test_catalog_reuses_openapi_tools_until_configuration_or_background_refresh(monkeypatch, tmp_path): - from litellm.proxy._experimental.mcp_server import openapi_to_mcp_generator, tool_registry + from litellm.proxy.mcp_server import openapi_to_mcp_generator, tool_registry registry = tool_registry.MCPToolRegistry() monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry) @@ -18793,7 +18797,7 @@ async def test_catalog_rejects_configuration_switch_before_client_creation(monke admitted = manager.get_mcp_server_by_id("catalog-server") read_rows.return_value = [_catalog_row("updated")] if change == "edit" else [] await asyncio.create_task(manager.reload_servers_from_database()) - with patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPClient") as factory: + with patch("litellm.proxy.mcp_server.mcp_server_manager.MCPClient") as factory: with pytest.raises(HTTPException) as exc: await manager._create_mcp_client(admitted) assert exc.value.status_code == 503 @@ -18804,8 +18808,8 @@ async def test_catalog_rejects_configuration_switch_before_client_creation(monke @pytest.mark.asyncio @pytest.mark.parametrize("dispatch", ["managed", "local"]) async def test_catalog_changed_openapi_handler_never_dispatches(monkeypatch, tmp_path, respx_mock, dispatch): - from litellm.proxy._experimental.mcp_server import operations, tool_registry - from litellm.proxy._experimental.mcp_server.utils import add_server_prefix_to_name, get_server_prefix + from litellm.proxy.mcp_server import operations, tool_registry + from litellm.proxy.mcp_server.utils import add_server_prefix_to_name, get_server_prefix registry = tool_registry.MCPToolRegistry() monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry) @@ -18912,7 +18916,7 @@ async def test_catalog_cancelled_config_hydration_preserves_published_credential raise asyncio.CancelledError() with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.hydrate_config_server_dcr_client", + "litellm.proxy.mcp_server.discoverable_endpoints.hydrate_config_server_dcr_client", side_effect=cancelled_hydration, ): with pytest.raises(asyncio.CancelledError): @@ -19029,7 +19033,7 @@ async def test_catalog_fresh_lookup_does_not_fall_back_to_stale_grants_when_data @pytest.mark.asyncio async def test_catalog_cancelled_registration_does_not_publish_partial_handlers(monkeypatch): - from litellm.proxy._experimental.mcp_server import tool_registry + from litellm.proxy.mcp_server import tool_registry manager = MCPServerManager() registry = tool_registry.MCPToolRegistry() @@ -19681,7 +19685,7 @@ async def test_close_all_upstream_sessions_logs_when_waiting_times_out( await self._never_close.wait() monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.MCP_UPSTREAM_SESSION_SHUTDOWN_TIMEOUT_SECONDS", + "litellm.proxy.mcp_server.mcp_server_manager.MCP_UPSTREAM_SESSION_SHUTDOWN_TIMEOUT_SECONDS", 0.05, ) caplog.set_level(logging.WARNING, logger="LiteLLM") @@ -19776,7 +19780,7 @@ class TestSharedIdentifierPrefixWarning: with ( patch("litellm.proxy.proxy_server.prisma_client", prisma), patch( - "litellm.proxy._experimental.mcp_server.db.MCPServerRepository", + "litellm.proxy.mcp_server.db.MCPServerRepository", return_value=repository, ), patch( @@ -19826,7 +19830,7 @@ async def test_reload_warns_once_about_a_blocked_stdio_row_that_is_rebuilt_every with ( patch("litellm.proxy.proxy_server.prisma_client", prisma), patch( - "litellm.proxy._experimental.mcp_server.db.MCPServerRepository", + "litellm.proxy.mcp_server.db.MCPServerRepository", return_value=repository, ), patch( @@ -20378,7 +20382,7 @@ class TestToolCatalogGuard: @pytest.mark.asyncio @pytest.mark.parametrize("add_prefix", [False, True]) async def test_openapi_catalog_is_scanned_and_pinned_like_an_upstream_listing(self, catalog_guardrail, add_prefix): - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry _, proxy_logging_obj = catalog_guardrail server = MCPServer( @@ -20495,7 +20499,7 @@ async def test_upstream_preparation_preserves_windows_command_and_caller_environ @pytest.mark.asyncio async def test_upstream_preparation_honors_case_sensitive_extra_command(monkeypatch: pytest.MonkeyPatch) -> None: - from litellm.proxy._experimental.mcp_server import upstream + from litellm.proxy.mcp_server import upstream monkeypatch.setenv("LITELLM_ENABLE_MCP_STDIO", "true") monkeypatch.setattr(upstream, "MCP_STDIO_ALLOWED_COMMANDS", frozenset({"CustomRunner"})) @@ -20560,8 +20564,8 @@ async def test_cached_discovery_cannot_authorize_another_servers_local_handler(m from fastapi import HTTPException from mcp.types import Tool - from litellm.proxy._experimental.mcp_server import operations - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server import operations + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry _catalog_database(monkeypatch, AsyncMock(return_value=[]), AsyncMock(return_value=_revision_row(7))) manager = MCPServerManager() @@ -20600,7 +20604,7 @@ async def test_cached_discovery_cannot_authorize_another_servers_local_handler(m def test_discovery_preserves_registered_tool_namespaces(monkeypatch, local_handler): from mcp.types import Tool - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry manager = MCPServerManager() allowed = MCPServer(server_id="allowed", name="allowed", transport=MCPTransport.http) @@ -20644,8 +20648,8 @@ async def test_overlapping_server_prefix_cannot_authorize_registered_openapi_han from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server import operations - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server import operations + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry manager = MCPServerManager() private = MCPServer(server_id="private", name="billing", alias="billing", transport=MCPTransport.http) @@ -20735,8 +20739,8 @@ async def test_catalog_revision_failure_preserves_state_and_recovers(monkeypatch @pytest.mark.asyncio @pytest.mark.parametrize("groups", [[], ["catalog-group"]]) async def test_access_group_resolution_uses_the_operation_catalog(monkeypatch, groups): - from litellm.proxy._experimental.mcp_server import mcp_server_manager - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + from litellm.proxy.mcp_server import mcp_server_manager + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler row = _catalog_row().model_copy(update={"mcp_access_groups": groups}) _catalog_database(monkeypatch, AsyncMock(return_value=[row])) @@ -20771,9 +20775,9 @@ async def test_empty_access_group_scope_cannot_inherit_unrelated_server( ): from types import SimpleNamespace - from litellm.proxy._experimental.mcp_server import mcp_server_manager - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth + from litellm.proxy.mcp_server import mcp_server_manager + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler row = _catalog_row() _catalog_database(monkeypatch, AsyncMock(return_value=[row])) @@ -20935,7 +20939,7 @@ async def test_paginated_listing_keeps_earlier_tool_metadata_and_caller_isolatio from mcp.types import ListToolsResult, PaginatedRequestParams from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import catalog, operations + from litellm.proxy.mcp_server import catalog, operations monkeypatch.setenv("LITELLM_SALT_KEY", "listed-tools-test") monkeypatch.setattr(proxy_server, "prisma_client", None) @@ -21026,9 +21030,9 @@ async def test_aggregate_bounds_each_paginated_server_by_its_own_listing_deadlin from mcp.types import JSONRPCRequest, PaginatedRequestParams from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import catalog, operations - from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module - from litellm.proxy._experimental.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR, get_server_prefix + from litellm.proxy.mcp_server import catalog, operations + from litellm.proxy.mcp_server import mcp_server_manager as manager_module + from litellm.proxy.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR, get_server_prefix monkeypatch.setenv("LITELLM_SALT_KEY", "paginated-listing-deadline-test") monkeypatch.setattr(proxy_server, "prisma_client", None) @@ -21093,7 +21097,7 @@ async def test_failed_aggregate_continuation_preserves_only_delivered_tool_metad from mcp.types import ListToolsResult, PaginatedRequestParams from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import catalog, operations + from litellm.proxy.mcp_server import catalog, operations monkeypatch.setenv("LITELLM_SALT_KEY", "failed-listing-test") monkeypatch.setattr(proxy_server, "prisma_client", None) @@ -21136,7 +21140,7 @@ async def test_aggregate_publishes_complete_bare_routes_only_after_delivering_a_ from mcp.types import ListToolsResult, PaginatedRequestParams from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import catalog, operations + from litellm.proxy.mcp_server import catalog, operations monkeypatch.delenv("LITELLM_SALT_KEY", raising=False) if can_seal: @@ -21202,7 +21206,7 @@ def test_discovery_keys_bind_static_auth_to_caller_and_configuration() -> None: @pytest.mark.parametrize("advertised", [False, True]) def test_untrusted_metadata_cannot_enable_cimd_without_matching_authorization_endpoint(advertised): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( _restrict_discovery_to_corroborated_authorization_server, ) @@ -21224,8 +21228,8 @@ async def test_manual_oauth_endpoints_discover_client_metadata_once( ) -> None: from starlette.requests import Request - from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module - from litellm.proxy._experimental.mcp_server.oauth_utils import get_cimd_client_id + from litellm.proxy.mcp_server import mcp_server_manager as manager_module + from litellm.proxy.mcp_server.oauth_utils import get_cimd_client_id monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") monkeypatch.setenv("LITELLM_MCP_OAUTH_DISCOVERY_ON_STARTUP", "1" if startup else "0") @@ -21347,7 +21351,7 @@ async def test_optional_client_metadata_discovery_failure_preserves_manual_endpo @pytest.mark.parametrize("capability", [True, False]) @pytest.mark.parametrize("rebuild", ["same", "repointed", "fresh_discovery", "anchored"]) def test_oauth_rebuild_retains_only_corroborated_cimd_capability(capability: bool, rebuild: str) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import carry_forward_resolved_oauth_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import carry_forward_resolved_oauth_endpoints previous: Final = MCPServer( server_id="cimd-rebuild", @@ -21437,10 +21441,12 @@ def test_openapi_extra_headers_exclude_gateway_admission_key(header_name: str) - @pytest.mark.asyncio async def test_legacy_upstream_elicitation_rejects_modern_downstream_without_consent() -> None: from types import SimpleNamespace + from mcp.types import ElicitRequestFormParams, ErrorData - from litellm.proxy._experimental.mcp_server import server as legacy_server - from litellm.proxy._experimental.mcp_server.legacy_callbacks import create_elicitation_callback - from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var + + from litellm.proxy.mcp_server import server as legacy_server + from litellm.proxy.mcp_server.legacy_callbacks import create_elicitation_callback + from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var session: Final = SimpleNamespace(client_params=None) session_token: Final = legacy_server.active_mcp_session_var.set(session) @@ -21450,7 +21456,7 @@ async def test_legacy_upstream_elicitation_rejects_modern_downstream_without_con relay: Final = AsyncMock() try: callback: Final = create_elicitation_callback() - with patch("litellm.proxy._experimental.mcp_server.elicitation_handler.handle_elicitation_request", relay): + with patch("litellm.proxy.mcp_server.elicitation_handler.handle_elicitation_request", relay): result: Final = await callback( None, ElicitRequestFormParams(message="Confirm", requested_schema={"type": "object"}) ) @@ -21536,7 +21542,7 @@ class TestMCPUpstreamRateLimitRetry: self, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.MCP_TOOL_LISTING_TIMEOUT", 0.5 + "litellm.proxy.mcp_server.mcp_server_manager.MCP_TOOL_LISTING_TIMEOUT", 0.5 ) manager: Final = MCPServerManager() client: Final = MagicMock() @@ -21579,7 +21585,7 @@ class TestMCPUpstreamRateLimitRetry: self, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.MCP_TOOL_LISTING_TIMEOUT", 0.05 + "litellm.proxy.mcp_server.mcp_server_manager.MCP_TOOL_LISTING_TIMEOUT", 0.05 ) manager: Final = MCPServerManager() server: Final = MCPServer(server_id="notes", name="notes", transport=MCPTransport.http) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py b/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py similarity index 91% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py rename to tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py index a873e2ab1ae..aa61deaf554 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py +++ b/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py @@ -1,4 +1,3 @@ -from litellm.proxy._experimental.mcp_server import operations as mcp_operations import asyncio import contextlib import contextvars @@ -33,14 +32,15 @@ from mcp_types.version import HANDSHAKE_PROTOCOL_VERSIONS, LATEST_HANDSHAKE_VERS from pydantic import JsonValue, TypeAdapter from starlette.types import Message, Receive, Scope, Send -from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var -from litellm.proxy._experimental.mcp_server.mcp_server_manager import ListedToolsCaller -from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing from litellm.proxy._types import ( LiteLLM_MCPServerTable, MCPTransport, UserAPIKeyAuth, ) +from litellm.proxy.mcp_server import operations as mcp_operations +from litellm.proxy.mcp_server.faults.list_outcomes import AggregateToolListing +from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var +from litellm.proxy.mcp_server.mcp_server_manager import ListedToolsCaller from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPOAuthMetadata, MCPServer, PinnedMCPTool @@ -50,7 +50,7 @@ def test_mcp_available_on_sdk2(): from packaging.version import Version - from litellm.proxy._experimental.mcp_server.server import MCP_AVAILABLE + from litellm.proxy.mcp_server.server import MCP_AVAILABLE assert Version("2.2.0") <= Version(version("mcp")) < Version("3") assert MCP_AVAILABLE is True @@ -71,7 +71,7 @@ def cleanup_mcp_global_state(): can leak between tests causing mock assertion failures. """ try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -109,7 +109,7 @@ def _paged_params(): async def test_mcp_server_tool_call_body_contains_request_data(_mcp_request_ctx): """Test that proxy_server_request body contains name and arguments""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( mcp_server_tool_call, set_auth_context, ) @@ -147,7 +147,7 @@ async def test_mcp_server_tool_call_body_contains_request_data(_mcp_request_ctx) mock_add_litellm_data_to_request, ): with patch( - "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool", + "litellm.proxy.mcp_server.operations.call_mcp_tool", mock_call_mcp_tool, ): with patch( @@ -171,7 +171,7 @@ async def test_mcp_server_tool_call_forwards_client_headers_to_logging(_mcp_requ """The MCP protocol path must hand the connection's client headers to the pre-call pipeline, so logging callbacks and guardrails see them the way the REST path does.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( mcp_server_tool_call, set_auth_context, ) @@ -203,7 +203,7 @@ async def test_mcp_server_tool_call_forwards_client_headers_to_logging(_mcp_requ mock_add_litellm_data_to_request, ): with patch( - "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool", + "litellm.proxy.mcp_server.operations.call_mcp_tool", mock_call_mcp_tool, ): with patch("litellm.proxy.proxy_server.proxy_config", MagicMock()): @@ -221,11 +221,11 @@ async def test_mcp_server_tool_call_strips_custom_litellm_key_header(_mcp_reques The pre-call pipeline only knows that name if it is passed in, so without it the virtual key reaches metadata.headers and proxy_server_request.headers in plaintext.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + from litellm.proxy.mcp_server.server import ( mcp_server_tool_call, set_auth_context, ) - from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request except ImportError: pytest.skip("MCP server not available") @@ -250,7 +250,7 @@ async def test_mcp_server_tool_call_strips_custom_litellm_key_header(_mcp_reques capturing_add_litellm_data_to_request, ): with patch( - "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool", + "litellm.proxy.mcp_server.operations.call_mcp_tool", mock_call_mcp_tool, ): with patch("litellm.proxy.proxy_server.proxy_config", MagicMock()): @@ -274,7 +274,7 @@ async def test_mcp_server_tool_call_relays_upstream_auth_error_as_iserror( ): from starlette.requests import Request - from litellm.proxy._experimental.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse + from litellm.proxy.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse status_code = 403 if required_scope is not None else 401 send = AsyncMock() @@ -282,8 +282,8 @@ async def test_mcp_server_tool_call_relays_upstream_auth_error_as_iserror( request = Request({"type": "http", "headers": [], SCOPE_RESPONSE_KEY: response}) headers: Final = asyncio.create_task(response({"type": "http.response.start", "status": 200, "headers": []})) try: - from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.exceptions import MCPUpstreamAuthError + from litellm.proxy.mcp_server.server import ( mcp_server_tool_call, set_auth_context, ) @@ -306,11 +306,11 @@ async def test_mcp_server_tool_call_relays_upstream_auth_error_as_iserror( mock_add_litellm_data_to_request, ): with patch( - "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool", + "litellm.proxy.mcp_server.operations.call_mcp_tool", mock_call_mcp_tool, ): with patch("litellm.proxy.proxy_server.proxy_config", MagicMock()): - with patch("litellm.proxy._experimental.mcp_server.operations.verbose_logger", mock_logger): + with patch("litellm.proxy.mcp_server.operations.verbose_logger", mock_logger): result = await mcp_server_tool_call( _mcp_request_ctx(request=request if http_transport else None), _call_tool_params("test_tool", {"param": "value"}), @@ -338,7 +338,7 @@ async def test_mcp_server_tool_call_relays_upstream_auth_error_as_iserror( def test_prepare_mcp_server_headers_case_insensitive_extra_headers(): try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _prepare_mcp_server_headers, ) except ImportError: @@ -368,7 +368,7 @@ def test_prepare_mcp_server_headers_case_insensitive_extra_headers(): def test_prepare_mcp_server_headers_group_header_defaults_for_members_only(): try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _prepare_mcp_server_headers, ) except ImportError: @@ -405,7 +405,7 @@ def test_prepare_mcp_server_headers_group_header_defaults_for_members_only(): def test_prepare_mcp_server_headers_passthrough_strips_authorization_without_admission_header(): try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _prepare_mcp_server_headers, ) except ImportError: @@ -441,7 +441,7 @@ def test_prepare_mcp_server_headers_passthrough_forwards_authorization_for_anony (``user_api_key_auth.api_key is None``) and the ``Authorization`` bearer is the upstream OAuth token — it must be forwarded, not stripped.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _prepare_mcp_server_headers, ) except ImportError: @@ -483,7 +483,7 @@ def test_prepare_mcp_server_headers_passthrough_strips_authorization_for_authent the bearer must still be stripped to avoid leaking the gateway key upstream.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _prepare_mcp_server_headers, ) except ImportError: @@ -519,7 +519,7 @@ def test_prepare_mcp_server_headers_passthrough_strips_authorization_for_authent def test_prepare_mcp_server_headers_oauth2_m2m_omits_litellm_caller_authorization(): """M2M OAuth must not put caller Bearer (LiteLLM API key) into extra_headers (#23652).""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _prepare_mcp_server_headers, ) except ImportError: @@ -553,7 +553,7 @@ def test_prepare_mcp_server_headers_oauth2_interactive_drops_caller_authorizatio caller-supplied bearer must not override another user's stored credential. Non-auth headers are still carried; only the credential is dropped.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _prepare_mcp_server_headers, ) except ImportError: @@ -583,10 +583,10 @@ def test_prepare_mcp_server_headers_oauth2_interactive_drops_caller_authorizatio def test_prepare_mcp_server_headers_legacy_delegate_strips_admission_authorization(): - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _prepare_mcp_server_headers, ) - from litellm.proxy._types import UserAPIKeyAuth server = MCPServer( server_id="legacy-delegate-admission", @@ -610,10 +610,10 @@ def test_prepare_mcp_server_headers_legacy_delegate_strips_admission_authorizati def test_prepare_mcp_server_headers_legacy_delegate_preserves_separate_upstream_authorization(): - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _prepare_mcp_server_headers, ) - from litellm.proxy._types import UserAPIKeyAuth server = MCPServer( server_id="legacy-delegate-dual-credential", @@ -640,10 +640,10 @@ def test_prepare_mcp_server_headers_legacy_delegate_preserves_separate_upstream_ def test_prepare_mcp_server_headers_legacy_delegate_strips_repeated_admission_key(): - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _prepare_mcp_server_headers, ) - from litellm.proxy._types import UserAPIKeyAuth server = MCPServer( server_id="legacy-delegate-repeated-key", @@ -672,7 +672,7 @@ def test_prepare_mcp_server_headers_legacy_delegate_strips_repeated_admission_ke def test_prepare_mcp_server_headers_m2m_skips_authorization_from_raw_extra_headers(): """M2M must not merge caller Authorization from raw_headers when extra_headers lists it.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _prepare_mcp_server_headers, ) except ImportError: @@ -715,7 +715,7 @@ def _client_forwarded_mode_server(server_id: str, auth_type) -> MCPServer: def _prepare_headers_in_scope(server: MCPServer, scope_servers): - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _prepare_mcp_server_headers, ) @@ -825,7 +825,7 @@ def test_prepare_mcp_server_headers_sole_consumer_still_forwards_via_extra_heade async def test_call_tool_m2m_skips_authorization_headers(): """M2M call_tool must not forward caller Authorization in oauth2/raw headers.""" try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) except ImportError: @@ -870,7 +870,7 @@ async def test_call_tool_m2m_skips_authorization_headers(): @pytest.mark.asyncio async def test_get_prompts_from_mcp_servers_success(): try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_prompts_from_mcp_servers, ) except ImportError: @@ -896,15 +896,15 @@ async def test_get_prompts_from_mcp_servers_success(): with ( patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", AsyncMock(return_value=[server_a, server_b]), ) as mock_allowed, patch( - "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers", + "litellm.proxy.mcp_server.operations._prepare_mcp_server_headers", return_value=(None, None), ) as mock_headers, patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, ): mock_manager.get_prompts_from_server = AsyncMock( @@ -930,7 +930,7 @@ async def test_get_prompts_from_mcp_servers_success(): @pytest.mark.asyncio async def test_get_resources_from_mcp_servers_success(): try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_resources_from_mcp_servers, ) except ImportError: @@ -956,15 +956,15 @@ async def test_get_resources_from_mcp_servers_success(): with ( patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", AsyncMock(return_value=[server_a, server_b]), ) as mock_allowed, patch( - "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers", + "litellm.proxy.mcp_server.operations._prepare_mcp_server_headers", return_value=(None, None), ) as mock_headers, patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, ): mock_manager.get_resources_from_server = AsyncMock( @@ -1003,7 +1003,7 @@ async def test_get_resources_from_mcp_servers_success(): @pytest.mark.asyncio async def test_get_resource_templates_from_mcp_servers_success(): try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_resource_templates_from_mcp_servers, ) except ImportError: @@ -1021,15 +1021,15 @@ async def test_get_resource_templates_from_mcp_servers_success(): with ( patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", AsyncMock(return_value=[server]), ) as mock_allowed, patch( - "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers", + "litellm.proxy.mcp_server.operations._prepare_mcp_server_headers", return_value=(None, None), ) as mock_headers, patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, ): mock_manager.get_resource_templates_from_server = AsyncMock( @@ -1058,7 +1058,7 @@ async def test_get_resource_templates_from_mcp_servers_success(): @pytest.mark.asyncio async def test_mcp_get_prompt_success(): try: - from litellm.proxy._experimental.mcp_server.server import mcp_get_prompt + from litellm.proxy.mcp_server.server import mcp_get_prompt except ImportError: pytest.skip("MCP server not available") @@ -1071,15 +1071,15 @@ async def test_mcp_get_prompt_success(): with ( patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", AsyncMock(return_value=[server]), ) as mock_allowed, patch( - "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers", + "litellm.proxy.mcp_server.operations._prepare_mcp_server_headers", return_value=({"Authorization": "token"}, {"X-Test": "1"}), ) as mock_headers, patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, ): mock_manager.get_prompt_from_server = AsyncMock(return_value=prompt_result) @@ -1118,7 +1118,7 @@ async def test_mcp_get_prompt_success(): @pytest.mark.asyncio async def test_mcp_read_resource_success(): try: - from litellm.proxy._experimental.mcp_server.server import mcp_read_resource + from litellm.proxy.mcp_server.server import mcp_read_resource except ImportError: pytest.skip("MCP server not available") @@ -1139,15 +1139,15 @@ async def test_mcp_read_resource_success(): with ( patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", AsyncMock(return_value=[server]), ) as mock_allowed, patch( - "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers", + "litellm.proxy.mcp_server.operations._prepare_mcp_server_headers", return_value=({"Authorization": "token"}, {"X-Test": "1"}), ) as mock_headers, patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, ): mock_manager.read_resource_from_server = AsyncMock(return_value=read_result) @@ -1188,7 +1188,8 @@ async def test_mcp_read_resource_success(): ) async def test_read_resource_preserves_content_metadata(_mcp_request_ctx, kind, metadata): from mcp.types import ReadResourceRequestParams - from litellm.proxy._experimental.mcp_server import operations, server + + from litellm.proxy.mcp_server import operations, server uri: Final = "https://example.com/resource" caller: Final = UserAPIKeyAuth(user_id="resource-caller") @@ -1231,7 +1232,7 @@ async def test_read_resource_preserves_content_metadata(_mcp_request_ctx, kind, @pytest.mark.asyncio async def test_mcp_read_resource_multiple_servers_error(): try: - from litellm.proxy._experimental.mcp_server.server import mcp_read_resource + from litellm.proxy.mcp_server.server import mcp_read_resource except ImportError: pytest.skip("MCP server not available") @@ -1243,7 +1244,7 @@ async def test_mcp_read_resource_multiple_servers_error(): server_b.name = "server_b" with patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", AsyncMock(return_value=[server_a, server_b]), ) as mock_allowed: with pytest.raises(HTTPException) as exc_info: @@ -1261,7 +1262,7 @@ async def test_mcp_read_resource_multiple_servers_error(): async def test_get_tools_from_mcp_servers_continues_when_one_server_fails(): """Test that _get_tools_from_mcp_servers continues when one server fails""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_tools_from_mcp_servers, set_auth_context, ) @@ -1333,14 +1334,14 @@ async def test_get_tools_from_mcp_servers_continues_when_one_server_fails(): mock_manager.get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", mock_manager, ): with ( patch( - "litellm.proxy._experimental.mcp_server.catalog.verbose_logger", + "litellm.proxy.mcp_server.catalog.verbose_logger", ) as mock_logger, - patch("litellm.proxy._experimental.mcp_server.operations.verbose_logger", mock_logger), + patch("litellm.proxy.mcp_server.operations.verbose_logger", mock_logger), ): # Test with server-specific auth headers mcp_server_auth_headers = { @@ -1374,7 +1375,7 @@ async def test_get_tools_from_mcp_servers_continues_when_one_server_fails(): async def test_get_tools_from_mcp_servers_handles_all_servers_failing(): """Test that _get_tools_from_mcp_servers handles all servers failing gracefully""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_tools_from_mcp_servers, set_auth_context, ) @@ -1432,14 +1433,14 @@ async def test_get_tools_from_mcp_servers_handles_all_servers_failing(): mock_manager.get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", mock_manager, ): with ( patch( - "litellm.proxy._experimental.mcp_server.catalog.verbose_logger", + "litellm.proxy.mcp_server.catalog.verbose_logger", ) as mock_logger, - patch("litellm.proxy._experimental.mcp_server.operations.verbose_logger", mock_logger), + patch("litellm.proxy.mcp_server.operations.verbose_logger", mock_logger), ): # Test with server-specific auth headers mcp_server_auth_headers = { @@ -1505,15 +1506,15 @@ async def _denied_scoped_list( mock_manager: MagicMock, resolver: AsyncMock, ) -> HTTPException: - from litellm.proxy._experimental.mcp_server.server import _get_tools_from_mcp_servers + from litellm.proxy.mcp_server.server import _get_tools_from_mcp_servers with ( patch( # test-quality-ok: the permission resolver is a module-level function; the suite's only seam - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", resolver, ), patch( # test-quality-ok: the server registry is a module-level singleton; the suite's only seam - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", mock_manager, ), ): @@ -1529,7 +1530,7 @@ async def _denied_scoped_list( @pytest.mark.asyncio async def test_scoped_list_denied_by_agent_binding_raises_403_naming_agent(): """The agent-binding veto must raise a 403 naming the agent, never a silent 200 with no tools.""" - pytest.importorskip("litellm.proxy._experimental.mcp_server.server") + pytest.importorskip("litellm.proxy.mcp_server.server") user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user", agent_id="agent-123") resolver = _scope_resolver(resolved_without_agent={"github": "srv-github"}) @@ -1552,19 +1553,19 @@ async def test_scoped_list_denied_by_agent_binding_raises_403_naming_agent(): @pytest.mark.asyncio async def test_empty_scope_lists_nothing_instead_of_raising_a_nameless_denial(): """An empty ``x-mcp-servers`` header scopes to no servers; that is an empty listing, not a 403.""" - pytest.importorskip("litellm.proxy._experimental.mcp_server.server") - from litellm.proxy._experimental.mcp_server.server import _get_tools_from_mcp_servers + pytest.importorskip("litellm.proxy.mcp_server.server") + from litellm.proxy.mcp_server.server import _get_tools_from_mcp_servers user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user") resolver = AsyncMock(return_value=[]) with ( patch( # test-quality-ok: the permission resolver is a module-level function; the suite's only seam - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", resolver, ), patch( # test-quality-ok: the server registry is a module-level singleton; the suite's only seam - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", _denied_scope_manager({"github": "srv-github"}), ), ): @@ -1579,7 +1580,7 @@ async def test_empty_scope_lists_nothing_instead_of_raising_a_nameless_denial(): @pytest.mark.asyncio async def test_scoped_list_denied_for_non_agent_key_raises_generic_403(): """A denial for a key with no agent binding stays generic and skips the agent-stripped rerun.""" - pytest.importorskip("litellm.proxy._experimental.mcp_server.server") + pytest.importorskip("litellm.proxy.mcp_server.server") user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user") resolver = AsyncMock(return_value=[]) @@ -1599,7 +1600,7 @@ async def test_scoped_list_denied_for_non_agent_key_raises_generic_403(): async def test_scoped_list_unknown_name_raises_same_generic_403_as_unauthorized(): """Unknown and registered-but-unauthorized names raise byte-identical generic 403s, so a caller cannot probe which server names exist.""" - pytest.importorskip("litellm.proxy._experimental.mcp_server.server") + pytest.importorskip("litellm.proxy.mcp_server.server") user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user", agent_id="agent-123") @@ -1623,7 +1624,7 @@ async def test_scoped_list_unknown_name_raises_same_generic_403_as_unauthorized( async def test_scoped_list_access_group_vetoed_by_agent_names_agent_and_group(): """An access-group scope vetoed by the agent binding raises the 403 naming the agent and the group instead of the silent empty list.""" - pytest.importorskip("litellm.proxy._experimental.mcp_server.server") + pytest.importorskip("litellm.proxy.mcp_server.server") user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user", agent_id="agent-123") @@ -1646,7 +1647,7 @@ async def test_scoped_list_mixed_unknown_and_vetoed_group_names_the_group_that_r """With an unknown name ahead of the agent-vetoed group in the scope, the 403 must name the group whose servers the key can reach, never the unknown name, or the admin is told to grant a group that does not exist.""" - pytest.importorskip("litellm.proxy._experimental.mcp_server.server") + pytest.importorskip("litellm.proxy.mcp_server.server") user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user", agent_id="agent-123") @@ -1668,7 +1669,7 @@ async def test_scoped_list_mixed_unknown_and_vetoed_group_names_the_group_that_r async def test_scoped_list_agent_key_denied_by_key_grants_raises_generic_403(): """When the agent-stripped rerun still resolves nothing, the 403 stays generic instead of blaming the agent binding.""" - pytest.importorskip("litellm.proxy._experimental.mcp_server.server") + pytest.importorskip("litellm.proxy.mcp_server.server") user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user", agent_id="agent-123") resolver = _scope_resolver(resolved_without_agent={}) @@ -1688,7 +1689,7 @@ async def test_scoped_list_agent_key_denied_by_key_grants_raises_generic_403(): async def test_scoped_list_agent_veto_attributed_for_differently_cased_server_name(): """The scope filter matches `/mcp/GitHub` to a server named `github` case-insensitively, so the agent-attributed 403 must match the same way instead of falling back to the generic denial.""" - pytest.importorskip("litellm.proxy._experimental.mcp_server.server") + pytest.importorskip("litellm.proxy.mcp_server.server") user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user", agent_id="agent-123") @@ -1711,7 +1712,7 @@ async def test_handle_list_tools_converts_permission_httpexception_to_mcp_error( """The MCP protocol handler surfaces a permission HTTPException as a clean JSON-RPC error (MCPError, INVALID_REQUEST) carrying the denial message, instead of a raw 500.""" try: - from litellm.proxy._experimental.mcp_server.server import handle_list_tools + from litellm.proxy.mcp_server.server import handle_list_tools except ImportError: pytest.skip("MCP server not available") @@ -1723,13 +1724,13 @@ async def test_handle_list_tools_converts_permission_httpexception_to_mcp_error( with ( patch( # test-quality-ok: the protocol handler reads auth from module context; no injection seam - "litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context", + "litellm.proxy.mcp_server.server.get_or_extract_auth_context", new=AsyncMock( return_value=(None, None, None, None, None, None, None), side_effect=denial if denial_at_auth else None ), ), patch( # test-quality-ok: the listing helper is the handler's only collaborator; the suite's seam - "litellm.proxy._experimental.mcp_server.operations._list_mcp_tools", + "litellm.proxy.mcp_server.operations._list_mcp_tools", new=AsyncMock(side_effect=denial), ), ): @@ -1793,7 +1794,7 @@ async def test_rate_limited_catalog_lists_return_mcp_errors(handler_name): @pytest.mark.asyncio async def test_mcp_server_tool_call_renders_denial_message_not_detail_dict(_mcp_request_ctx): try: - from litellm.proxy._experimental.mcp_server.server import mcp_server_tool_call + from litellm.proxy.mcp_server.server import mcp_server_tool_call except ImportError: pytest.skip("MCP server not available") @@ -1802,11 +1803,11 @@ async def test_mcp_server_tool_call_renders_denial_message_not_detail_dict(_mcp_ with ( patch( # test-quality-ok: the protocol handler reads auth from module context; no injection seam - "litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context", + "litellm.proxy.mcp_server.server.get_or_extract_auth_context", new=AsyncMock(return_value=(None, None, None, None, None, None, None)), ), patch( # test-quality-ok: the tool-call helper is the handler's only collaborator; the suite's seam - "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool", + "litellm.proxy.mcp_server.operations.call_mcp_tool", new=AsyncMock(side_effect=denial), ), ): @@ -1820,7 +1821,7 @@ async def test_mcp_server_tool_call_renders_denial_message_not_detail_dict(_mcp_ async def test_mcp_server_tool_call_body_with_none_arguments(_mcp_request_ctx): """Test that proxy_server_request body handles None arguments correctly""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( mcp_server_tool_call, set_auth_context, ) @@ -1857,7 +1858,7 @@ async def test_mcp_server_tool_call_body_with_none_arguments(_mcp_request_ctx): mock_add_litellm_data_to_request, ): with patch( - "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool", + "litellm.proxy.mcp_server.operations.call_mcp_tool", mock_call_mcp_tool, ): with patch( @@ -1880,14 +1881,14 @@ async def test_mcp_server_tool_call_body_with_none_arguments(_mcp_request_ctx): async def test_concurrent_initialize_session_managers(): """Test that concurrent calls to initialize_session_managers don't cause race conditions.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( initialize_session_managers, ) except ImportError: pytest.skip("MCP server not available") # Import the module to reset state - import litellm.proxy._experimental.mcp_server.server as mcp_server + import litellm.proxy.mcp_server.server as mcp_server # Reset state before test original_initialized = mcp_server._SESSION_MANAGERS_INITIALIZED @@ -1920,7 +1921,7 @@ async def test_concurrent_initialize_session_managers(): "run", return_value=mock_cm_stateful, ) as mock_stateful_run, - patch("litellm.proxy._experimental.mcp_server.operations.verbose_logger"), + patch("litellm.proxy.mcp_server.operations.verbose_logger"), ): # Create multiple concurrent tasks that call initialize_session_managers async def init_task(): @@ -1981,7 +1982,7 @@ async def test_streamable_http_session_manager_is_stateless(): - stateful manager for clients with session IDs (Claude Code, Cursor, VSCode) """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( session_manager_stateful, session_manager_stateless, ) @@ -2021,9 +2022,9 @@ async def test_mcp_routing_initialize_to_stateful_no_session_to_stateless( from starlette.requests import Request from starlette.types import Message, Receive, Scope, Send - from litellm.proxy._experimental.mcp_server import server as mcp_module - from litellm.proxy._experimental.mcp_server.mcp_debug import record_auth_resolution - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import AuthResolution + from litellm.proxy.mcp_server import server as mcp_module + from litellm.proxy.mcp_server.mcp_debug import record_auth_resolution + from litellm.proxy.mcp_server.outbound_credentials.types import AuthResolution scope: Final[Scope] = {"type": "http", "method": method, "path": "/mcp", "headers": []} receive: Final = AsyncMock(return_value={"type": "http.request", "body": request_body, "more_body": False}) @@ -2046,7 +2047,7 @@ async def test_mcp_routing_initialize_to_stateful_no_session_to_stateless( stateful_handle: Final = AsyncMock(side_effect=handle_request) with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=( UserAPIKeyAuth(user_id="debug-user"), @@ -2057,13 +2058,13 @@ async def test_mcp_routing_initialize_to_stateful_no_session_to_stateless( {"x-litellm-mcp-debug": "true"} if debug else {}, ), ), - patch("litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True), + patch("litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True), patch( - "litellm.proxy._experimental.mcp_server.server.session_manager_stateless", + "litellm.proxy.mcp_server.server.session_manager_stateless", SimpleNamespace(handle_request=stateless_handle), ), patch( - "litellm.proxy._experimental.mcp_server.server.session_manager_stateful", + "litellm.proxy.mcp_server.server.session_manager_stateful", SimpleNamespace(handle_request=stateful_handle), ), ): @@ -2114,14 +2115,14 @@ def _client_allowlist_patches( stack: Final = contextlib.ExitStack() stack.enter_context( patch( # test-quality-ok: the ASGI handler resolves auth through a module-level function; no injection seam - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(UserAPIKeyAuth(user_id="allowlist-user", jwt_claims=jwt_claims), None, None, None, None, {}), ) ) stack.enter_context( patch( # test-quality-ok: module flag guarding lazy session-manager startup; no injection seam - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True ) ) stack.enter_context( @@ -2154,7 +2155,7 @@ async def test_streamable_http_rejects_unlisted_client_before_any_session_work( request_body: bytes, expected_fragment: str, ) -> None: - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module scope: Final[Scope] = {"type": "http", "method": "POST", "path": "/mcp", "headers": headers} receive: Final = AsyncMock(return_value={"type": "http.request", "body": request_body, "more_body": False}) @@ -2166,15 +2167,15 @@ async def test_streamable_http_rejects_unlisted_client_before_any_session_work( with ( _client_allowlist_patches(_ALLOWLIST_SETTINGS, jwt_claims), patch( # test-quality-ok: session managers are module singletons; the downstream call is the observable - "litellm.proxy._experimental.mcp_server.server.session_manager_stateful", + "litellm.proxy.mcp_server.server.session_manager_stateful", SimpleNamespace(handle_request=stateful_handle), ), patch( # test-quality-ok: session managers are module singletons; the downstream call is the observable - "litellm.proxy._experimental.mcp_server.server.session_manager_stateless", + "litellm.proxy.mcp_server.server.session_manager_stateless", SimpleNamespace(handle_request=stateless_handle), ), patch( # test-quality-ok: module-level cap check; asserting it is never reached is the point - "litellm.proxy._experimental.mcp_server.server._enforce_stateful_session_cap_for_owner", session_cap + "litellm.proxy.mcp_server.server._enforce_stateful_session_cap_for_owner", session_cap ), pytest.raises(HTTPException) as denied, ): @@ -2206,7 +2207,7 @@ async def test_streamable_http_rejects_unlisted_client_before_any_session_work( async def test_streamable_http_admits_listed_or_unrestricted_clients_and_hands_the_body_downstream( settings: dict[str, object], jwt_claims: dict[str, object] | None, headers: list[tuple[bytes, bytes]] ) -> None: - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module scope: Final[Scope] = {"type": "http", "method": "POST", "path": "/mcp", "headers": headers} receive: Final = AsyncMock( @@ -2227,11 +2228,11 @@ async def test_streamable_http_admits_listed_or_unrestricted_clients_and_hands_t with ( _client_allowlist_patches(settings, jwt_claims), patch( # test-quality-ok: session managers are module singletons; the downstream call is the observable - "litellm.proxy._experimental.mcp_server.server.session_manager_stateful", + "litellm.proxy.mcp_server.server.session_manager_stateful", SimpleNamespace(handle_request=stateful_handle), ), patch( # test-quality-ok: session managers are module singletons; the downstream call is the observable - "litellm.proxy._experimental.mcp_server.server.session_manager_stateless", + "litellm.proxy.mcp_server.server.session_manager_stateless", SimpleNamespace(handle_request=stateless_handle), ), ): @@ -2256,7 +2257,7 @@ async def test_streamable_http_admits_listed_or_unrestricted_clients_and_hands_t async def test_sse_endpoint_applies_the_same_client_allowlist( jwt_claims: dict[str, object] | None, headers: list[tuple[bytes, bytes]], admitted: bool ) -> None: - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module scope: Final[Scope] = {"type": "http", "method": "POST", "path": "/mcp/sse", "headers": headers} receive: Final = AsyncMock(return_value={"type": "http.request", "body": _INITIALIZE, "more_body": False}) @@ -2269,11 +2270,11 @@ async def test_sse_endpoint_applies_the_same_client_allowlist( with ( _client_allowlist_patches(_ALLOWLIST_SETTINGS, jwt_claims), patch( # test-quality-ok: module-level pre-auth probe unrelated to the allowlist under test; no injection seam - "litellm.proxy._experimental.mcp_server.server._raise_preemptive_401_for_unauthenticated_servers", + "litellm.proxy.mcp_server.server._raise_preemptive_401_for_unauthenticated_servers", new_callable=AsyncMock, ), patch( # test-quality-ok: module-level upstream auth probe unrelated to the allowlist under test; no injection seam - "litellm.proxy._experimental.mcp_server.server._check_passthrough_upstream_auth", + "litellm.proxy.mcp_server.server._check_passthrough_upstream_auth", new_callable=AsyncMock, ), patch.object( # test-quality-ok: SSE manager is a module singleton; the downstream call is the observable @@ -2302,7 +2303,7 @@ async def test_mcp_routing_chunked_initialize_to_stateful(): Test that chunked initialize requests route to the stateful manager. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, session_manager_stateless, @@ -2344,20 +2345,20 @@ async def test_mcp_routing_chunked_initialize_to_stateful(): with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(UserAPIKeyAuth(), None, ["progress_test"], None, None, None), ), patch( - "litellm.proxy._experimental.mcp_server.server.set_auth_context", + "litellm.proxy.mcp_server.server.set_auth_context", ), patch( # test-quality-ok: registry is empty in unit tests; key owns one server - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[MagicMock()], ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object( @@ -2398,8 +2399,8 @@ async def test_mcp_routing_caps_body_peek_for_oversized_chunked_body(): receive into the downstream handler. """ try: - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, session_manager_stateless, @@ -2456,13 +2457,13 @@ async def test_mcp_routing_caps_body_peek_for_oversized_chunked_body(): with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(UserAPIKeyAuth(), None, ["progress_test"], None, None, None), ), - patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), + patch("litellm.proxy.mcp_server.server.set_auth_context"), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object(session_manager_stateless, "handle_request", side_effect=stateless_handle), @@ -2496,8 +2497,8 @@ async def test_mcp_routing_peek_survives_multibyte_char_split_at_cap( ) -> None: from starlette.requests import Request - from litellm.proxy._experimental.mcp_server import server as mcp_module - from litellm.proxy._experimental.mcp_server.scope_challenge import finish_scope_response + from litellm.proxy.mcp_server import server as mcp_module + from litellm.proxy.mcp_server.scope_challenge import finish_scope_response params: Final = ( { @@ -2538,11 +2539,11 @@ async def test_mcp_routing_peek_survives_multibyte_char_split_at_cap( with ( _client_allowlist_patches({}, None), patch( - "litellm.proxy._experimental.mcp_server.server.session_manager_stateless", + "litellm.proxy.mcp_server.server.session_manager_stateless", SimpleNamespace(handle_request=stateless_handle), ), patch( - "litellm.proxy._experimental.mcp_server.server.session_manager_stateful", + "litellm.proxy.mcp_server.server.session_manager_stateful", SimpleNamespace(handle_request=stateful_handle), ), ): @@ -2562,8 +2563,8 @@ async def test_enforce_stateful_session_cap_evicts_oldest_idle_then_rejects(): every one of its sessions is in flight (nothing safe to evict). """ try: - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import ( session_manager_stateful, ) except ImportError: @@ -2635,8 +2636,8 @@ async def test_mcp_routing_initialize_rejected_when_owner_at_session_cap(): and must not reach the stateful session manager. """ try: - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, session_manager_stateless, @@ -2675,13 +2676,13 @@ async def test_mcp_routing_initialize_rejected_when_owner_at_session_cap(): with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(UserAPIKeyAuth(), None, ["progress_test"], None, None, None), ), - patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), + patch("litellm.proxy.mcp_server.server.set_auth_context"), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object(mcp_server, "_owner_fingerprint_for", return_value="owner-X"), @@ -2715,8 +2716,8 @@ async def test_stateful_mcp_requests_refresh_session_auth_context(): stored auth object must be refreshed for each mcp-session-id request. """ try: - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import ( get_auth_context, handle_streamable_http_mcp, session_manager_stateful, @@ -2767,7 +2768,7 @@ async def test_stateful_mcp_requests_refresh_session_auth_context(): with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=( current_auth, @@ -2779,7 +2780,7 @@ async def test_stateful_mcp_requests_refresh_session_auth_context(): ), ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object( @@ -2810,7 +2811,7 @@ async def test_stateful_mcp_requests_refresh_session_auth_context(): @pytest.mark.asyncio async def test_initialize_response_capture_accepts_str_headers_and_sets_auth_context(): try: - from litellm.proxy._experimental.mcp_server import server as mcp_server + from litellm.proxy.mcp_server import server as mcp_server except ImportError: pytest.skip("MCP server not available") @@ -2858,8 +2859,8 @@ async def test_initialize_response_capture_accepts_str_headers_and_sets_auth_con @pytest.mark.asyncio async def test_initialize_request_tracks_active_session_after_response_header(): try: - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, session_manager_stateless, @@ -2907,17 +2908,17 @@ async def test_initialize_request_tracks_active_session_after_response_header(): try: with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(owner_auth, None, None, None, None, None), ), patch( # test-quality-ok: registry is empty in unit tests; key owns one server - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[MagicMock()], ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object( @@ -2960,7 +2961,7 @@ _INITIALIZE_WITH_CLIENT_INFO: Final = ( ) def test_extract_initialize_client_info_reads_client_name_and_version(body, expected_name, expected_version): try: - from litellm.proxy._experimental.mcp_server import server as mcp_server + from litellm.proxy.mcp_server import server as mcp_server except ImportError: pytest.skip("MCP server not available") @@ -2982,7 +2983,7 @@ def test_extract_initialize_client_info_reads_client_name_and_version(body, expe ) def test_extract_initialize_client_info_returns_none_without_client_info(body): try: - from litellm.proxy._experimental.mcp_server import server as mcp_server + from litellm.proxy.mcp_server import server as mcp_server except ImportError: pytest.skip("MCP server not available") @@ -2994,7 +2995,7 @@ def test_oversized_initialize_peek_neither_routes_stateful_nor_attributes_client an initialize larger than the peek can never become a tracked session that then reports an unknown client.""" try: - from litellm.proxy._experimental.mcp_server import server as mcp_server + from litellm.proxy.mcp_server import server as mcp_server except ImportError: pytest.skip("MCP server not available") @@ -3016,8 +3017,8 @@ async def test_initialize_request_records_client_name_in_gateway_sessions_report """The real initialize body's clientInfo is attributed to the session the stateful manager creates, together with the authenticated user.""" try: - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, session_manager_stateless, @@ -3060,17 +3061,17 @@ async def test_initialize_request_records_client_name_in_gateway_sessions_report try: with ( patch( # test-quality-ok: admission auth is resolved by a module-level function; the suite's only seam - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(owner_auth, None, None, None, None, None), ), patch( # test-quality-ok: registry is empty in unit tests; key owns one server - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[MagicMock()], ), patch( # test-quality-ok: session manager init is a module-level flag; the suite's only seam - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object( # test-quality-ok: the transports are module-level singletons; the suite's only seam @@ -3117,8 +3118,8 @@ async def test_initialize_request_records_client_name_in_gateway_sessions_report def test_gateway_sessions_report_groups_live_sessions_by_client_and_user(): try: - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import session_manager_stateful + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import session_manager_stateful except ImportError: pytest.skip("MCP server not available") from mcp.types import Implementation @@ -3187,7 +3188,7 @@ def test_gateway_sessions_report_groups_live_sessions_by_client_and_user(): def test_remove_stateful_session_tracking_drops_client_info(): try: - from litellm.proxy._experimental.mcp_server import server as mcp_server + from litellm.proxy.mcp_server import server as mcp_server except ImportError: pytest.skip("MCP server not available") from mcp.types import Implementation @@ -3225,8 +3226,8 @@ def _admin_terminate_fixture(mcp_server): @pytest.mark.asyncio async def test_terminate_mcp_gateway_sessions_by_user_closes_every_live_session_of_that_user(): try: - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import session_manager_stateful + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import session_manager_stateful except ImportError: pytest.skip("MCP server not available") @@ -3284,8 +3285,8 @@ async def test_terminate_mcp_gateway_sessions_by_user_closes_every_live_session_ @pytest.mark.asyncio async def test_terminate_mcp_gateway_sessions_prefix_and_user_must_both_match(): try: - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import session_manager_stateful + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import session_manager_stateful except ImportError: pytest.skip("MCP server not available") @@ -3322,8 +3323,8 @@ async def test_admin_terminated_session_id_gets_404_instead_of_a_fresh_stateless try: from starlette.types import Scope - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import session_manager_stateful + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import session_manager_stateful except ImportError: pytest.skip("MCP server not available") @@ -3388,8 +3389,8 @@ async def test_admin_terminated_session_id_stays_refused_while_replayed_and_is_f try: from starlette.types import Scope - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import session_manager_stateful + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import session_manager_stateful except ImportError: pytest.skip("MCP server not available") @@ -3454,8 +3455,8 @@ async def test_admin_terminated_session_id_stays_refused_while_replayed_and_is_f @pytest.mark.asyncio async def test_initialize_request_with_existing_session_tracks_new_session(): try: - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, session_manager_stateless, @@ -3528,7 +3529,7 @@ async def test_initialize_request_with_existing_session_tracks_new_session(): with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=( owner_auth, @@ -3540,7 +3541,7 @@ async def test_initialize_request_with_existing_session_tracks_new_session(): ), ), patch( # test-quality-ok: registry is empty in unit tests; key owns one server - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[ MCPServer( @@ -3553,7 +3554,7 @@ async def test_initialize_request_with_existing_session_tracks_new_session(): ], ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object( @@ -3589,7 +3590,7 @@ async def test_initialize_request_with_existing_session_tracks_new_session(): async def test_stateful_mcp_auth_contexts_expire_with_idle_sessions(): """Expired session auth contexts should not remain in memory indefinitely.""" try: - from litellm.proxy._experimental.mcp_server import server as mcp_server + from litellm.proxy.mcp_server import server as mcp_server except ImportError: pytest.skip("MCP server not available") @@ -3620,7 +3621,7 @@ async def test_stateful_mcp_auth_contexts_expire_with_idle_sessions(): async def test_stateful_mcp_auth_contexts_do_not_expire_active_sessions(): """Active stateful sessions should not be terminated by idle cleanup.""" try: - from litellm.proxy._experimental.mcp_server import server as mcp_server + from litellm.proxy.mcp_server import server as mcp_server except ImportError: pytest.skip("MCP server not available") @@ -3657,7 +3658,7 @@ async def test_stateful_mcp_auth_contexts_do_not_expire_active_sessions(): async def test_stateful_mcp_auth_context_cleanup_respects_zero_now(): """Explicit now=0 should be used as-is instead of falling back to monotonic.""" try: - from litellm.proxy._experimental.mcp_server import server as mcp_server + from litellm.proxy.mcp_server import server as mcp_server except ImportError: pytest.skip("MCP server not available") @@ -3696,7 +3697,7 @@ async def test_stateful_mcp_auth_context_cleanup_respects_zero_now(): async def test_stateful_mcp_cleanup_loop_survives_purge_errors(): """Cleanup loop should keep running after one purge attempt fails.""" try: - from litellm.proxy._experimental.mcp_server import server as mcp_server + from litellm.proxy.mcp_server import server as mcp_server except ImportError: pytest.skip("MCP server not available") @@ -3721,7 +3722,7 @@ async def test_owner_fingerprint_distinguishes_oauth_callers(): user could hijack another's mcp-session-id. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _owner_fingerprint_for, ) except ImportError: @@ -3757,7 +3758,7 @@ async def test_owner_fingerprint_distinguishes_oauth_callers(): async def test_owner_fingerprint_hashes_custom_api_keys(): """Custom API key formats should not appear in owner fingerprints.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _owner_fingerprint_for, ) except ImportError: @@ -3788,8 +3789,8 @@ async def test_stateful_mcp_session_owner_mismatch_returns_403(body: bytes): with 403, and the stateful manager must never be invoked. """ try: - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, ) @@ -3829,12 +3830,12 @@ async def test_stateful_mcp_session_owner_mismatch_returns_403(body: bytes): with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(intruder_auth, None, None, None, None, None), ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object( @@ -3866,8 +3867,8 @@ async def test_stateful_mcp_session_serializes_concurrent_requests(): MCPAuthenticatedUser while in-flight callbacks are still running. """ try: - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, ) @@ -3912,12 +3913,12 @@ async def test_stateful_mcp_session_serializes_concurrent_requests(): try: with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(owner_auth, None, None, None, None, None), ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object(session_manager_stateful, "handle_request", side_effect=slow_handle), @@ -3949,8 +3950,8 @@ async def test_stateful_mcp_lock_does_not_leak_when_auth_context_missing(): leaked lock would otherwise live forever. """ try: - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, ) @@ -3980,12 +3981,12 @@ async def test_stateful_mcp_lock_does_not_leak_when_auth_context_missing(): try: with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(owner_auth, None, None, None, None, None), ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object(session_manager_stateful, "handle_request", side_effect=handle), @@ -4016,8 +4017,8 @@ async def test_stateful_mcp_get_stream_does_not_block_post(): same mcp-session-id hang for the lifetime of the stream. """ try: - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, ) @@ -4059,12 +4060,12 @@ async def test_stateful_mcp_get_stream_does_not_block_post(): try: with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(owner_auth, None, None, None, None, None), ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object(session_manager_stateful, "handle_request", side_effect=handle), @@ -4099,7 +4100,7 @@ def test_jsonrpc_text_has_top_level_method_ignores_nested_method(): """The top-level-key scan must not be fooled by a ``method`` field nested inside a JSON-RPC response's ``result`` payload — a flat substring search would, and that misread is what deadlocks the session lock.""" - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _jsonrpc_text_has_top_level_method, ) @@ -4131,8 +4132,8 @@ async def test_truncated_jsonrpc_response_with_nested_method_skips_lock( deadlock behind the in-flight request POST that is holding the lock while it awaits this very response (e.g. sampling/createMessage).""" try: - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, ) @@ -4207,12 +4208,12 @@ async def test_truncated_jsonrpc_response_with_nested_method_skips_lock( try: with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(owner_auth, None, None, None, None, None), ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object(session_manager_stateful, "handle_request", side_effect=handle), @@ -4265,14 +4266,14 @@ async def test_mcp_routing_with_conflicting_alias_and_group_name(): conflicts with an access group name (e.g., "group"). """ try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_mcp_servers_in_path, _get_tools_from_mcp_servers, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: pytest.skip("MCP server not available") @@ -4310,15 +4311,15 @@ async def test_mcp_routing_with_conflicting_alias_and_group_name(): with ( patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_allowed_mcp_servers", mock_get_allowed, ), patch( - "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.server.MCPRequestHandler.get_mcp_servers_from_access_groups", mock_db_lookup, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_tools_from_server", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_tools_from_server", mock_get_tools_spy, ), ): @@ -4346,14 +4347,14 @@ async def test_oauth2_caller_headers_not_forwarded_for_migrated_server(): caller's oauth2 Authorization to the MCP client — the resolver injects the stored per-user token, so a caller-supplied bearer cannot override another user's credential.""" try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_tools_from_mcp_servers, set_auth_context, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: @@ -4427,16 +4428,16 @@ async def test_oauth2_caller_headers_not_forwarded_for_migrated_server(): side_effect=mock_fetch_tools_with_timeout, ), patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", AsyncMock(return_value=[oauth2_server]), ), patch( - "litellm.proxy._experimental.mcp_server.operations._prefetch_oauth_creds_for_user", + "litellm.proxy.mcp_server.operations._prefetch_oauth_creds_for_user", new_callable=AsyncMock, return_value={}, ), patch( - "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db", + "litellm.proxy.mcp_server.operations._get_user_oauth_extra_headers_from_db", new_callable=AsyncMock, return_value=None, ), @@ -4469,7 +4470,7 @@ async def test_oauth2_caller_headers_not_forwarded_for_migrated_server(): async def test_list_tools_single_server_unprefixed_names(): """When only one MCP server is allowed, list tools should return prefixed names (server prefix is always added).""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_tools_from_mcp_servers, set_auth_context, ) @@ -4518,7 +4519,7 @@ async def test_list_tools_single_server_unprefixed_names(): mock_manager.get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", mock_manager, ): listing = await _get_tools_from_mcp_servers( @@ -4537,7 +4538,7 @@ async def test_list_tools_single_server_unprefixed_names(): async def test_list_tools_multiple_servers_prefixed_names(): """When multiple MCP servers are allowed, list tools should return prefixed names.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_tools_from_mcp_servers, set_auth_context, ) @@ -4597,7 +4598,7 @@ async def test_list_tools_multiple_servers_prefixed_names(): mock_manager.get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", mock_manager, ): listing = await _get_tools_from_mcp_servers( @@ -4615,10 +4616,10 @@ async def test_list_tools_multiple_servers_prefixed_names(): @pytest.mark.asyncio async def test_mcp_manager_allows_public_servers_without_permissions(): try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: pytest.skip("MCP server not available") @@ -4638,7 +4639,7 @@ async def test_mcp_manager_allows_public_servers_without_permissions(): return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPRequestHandler.get_allowed_mcp_servers", + "litellm.proxy.mcp_server.mcp_server_manager.MCPRequestHandler.get_allowed_mcp_servers", AsyncMock(return_value=[]), ), ): @@ -4650,10 +4651,10 @@ async def test_mcp_manager_allows_public_servers_without_permissions(): @pytest.mark.asyncio async def test_mcp_manager_returns_public_when_permission_lookup_fails(): try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: pytest.skip("MCP server not available") @@ -4673,7 +4674,7 @@ async def test_mcp_manager_returns_public_when_permission_lookup_fails(): return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPRequestHandler.get_allowed_mcp_servers", + "litellm.proxy.mcp_server.mcp_server_manager.MCPRequestHandler.get_allowed_mcp_servers", AsyncMock(side_effect=Exception("boom")), ), ): @@ -4685,11 +4686,11 @@ async def test_mcp_manager_returns_public_when_permission_lookup_fails(): @pytest.mark.asyncio async def test_mcp_manager_merges_public_and_restricted_servers(): try: - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPServerAccess - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPServerAccess + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: pytest.skip("MCP server not available") @@ -4717,7 +4718,7 @@ async def test_mcp_manager_merges_public_and_restricted_servers(): return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPRequestHandler.get_mcp_server_access", + "litellm.proxy.mcp_server.mcp_server_manager.MCPRequestHandler.get_mcp_server_access", AsyncMock(return_value=MCPServerAccess(server_ids=("restricted",), scope="scoped")), ), ): @@ -4731,8 +4732,8 @@ async def test_call_mcp_tool_user_unauthorized_access(): """Test that a user cannot call a tool from a server they don't have access to""" from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server.server import call_mcp_tool from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import call_mcp_tool # Create a mock user without access to the server mock_user_auth = UserAPIKeyAuth( @@ -4773,11 +4774,11 @@ async def test_call_mcp_tool_user_unauthorized_access(): with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_allowed_mcp_servers", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_allowed_mcp_servers", AsyncMock(return_value=["allowed_server", "another_server"]), ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_id", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_id", side_effect=mock_get_server_by_id, ), ): @@ -4801,17 +4802,17 @@ async def test_call_mcp_tool_user_unauthorized_access(): @pytest.mark.asyncio async def test_call_mcp_tool_scoped_denial_names_the_binding_agent(): - from litellm.proxy._experimental.mcp_server.server import call_mcp_tool + from litellm.proxy.mcp_server.server import call_mcp_tool agent_bound_key = UserAPIKeyAuth(api_key="test-key", user_id="test-user", agent_id="agent-123") with ( patch( # test-quality-ok: the server registry is a module-level singleton; the suite's only seam - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_allowed_mcp_servers", AsyncMock(return_value=[]), ), patch( # test-quality-ok: the permission resolver is a module-level function; the suite's only seam - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", _scope_resolver({"github": "srv-github"}), ), ): @@ -4840,8 +4841,8 @@ async def test_call_mcp_tool_unauthorized_403_does_not_leak_server_credentials() """ from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server.server import call_mcp_tool from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import call_mcp_tool from litellm.types.mcp_server.mcp_server_manager import MCPServer mock_user_auth = UserAPIKeyAuth( @@ -4879,11 +4880,11 @@ async def test_call_mcp_tool_unauthorized_403_does_not_leak_server_credentials() with ( patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_allowed_mcp_servers", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_allowed_mcp_servers", AsyncMock(return_value=["allowed_server"]), ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_id", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_id", side_effect=mock_get_server_by_id, ), ): @@ -4947,11 +4948,11 @@ def test_mcpserver_repr_and_str_mask_credentials(): async def test_list_tools_filters_by_key_team_permissions(): """Test that list_tools filters tools based on key/team mcp_tool_permissions""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _get_tools_from_mcp_servers, set_auth_context, ) - from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth except ImportError: pytest.skip("MCP server not available") @@ -5026,7 +5027,7 @@ async def test_list_tools_filters_by_key_team_permissions(): mock_manager.get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", mock_manager, ): listing = await _get_tools_from_mcp_servers( @@ -5046,14 +5047,14 @@ async def test_list_tools_filters_by_key_team_permissions(): async def test_list_tools_with_team_tool_permissions_inheritance(): """Test that list_tools correctly applies key/team tool permissions inheritance logic""" try: - from litellm.proxy._experimental.mcp_server.server import ( - _get_tools_from_mcp_servers, - set_auth_context, - ) from litellm.proxy._types import ( LiteLLM_ObjectPermissionTable, UserAPIKeyAuth, ) + from litellm.proxy.mcp_server.server import ( + _get_tools_from_mcp_servers, + set_auth_context, + ) except ImportError: pytest.skip("MCP server not available") @@ -5137,12 +5138,12 @@ async def test_list_tools_with_team_tool_permissions_inheritance(): mock_manager.get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", mock_manager, ): # Mock the team object permission retrieval with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_team_object_permission", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_team_object_permission", AsyncMock(return_value=team_object_permission), ): listing = await _get_tools_from_mcp_servers( @@ -5162,11 +5163,11 @@ async def test_list_tools_with_team_tool_permissions_inheritance(): async def test_list_tools_with_no_tool_permissions_shows_all(): """Test that list_tools shows all tools when no mcp_tool_permissions are set""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _get_tools_from_mcp_servers, set_auth_context, ) - from litellm.proxy._types import UserAPIKeyAuth except ImportError: pytest.skip("MCP server not available") @@ -5229,7 +5230,7 @@ async def test_list_tools_with_no_tool_permissions_shows_all(): mock_manager.get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", mock_manager, ): listing = await _get_tools_from_mcp_servers( @@ -5255,11 +5256,11 @@ async def test_list_tools_strips_prefix_when_matching_permissions(): The filtering should strip the prefix before comparing. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _get_tools_from_mcp_servers, set_auth_context, ) - from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth except ImportError: pytest.skip("MCP server not available") @@ -5335,7 +5336,7 @@ async def test_list_tools_strips_prefix_when_matching_permissions(): mock_manager.get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", mock_manager, ): listing = await _get_tools_from_mcp_servers( @@ -5359,7 +5360,7 @@ def test_filter_tools_by_allowed_tools(): """Test that filter_tools_by_allowed_tools filters tools correctly""" from mcp.types import Tool - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( filter_tools_by_allowed_tools, ) from litellm.types.mcp import MCPTransport @@ -5431,7 +5432,7 @@ def test_filter_tools_by_allowed_tools(): def test_apply_display_name_overrides_leaves_descriptions_to_the_catalog_guard(): from mcp.types import Tool - from litellm.proxy._experimental.mcp_server.server import apply_display_name_overrides + from litellm.proxy.mcp_server.server import apply_display_name_overrides from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -5472,7 +5473,7 @@ def test_apply_display_name_overrides_leaves_descriptions_to_the_catalog_guard() def test_apply_display_name_overrides_no_overrides(): from mcp.types import Tool - from litellm.proxy._experimental.mcp_server.server import apply_display_name_overrides + from litellm.proxy.mcp_server.server import apply_display_name_overrides from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -5514,7 +5515,7 @@ class TestMCPServerManagerReload: @pytest.mark.asyncio async def test_reuses_existing_server_when_updated_at_matches(self): try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) except ImportError: @@ -5551,7 +5552,7 @@ class TestMCPServerManagerReload: @pytest.mark.asyncio async def test_rebuilds_server_when_updated_at_changes(self): try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) except ImportError: @@ -5599,7 +5600,7 @@ class TestMCPServerManagerReload: @pytest.mark.asyncio async def test_skips_server_when_build_from_database_fails(self, caplog): try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) except ImportError: @@ -5660,7 +5661,7 @@ class TestMCPServerManagerReload: @pytest.mark.asyncio async def test_skips_server_when_openapi_registration_fails(self, caplog): try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) except ImportError: @@ -5744,11 +5745,11 @@ async def test_call_mcp_tool_logs_failure_via_post_call_failure_hook(): Ensure proxy-side `call_mcp_tool` logs failures via `proxy_logging_obj.post_call_failure_hook`. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import MCPTransport, UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( call_mcp_tool, global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport, UserAPIKeyAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: pytest.skip("MCP server not available") @@ -5781,12 +5782,12 @@ async def test_call_mcp_tool_logs_failure_via_post_call_failure_hook(): return_value=mock_server, ), patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers_from_mcp_server_names", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers_from_mcp_server_names", new_callable=AsyncMock, return_value=[mock_server], ), patch( - "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool", + "litellm.proxy.mcp_server.operations.execute_mcp_tool", new_callable=AsyncMock, side_effect=Exception("boom"), ), @@ -5816,10 +5817,10 @@ async def test_get_tools_from_mcp_servers_logs_list_tools_to_spendlogs_when_enab try: from mcp.types import Tool as MCPTool - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _get_tools_from_mcp_servers, ) - from litellm.proxy._types import UserAPIKeyAuth except ImportError: pytest.skip("MCP server not available") @@ -5850,26 +5851,26 @@ async def test_get_tools_from_mcp_servers_logs_list_tools_to_spendlogs_when_enab with ( patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[server_a]), ), patch( - "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers", + "litellm.proxy.mcp_server.operations._prepare_mcp_server_headers", return_value=(None, None), ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, patch( - "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_allowed_tools", + "litellm.proxy.mcp_server.operations.filter_tools_by_allowed_tools", side_effect=lambda tools, _server: tools, ), patch( - "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_key_team_permissions", + "litellm.proxy.mcp_server.operations.filter_tools_by_key_team_permissions", new=AsyncMock(side_effect=lambda tools, **_: tools), ), patch( - "litellm.proxy._experimental.mcp_server.operations.function_setup", + "litellm.proxy.mcp_server.operations.function_setup", side_effect=_capture_function_setup, ), ): @@ -5928,21 +5929,21 @@ async def test_get_tools_from_mcp_servers_all_failed_listing_never_runs_proxy_fa with ( patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=servers), ), patch( - "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers", + "litellm.proxy.mcp_server.operations._prepare_mcp_server_headers", return_value=(None, None), ), - patch("litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", manager), - patch("litellm.proxy._experimental.mcp_server.operations.filter_tools_by_allowed_tools", return_value=[]), + patch("litellm.proxy.mcp_server.operations.global_mcp_server_manager", manager), + patch("litellm.proxy.mcp_server.operations.filter_tools_by_allowed_tools", return_value=[]), patch( - "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_key_team_permissions", + "litellm.proxy.mcp_server.operations.filter_tools_by_key_team_permissions", new=AsyncMock(return_value=[]), ), patch( - "litellm.proxy._experimental.mcp_server.operations.function_setup", + "litellm.proxy.mcp_server.operations.function_setup", return_value=(logging_obj, None), ), patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_obj), @@ -5974,10 +5975,10 @@ async def test_get_tools_from_mcp_servers_returns_tools_when_success_logging_fai async_success_handler raises (e.g. serialization errors in logging path). """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _get_tools_from_mcp_servers, ) - from litellm.proxy._types import UserAPIKeyAuth except ImportError: pytest.skip("MCP server not available") @@ -6002,26 +6003,26 @@ async def test_get_tools_from_mcp_servers_returns_tools_when_success_logging_fai with ( patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[server_a]), ), patch( - "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers", + "litellm.proxy.mcp_server.operations._prepare_mcp_server_headers", return_value=(None, None), ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, patch( - "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_allowed_tools", + "litellm.proxy.mcp_server.operations.filter_tools_by_allowed_tools", side_effect=lambda tools, _server: tools, ), patch( - "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_key_team_permissions", + "litellm.proxy.mcp_server.operations.filter_tools_by_key_team_permissions", new=AsyncMock(side_effect=lambda tools, **_: tools), ), patch( - "litellm.proxy._experimental.mcp_server.operations.function_setup", + "litellm.proxy.mcp_server.operations.function_setup", return_value=(dummy_logging_obj, None), ), ): @@ -6051,7 +6052,7 @@ def test_tool_name_matches_case_insensitive(): Without case-insensitive matching, all tools would be filtered out. """ try: - from litellm.proxy._experimental.mcp_server.server import _tool_name_matches + from litellm.proxy.mcp_server.server import _tool_name_matches except ImportError: pytest.skip("MCP server not available") @@ -6095,7 +6096,7 @@ def test_filter_tools_by_allowed_tools_case_insensitive(): camelCase allowed_tools configuration from the OpenAPI spec. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( filter_tools_by_allowed_tools, ) from litellm.types.mcp_server.tool_registry import MCPTool @@ -6158,7 +6159,7 @@ def test_filter_tools_by_allowed_tools_case_insensitive(): def test_filter_tools_by_allowed_tools_no_filter(): """Test that filter_tools_by_allowed_tools returns all tools when no filter is set.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( filter_tools_by_allowed_tools, ) from litellm.types.mcp_server.tool_registry import MCPTool @@ -6201,7 +6202,7 @@ def test_filter_tools_by_allowed_tools_no_filter(): def test_filter_tools_enforced_empty_allowlist_blocks_all(): from mcp.types import Tool - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( filter_tools_by_allowed_tools, ) from litellm.types.mcp import MCPTransport @@ -6231,7 +6232,7 @@ def test_filter_tools_enforced_empty_allowlist_blocks_all(): def test_filter_tools_legacy_empty_allowlist_allows_all(): from mcp.types import Tool - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( filter_tools_by_allowed_tools, ) from litellm.types.mcp import MCPTransport @@ -6259,7 +6260,7 @@ def test_filter_tools_legacy_empty_allowlist_allows_all(): def test_check_allowed_or_banned_tools_enforced_empty_denies_calls(): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp import MCPTransport @@ -6292,10 +6293,10 @@ async def test_get_tools_from_mcp_servers_injects_stored_oauth2_token(): API because the stored credential was never injected. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _get_tools_from_mcp_servers, ) - from litellm.proxy._types import UserAPIKeyAuth from litellm.types.mcp import MCPAuth except ImportError: pytest.skip("MCP server not available") @@ -6322,23 +6323,23 @@ async def test_get_tools_from_mcp_servers_injects_stored_oauth2_token(): with ( patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[oauth2_server]), ), patch( # Patch the bulk prefetch so no real DB connection is needed - "litellm.proxy._experimental.mcp_server.operations._prefetch_oauth_creds_for_user", + "litellm.proxy.mcp_server.operations._prefetch_oauth_creds_for_user", new=AsyncMock(return_value=prefetched_creds), ) as mock_prefetch, patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, patch( - "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_allowed_tools", + "litellm.proxy.mcp_server.operations.filter_tools_by_allowed_tools", side_effect=lambda tools, _server: tools, ), patch( - "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_key_team_permissions", + "litellm.proxy.mcp_server.operations.filter_tools_by_key_team_permissions", new=AsyncMock(side_effect=lambda tools, **_: tools), ), ): @@ -6395,7 +6396,7 @@ class TestMergeGatewayInitializeInstructions: def _merge(self, servers): try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _merge_gateway_initialize_instructions, ) except ImportError: @@ -6418,7 +6419,7 @@ class TestMergeGatewayInitializeInstructions: def test_yaml_override_beats_upstream_cache(self): """YAML/DB instructions take precedence over upstream cache.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -6431,7 +6432,7 @@ class TestMergeGatewayInitializeInstructions: def test_upstream_cache_used_when_no_yaml(self): """Upstream cached instructions are used when no YAML override is set.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -6471,7 +6472,7 @@ class TestMergeGatewayInitializeInstructions: def test_mixed_yaml_cache_specpath(self): """YAML, upstream-cache, and spec_path servers are handled correctly together.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -6493,7 +6494,7 @@ class TestEnsureUpstreamInitializeInstructionsCached: async def test_skips_when_yaml_instructions_set(self): from unittest.mock import AsyncMock, patch - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -6506,7 +6507,7 @@ class TestEnsureUpstreamInitializeInstructionsCached: async def test_skips_when_already_cached(self): from unittest.mock import AsyncMock, patch - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -6523,7 +6524,7 @@ class TestEnsureUpstreamInitializeInstructionsCached: async def test_skips_when_spec_path_set(self): from unittest.mock import AsyncMock, patch - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -6536,7 +6537,7 @@ class TestEnsureUpstreamInitializeInstructionsCached: async def test_runs_upstream_session_and_caches(self): from unittest.mock import AsyncMock, MagicMock, patch - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -6565,7 +6566,7 @@ class TestEnsureUpstreamInitializeInstructionsCached: """Upstream returns no instructions → next call within cooldown must not reconnect.""" from unittest.mock import AsyncMock, MagicMock, patch - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -6590,7 +6591,7 @@ class TestEnsureUpstreamInitializeInstructionsCached: """run_with_session raises → cooldown applies, no immediate retry.""" from unittest.mock import AsyncMock, MagicMock, patch - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -6613,7 +6614,7 @@ class TestEnsureUpstreamInitializeInstructionsCached: @pytest.mark.asyncio async def test_reload_resets_probe_cooldown(self): """load_servers_from_config clears the negative-cache map so reloads re-probe.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -6631,11 +6632,11 @@ class TestGatewayCreateInitializationOptions: def test_no_contextvar_returns_default_options(self): """When ContextVar is None, instructions are absent.""" try: - from litellm.proxy._experimental.mcp_server.mcp_context import ( + from litellm.proxy.mcp_server.mcp_context import ( mcp_gateway_initialize_instructions, mcp_gateway_server_name, ) - from litellm.proxy._experimental.mcp_server.server import server + from litellm.proxy.mcp_server.server import server except ImportError: pytest.skip("MCP server not available") @@ -6652,7 +6653,7 @@ class TestGatewayCreateInitializationOptions: @pytest.mark.asyncio async def test_scoped_request_uses_configured_server_alias(self): try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _gateway_initialize_instructions_request_scope, global_mcp_server_manager, server, @@ -6670,7 +6671,7 @@ class TestGatewayCreateInitializationOptions: with ( patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[scoped_server], ), @@ -6694,13 +6695,13 @@ class TestGatewayCreateInitializationOptions: async def test_initialize_with_no_granted_servers_returns_403(self): from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _gateway_initialize_instructions_request_scope, ) - from litellm.proxy._types import UserAPIKeyAuth with patch( # test-quality-ok: grant resolution is the input under test - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[], ): @@ -6720,13 +6721,13 @@ class TestGatewayCreateInitializationOptions: async def test_initialize_with_no_granted_scoped_servers_returns_scoped_denial(self): from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _gateway_initialize_instructions_request_scope, ) - from litellm.proxy._types import UserAPIKeyAuth with patch( # test-quality-ok: grant resolution is the input under test - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[], ): @@ -6744,14 +6745,14 @@ class TestGatewayCreateInitializationOptions: @pytest.mark.asyncio async def test_non_initialize_request_with_no_granted_servers_is_not_rejected_here(self): - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _gateway_initialize_instructions_request_scope, mcp_gateway_initialize_instructions, ) - from litellm.proxy._types import UserAPIKeyAuth with patch( # test-quality-ok: grant resolution is the input under test - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[], ): @@ -6765,8 +6766,8 @@ class TestGatewayCreateInitializationOptions: @pytest.mark.asyncio async def test_sse_handler_scopes_server_name_from_single_server_path(self): try: - from litellm.proxy._experimental.mcp_server import server as mcp_server - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.server import ( global_mcp_server_manager, handle_sse_mcp, server, @@ -6807,7 +6808,7 @@ class TestGatewayCreateInitializationOptions: with ( patch.object(mcp_server.sse, "connect_sse", connect_sse), patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=( UserAPIKeyAuth(api_key="sk-test"), @@ -6819,7 +6820,7 @@ class TestGatewayCreateInitializationOptions: ), ), patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[scoped_server], ), @@ -6829,15 +6830,15 @@ class TestGatewayCreateInitializationOptions: new_callable=AsyncMock, ), patch( - "litellm.proxy._experimental.mcp_server.server._raise_preemptive_401_for_unauthenticated_servers", + "litellm.proxy.mcp_server.server._raise_preemptive_401_for_unauthenticated_servers", new_callable=AsyncMock, ), patch( - "litellm.proxy._experimental.mcp_server.server._check_passthrough_upstream_auth", + "litellm.proxy.mcp_server.server._check_passthrough_upstream_auth", new_callable=AsyncMock, ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object( @@ -6854,10 +6855,10 @@ class TestGatewayCreateInitializationOptions: def test_contextvar_set_injects_instructions(self): """When ContextVar has a value, it appears in InitializationOptions.""" try: - from litellm.proxy._experimental.mcp_server.mcp_context import ( + from litellm.proxy.mcp_server.mcp_context import ( mcp_gateway_initialize_instructions, ) - from litellm.proxy._experimental.mcp_server.server import server + from litellm.proxy.mcp_server.server import server except ImportError: pytest.skip("MCP server not available") @@ -6871,10 +6872,10 @@ class TestGatewayCreateInitializationOptions: def test_contextvar_reset_removes_instructions(self): """After resetting the ContextVar, instructions disappear.""" try: - from litellm.proxy._experimental.mcp_server.mcp_context import ( + from litellm.proxy.mcp_server.mcp_context import ( mcp_gateway_initialize_instructions, ) - from litellm.proxy._experimental.mcp_server.server import server + from litellm.proxy.mcp_server.server import server except ImportError: pytest.skip("MCP server not available") @@ -6894,10 +6895,10 @@ async def test_list_tools_with_legacy_db_m2m_server_resolves_oauth2_flow(): Authorization header is forwarded upstream instead of being blocked. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _get_tools_from_mcp_servers, ) - from litellm.proxy._types import UserAPIKeyAuth from litellm.types.mcp import MCPAuth except ImportError: pytest.skip("MCP server not available") @@ -6954,14 +6955,14 @@ async def test_list_tools_with_legacy_db_m2m_server_resolves_oauth2_flow(): with ( patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, patch( - "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_allowed_tools", + "litellm.proxy.mcp_server.operations.filter_tools_by_allowed_tools", side_effect=lambda tools, _server: tools, ), patch( - "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_key_team_permissions", + "litellm.proxy.mcp_server.operations.filter_tools_by_key_team_permissions", new=AsyncMock(side_effect=lambda tools, **_: tools), ), ): @@ -6998,7 +6999,7 @@ async def test_call_tool_empty_extra_headers_returns_none(): differently if an empty dict is passed instead. """ try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp import MCPAuth @@ -7073,7 +7074,7 @@ async def test_call_tool_empty_extra_headers_returns_none(): @pytest.mark.asyncio async def test_probe_upstream_auth_returns_upstream_status(): """_probe_upstream_auth forwards the status code from the upstream server.""" - from litellm.proxy._experimental.mcp_server.server import _probe_upstream_auth + from litellm.proxy.mcp_server.server import _probe_upstream_auth mock_response = MagicMock() mock_response.status_code = 401 @@ -7083,7 +7084,7 @@ async def test_probe_upstream_auth_returns_upstream_status(): mock_client.post = AsyncMock(return_value=mock_response) with patch( - "litellm.proxy._experimental.mcp_server.server.get_async_httpx_client", + "litellm.proxy.mcp_server.server.get_async_httpx_client", return_value=mock_client, ): status, www_auth = await _probe_upstream_auth("http://upstream/mcp", "Bearer some-token") @@ -7106,7 +7107,7 @@ async def test_probe_upstream_auth_surfaces_httpx_status_error(): fail-open `except Exception`) so the auth check is not silently defeated. """ - from litellm.proxy._experimental.mcp_server.server import _probe_upstream_auth + from litellm.proxy.mcp_server.server import _probe_upstream_auth mock_response = MagicMock() mock_response.status_code = 401 @@ -7118,7 +7119,7 @@ async def test_probe_upstream_auth_surfaces_httpx_status_error(): mock_client.post = AsyncMock(side_effect=error) with patch( - "litellm.proxy._experimental.mcp_server.server.get_async_httpx_client", + "litellm.proxy.mcp_server.server.get_async_httpx_client", return_value=mock_client, ): status, www_auth = await _probe_upstream_auth("http://upstream/mcp", "Bearer some-token") @@ -7130,13 +7131,13 @@ async def test_probe_upstream_auth_surfaces_httpx_status_error(): @pytest.mark.asyncio async def test_probe_upstream_auth_fails_open_on_network_error(): """_probe_upstream_auth returns (200, None) when the network call fails.""" - from litellm.proxy._experimental.mcp_server.server import _probe_upstream_auth + from litellm.proxy.mcp_server.server import _probe_upstream_auth mock_client = MagicMock() mock_client.post = AsyncMock(side_effect=Exception("connection refused")) with patch( - "litellm.proxy._experimental.mcp_server.server.get_async_httpx_client", + "litellm.proxy.mcp_server.server.get_async_httpx_client", return_value=mock_client, ): status, www_auth = await _probe_upstream_auth("http://upstream/mcp", "Bearer some-token") @@ -7147,7 +7148,7 @@ async def test_probe_upstream_auth_fails_open_on_network_error(): def test_get_forwarded_auth_from_scope_extracts_header(): """Returns Authorization value when x-litellm-api-key is also present.""" - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_forwarded_auth_from_scope, ) @@ -7162,7 +7163,7 @@ def test_get_forwarded_auth_from_scope_extracts_header(): def test_get_forwarded_auth_from_scope_returns_none_when_missing(): - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_forwarded_auth_from_scope, ) @@ -7177,7 +7178,7 @@ def test_get_forwarded_auth_from_scope_skips_when_no_litellm_key_header(): leak the proxy key, so the helper must return None and the probe must not fire. """ - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_forwarded_auth_from_scope, ) @@ -7223,7 +7224,7 @@ def _patch_delegate_resolver(server: MCPServer, *resolvable_names: str): return server if name in resolvable_names else None return patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", side_effect=_resolve, ) @@ -7231,10 +7232,10 @@ def _patch_delegate_resolver(server: MCPServer, *resolvable_names: str): @pytest.mark.asyncio async def test_legacy_delegate_bare_token_is_not_probed_upstream(): # test-quality-ok: this removed security-sensitive egress has no return value; non-invocation is the contract """A bare bearer is an admission credential and must never reach upstream.""" - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _check_passthrough_upstream_auth, ) - from litellm.proxy._types import UserAPIKeyAuth server = _delegate_auth_mcp_server() scope = _delegate_scope([(b"authorization", b"Bearer bogus-token")]) @@ -7242,11 +7243,11 @@ async def test_legacy_delegate_bare_token_is_not_probed_upstream(): # test-qual with ( _patch_delegate_resolver(server, "delegate_test"), patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[server]), ), patch( - "litellm.proxy._experimental.mcp_server.server._probe_upstream_auth", + "litellm.proxy.mcp_server.server._probe_upstream_auth", new=AsyncMock(return_value=(401, 'Bearer realm="upstream", error="invalid_token"')), ) as probe, ): @@ -7266,10 +7267,10 @@ async def test_legacy_delegate_bare_token_is_not_probed_upstream(): # test-qual @pytest.mark.asyncio async def test_legacy_delegate_dual_credentials_are_not_probed_upstream(): # test-quality-ok: this removed security-sensitive egress has no return value; non-invocation is the contract """A separate upstream bearer never triggers the removed legacy probe.""" - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _check_passthrough_upstream_auth, ) - from litellm.proxy._types import UserAPIKeyAuth server = _delegate_auth_mcp_server() scope = _delegate_scope( @@ -7281,11 +7282,11 @@ async def test_legacy_delegate_dual_credentials_are_not_probed_upstream(): # te with ( patch( # test-quality-ok: isolate authorized-server resolution so this test targets the preflight boundary - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[server]), ), patch( # test-quality-ok: the removed probe call is the security regression under test - "litellm.proxy._experimental.mcp_server.server._probe_upstream_auth", + "litellm.proxy.mcp_server.server._probe_upstream_auth", new=AsyncMock(), ) as probe, ): @@ -7308,10 +7309,10 @@ async def test_legacy_delegate_dual_credentials_are_not_probed_upstream(): # te ) @pytest.mark.asyncio async def test_oauth_passthrough_preflight_preserves_status_contract(probe_status, expected_status): - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _check_passthrough_upstream_auth, ) - from litellm.proxy._types import UserAPIKeyAuth server = MCPServer( server_id="passthrough-id", @@ -7331,11 +7332,11 @@ async def test_oauth_passthrough_preflight_preserves_status_contract(probe_statu with ( patch( # test-quality-ok: isolate authorized-server resolution so this test exercises the preflight contract - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[server]), ), patch( # test-quality-ok: the upstream transport boundary is the behavior being mapped to an HTTP response - "litellm.proxy._experimental.mcp_server.server._probe_upstream_auth", + "litellm.proxy.mcp_server.server._probe_upstream_auth", new=AsyncMock(return_value=(probe_status, None)), ) as probe, ): @@ -7372,10 +7373,10 @@ async def test_oauth_passthrough_preflight_preserves_status_contract(probe_statu async def test_delegate_tokenless_request_not_probed(): """Tokenless delegate requests are the preemptive challenge's job; the preflight must not probe upstream with an empty credential.""" - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _check_passthrough_upstream_auth, ) - from litellm.proxy._types import UserAPIKeyAuth server = _delegate_auth_mcp_server() scope = _delegate_scope([(b"content-type", b"application/json")]) @@ -7383,11 +7384,11 @@ async def test_delegate_tokenless_request_not_probed(): with ( _patch_delegate_resolver(server, "delegate_test"), patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[server]), ), patch( - "litellm.proxy._experimental.mcp_server.server._probe_upstream_auth", + "litellm.proxy.mcp_server.server._probe_upstream_auth", new=AsyncMock(return_value=(401, None)), ) as probe, ): @@ -7408,10 +7409,10 @@ async def test_delegate_tokenless_request_not_probed(): async def test_delegate_preflight_skipped_on_multi_server_routes(): """The delegate probe is gated to single-server routes so one rejected token cannot 401 a multi-server aggregate connect (matching the OBO preflight).""" - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _check_passthrough_upstream_auth, ) - from litellm.proxy._types import UserAPIKeyAuth servers = [_delegate_auth_mcp_server("delegate-1"), _delegate_auth_mcp_server("delegate-2")] scope = _delegate_scope([(b"authorization", b"Bearer bogus-token")]) @@ -7419,11 +7420,11 @@ async def test_delegate_preflight_skipped_on_multi_server_routes(): with ( _patch_delegate_resolver(servers[0], "delegate_test", "other_server"), patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=servers), ), patch( - "litellm.proxy._experimental.mcp_server.server._probe_upstream_auth", + "litellm.proxy.mcp_server.server._probe_upstream_auth", new=AsyncMock(return_value=(401, None)), ) as probe, ): @@ -7446,10 +7447,10 @@ async def test_bare_authorization_never_probes_passthrough_servers(): only delegate servers (where admission classified it as an upstream token) may be probed with it; ``is_oauth_passthrough`` servers still require the unambiguous ``x-litellm-api-key`` + ``Authorization`` pair.""" - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _check_passthrough_upstream_auth, ) - from litellm.proxy._types import UserAPIKeyAuth passthrough_server = MCPServer( server_id="pt-1", @@ -7465,11 +7466,11 @@ async def test_bare_authorization_never_probes_passthrough_servers(): with ( _patch_delegate_resolver(passthrough_server, "pt_server"), patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[passthrough_server]), ), patch( - "litellm.proxy._experimental.mcp_server.server._probe_upstream_auth", + "litellm.proxy.mcp_server.server._probe_upstream_auth", new=AsyncMock(return_value=(401, None)), ) as probe, ): @@ -7494,10 +7495,10 @@ async def test_delegate_not_probed_when_named_only_via_server_id(): so the bare ``Authorization`` header is that LiteLLM key. The probe must resolve the target through the SAME resolver and therefore skip it, never forwarding the key upstream, even though the widened allowed-server set still contains it.""" - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _check_passthrough_upstream_auth, ) - from litellm.proxy._types import UserAPIKeyAuth server = _delegate_auth_mcp_server(server_id="delegate-secret-id") # Admission's resolver matches alias/server_name/name only, never server_id: the @@ -7514,11 +7515,11 @@ async def test_delegate_not_probed_when_named_only_via_server_id(): with ( _patch_delegate_resolver(server, "delegate_test"), patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[server]), ), patch( - "litellm.proxy._experimental.mcp_server.server._probe_upstream_auth", + "litellm.proxy.mcp_server.server._probe_upstream_auth", new=AsyncMock(return_value=(401, None)), ) as probe, ): @@ -7540,21 +7541,21 @@ async def test_delegate_probe_not_fanned_out_to_access_group_members(): """A single access-group name passes the one-target route gate but must not fan the delegate probe out to group-expanded member servers; the group name resolves to no server under admission's resolver, so no probe fires.""" - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( _check_passthrough_upstream_auth, ) - from litellm.proxy._types import UserAPIKeyAuth group_member = _delegate_auth_mcp_server() with ( _patch_delegate_resolver(group_member, "delegate_test"), patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[group_member]), ), patch( - "litellm.proxy._experimental.mcp_server.server._probe_upstream_auth", + "litellm.proxy.mcp_server.server._probe_upstream_auth", new=AsyncMock(return_value=(401, None)), ) as probe, ): @@ -7574,7 +7575,7 @@ async def test_delegate_probe_not_fanned_out_to_access_group_members(): @pytest.mark.asyncio async def test_create_mcp_client_sampling_disabled_by_default(): """Sampling callback must be None when allow_sampling is not set (default False).""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -7593,7 +7594,7 @@ async def test_create_mcp_client_sampling_disabled_by_default(): @pytest.mark.asyncio async def test_create_mcp_client_sampling_enabled(): """Sampling callback must be set when allow_sampling=True.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -7615,7 +7616,7 @@ async def test_execute_mcp_tool_rest_server_id_authoritative_for_unprefixed_tool """REST server_id + unprefixed tool name must not use global tool-name mapping.""" from mcp.types import TextContent - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module api_key_server = MCPServer( server_id="api-key-server-id", @@ -7712,7 +7713,7 @@ def _worker_that_never_listed(server: MCPServer, upstream_tools: tuple[str, ...] tools/list with ``upstream_tools`` and a managed dispatch that records what reaches it.""" from mcp.types import Tool as MCPTool - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module mcp_operations.global_mcp_server_manager.registry[server.server_id] = server dispatched: dict[str, object] = {} @@ -7746,7 +7747,7 @@ def _worker_that_never_listed(server: MCPServer, upstream_tools: tuple[str, ...] async def test_execute_mcp_tool_lists_never_listed_passthrough_server_with_caller_token_first(): """A prefixed tools/call on a worker that has not served tools/list must list that server once with the caller's own credentials and then dispatch, instead of answering 404.""" - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module server = _never_listed_passthrough_server() with _worker_that_never_listed(server, upstream_tools=("add",)) as worker: @@ -7767,7 +7768,7 @@ async def test_execute_mcp_tool_lists_never_listed_passthrough_server_with_calle @pytest.mark.asyncio async def test_execute_mcp_tool_rest_server_id_lists_never_listed_server_first(): - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module server = _never_listed_passthrough_server() with _worker_that_never_listed(server, upstream_tools=("add",)) as worker: @@ -7787,7 +7788,7 @@ async def test_execute_mcp_tool_rest_server_id_lists_never_listed_server_first() @pytest.mark.asyncio async def test_execute_mcp_tool_unknown_tool_on_never_listed_server_lists_once_then_404s(): - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module server = _never_listed_passthrough_server() with ( @@ -7811,7 +7812,7 @@ async def test_execute_mcp_tool_unknown_tool_on_never_listed_server_lists_once_t async def test_execute_mcp_tool_does_not_relist_a_server_this_worker_already_listed(): from mcp.types import Tool as MCPTool - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module server = _never_listed_passthrough_server() with _worker_that_never_listed(server, upstream_tools=("add",)) as worker: @@ -7834,7 +7835,7 @@ async def test_execute_mcp_tool_lists_a_tool_this_worker_has_not_yet_seen_on_a_l for a different tool it has not cached, so callers with wider upstream catalogs are not 404ed.""" from mcp.types import Tool as MCPTool - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module server = _never_listed_passthrough_server() with _worker_that_never_listed(server, upstream_tools=("add", "multiply")) as worker: @@ -7854,7 +7855,7 @@ async def test_execute_mcp_tool_lists_a_tool_this_worker_has_not_yet_seen_on_a_l @pytest.mark.asyncio async def test_execute_mcp_tool_never_lists_a_server_the_caller_cannot_access(): - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module server = _never_listed_passthrough_server() other_server = MCPServer(server_id="other-1", name="other", transport=MCPTransport.http) @@ -7885,7 +7886,7 @@ async def test_execute_mcp_tool_strips_a_prefix_that_contains_the_separator(): """ from mcp.types import TextContent - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module server_id = "117c814c-1a2b-4c4d-8e8f-0a1b2c3d4e5f" alias_less_server = MCPServer( @@ -7944,7 +7945,7 @@ async def test_execute_mcp_tool_rest_server_id_injects_requested_server_credenti """REST server_id must inject the requested server's auth, not a URL-collision peer's.""" from mcp.types import TextContent - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module requested_server = MCPServer( server_id="requested-server-id", @@ -8028,7 +8029,7 @@ async def test_execute_mcp_tool_rest_server_id_injects_requested_server_credenti @pytest.mark.asyncio async def test_execute_mcp_tool_rest_prefixed_tool_still_validates_server_id(): """Prefixed REST tool names must still match the requested server_id.""" - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module api_key_server = MCPServer( server_id="api-key-server-id", @@ -8092,7 +8093,7 @@ async def test_execute_mcp_tool_rest_prefixed_tool_still_validates_server_id(): @pytest.mark.asyncio async def test_execute_mcp_tool_rest_unauthorized_prefix_still_mismatches(): """Prefixed name for a registry server the caller cannot access must 403.""" - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module api_key_server = MCPServer( server_id="api-key-server-id", @@ -8156,7 +8157,7 @@ async def test_execute_mcp_tool_rest_hyphenated_upstream_tool_name_routes_to_req """REST server_id + hyphenated upstream tool name (no registry prefix) must route, not 400.""" from mcp.types import TextContent - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module api_key_server = MCPServer( server_id="api-key-server-id", @@ -8227,8 +8228,8 @@ async def test_execute_mcp_tool_sets_model_in_model_call_details(): import uuid from datetime import timezone - from litellm.proxy._experimental.mcp_server import server as mcp_module from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.mcp_server import server as mcp_module from litellm.utils import Rules, function_setup user = UserAPIKeyAuth( @@ -8282,11 +8283,11 @@ async def test_execute_mcp_tool_sets_model_in_model_call_details(): return_value=fake_tool, ), patch( - "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool", + "litellm.proxy.mcp_server.operations._handle_local_mcp_tool", new=AsyncMock(return_value=CallToolResult(content=[], is_error=False)), ), patch( - "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed", + "litellm.proxy.mcp_server.server.MCPRequestHandler.is_tool_allowed", return_value=True, ), ): @@ -8309,7 +8310,7 @@ async def test_execute_mcp_tool_hands_openapi_hooks_the_listed_entry_and_nothing only, as before this metadata existed, so a pre_mcp_call policy never scans a description the caller was not served. Once the caller has listed, the same call hands the entry that listing served.""" from litellm.caching.caching import DualCache - from litellm.proxy._experimental.mcp_server import operations as mcp_module + from litellm.proxy.mcp_server import operations as mcp_module from litellm.proxy.utils import ProxyLogging petstore = MCPServer( @@ -8369,7 +8370,7 @@ async def test_execute_mcp_tool_hands_openapi_hooks_the_listed_entry_and_nothing async def test_execute_mcp_tool_hands_openapi_hooks_the_guarded_catalog_entry_clients_saw(): """When tools/list pinned the schema and masked the description of an OpenAPI tool, the local-registry call path must hand the pre-call hooks that served entry, not the raw registry one.""" - from litellm.proxy._experimental.mcp_server import operations as mcp_module + from litellm.proxy.mcp_server import operations as mcp_module petstore = MCPServer( server_id="petstore-id", @@ -8423,7 +8424,7 @@ async def test_execute_mcp_tool_hands_openapi_hooks_the_guarded_catalog_entry_cl async def test_execute_mcp_tool_hands_openapi_hooks_each_callers_own_listed_entry(): """Two keys can be shown differently guarded OpenAPI catalogs. The call path must evaluate each key against the entry its own tools/list served, not the entry the most recent listing left behind.""" - from litellm.proxy._experimental.mcp_server import operations as mcp_module + from litellm.proxy.mcp_server import operations as mcp_module petstore = MCPServer( server_id="petstore-id", @@ -8479,7 +8480,7 @@ async def test_execute_mcp_tool_hands_openapi_hooks_each_callers_own_listed_entr async def test_execute_mcp_tool_runs_the_longer_colliding_operation_and_hands_hooks_no_registry_metadata(): """An OpenAPI operation whose name starts with its own server prefix runs instead of the shorter one, and with no prior listing the pre-call hooks get name and arguments only, never either registry entry.""" - from litellm.proxy._experimental.mcp_server import operations as mcp_module + from litellm.proxy.mcp_server import operations as mcp_module petstore = MCPServer( server_id="petstore-id", @@ -8523,7 +8524,7 @@ async def test_execute_mcp_tool_runs_the_longer_colliding_operation_and_hands_ho async def test_execute_mcp_tool_hands_hooks_nothing_for_a_never_listed_operation_named_after_a_listed_one(): """After the caller listed ``get_pet``, a call to the never-listed ``petstore-get_pet`` operation hands the pre-call hooks name and arguments only, not the listed sibling's description and schema.""" - from litellm.proxy._experimental.mcp_server import operations as mcp_module + from litellm.proxy.mcp_server import operations as mcp_module petstore = MCPServer( server_id="petstore-id", @@ -8613,7 +8614,7 @@ async def test_fetch_pinnable_tool_catalog_records_no_listed_catalog_for_the_adm """The pin snapshot lists the raw upstream catalog, without the catalog guard or the admin's description overrides, so it must not become what the admin's own later tools/call is evaluated against.""" from litellm.caching.caching import DualCache - from litellm.proxy._experimental.mcp_server.rest_endpoints import fetch_pinnable_tool_catalog + from litellm.proxy.mcp_server.rest_endpoints import fetch_pinnable_tool_catalog from litellm.proxy.utils import ProxyLogging manager = mcp_operations.global_mcp_server_manager @@ -8656,7 +8657,7 @@ async def test_execute_mcp_tool_rest_unresolved_prefixed_name_routes_to_requeste """ from mcp.types import TextContent - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module requested_server = MCPServer( server_id="rest-target-id", @@ -8740,7 +8741,7 @@ async def test_execute_mcp_tool_rest_unresolved_prefixed_name_routes_to_requeste @pytest.mark.asyncio async def test_execute_mcp_tool_rest_prefix_retry_resolution_still_enforces_server_id(): """A managed tool resolved via the requested server's prefix must still honor the server_id guard.""" - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module requested_server = MCPServer( server_id="api-key-server-id", @@ -8832,7 +8833,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_unknown_name_fails_ NOT silently fall back to the caller's full allowed-server set. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_allowed_mcp_servers_from_mcp_server_names, ) except ImportError: @@ -8844,7 +8845,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_unknown_name_fails_ ] with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp." + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp." "MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], @@ -8865,7 +8866,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_none_returns_all(): path that the fail-closed fix must not break. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_allowed_mcp_servers_from_mcp_server_names, ) except ImportError: @@ -8891,7 +8892,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_known_alias_returns server. Guards against the fix accidentally narrowing the happy path. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_allowed_mcp_servers_from_mcp_server_names, ) except ImportError: @@ -8903,7 +8904,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_known_alias_returns ] with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp." + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp." "MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], @@ -8924,7 +8925,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_mixed_known_and_unk when NOTHING resolves. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_allowed_mcp_servers_from_mcp_server_names, ) except ImportError: @@ -8936,7 +8937,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_mixed_known_and_unk ] with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp." + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp." "MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], @@ -8957,7 +8958,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_access_group_resolv as unresolved). """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_allowed_mcp_servers_from_mcp_server_names, ) except ImportError: @@ -8969,7 +8970,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_access_group_resolv ] with patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp." + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp." "MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=["id-b"], @@ -8989,7 +8990,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_empty_list_fails_cl explicit filter request. Fail closed rather than returning everything. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_allowed_mcp_servers_from_mcp_server_names, ) except ImportError: @@ -9017,10 +9018,10 @@ class TestProxyExceptionToHttpException: """ def test_preserves_401_status_and_www_authenticate_header(self): - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import ProxyException + from litellm.proxy.mcp_server.server import ( _proxy_exception_to_http_exception, ) - from litellm.proxy._types import ProxyException exc = ProxyException( message="Authentication Error, invalid token", @@ -9037,10 +9038,10 @@ class TestProxyExceptionToHttpException: assert http_exc.headers["WWW-Authenticate"] == 'Bearer resource_metadata="/x"' def test_preserves_403_status(self): - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import ProxyException + from litellm.proxy.mcp_server.server import ( _proxy_exception_to_http_exception, ) - from litellm.proxy._types import ProxyException http_exc = _proxy_exception_to_http_exception( ProxyException(message="Forbidden", type="auth_error", param="key", code=403) @@ -9049,10 +9050,10 @@ class TestProxyExceptionToHttpException: assert http_exc.status_code == 403 def test_non_numeric_code_falls_back_to_500(self): - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import ProxyException + from litellm.proxy.mcp_server.server import ( _proxy_exception_to_http_exception, ) - from litellm.proxy._types import ProxyException # ProxyException normalises code to the string "None" when unset. http_exc = _proxy_exception_to_http_exception( @@ -9070,8 +9071,8 @@ class TestStreamableHttpAuthErrorMapping: @pytest.mark.asyncio async def test_streamable_http_propagates_proxy_exception_as_401(self): - from litellm.proxy._experimental.mcp_server import server as mcp_module from litellm.proxy._types import ProxyException + from litellm.proxy.mcp_server import server as mcp_module scope = { "type": "http", @@ -9111,8 +9112,8 @@ class TestStreamableHttpAuthErrorMapping: @pytest.mark.asyncio async def test_sse_propagates_proxy_exception_as_401(self): - from litellm.proxy._experimental.mcp_server import server as mcp_module from litellm.proxy._types import ProxyException + from litellm.proxy.mcp_server import server as mcp_module scope = { "type": "http", @@ -9167,7 +9168,7 @@ class TestMCPMetaTraceCarrier: from mcp.types import CallToolRequestParams - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _mcp_meta_trace_carrier, ) @@ -9195,7 +9196,7 @@ class TestMCPMetaTraceCarrier: from mcp.types import CallToolRequestParams - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _mcp_meta_trace_carrier, ) @@ -9217,15 +9218,15 @@ async def test_stateful_mcp_tool_call_uses_current_requests_otel_destinations(_m reset_request_destinations, set_request_destinations, ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _MCP_DESTINATIONS_SCOPE_KEY, mcp_server_tool_call, set_auth_context, ) - from litellm.proxy._experimental.mcp_server.tool_registry import ( + from litellm.proxy.mcp_server.tool_registry import ( global_mcp_tool_registry, ) @@ -9271,10 +9272,10 @@ async def test_stateful_mcp_tool_call_uses_current_requests_otel_destinations(_m @pytest.mark.asyncio async def test_get_allowed_mcp_servers_includes_active_servers_submitted_by_user(): """BYOM submitters can see approved servers they submitted without allow_all_keys.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth submitted_server = _make_mcp_server_for_scope_filter("submitted-1", "user_mcp") submitter = UserAPIKeyAuth( @@ -9298,13 +9299,13 @@ async def test_get_allowed_mcp_servers_includes_active_servers_submitted_by_user return_value={"submitted-1": submitted_server}, ), patch( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp." + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp." "MCPRequestHandler.get_allowed_mcp_servers", AsyncMock(return_value=[]), ), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), patch( - "litellm.proxy._experimental.mcp_server.db.get_active_submitted_mcp_server_ids_for_user", + "litellm.proxy.mcp_server.db.get_active_submitted_mcp_server_ids_for_user", side_effect=_submitted_ids, ), ): @@ -9317,10 +9318,10 @@ async def test_get_allowed_mcp_servers_includes_active_servers_submitted_by_user @pytest.mark.asyncio async def test_get_active_submitted_mcp_server_ids_for_user_queries_active_rows(): - from litellm.proxy._experimental.mcp_server.db import ( + from litellm.proxy._types import MCPApprovalStatus + from litellm.proxy.mcp_server.db import ( get_active_submitted_mcp_server_ids_for_user, ) - from litellm.proxy._types import MCPApprovalStatus row = MagicMock() row.server_id = "submitted-1" @@ -9341,7 +9342,7 @@ async def test_get_active_submitted_mcp_server_ids_for_user_queries_active_rows( @pytest.mark.asyncio async def test_get_active_submitted_mcp_server_ids_for_user_empty_user_id_skips_db(): - from litellm.proxy._experimental.mcp_server.db import ( + from litellm.proxy.mcp_server.db import ( get_active_submitted_mcp_server_ids_for_user, ) @@ -9381,7 +9382,7 @@ def _mock_mcp_proxy_logging() -> MagicMock: def test_extract_mcp_tool_result_error_message(): - from litellm.proxy._experimental.mcp_server.utils import ( + from litellm.proxy.mcp_server.utils import ( extract_mcp_tool_result_error_message, ) @@ -9404,8 +9405,8 @@ async def test_fire_mcp_tool_call_logging_iserror_logs_failure(): """Regression test: a CallToolResult with is_error=True must go down the failure logging path (async_failure_handler + post_call_failure_hook), never async_success_handler.""" - from litellm.proxy._experimental.mcp_server.exceptions import MCPToolResultError - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.exceptions import MCPToolResultError + from litellm.proxy.mcp_server.server import ( _fire_mcp_tool_call_logging, ) @@ -9443,7 +9444,7 @@ async def test_fire_mcp_tool_call_logging_iserror_logs_failure(): async def test_fire_mcp_tool_call_logging_success_path_unchanged(): """is_error=False must keep today's behavior: success handler fires, no failure logging, no post_call_failure_hook.""" - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _fire_mcp_tool_call_logging, ) @@ -9472,7 +9473,7 @@ async def test_fire_mcp_tool_call_logging_success_path_unchanged(): async def test_fire_mcp_tool_call_logging_applies_hook_content(): from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _fire_mcp_tool_call_logging, ) from litellm.types.mcp import MCPPostCallResponseObject @@ -9530,7 +9531,7 @@ async def test_fire_mcp_tool_call_logging_applies_hook_content(): async def test_fire_mcp_tool_call_logging_iserror_without_auth_skips_failure_hook(): """Without a UserAPIKeyAuth the failure handlers still fire but the proxy post_call_failure_hook (which requires one) is skipped.""" - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _fire_mcp_tool_call_logging, ) @@ -9556,7 +9557,7 @@ async def test_fire_mcp_tool_call_logging_strips_credentials_from_failure_hook() """Credential-bearing request_data fields (raw request headers, upstream MCP auth headers, OAuth tokens) must never reach post_call_failure_hook callbacks; non-credential fields must survive untouched.""" - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _fire_mcp_tool_call_logging, ) @@ -9622,7 +9623,7 @@ async def test_fire_mcp_tool_call_logging_iserror_builds_failure_payload(monkeyp status='failure' with the tool's error text, so OTel (whose _parse_error keys off status) marks the MCP span ERROR.""" import litellm - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _fire_mcp_tool_call_logging, ) @@ -9651,7 +9652,7 @@ async def test_fire_mcp_tool_call_logging_iserror_builds_failure_payload(monkeyp async def test_fire_mcp_tool_call_logging_success_builds_success_payload(monkeypatch): """is_error=False still produces a status='success' payload.""" import litellm - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _fire_mcp_tool_call_logging, ) @@ -9688,7 +9689,7 @@ async def test_fire_mcp_tool_call_logging_iserror_emits_otel_error_span(monkeypa from litellm.integrations.otel import OpenTelemetryV2Config from litellm.integrations.otel.logger import OpenTelemetryV2 from litellm.integrations.otel.plumbing import providers - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _fire_mcp_tool_call_logging, ) @@ -9729,8 +9730,8 @@ async def test_call_tool_with_legacy_db_m2m_server_resolves_oauth2_flow(): call path was not). """ try: - from litellm.proxy._experimental.mcp_server.server import call_mcp_tool from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import call_mcp_tool from litellm.types.mcp import MCPAuth except ImportError: pytest.skip("MCP server not available") @@ -9759,14 +9760,14 @@ async def test_call_tool_with_legacy_db_m2m_server_resolves_oauth2_flow(): with ( patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, patch( - "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool", + "litellm.proxy.mcp_server.operations.execute_mcp_tool", side_effect=capture_execute, ), patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers_from_mcp_server_names", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers_from_mcp_server_names", new=AsyncMock(side_effect=lambda mcp_servers, allowed_mcp_servers: allowed_mcp_servers), ), ): @@ -9804,7 +9805,7 @@ def test_redact_mcp_resource_url_strips_credentials(url, expected): """The MCP tool-call log records the upstream resource, so the URL must be redacted to scheme+host+path: userinfo, query string, and fragment (which can carry embedded tokens or secret parameters) must never reach spend-log metadata or logging callbacks.""" - from litellm.proxy._experimental.mcp_server.server import redact_mcp_resource_url + from litellm.proxy.mcp_server.server import redact_mcp_resource_url assert redact_mcp_resource_url(url) == expected @@ -9815,12 +9816,12 @@ async def test_call_mcp_tool_skips_failure_hook_for_upstream_auth_error(): caller-must-reauth signal, not a failed call, so call_mcp_tool must re-raise it WITHOUT firing post_call_failure_hook (which records a failure and can trip LLM exception alerts). The streamable handler downgrades it to an informational isError result afterward.""" - from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import MCPTransport, UserAPIKeyAuth + from litellm.proxy.mcp_server.exceptions import MCPUpstreamAuthError + from litellm.proxy.mcp_server.server import ( call_mcp_tool, global_mcp_server_manager, ) - from litellm.proxy._types import MCPTransport, UserAPIKeyAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer mock_server = MCPServer( @@ -9844,12 +9845,12 @@ async def test_call_mcp_tool_skips_failure_hook_for_upstream_auth_error(): ), patch.object(global_mcp_server_manager, "get_mcp_server_by_id", return_value=mock_server), patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers_from_mcp_server_names", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers_from_mcp_server_names", new_callable=AsyncMock, return_value=[mock_server], ), patch( - "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool", + "litellm.proxy.mcp_server.operations.execute_mcp_tool", new_callable=AsyncMock, side_effect=MCPUpstreamAuthError(status_code=401, www_authenticate="Bearer", server_name="test_server"), ), @@ -9885,7 +9886,7 @@ def _interim_input_required_result(): @contextlib.contextmanager def _managed_tool_returning(server, upstream_result, proxy_logging_mock): - from litellm.proxy._experimental.mcp_server.server import global_mcp_server_manager + from litellm.proxy.mcp_server.server import global_mcp_server_manager with ( patch.object( @@ -9898,20 +9899,20 @@ def _managed_tool_returning(server, upstream_result, proxy_logging_mock): patch.object(global_mcp_server_manager, "get_mcp_server_from_tool_name", return_value=server), patch.object(global_mcp_server_manager, "server_owning_tool_name_prefix", return_value=server), patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers_from_mcp_server_names", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers_from_mcp_server_names", new_callable=AsyncMock, return_value=[server], ), patch( - "litellm.proxy._experimental.mcp_server.operations._list_tools_before_first_call", + "litellm.proxy.mcp_server.operations._list_tools_before_first_call", new_callable=AsyncMock, ), patch( - "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers", + "litellm.proxy.mcp_server.operations._prepare_mcp_server_headers", return_value=(None, None), ), patch( - "litellm.proxy._experimental.mcp_server.operations._handle_managed_mcp_tool", + "litellm.proxy.mcp_server.operations._handle_managed_mcp_tool", new_callable=AsyncMock, return_value=upstream_result, ) as managed_call, @@ -9926,12 +9927,12 @@ async def test_call_mcp_tool_legacy_interim_result_is_rejected_into_failure_acco must come back as isError and go through the same failure accounting as any other errored call.""" from mcp.types import CallToolResult - from litellm.proxy._experimental.mcp_server.result_conversion import ( + from litellm.proxy._types import MCPTransport, UserAPIKeyAuth + from litellm.proxy.mcp_server.result_conversion import ( INPUT_REQUIRED_UNSUPPORTED_MESSAGE, WireCompat, ) - from litellm.proxy._experimental.mcp_server.server import call_mcp_tool - from litellm.proxy._types import MCPTransport, UserAPIKeyAuth + from litellm.proxy.mcp_server.server import call_mcp_tool from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -9970,9 +9971,9 @@ async def test_call_mcp_tool_modern_interim_result_passes_through_without_comple logged as a completed success nor run through the post-call guardrail and success hooks.""" from mcp.types import InputRequiredResult - from litellm.proxy._experimental.mcp_server.result_conversion import WireCompat - from litellm.proxy._experimental.mcp_server.server import call_mcp_tool from litellm.proxy._types import MCPTransport, UserAPIKeyAuth + from litellm.proxy.mcp_server.result_conversion import WireCompat + from litellm.proxy.mcp_server.server import call_mcp_tool from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( @@ -10017,15 +10018,15 @@ async def test_aggregate_listing_reports_per_server_outcomes(): """A failed server must contribute a classified outcome, not just silently shrink the list: without the outcome a broken upstream is indistinguishable from a healthy server with no tools.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_tools_from_mcp_servers, set_auth_context, ) except ImportError: pytest.skip("MCP server not available") - from litellm.proxy._experimental.mcp_server.exceptions import MCPServerListError - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ServerListFault + from litellm.proxy.mcp_server.exceptions import MCPServerListError + from litellm.proxy.mcp_server.faults.list_outcomes import ServerListFault user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user") set_auth_context(user_api_key_auth) @@ -10075,7 +10076,7 @@ async def test_aggregate_listing_reports_per_server_outcomes(): mock_manager.get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", mock_manager, ): listing = await _get_tools_from_mcp_servers( @@ -10100,7 +10101,7 @@ async def test_outcome_keys_use_display_prefix_never_canonical_names(): the caller already sees on tool names: keying them by canonical server_name would let any authenticated caller enumerate internal server names the alias scheme deliberately hides.""" try: - from litellm.proxy._experimental.mcp_server.server import _aggregate_server_key + from litellm.proxy.mcp_server.server import _aggregate_server_key except ImportError: pytest.skip("MCP server not available") @@ -10121,13 +10122,13 @@ async def test_handle_list_tools_attaches_outcome_meta(_mcp_request_ctx): """The protocol handler returns a ListToolsResult whose _meta carries the per-server outcomes, so MCP clients can tell a degraded listing from a genuinely empty one.""" try: - from litellm.proxy._experimental.mcp_server.server import handle_list_tools + from litellm.proxy.mcp_server.server import handle_list_tools except ImportError: pytest.skip("MCP server not available") from mcp.types import ListToolsResult, Tool - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( + from litellm.proxy.mcp_server.faults.list_outcomes import ( SERVER_OUTCOMES_META_KEY, AggregateToolListing, ServerListFault, @@ -10145,11 +10146,11 @@ async def test_handle_list_tools_attaches_outcome_meta(_mcp_request_ctx): with ( patch( - "litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context", + "litellm.proxy.mcp_server.server.get_or_extract_auth_context", new=AsyncMock(return_value=(None, None, None, None, None, None, None)), ), patch( - "litellm.proxy._experimental.mcp_server.operations._list_mcp_tools", + "litellm.proxy.mcp_server.operations._list_mcp_tools", new=AsyncMock(return_value=listing), ), ): @@ -10168,17 +10169,17 @@ async def _list_tools_with_listing( params: PaginatedRequestParams, ) -> ListToolsResult: try: - from litellm.proxy._experimental.mcp_server.server import handle_list_tools + from litellm.proxy.mcp_server.server import handle_list_tools except ImportError: pytest.skip("MCP server not available") with ( patch( - "litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context", + "litellm.proxy.mcp_server.server.get_or_extract_auth_context", new=AsyncMock(return_value=(None, None, None, None, None, None, None)), ), patch( - "litellm.proxy._experimental.mcp_server.operations._list_mcp_tools", + "litellm.proxy.mcp_server.operations._list_mcp_tools", new=AsyncMock(return_value=listing), ), ): @@ -10192,7 +10193,7 @@ async def test_handle_list_tools_errors_when_every_server_failed( from mcp.shared.exceptions import MCPError from mcp.types import INTERNAL_ERROR - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( + from litellm.proxy.mcp_server.faults.list_outcomes import ( SERVER_OUTCOMES_META_KEY, AggregateToolListing, ServerListFault, @@ -10223,7 +10224,7 @@ async def test_handle_list_tools_auth_fault_keeps_degraded_result( ) -> None: from mcp.types import ListToolsResult - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( + from litellm.proxy.mcp_server.faults.list_outcomes import ( SERVER_OUTCOMES_META_KEY, AggregateToolListing, ServerListFault, @@ -10248,7 +10249,7 @@ async def test_handle_list_tools_all_faults_on_continuation_stays_degraded( ) -> None: from mcp.types import ListToolsResult, PaginatedRequestParams - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( + from litellm.proxy.mcp_server.faults.list_outcomes import ( SERVER_OUTCOMES_META_KEY, AggregateToolListing, ServerListFault, @@ -10275,7 +10276,7 @@ async def test_handle_list_tools_empty_outcomes_is_a_clean_empty_result( ) -> None: from mcp.types import ListToolsResult - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing + from litellm.proxy.mcp_server.faults.list_outcomes import AggregateToolListing listing: Final = AggregateToolListing(tools=[], outcomes={}) @@ -10336,7 +10337,7 @@ class TestPreemptive401ModeAware: return {"type": "http", "method": "POST", "path": f"/mcp/{alias}", "headers": []} async def _run(self, server, oauth2_headers, has_stored_token: bool): - from litellm.proxy._experimental.mcp_server import server as server_module + from litellm.proxy.mcp_server import server as server_module with ( patch.object( @@ -10362,7 +10363,7 @@ class TestPreemptive401ModeAware: @pytest.mark.asyncio async def test_deferred_discovery_runs_before_delegate_challenge(self): - from litellm.proxy._experimental.mcp_server import server as server_module + from litellm.proxy.mcp_server import server as server_module manager = mcp_operations.global_mcp_server_manager server = _make_oauth2_server( @@ -10394,7 +10395,7 @@ class TestPreemptive401ModeAware: @pytest.mark.asyncio async def test_stamped_m2m_challenge_skips_deferred_discovery(self): - from litellm.proxy._experimental.mcp_server import server as server_module + from litellm.proxy.mcp_server import server as server_module manager = mcp_operations.global_mcp_server_manager server = _make_oauth2_server("stamped_m2m", oauth2_flow="client_credentials") @@ -10427,7 +10428,7 @@ class TestPreemptive401ModeAware: async def test_gateway_as_metadata_challenge_under_server_root_path(self, original_path, expected_as_path): """Under SERVER_ROOT_PATH the challenge must keep the spelling the client called and point at a route the proxy registered, so it has to compare a route-relative path and carry the root suffix.""" - from litellm.proxy._experimental.mcp_server import server as server_module + from litellm.proxy.mcp_server import server as server_module server = _make_oauth2_server("interactive", oauth2_flow="authorization_code") scope = { @@ -10542,7 +10543,7 @@ class TestSingleServerPreflightReachesIdJag: ) async def _run(self, server: MCPServer, mcp_servers: list[str], preflight: AsyncMock) -> None: - from litellm.proxy._experimental.mcp_server import server as server_module + from litellm.proxy.mcp_server import server as server_module with ( patch.object( # test-quality-ok: route wiring must use the manager's configured server @@ -10587,7 +10588,7 @@ class TestSingleServerPreflightReachesIdJag: RFC 9728 discovery challenge from the block above and the preflight is never reached; pushing a subject-less exchange through the resolver would turn that challenge into some other status and strand a client that only had to SSO and retry.""" - from litellm.proxy._experimental.mcp_server import server as server_module + from litellm.proxy.mcp_server import server as server_module token_exchange = MCPServer( server_id="id-obo", @@ -10666,7 +10667,7 @@ class TestOboPreflightScopedToAllowedServers: SUBJECT_HEADERS = {"Authorization": "Bearer upstream-subject-token"} async def _run(self, requested: MCPServer, allowed: list[MCPServer], user_api_key_auth: UserAPIKeyAuth | None): - from litellm.proxy._experimental.mcp_server import server as server_module + from litellm.proxy.mcp_server import server as server_module allowed_lookup = AsyncMock(return_value=allowed) preflight = AsyncMock() @@ -10720,7 +10721,7 @@ class TestOboPreflightScopedToAllowedServers: @pytest.mark.asyncio async def test_post_mcp_call_guardrails_return_the_rewritten_result(): """The result a post_mcp_call guardrail rewrote must be what the caller sends back.""" - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _run_post_mcp_call_guardrails, ) @@ -10754,7 +10755,7 @@ async def test_post_mcp_call_guardrails_run_without_a_logging_object(): caller that omits it) would otherwise skip the guardrail entirely and return the unscanned tool output to the client. """ - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _run_post_mcp_call_guardrails, ) @@ -10782,7 +10783,7 @@ async def test_post_mcp_call_guardrails_run_without_a_logging_object(): async def test_post_mcp_call_guardrails_propagate_a_block(): """A post_mcp_call guardrail rejection must propagate instead of returning the result.""" from litellm.exceptions import BlockedPiiEntityError - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _run_post_mcp_call_guardrails, ) @@ -10806,8 +10807,8 @@ async def test_post_mcp_call_guardrails_propagate_a_block(): async def test_post_mcp_call_guardrails_preserve_and_deduplicate_mapping_entries( monkeypatch: pytest.MonkeyPatch, ) -> None: - from litellm.proxy._experimental.mcp_server.server import _run_post_mcp_call_guardrails from litellm.proxy import proxy_server + from litellm.proxy.mcp_server.server import _run_post_mcp_call_guardrails monkeypatch.setattr("litellm.proxy.utils.premium_user_check", MagicMock()) monkeypatch.setattr(proxy_server, "premium_user", True, raising=False) @@ -10858,7 +10859,7 @@ class TestListFiltersHonorThePrefixBoundary: @staticmethod def _alias_less_server(**overrides): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServer + from litellm.proxy.mcp_server.mcp_server_manager import MCPServer return MCPServer( server_id=TestListFiltersHonorThePrefixBoundary.SERVER_ID, @@ -10877,7 +10878,7 @@ class TestListFiltersHonorThePrefixBoundary: ] def test_bare_allowlist_entry_keeps_the_published_tool(self): - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( filter_tools_by_allowed_tools, ) @@ -10889,7 +10890,7 @@ class TestListFiltersHonorThePrefixBoundary: assert [tool.name for tool in kept] == [f"{self.SERVER_ID}-read_wiki_contents"] def test_bare_blocklist_entry_excludes_the_published_tool(self): - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( filter_tools_by_allowed_tools, ) @@ -10901,7 +10902,7 @@ class TestListFiltersHonorThePrefixBoundary: assert [tool.name for tool in kept] == [f"{self.SERVER_ID}-read_wiki_contents"] def test_unrelated_entry_does_not_match(self): - from litellm.proxy._experimental.mcp_server.server import _tool_name_matches + from litellm.proxy.mcp_server.server import _tool_name_matches server = self._alias_less_server() @@ -10912,7 +10913,7 @@ class TestListFiltersHonorThePrefixBoundary: # folding recovers a spec-spelled entry on an OpenAPI server. A native server # gets none of it: routing dispatches two names differing only in case as two # tools, so one policy must not decide both. - from litellm.proxy._experimental.mcp_server.server import _tool_name_matches + from litellm.proxy.mcp_server.server import _tool_name_matches native = self._alias_less_server() openapi = self._alias_less_server(spec_path="/specs/petstore.yaml") @@ -10925,8 +10926,8 @@ class TestListFiltersHonorThePrefixBoundary: # Routing accepts the alias form, so an entry stored before short # prefixes were turned on still governs the tool. Matching only the # published spelling left it advertised while dispatch refused it. - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServer - from litellm.proxy._experimental.mcp_server.server import _tool_name_matches + from litellm.proxy.mcp_server.mcp_server_manager import MCPServer + from litellm.proxy.mcp_server.server import _tool_name_matches monkeypatch.setenv("LITELLM_USE_SHORT_MCP_TOOL_PREFIX", "true") server = MCPServer( @@ -10952,11 +10953,11 @@ class TestListFiltersHonorThePrefixBoundary: """ from mcp.types import Tool as MCPTool - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServer, MCPServerManager, ) - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( filter_tools_by_allowed_tools, ) @@ -11017,13 +11018,13 @@ class TestListFiltersHonorThePrefixBoundary: """ from mcp.types import Tool as MCPTool - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( filter_tools_by_key_team_permissions, ) - from litellm.proxy._types import UserAPIKeyAuth server = MCPServer( server_id=self.SERVER_ID, @@ -11036,7 +11037,7 @@ class TestListFiltersHonorThePrefixBoundary: with ( patch.object(MCPRequestHandler, "get_allowed_tools_for_server", AsyncMock(return_value=grants)), - patch("litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager") as mock_manager, + patch("litellm.proxy.mcp_server.operations.global_mcp_server_manager") as mock_manager, ): mock_manager.get_mcp_server_by_id.return_value = server @@ -11058,7 +11059,7 @@ async def test_list_tools_injects_byok_credential_for_non_oauth2_auth_types(auth """Regression for BYOK servers on a non-oauth2 auth_type: the stored per-user credential must be attached when listing tools, otherwise the upstream 401 is absorbed and the server lists nothing.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _get_tools_from_mcp_servers, set_auth_context, ) @@ -11099,11 +11100,11 @@ async def test_list_tools_injects_byok_credential_for_non_oauth2_auth_types(auth with ( patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", mock_manager, ), patch( - "litellm.proxy._experimental.mcp_server.operations._get_byok_credential", + "litellm.proxy.mcp_server.operations._get_byok_credential", AsyncMock(return_value="personal-api-key"), ), ): @@ -11120,7 +11121,7 @@ async def test_list_tools_injects_byok_credential_for_non_oauth2_auth_types(auth @pytest.mark.asyncio async def test_active_request_ctx_var_feeds_get_current_session(_mcp_request_ctx) -> None: - from litellm.proxy._experimental.mcp_server.server import _get_current_session + from litellm.proxy.mcp_server.server import _get_current_session session = SimpleNamespace() ctx = _mcp_request_ctx(session=session) @@ -11171,7 +11172,7 @@ async def test_mcp_origin_admission_precedes_authentication( ) -> None: import httpx - from litellm.proxy._experimental.mcp_server import server + from litellm.proxy.mcp_server import server authenticate: Final = AsyncMock(side_effect=HTTPException(status_code=401, detail="authentication required")) with ( @@ -11196,12 +11197,12 @@ async def test_mcp_origin_admission_precedes_authentication( async def test_active_request_ctx_var_feeds_auth_resolution_recording(_mcp_request_ctx) -> None: from starlette.requests import Request - from litellm.proxy._experimental.mcp_server.mcp_debug import ( + from litellm.proxy.mcp_server.mcp_debug import ( MCP_AUTH_DIAGNOSTICS_SCOPE_KEY, MCPAuthDiagnostics, record_auth_resolution, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.types import AuthResolution + from litellm.proxy.mcp_server.outbound_credentials.types import AuthResolution diagnostics = MCPAuthDiagnostics() ctx = _mcp_request_ctx(request=Request({"type": "http", MCP_AUTH_DIAGNOSTICS_SCOPE_KEY: diagnostics})) @@ -11228,8 +11229,8 @@ async def test_active_request_ctx_var_feeds_auth_resolution_recording(_mcp_reque async def test_streamable_http_rejects_modern_protocol_version( header_value: str, expected_rejected: bool, handler: str, monkeypatch: pytest.MonkeyPatch ) -> None: - from litellm.proxy._experimental.mcp_server import server as mcp_module - from litellm.proxy._experimental.mcp_server.server import unsupported_protocol_version + from litellm.proxy.mcp_server import server as mcp_module + from litellm.proxy.mcp_server.server import unsupported_protocol_version monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) scope: Scope = { @@ -11273,7 +11274,7 @@ async def test_streamable_http_rejects_modern_protocol_version( ], ) async def test_native_listing_preserves_empty_result_on_auth_failure(_mcp_request_ctx, handler_name, field): - from litellm.proxy._experimental.mcp_server import server + from litellm.proxy.mcp_server import server with patch.object(server, "get_or_extract_auth_context", AsyncMock(side_effect=RuntimeError("auth failure"))): result = await getattr(server, handler_name)(_mcp_request_ctx(), _paged_params()) @@ -11283,8 +11284,8 @@ async def test_native_listing_preserves_empty_result_on_auth_failure(_mcp_reques @pytest.mark.asyncio @pytest.mark.parametrize("failure_hook_raises", [False, True]) async def test_tool_listing_preserves_permission_denial_when_failure_logging_fails(failure_hook_raises): - from litellm.proxy._experimental.mcp_server import operations from litellm.proxy import proxy_server + from litellm.proxy.mcp_server import operations auth = UserAPIKeyAuth(user_id="denied-caller") denial = HTTPException(status_code=403, detail="scope denied") @@ -11320,8 +11321,9 @@ async def test_legacy_sse_mount_emits_message_endpoint( ) -> None: from starlette.applications import Starlette from starlette.routing import Mount - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing - from litellm.proxy._experimental.mcp_server import server as mcp_server + + from litellm.proxy.mcp_server import server as mcp_server + from litellm.proxy.mcp_server.faults.list_outcomes import AggregateToolListing app: Final = Starlette(routes=[Mount("/mcp", app=mcp_server.app)]) incoming: Final[asyncio.Queue[Message]] = asyncio.Queue() @@ -11469,7 +11471,7 @@ async def test_legacy_sse_mount_emits_message_endpoint( @pytest.mark.parametrize("revision,rejected", [("2024-11-05", False), ("2025-11-25", True), ("2026-07-28", True)]) def test_protocol_header_respects_configured_advertisement(revision, rejected): from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.server import unsupported_protocol_version + from litellm.proxy.mcp_server.server import unsupported_protocol_version with patch.dict(proxy_server.general_settings, {"mcp_advertised_versions": ["2024-11-05"]}): result = unsupported_protocol_version({"headers": [(b"mcp-protocol-version", revision.encode())]}) @@ -11479,7 +11481,8 @@ def test_protocol_header_respects_configured_advertisement(revision, rejected): @pytest.mark.asyncio async def test_discovery_adapter_preserves_authenticated_context(_mcp_request_ctx): from mcp.types import DiscoverResult, RequestParams, ServerCapabilities - from litellm.proxy._experimental.mcp_server import server + + from litellm.proxy.mcp_server import server expected = DiscoverResult(supported_versions=["2025-11-25"], capabilities=ServerCapabilities()) dispatched = AsyncMock(return_value=expected) @@ -11522,11 +11525,13 @@ async def test_modern_oauth_challenge_follows_continuation_authorization( monkeypatch: pytest.MonkeyPatch, ) -> None: import time - from pydantic import TypeAdapter - from litellm.proxy._experimental.mcp_server import server - from litellm.proxy._experimental.mcp_server.contracts import OperationContext - from litellm.proxy._experimental.mcp_server.interactions import InteractionOperation, bind_target, seal_continuation + from mcp.types import InputRequiredResult + from pydantic import TypeAdapter + + from litellm.proxy.mcp_server import server + from litellm.proxy.mcp_server.contracts import OperationContext + from litellm.proxy.mcp_server.interactions import InteractionOperation, bind_target, seal_continuation monkeypatch.setenv("LITELLM_SALT_KEY", "challenge-test-salt") monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"mcp_advertised_versions": ["2026-07-28"]}) @@ -11674,7 +11679,7 @@ async def test_modern_oauth_challenge_follows_continuation_authorization( async def test_modern_preflight_enforces_sdk_body_limit( method: str, size_delta: int, chunked: bool, monkeypatch: pytest.MonkeyPatch ) -> None: - from litellm.proxy._experimental.mcp_server import server + from litellm.proxy.mcp_server import server monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"mcp_advertised_versions": ["2026-07-28"]}) limit: Final = server.session_manager_stateless.max_request_body_size @@ -11747,8 +11752,8 @@ async def test_modern_preflight_enforces_sdk_body_limit( async def test_modern_preflight_leaves_invalid_envelopes_to_sdk_without_upstream_work( invalid_request: Literal["malformed", "other_method", "header_mismatch", "duplicate_header"], ) -> None: - from litellm.proxy._experimental.mcp_server import server - from litellm.proxy._experimental.mcp_server.contracts import OperationContext + from litellm.proxy.mcp_server import server + from litellm.proxy.mcp_server.contracts import OperationContext envelope: Final = { "jsonrpc": "2.0", @@ -11787,9 +11792,10 @@ async def test_modern_preflight_leaves_invalid_envelopes_to_sdk_without_upstream @pytest.mark.parametrize("forwarded_token", (False, True)) async def test_modern_passthrough_probe_uses_scrubbed_server_credential(binding: str, forwarded_token: bool): import respx + from litellm.proxy._types import LiteLLM_ObjectPermissionTable - from litellm.proxy._experimental.mcp_server import server - from litellm.proxy._experimental.mcp_server.contracts import OperationContext + from litellm.proxy.mcp_server import server + from litellm.proxy.mcp_server.contracts import OperationContext target: Final = MCPServer( server_id="probe-id", @@ -11860,8 +11866,9 @@ async def test_modern_passthrough_probe_uses_scrubbed_server_credential(binding: @pytest.mark.parametrize("header_shape", ("mapping", "non_authorization")) async def test_passthrough_probes_bind_each_token_to_its_authorized_server(allowed: bool, header_shape: str): import respx + from litellm.proxy._types import LiteLLM_ObjectPermissionTable - from litellm.proxy._experimental.mcp_server import server + from litellm.proxy.mcp_server import server targets: Final = [ MCPServer( @@ -11913,7 +11920,7 @@ async def test_passthrough_probes_bind_each_token_to_its_authorized_server(allow @pytest.mark.asyncio async def test_streamable_http_admission_auth_sees_only_peeked_jsonrpc_method() -> None: from litellm.constants import MCP_PEEKED_BODY_SCOPE_KEY - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module scope: Final[Scope] = {"type": "http", "method": "POST", "path": "/mcp", "headers": []} receive: Final = AsyncMock( @@ -11938,22 +11945,22 @@ async def test_streamable_http_admission_auth_sees_only_peeked_jsonrpc_method() with ( patch( # test-quality-ok: the ASGI handler resolves auth through a module-level function; no injection seam - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, side_effect=extract, ), patch( # test-quality-ok: module flag guarding lazy session-manager startup; no injection seam - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True ), patch( # test-quality-ok: the allowlist is read off this module global; no injection seam "litellm.proxy.proxy_server.general_settings", _ALLOWLIST_SETTINGS ), patch( # test-quality-ok: session managers are module singletons; the downstream call is the observable - "litellm.proxy._experimental.mcp_server.server.session_manager_stateful", + "litellm.proxy.mcp_server.server.session_manager_stateful", SimpleNamespace(handle_request=stateful_handle), ), patch( # test-quality-ok: session managers are module singletons; the downstream call is the observable - "litellm.proxy._experimental.mcp_server.server.session_manager_stateless", + "litellm.proxy.mcp_server.server.session_manager_stateless", SimpleNamespace(handle_request=stateless_handle), ), ): @@ -11980,7 +11987,7 @@ async def test_streamable_http_admission_auth_sees_only_peeked_jsonrpc_method() ], ) async def test_stateful_cancellation_does_not_wait_for_call_or_change_its_auth(payload, valid): - from litellm.proxy._experimental.mcp_server import server as mcp_server + from litellm.proxy.mcp_server import server as mcp_server session_id = "cancellation-isolation-session" owner = UserAPIKeyAuth(api_key="owner-key", user_id="owner") @@ -12048,11 +12055,10 @@ async def test_stateful_cancellation_does_not_wait_for_call_or_change_its_auth(p @pytest.mark.parametrize("selected", [False, True]) async def test_progress_text_obeys_request_output_guardrails(monkeypatch, action, message, selected): import litellm - from litellm.caching.caching import DualCache from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.operations import _guarded_progress_callback from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail + from litellm.proxy.mcp_server.operations import _guarded_progress_callback from litellm.proxy.utils import ProxyLogging from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks @@ -12084,7 +12090,7 @@ async def test_progress_text_obeys_request_output_guardrails(monkeypatch, action async def test_progress_callback_preserves_absence_and_cancellation(): import asyncio - from litellm.proxy._experimental.mcp_server.operations import _guarded_progress_callback + from litellm.proxy.mcp_server.operations import _guarded_progress_callback assert _guarded_progress_callback(None, None, None, None, {}) is None callback = _guarded_progress_callback(AsyncMock(side_effect=asyncio.CancelledError), None, None, None, {}) @@ -12098,13 +12104,12 @@ async def test_execute_mcp_tool_derives_team_guardrail_auth_for_personal_keys( monkeypatch: pytest.MonkeyPatch, ) -> None: import litellm - from litellm.caching.caching import DualCache from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager - from litellm.proxy._experimental.mcp_server.utils import get_server_prefix - from litellm.proxy.auth import auth_checks from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, LiteLLM_UserTable + from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.utils import get_server_prefix from litellm.proxy.utils import ProxyLogging user_api_key_auth: Final = UserAPIKeyAuth(api_key="sk-personal", user_id="owner") diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_session_logging.py b/tests/unit/proxy/mcp_server/test_mcp_session_logging.py similarity index 91% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_session_logging.py rename to tests/unit/proxy/mcp_server/test_mcp_session_logging.py index 790937cc1de..83bd4a29908 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_session_logging.py +++ b/tests/unit/proxy/mcp_server/test_mcp_session_logging.py @@ -1,7 +1,7 @@ """The MCP ``mcp-session-id`` is captured for tool-call logging so the otel span can carry ``mcp.session.id``. Guards the header read against casing and absence.""" -from litellm.proxy._experimental.mcp_server.server import _mcp_session_id_from_headers +from litellm.proxy.mcp_server.server import _mcp_session_id_from_headers def test_reads_session_id_case_insensitively(): diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py b/tests/unit/proxy/mcp_server/test_mcp_sigv4_auth.py similarity index 93% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py rename to tests/unit/proxy/mcp_server/test_mcp_sigv4_auth.py index 7b9fc17840a..5350e8a9f41 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py +++ b/tests/unit/proxy/mcp_server/test_mcp_sigv4_auth.py @@ -15,7 +15,7 @@ import httpx2 import pytest from prisma import models -from litellm.experimental_mcp_client.client import MCPClient, MCPSigV4Auth +from litellm.mcp_client.client import MCPClient, MCPSigV4Auth from litellm.types.mcp import MCPAuth, MCPTransport _FUTURE_EXPIRATION: Final = datetime.now(timezone.utc) + timedelta(hours=1) @@ -444,7 +444,7 @@ class TestMCPServerManagerSigV4: @pytest.mark.asyncio async def test_create_mcp_client_with_sigv4(self): """_create_mcp_client creates client with SigV4 auth when auth_type is aws_sigv4.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -471,7 +471,7 @@ class TestMCPServerManagerSigV4: @pytest.mark.asyncio async def test_create_mcp_client_without_sigv4(self): """_create_mcp_client does not create SigV4 auth for non-SigV4 servers.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -494,7 +494,7 @@ class TestMCPServerManagerSigV4: @pytest.mark.asyncio async def test_load_config_with_aws_role_name(self): """Config loading correctly parses aws_role_name and aws_session_name.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -519,7 +519,7 @@ class TestMCPServerManagerSigV4: @pytest.mark.asyncio async def test_create_mcp_client_with_role_assumption(self): """_create_mcp_client passes aws_role_name to MCPSigV4Auth.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -555,7 +555,7 @@ class TestMCPServerManagerSigV4: def test_extract_aws_credentials_includes_role_fields(self): """_extract_aws_credentials extracts aws_role_name and aws_session_name.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -577,7 +577,7 @@ class TestSigV4CredentialEncryption: def test_encrypt_credentials_handles_aws_fields(self): """AWS credential fields are encrypted in the credentials dict.""" - from litellm.proxy._experimental.mcp_server.db import encrypt_credentials + from litellm.proxy.mcp_server.db import encrypt_credentials creds = { "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", @@ -588,7 +588,7 @@ class TestSigV4CredentialEncryption: } with patch( - "litellm.proxy._experimental.mcp_server.db.encrypt_value_helper", + "litellm.proxy.mcp_server.db.encrypt_value_helper", side_effect=lambda value, new_encryption_key: f"enc:{value}", ): result = encrypt_credentials(credentials=creds, encryption_key="test-key") @@ -603,12 +603,12 @@ class TestSigV4CredentialEncryption: def test_encrypt_credentials_skips_absent_aws_fields(self): """encrypt_credentials does not fail when AWS fields are absent.""" - from litellm.proxy._experimental.mcp_server.db import encrypt_credentials + from litellm.proxy.mcp_server.db import encrypt_credentials creds = {"auth_value": "some-token"} with patch( - "litellm.proxy._experimental.mcp_server.db.encrypt_value_helper", + "litellm.proxy.mcp_server.db.encrypt_value_helper", side_effect=lambda value, new_encryption_key: f"enc:{value}", ): result = encrypt_credentials(credentials=creds, encryption_key="test-key") @@ -623,8 +623,8 @@ class TestCredentialMergeOnUpdate: @pytest.mark.asyncio async def test_partial_update_preserves_existing_credentials(self): """Updating only aws_region_name should not wipe aws_secret_access_key.""" - from litellm.proxy._experimental.mcp_server.db import update_mcp_server from litellm.proxy._types import UpdateMCPServerRequest + from litellm.proxy.mcp_server.db import update_mcp_server existing_record = MagicMock() existing_record.auth_type = "aws_sigv4" @@ -648,11 +648,11 @@ class TestCredentialMergeOnUpdate: with ( patch( - "litellm.proxy._experimental.mcp_server.db.get_salt_key", + "litellm.proxy.mcp_server.db.get_salt_key", return_value=None, ), patch( - "litellm.proxy._experimental.mcp_server.db.encrypt_value_helper", + "litellm.proxy.mcp_server.db.encrypt_value_helper", side_effect=lambda value, new_encryption_key: value, ), ): @@ -673,8 +673,8 @@ class TestCredentialMergeOnUpdate: @pytest.mark.asyncio async def test_update_without_credentials_preserves_all(self): """Update with no credentials field should not touch existing credentials.""" - from litellm.proxy._experimental.mcp_server.db import update_mcp_server from litellm.proxy._types import UpdateMCPServerRequest + from litellm.proxy.mcp_server.db import update_mcp_server mock_prisma = MagicMock() mock_prisma.db.litellm_mcpservertable.update = AsyncMock(return_value=_updated_row()) @@ -685,7 +685,7 @@ class TestCredentialMergeOnUpdate: ) with patch( - "litellm.proxy._experimental.mcp_server.db.get_salt_key", + "litellm.proxy.mcp_server.db.get_salt_key", return_value=None, ): await update_mcp_server(mock_prisma, data, "test-user") @@ -696,8 +696,8 @@ class TestCredentialMergeOnUpdate: @pytest.mark.asyncio async def test_update_new_server_no_merge(self): """Update with credentials on a server that has no existing credentials.""" - from litellm.proxy._experimental.mcp_server.db import update_mcp_server from litellm.proxy._types import UpdateMCPServerRequest + from litellm.proxy.mcp_server.db import update_mcp_server existing_record = MagicMock() existing_record.auth_type = "aws_sigv4" @@ -715,11 +715,11 @@ class TestCredentialMergeOnUpdate: with ( patch( - "litellm.proxy._experimental.mcp_server.db.get_salt_key", + "litellm.proxy.mcp_server.db.get_salt_key", return_value=None, ), patch( - "litellm.proxy._experimental.mcp_server.db.encrypt_value_helper", + "litellm.proxy.mcp_server.db.encrypt_value_helper", side_effect=lambda value, new_encryption_key: value, ), ): @@ -732,8 +732,8 @@ class TestCredentialMergeOnUpdate: @pytest.mark.asyncio async def test_auth_type_change_replaces_credentials_entirely(self): """Switching auth_type should replace credentials, not merge.""" - from litellm.proxy._experimental.mcp_server.db import update_mcp_server from litellm.proxy._types import UpdateMCPServerRequest + from litellm.proxy.mcp_server.db import update_mcp_server existing_record = MagicMock() existing_record.auth_type = "aws_sigv4" @@ -757,11 +757,11 @@ class TestCredentialMergeOnUpdate: with ( patch( - "litellm.proxy._experimental.mcp_server.db.get_salt_key", + "litellm.proxy.mcp_server.db.get_salt_key", return_value=None, ), patch( - "litellm.proxy._experimental.mcp_server.db.encrypt_value_helper", + "litellm.proxy.mcp_server.db.encrypt_value_helper", side_effect=lambda value, new_encryption_key: f"enc:{value}", ), ): @@ -775,8 +775,8 @@ class TestCredentialMergeOnUpdate: @pytest.mark.asyncio async def test_same_auth_type_merges_credentials(self): """Same auth_type should merge credentials (preserve untouched fields).""" - from litellm.proxy._experimental.mcp_server.db import update_mcp_server from litellm.proxy._types import UpdateMCPServerRequest + from litellm.proxy.mcp_server.db import update_mcp_server existing_record = MagicMock() existing_record.auth_type = "oauth2" @@ -800,11 +800,11 @@ class TestCredentialMergeOnUpdate: with ( patch( - "litellm.proxy._experimental.mcp_server.db.get_salt_key", + "litellm.proxy.mcp_server.db.get_salt_key", return_value=None, ), patch( - "litellm.proxy._experimental.mcp_server.db.encrypt_value_helper", + "litellm.proxy.mcp_server.db.encrypt_value_helper", side_effect=lambda value, new_encryption_key: value, ), ): @@ -823,7 +823,7 @@ class TestSigV4BuildFromTable: @pytest.mark.asyncio async def test_build_mcp_server_from_table_with_sigv4_credentials(self): """SigV4 credentials from DB are decrypted and mapped to MCPServer fields.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -878,7 +878,7 @@ class TestSigV4BuildFromTable: manager = MCPServerManager() with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.decrypt_value_helper", + "litellm.proxy.mcp_server.mcp_server_manager.decrypt_value_helper", side_effect=lambda value, key, exception_type, return_original_value: value.replace("enc:", ""), ): server = await manager.build_mcp_server_from_table(table_record) @@ -893,7 +893,7 @@ class TestSigV4BuildFromTable: @pytest.mark.asyncio async def test_build_mcp_server_from_table_without_sigv4_credentials(self): """Non-SigV4 servers still work — AWS fields default to None.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -940,7 +940,7 @@ class TestSigV4BuildFromTable: manager = MCPServerManager() with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.decrypt_value_helper", + "litellm.proxy.mcp_server.mcp_server_manager.decrypt_value_helper", side_effect=lambda value, key, exception_type, return_original_value: value.replace("enc:", ""), ): server = await manager.build_mcp_server_from_table(table_record) @@ -958,7 +958,7 @@ class TestDecryptCredentials: def test_decrypt_credentials_handles_all_secret_fields(self): """All secret fields are decrypted; non-secret fields are left as-is.""" - from litellm.proxy._experimental.mcp_server.db import decrypt_credentials + from litellm.proxy.mcp_server.db import decrypt_credentials creds = { "auth_value": "enc:tok", @@ -972,7 +972,7 @@ class TestDecryptCredentials: } with patch( - "litellm.proxy._experimental.mcp_server.db.decrypt_value_helper", + "litellm.proxy.mcp_server.db.decrypt_value_helper", side_effect=lambda value, key, exception_type="error", return_original_value=False: value.replace( "enc:", "" ), @@ -991,12 +991,12 @@ class TestDecryptCredentials: def test_decrypt_credentials_skips_absent_fields(self): """Absent fields are not touched.""" - from litellm.proxy._experimental.mcp_server.db import decrypt_credentials + from litellm.proxy.mcp_server.db import decrypt_credentials creds = {"aws_access_key_id": "enc:AKI"} with patch( - "litellm.proxy._experimental.mcp_server.db.decrypt_value_helper", + "litellm.proxy.mcp_server.db.decrypt_value_helper", side_effect=lambda value, key, exception_type="error", return_original_value=False: value.replace( "enc:", "" ), @@ -1013,7 +1013,7 @@ class TestRotateCredentials: @pytest.mark.asyncio async def test_rotation_decrypts_then_reencrypts(self): """Key rotation should decrypt with old key then encrypt with new key.""" - from litellm.proxy._experimental.mcp_server.db import ( + from litellm.proxy.mcp_server.db import ( rotate_mcp_server_credentials_master_key, ) @@ -1034,17 +1034,17 @@ class TestRotateCredentials: with ( patch( - "litellm.proxy._experimental.mcp_server.db.get_salt_key", + "litellm.proxy.mcp_server.db.get_salt_key", return_value="old-key", ), patch( - "litellm.proxy._experimental.mcp_server.db.decrypt_value_helper", + "litellm.proxy.mcp_server.db.decrypt_value_helper", side_effect=lambda value, key, exception_type="error", return_original_value=False: value.replace( "enc_old:", "" ), ), patch( - "litellm.proxy._experimental.mcp_server.db.encrypt_value_helper", + "litellm.proxy.mcp_server.db.encrypt_value_helper", side_effect=lambda value, new_encryption_key: f"enc_new:{value}", ), ): @@ -1063,7 +1063,7 @@ class TestRotateCredentials: async def test_rotation_reencrypts_global_env_vars(self): """Global env var values are re-encrypted under the new key; user-scope placeholders are left untouched.""" - from litellm.proxy._experimental.mcp_server.db import ( + from litellm.proxy.mcp_server.db import ( rotate_mcp_server_credentials_master_key, ) @@ -1083,17 +1083,17 @@ class TestRotateCredentials: with ( patch( - "litellm.proxy._experimental.mcp_server.db.get_salt_key", + "litellm.proxy.mcp_server.db.get_salt_key", return_value="old-key", ), patch( - "litellm.proxy._experimental.mcp_server.db.decrypt_value_helper", + "litellm.proxy.mcp_server.db.decrypt_value_helper", side_effect=lambda value, key, exception_type="error", return_original_value=False: value.replace( "enc_old:", "" ), ), patch( - "litellm.proxy._experimental.mcp_server.db.encrypt_value_helper", + "litellm.proxy.mcp_server.db.encrypt_value_helper", side_effect=lambda value, new_encryption_key: f"enc_new:{value}", ), ): @@ -1116,8 +1116,8 @@ class TestAuthTypeSwitchClearsCredentials: @pytest.mark.asyncio async def test_auth_type_change_without_credentials_clears_stale(self): """Changing auth_type without providing credentials should clear old ones.""" - from litellm.proxy._experimental.mcp_server.db import update_mcp_server from litellm.proxy._types import UpdateMCPServerRequest + from litellm.proxy.mcp_server.db import update_mcp_server existing_record = MagicMock() existing_record.auth_type = "oauth2" @@ -1134,7 +1134,7 @@ class TestAuthTypeSwitchClearsCredentials: ) with patch( - "litellm.proxy._experimental.mcp_server.db.get_salt_key", + "litellm.proxy.mcp_server.db.get_salt_key", return_value=None, ): await update_mcp_server(mock_prisma, data, "test-user") diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_stale_session.py b/tests/unit/proxy/mcp_server/test_mcp_stale_session.py similarity index 84% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_stale_session.py rename to tests/unit/proxy/mcp_server/test_mcp_stale_session.py index 66d853bdd44..88f18b609a4 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_stale_session.py +++ b/tests/unit/proxy/mcp_server/test_mcp_stale_session.py @@ -25,7 +25,7 @@ class TestHandleStaleMcpSession: async def test_strips_stale_session_id_for_non_delete(self): """Non-DELETE requests should have stale session IDs stripped.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _handle_stale_mcp_session, ) except ImportError: @@ -55,7 +55,7 @@ class TestHandleStaleMcpSession: async def test_delete_stale_session_returns_success(self): """DELETE requests for non-existent sessions should return success (idempotent).""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _handle_stale_mcp_session, _stateful_session_active_request_counts, _stateful_session_auth_context_last_seen, @@ -111,7 +111,7 @@ class TestHandleStaleMcpSession: async def test_preserves_valid_session_id(self): """Valid session IDs should not be modified.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _handle_stale_mcp_session, ) except ImportError: @@ -141,7 +141,7 @@ class TestHandleStaleMcpSession: async def test_no_op_when_no_session_header(self): """No session header should result in no-op.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _handle_stale_mcp_session, ) except ImportError: @@ -167,7 +167,7 @@ class TestHandleStaleMcpSession: async def test_no_op_when_server_instances_missing(self): """If _server_instances attr doesn't exist, don't crash.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _handle_stale_mcp_session, ) except ImportError: @@ -194,7 +194,7 @@ class TestHandleStaleMcpSession: async def test_delete_valid_session_not_handled(self): """DELETE requests for existing sessions should not be intercepted.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _handle_stale_mcp_session, ) except ImportError: @@ -227,7 +227,7 @@ async def test_stale_mcp_session_id_is_stripped(): request to the session manager so a fresh session is created. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, session_manager_stateless, @@ -263,15 +263,15 @@ async def test_stale_mcp_session_id_is_stripped(): with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(UserAPIKeyAuth(), None, None, None, None, None), ), patch( - "litellm.proxy._experimental.mcp_server.server.set_auth_context", + "litellm.proxy.mcp_server.server.set_auth_context", ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object( @@ -312,7 +312,7 @@ async def test_delete_stale_mcp_session_returns_success(): without forwarding to the session manager (idempotent DELETE). """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager, ) @@ -340,15 +340,15 @@ async def test_delete_stale_mcp_session_returns_success(): with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(UserAPIKeyAuth(), None, None, None, None, None), ), patch( - "litellm.proxy._experimental.mcp_server.server.set_auth_context", + "litellm.proxy.mcp_server.server.set_auth_context", ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object( @@ -378,7 +378,7 @@ async def test_failed_delete_preserves_stateful_session_tracking(): owner/auth tracking so the session cannot be hijacked or hidden from cleanup. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _owner_fingerprint_for, _stateful_session_auth_context_last_seen, _stateful_session_auth_contexts, @@ -419,12 +419,12 @@ async def test_failed_delete_preserves_stateful_session_tracking(): try: with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(user_auth, None, None, None, None, None), ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object( @@ -461,7 +461,7 @@ async def test_valid_mcp_session_id_is_preserved(): handle_streamable_http_mcp should NOT strip the header. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, ) @@ -494,15 +494,15 @@ async def test_valid_mcp_session_id_is_preserved(): with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(UserAPIKeyAuth(), None, None, None, None, None), ), patch( - "litellm.proxy._experimental.mcp_server.server.set_auth_context", + "litellm.proxy.mcp_server.server.set_auth_context", ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object( @@ -530,7 +530,7 @@ async def test_no_mcp_session_id_header_works_normally(): handle_streamable_http_mcp should work without any issues. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager, ) @@ -557,15 +557,15 @@ async def test_no_mcp_session_id_header_works_normally(): with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(UserAPIKeyAuth(), None, None, None, None, None), ), patch( - "litellm.proxy._experimental.mcp_server.server.set_auth_context", + "litellm.proxy.mcp_server.server.set_auth_context", ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch.object( @@ -596,7 +596,7 @@ async def test_per_user_oauth_missing_stored_token_returns_preemptive_401(): from fastapi import HTTPException try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateless, ) @@ -627,29 +627,29 @@ async def test_per_user_oauth_missing_stored_token_returns_preemptive_401(): with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(user_auth, None, ["repro_oauth_server"], None, None, None), ), patch( - "litellm.proxy._experimental.mcp_server.server.set_auth_context", + "litellm.proxy.mcp_server.server.set_auth_context", ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + "litellm.proxy.mcp_server.server._handle_stale_mcp_session", new_callable=AsyncMock, return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token", new_callable=AsyncMock, return_value=False, ) as mock_has_token, patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=oauth_server, ), patch.object( @@ -683,7 +683,7 @@ async def test_admitted_subject_missing_stored_token_challenged_with_resource_me from fastapi import HTTPException try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateless, ) @@ -715,29 +715,29 @@ async def test_admitted_subject_missing_stored_token_challenged_with_resource_me with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(user_auth, None, ["repro_oauth_server"], None, None, None), ), patch( - "litellm.proxy._experimental.mcp_server.server.set_auth_context", + "litellm.proxy.mcp_server.server.set_auth_context", ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + "litellm.proxy.mcp_server.server._handle_stale_mcp_session", new_callable=AsyncMock, return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token", new_callable=AsyncMock, return_value=False, ) as mock_has_token, patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=oauth_server, ), patch.object( @@ -781,11 +781,11 @@ async def test_client_credentials_server_is_not_preemptively_challenged(m2m_fiel never look for. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateless, ) - from litellm.proxy._types import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer except ImportError: pytest.skip("MCP server not available") @@ -826,23 +826,23 @@ async def test_client_credentials_server_is_not_preemptively_challenged(m2m_fiel with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(user_auth, None, ["m2m_server"], None, None, None), ), - patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), - patch("litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True), + patch("litellm.proxy.mcp_server.server.set_auth_context"), + patch("litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True), patch( - "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + "litellm.proxy.mcp_server.server._handle_stale_mcp_session", new_callable=AsyncMock, return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token", new_callable=AsyncMock, ) as mock_has_token, patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=m2m_server, ), patch.object(session_manager_stateless, "handle_request", new_callable=AsyncMock) as mock_handle_request, @@ -866,10 +866,10 @@ async def test_handle_streamable_http_mcp_delegated_server_surfaces_upstream_cha from fastapi import HTTPException try: - from litellm.proxy._experimental.mcp_server.exceptions import ( + from litellm.proxy.mcp_server.exceptions import ( MCPUpstreamAuthError, ) - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, ) @@ -912,7 +912,7 @@ async def test_handle_streamable_http_mcp_delegated_server_surfaces_upstream_cha with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=( user_auth, @@ -923,27 +923,27 @@ async def test_handle_streamable_http_mcp_delegated_server_surfaces_upstream_cha None, ), ), - patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), + patch("litellm.proxy.mcp_server.server.set_auth_context"), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + "litellm.proxy.mcp_server.server._handle_stale_mcp_session", new_callable=AsyncMock, return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db", + "litellm.proxy.mcp_server.operations._get_user_oauth_extra_headers_from_db", new_callable=AsyncMock, return_value=None, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=delegated_server, ), patch( # test-quality-ok: registry is empty in unit tests; key owns the delegated server - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[delegated_server], ), @@ -973,7 +973,7 @@ async def test_per_user_oauth_with_stored_token_skips_preemptive_401(): 401 and continue to session manager request handling. """ try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateless, ) @@ -1010,29 +1010,29 @@ async def test_per_user_oauth_with_stored_token_skips_preemptive_401(): with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(user_auth, None, ["repro_oauth_server"], None, None, None), ), patch( - "litellm.proxy._experimental.mcp_server.server.set_auth_context", + "litellm.proxy.mcp_server.server.set_auth_context", ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + "litellm.proxy.mcp_server.server._handle_stale_mcp_session", new_callable=AsyncMock, return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token", new_callable=AsyncMock, return_value=True, ) as mock_has_token, patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=oauth_server, ), patch.object( @@ -1068,7 +1068,7 @@ async def test_handle_streamable_http_mcp_delegated_server_without_token_returns from fastapi import HTTPException try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, ) @@ -1107,7 +1107,7 @@ async def test_handle_streamable_http_mcp_delegated_server_without_token_returns with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=( user_auth, @@ -1119,23 +1119,23 @@ async def test_handle_streamable_http_mcp_delegated_server_without_token_returns ), ), patch( - "litellm.proxy._experimental.mcp_server.server.set_auth_context", + "litellm.proxy.mcp_server.server.set_auth_context", ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + "litellm.proxy.mcp_server.server._handle_stale_mcp_session", new_callable=AsyncMock, return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token", new_callable=AsyncMock, ) as mock_has_token, patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=delegated_server, ), patch.object( @@ -1168,7 +1168,7 @@ async def test_handle_streamable_http_mcp_token_exchange_without_subject_returns from fastapi import HTTPException try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, ) @@ -1208,22 +1208,22 @@ async def test_handle_streamable_http_mcp_token_exchange_without_subject_returns with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(user_auth, None, ["obo_server"], None, None, None), ), - patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), + patch("litellm.proxy.mcp_server.server.set_auth_context"), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + "litellm.proxy.mcp_server.server._handle_stale_mcp_session", new_callable=AsyncMock, return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=obo_server, ), patch.object( @@ -1290,7 +1290,7 @@ async def test_handle_streamable_http_mcp_oauth_delegate_without_token_returns_g from fastapi import HTTPException try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, ) @@ -1312,17 +1312,17 @@ async def test_handle_streamable_http_mcp_oauth_delegate_without_token_returns_g with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(user_auth, None, ["od_server"], None, None, None), ), - patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), + patch("litellm.proxy.mcp_server.server.set_auth_context"), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=od_server, ), patch.object( @@ -1349,7 +1349,7 @@ async def test_handle_streamable_http_mcp_oauth_delegate_with_forwarded_token_sk re-challenge. Guards the ``_get_forwarded_auth_from_scope(...) is None`` condition: dropping it would 401 even a fully-authenticated request.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateless, ) @@ -1377,26 +1377,26 @@ async def test_handle_streamable_http_mcp_oauth_delegate_with_forwarded_token_sk with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(user_auth, None, ["od_server"], None, None, None), ), - patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), + patch("litellm.proxy.mcp_server.server.set_auth_context"), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + "litellm.proxy.mcp_server.server._handle_stale_mcp_session", new_callable=AsyncMock, return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.server._check_passthrough_upstream_auth", + "litellm.proxy.mcp_server.server._check_passthrough_upstream_auth", new_callable=AsyncMock, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=od_server, ), patch.object( @@ -1422,7 +1422,7 @@ async def _run_passthrough_connect( challenged (raised) or forwarded to the session manager. Returns (challenged, www_authenticate).""" from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateless, ) @@ -1442,23 +1442,23 @@ async def _run_passthrough_connect( with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(user_auth, None, server_names, mcp_server_auth_headers, None, None), ), - patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), - patch("litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True), + patch("litellm.proxy.mcp_server.server.set_auth_context"), + patch("litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True), patch( - "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + "litellm.proxy.mcp_server.server._handle_stale_mcp_session", new_callable=AsyncMock, return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.server._check_passthrough_upstream_auth", + "litellm.proxy.mcp_server.server._check_passthrough_upstream_auth", new_callable=AsyncMock, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=server, ), patch.object(session_manager_stateless, "handle_request", new_callable=AsyncMock) as mock_handle_request, @@ -1478,7 +1478,7 @@ async def test_handle_streamable_http_mcp_per_server_header_skips_preemptive_cha connect gate must recognize it and forward instead of spuriously 401-ing, since egress already honors it. Without this, the mandatory multi-server binding is unusable at connect.""" try: - from litellm.proxy._experimental.mcp_server.server import handle_streamable_http_mcp # noqa: F401 + from litellm.proxy.mcp_server.server import handle_streamable_http_mcp # noqa: F401 except ImportError: pytest.skip("MCP server not available") @@ -1497,7 +1497,7 @@ async def test_handle_streamable_http_mcp_sanitized_per_server_header_skips_pree alias 'pt-server' arrives as the header key 'pt_server'. Egress resolves that via the sanitized alias, so the connect gate must too, or it 401s a token egress would forward.""" try: - from litellm.proxy._experimental.mcp_server.server import handle_streamable_http_mcp # noqa: F401 + from litellm.proxy.mcp_server.server import handle_streamable_http_mcp # noqa: F401 except ImportError: pytest.skip("MCP server not available") @@ -1515,7 +1515,7 @@ async def test_handle_streamable_http_mcp_aggregate_does_not_preemptively_challe """A multi-server aggregate must degrade gracefully: the preemptive 401 is single-server only, so one server missing a token cannot 401 the whole connect (the listing absorbs per-server failures).""" try: - from litellm.proxy._experimental.mcp_server.server import handle_streamable_http_mcp # noqa: F401 + from litellm.proxy.mcp_server.server import handle_streamable_http_mcp # noqa: F401 except ImportError: pytest.skip("MCP server not available") @@ -1536,7 +1536,7 @@ async def test_handle_streamable_http_mcp_true_passthrough_without_token_surface from fastapi import HTTPException try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, ) @@ -1565,21 +1565,21 @@ async def test_handle_streamable_http_mcp_true_passthrough_without_token_surface with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(user_auth, None, ["tp_server"], None, None, None), ), - patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), + patch("litellm.proxy.mcp_server.server.set_auth_context"), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.server.get_async_httpx_client", + "litellm.proxy.mcp_server.server.get_async_httpx_client", return_value=probe_client, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=tp_server, ), patch.object( @@ -1606,7 +1606,7 @@ async def test_handle_streamable_http_mcp_true_passthrough_dcr_bridge_challenges from fastapi import HTTPException try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateful, ) @@ -1633,21 +1633,21 @@ async def test_handle_streamable_http_mcp_true_passthrough_dcr_bridge_challenges with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(user_auth, None, ["tp_bridge_server"], None, None, None), ), - patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), + patch("litellm.proxy.mcp_server.server.set_auth_context"), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.server.get_async_httpx_client", + "litellm.proxy.mcp_server.server.get_async_httpx_client", return_value=probe_client, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=bridge_server, ), patch.object( @@ -1674,7 +1674,7 @@ async def test_handle_streamable_http_mcp_true_passthrough_with_token_skips_prob ``not _scope_has_authorization_header(scope)`` condition and the no-probe fast path.""" try: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateless, ) @@ -1702,30 +1702,30 @@ async def test_handle_streamable_http_mcp_true_passthrough_with_token_skips_prob with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(user_auth, None, ["tp_server"], None, None, None), ), - patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), + patch("litellm.proxy.mcp_server.server.set_auth_context"), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + "litellm.proxy.mcp_server.server._handle_stale_mcp_session", new_callable=AsyncMock, return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.server._check_passthrough_upstream_auth", + "litellm.proxy.mcp_server.server._check_passthrough_upstream_auth", new_callable=AsyncMock, ), patch( - "litellm.proxy._experimental.mcp_server.server.get_async_httpx_client", + "litellm.proxy.mcp_server.server.get_async_httpx_client", return_value=probe_client, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=tp_server, ), patch.object( @@ -1806,7 +1806,7 @@ async def test_streamable_http_skips_preflight_for_disabled_methods( # test-qua monkeypatch: pytest.MonkeyPatch, ) -> None: from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, session_manager_stateless, ) @@ -1838,39 +1838,39 @@ async def test_streamable_http_skips_preflight_for_disabled_methods( # test-qua with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, return_value=(user_auth, None, ["tp_server"], None, None, None), ), - patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), + patch("litellm.proxy.mcp_server.server.set_auth_context"), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + "litellm.proxy.mcp_server.server._handle_stale_mcp_session", new_callable=AsyncMock, return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.server._raise_preemptive_401_for_unauthenticated_servers", + "litellm.proxy.mcp_server.server._raise_preemptive_401_for_unauthenticated_servers", new_callable=AsyncMock, ) as mock_preemptive_401, patch( - "litellm.proxy._experimental.mcp_server.server._check_passthrough_upstream_auth", + "litellm.proxy.mcp_server.server._check_passthrough_upstream_auth", new_callable=AsyncMock, ) as mock_passthrough_auth, patch( - "litellm.proxy._experimental.mcp_server.server._preflight_modern_interaction", + "litellm.proxy.mcp_server.server._preflight_modern_interaction", new_callable=AsyncMock, return_value=None, ) as mock_modern_preflight, patch( - "litellm.proxy._experimental.mcp_server.server.get_async_httpx_client", + "litellm.proxy.mcp_server.server.get_async_httpx_client", return_value=probe_client, ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=tp_server, ), patch.object( diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_tool_search.py b/tests/unit/proxy/mcp_server/test_mcp_tool_search.py similarity index 90% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_tool_search.py rename to tests/unit/proxy/mcp_server/test_mcp_tool_search.py index ea0dd70cd22..b21202ba227 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_tool_search.py +++ b/tests/unit/proxy/mcp_server/test_mcp_tool_search.py @@ -1,4 +1,5 @@ -from litellm.proxy._experimental.mcp_server import operations as mcp_operations +from litellm.proxy.mcp_server import operations as mcp_operations + """ Tests for MCP tool search feature. @@ -20,14 +21,16 @@ from typing import Any, Final from unittest.mock import AsyncMock, MagicMock, patch import pytest -from pydantic import JsonValue from mcp.types import Tool +from pydantic import JsonValue import litellm from litellm.models.object_permission import LiteLLM_ObjectPermissionTable -from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing -from litellm.proxy._experimental.mcp_server.mcp_server_manager import ListedToolsCaller -from litellm.proxy._experimental.mcp_server.tool_search import ( +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.common_utils.semantic_text_index import EmbeddingFailed, SemanticTextIndex, Vector +from litellm.proxy.mcp_server.faults.list_outcomes import AggregateToolListing +from litellm.proxy.mcp_server.mcp_server_manager import ListedToolsCaller +from litellm.proxy.mcp_server.tool_search import ( AGENT_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME, MCP_TOOL_SEARCH_TOOL_NAME, @@ -41,8 +44,6 @@ from litellm.proxy._experimental.mcp_server.tool_search import ( search_mcp_tools, search_tools, ) -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth -from litellm.proxy.common_utils.semantic_text_index import EmbeddingFailed, SemanticTextIndex, Vector from litellm.types.mcp import MCPToolSearchSettings, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -97,7 +98,7 @@ SAMPLE_TOOLS = _make_tools( def test_offline_schema_validation_preserves_supported_constraints( schema: Mapping[str, JsonValue], arguments: Mapping[str, JsonValue], error: str | None ) -> None: - from litellm.proxy._experimental.mcp_server.tool_search import _validate_tool_arguments + from litellm.proxy.mcp_server.tool_search import _validate_tool_arguments result: Final = _validate_tool_arguments(schema, arguments) if error is None: @@ -108,7 +109,7 @@ def test_offline_schema_validation_preserves_supported_constraints( @pytest.mark.parametrize(("count", "allowed"), ((4_999, True), (5_000, False))) def test_validation_node_limit(count: int, allowed: bool) -> None: - from litellm.proxy._experimental.mcp_server.tool_search import _validation_limit_error + from litellm.proxy.mcp_server.tool_search import _validation_limit_error result: Final = _validation_limit_error({}, {str(index): None for index in range(count)}) assert (result is None) is allowed @@ -116,7 +117,7 @@ def test_validation_node_limit(count: int, allowed: bool) -> None: @pytest.mark.parametrize(("size", "allowed"), ((1_048_575, True), (1_048_576, False))) def test_validation_text_limit(size: int, allowed: bool) -> None: - from litellm.proxy._experimental.mcp_server.tool_search import _validation_limit_error + from litellm.proxy.mcp_server.tool_search import _validation_limit_error assert (_validation_limit_error({}, {"x": "a" * size}) is None) is allowed @@ -125,7 +126,7 @@ def test_validation_text_limit(size: int, allowed: bool) -> None: def test_validation_depth_limit(depth: int, allowed: bool) -> None: from functools import reduce - from litellm.proxy._experimental.mcp_server.tool_search import _validation_limit_error + from litellm.proxy.mcp_server.tool_search import _validation_limit_error nested: Final = reduce(lambda value, _: {"x": value}, range(depth), {}) assert (_validation_limit_error({}, nested) is None) is allowed @@ -133,7 +134,7 @@ def test_validation_depth_limit(depth: int, allowed: bool) -> None: @pytest.mark.asyncio async def test_oversized_arguments_are_rejected_before_worker_submission() -> None: - from litellm.proxy._experimental.mcp_server.tool_search import _tool_argument_validation_error + from litellm.proxy.mcp_server.tool_search import _tool_argument_validation_error with patch("anyio.to_process.run_sync", new_callable=AsyncMock) as submit: result: Final = await _tool_argument_validation_error({}, {"value": "x" * 1_048_576}) @@ -145,7 +146,7 @@ async def test_oversized_arguments_are_rejected_before_worker_submission() -> No async def test_failed_validation_worker_returns_tool_error() -> None: from anyio import BrokenWorkerProcess - from litellm.proxy._experimental.mcp_server.tool_search import _tool_argument_validation_error + from litellm.proxy.mcp_server.tool_search import _tool_argument_validation_error with patch("anyio.to_process.run_sync", side_effect=BrokenWorkerProcess): result: Final = await _tool_argument_validation_error({}, {}) @@ -472,7 +473,7 @@ class TestGetVirtualToolDefinitions: class TestListToolRestApiWithToolSearch: @pytest.mark.asyncio async def test_returns_only_virtual_tools_when_flag_enabled(self) -> None: - from litellm.proxy._experimental.mcp_server.rest_endpoints import router + from litellm.proxy.mcp_server.rest_endpoints import router user_api_key_dict = UserAPIKeyAuth( api_key="test_key", @@ -509,7 +510,7 @@ class TestListToolRestApiWithToolSearch: @pytest.mark.asyncio async def test_returns_full_catalog_when_flag_disabled(self) -> None: - from litellm.proxy._experimental.mcp_server.rest_endpoints import router + from litellm.proxy.mcp_server.rest_endpoints import router user_api_key_dict = UserAPIKeyAuth( api_key="test_key", @@ -538,32 +539,32 @@ class TestListToolRestApiWithToolSearch: with ( patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints.build_effective_auth_contexts", + "litellm.proxy.mcp_server.rest_endpoints.build_effective_auth_contexts", new_callable=AsyncMock, return_value=[user_api_key_dict], ), - patch("litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager") as mock_manager, + patch("litellm.proxy.mcp_server.rest_endpoints.global_mcp_server_manager") as mock_manager, patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._get_tools_for_single_server", + "litellm.proxy.mcp_server.rest_endpoints._get_tools_for_single_server", new_callable=AsyncMock, return_value=fake_tools, ), - patch("litellm.proxy._experimental.mcp_server.rest_endpoints.IPAddressUtils"), + patch("litellm.proxy.mcp_server.rest_endpoints.IPAddressUtils"), patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._prefetch_user_oauth_creds", + "litellm.proxy.mcp_server.rest_endpoints._prefetch_user_oauth_creds", new_callable=AsyncMock, return_value={}, ), patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._v1_resolved_oauth2_server_ids", + "litellm.proxy.mcp_server.rest_endpoints._v1_resolved_oauth2_server_ids", return_value=[], ), patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._get_server_auth_header", + "litellm.proxy.mcp_server.rest_endpoints._get_server_auth_header", return_value=None, ), patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._get_user_oauth_extra_headers", + "litellm.proxy.mcp_server.rest_endpoints._get_user_oauth_extra_headers", new_callable=AsyncMock, return_value=None, ), @@ -587,7 +588,7 @@ class TestListToolRestApiWithToolSearch: """Regression: an admin listing with include_disabled_tools must see the real catalog (to configure allowlists) even when mcp_tool_search_enabled is set, instead of the two virtual tools.""" - from litellm.proxy._experimental.mcp_server.rest_endpoints import router + from litellm.proxy.mcp_server.rest_endpoints import router user_api_key_dict = UserAPIKeyAuth( api_key="admin_key", @@ -617,32 +618,32 @@ class TestListToolRestApiWithToolSearch: with ( patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints.build_effective_auth_contexts", + "litellm.proxy.mcp_server.rest_endpoints.build_effective_auth_contexts", new_callable=AsyncMock, return_value=[user_api_key_dict], ), - patch("litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager") as mock_manager, + patch("litellm.proxy.mcp_server.rest_endpoints.global_mcp_server_manager") as mock_manager, patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._get_tools_for_single_server", + "litellm.proxy.mcp_server.rest_endpoints._get_tools_for_single_server", new_callable=AsyncMock, return_value=fake_tools, ), - patch("litellm.proxy._experimental.mcp_server.rest_endpoints.IPAddressUtils"), + patch("litellm.proxy.mcp_server.rest_endpoints.IPAddressUtils"), patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._prefetch_user_oauth_creds", + "litellm.proxy.mcp_server.rest_endpoints._prefetch_user_oauth_creds", new_callable=AsyncMock, return_value={}, ), patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._v1_resolved_oauth2_server_ids", + "litellm.proxy.mcp_server.rest_endpoints._v1_resolved_oauth2_server_ids", return_value=[], ), patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._get_server_auth_header", + "litellm.proxy.mcp_server.rest_endpoints._get_server_auth_header", return_value=None, ), patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._get_user_oauth_extra_headers", + "litellm.proxy.mcp_server.rest_endpoints._get_user_oauth_extra_headers", new_callable=AsyncMock, return_value=None, ), @@ -672,7 +673,7 @@ class TestCallToolRestApiVirtualTools: return mock_request def _get_call_fn(self) -> Any: - from litellm.proxy._experimental.mcp_server.rest_endpoints import router + from litellm.proxy.mcp_server.rest_endpoints import router return next( r.endpoint @@ -698,7 +699,7 @@ class TestCallToolRestApiVirtualTools: mock_tool.input_schema = {"type": "object", "properties": {}} with patch( - "litellm.proxy._experimental.mcp_server.operations._list_mcp_tools", + "litellm.proxy.mcp_server.operations._list_mcp_tools", new_callable=AsyncMock, return_value=AggregateToolListing(tools=[mock_tool], outcomes={}), ): @@ -742,17 +743,17 @@ class TestCallToolRestApiVirtualTools: with ( patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[MagicMock()], ), patch( - "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool", + "litellm.proxy.mcp_server.operations.execute_mcp_tool", new_callable=AsyncMock, return_value=fake_result, ) as mock_execute, patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._fire_mcp_tool_call_logging", + "litellm.proxy.mcp_server.rest_endpoints._fire_mcp_tool_call_logging", new_callable=AsyncMock, side_effect=RuntimeError("logging failed"), ) as mock_fire_logging, @@ -791,16 +792,16 @@ class TestCallToolRestApiVirtualTools: with ( patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints.IPAddressUtils.get_mcp_client_ip", + "litellm.proxy.mcp_server.rest_endpoints.IPAddressUtils.get_mcp_client_ip", return_value="203.0.113.7", ), patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[MagicMock()], ) as mock_allowed, patch( - "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool", + "litellm.proxy.mcp_server.operations.execute_mcp_tool", new_callable=AsyncMock, return_value=fake_result, ), @@ -821,11 +822,11 @@ class TestCallToolRestApiVirtualTools: with ( patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints.IPAddressUtils.get_mcp_client_ip", + "litellm.proxy.mcp_server.rest_endpoints.IPAddressUtils.get_mcp_client_ip", return_value="203.0.113.7", ), patch( - "litellm.proxy._experimental.mcp_server.operations._list_mcp_tools", + "litellm.proxy.mcp_server.operations._list_mcp_tools", new_callable=AsyncMock, return_value=AggregateToolListing(tools=[], outcomes={}), ) as mock_list, @@ -893,7 +894,7 @@ class TestCallToolRestApiVirtualTools: ) fake_result = CallToolResult(content=[TextContent(type="text", text="[]")], isError=False) with patch( # test-quality-ok: the embedding router only resolves via proxy_server globals, no injection seam - "litellm.proxy._experimental.mcp_server.tool_search.handle_skill_search", + "litellm.proxy.mcp_server.tool_search.handle_skill_search", new_callable=AsyncMock, return_value=fake_result, ) as mock_search: @@ -958,7 +959,7 @@ class TestCallToolRestApiVirtualTools: "litellm.proxy.proxy_server.proxy_logging_obj", key_limits ), patch( # test-quality-ok: the authorized catalog is the seam every virtual tool shares; the ranking under test stays real - "litellm.proxy._experimental.mcp_server.operations._list_mcp_tools", + "litellm.proxy.mcp_server.operations._list_mcp_tools", new_callable=AsyncMock, return_value=AggregateToolListing(tools=list(CATALOG), outcomes={}), ) as mock_list, @@ -1027,7 +1028,7 @@ class TestDispatchVirtualMcpTool: @pytest.mark.asyncio async def test_returns_none_for_non_virtual_tool(self) -> None: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _dispatch_virtual_mcp_tool, ) @@ -1041,7 +1042,7 @@ class TestDispatchVirtualMcpTool: @pytest.mark.asyncio async def test_rejects_when_flag_disabled(self) -> None: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( _dispatch_virtual_mcp_tool, ) @@ -1057,11 +1058,11 @@ class TestDispatchVirtualMcpTool: @pytest.mark.asyncio async def test_routes_search_with_client_ip(self) -> None: - from litellm.proxy._experimental.mcp_server import server as srv + from litellm.proxy.mcp_server import server as srv uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) with patch( - "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", + "litellm.proxy.mcp_server.tool_search.handle_mcp_tool_search", new_callable=AsyncMock, return_value="SEARCH_RESULT", ) as mock_search: @@ -1079,11 +1080,11 @@ class TestDispatchVirtualMcpTool: @pytest.mark.asyncio async def test_routes_agent_search_to_its_handler(self) -> None: - from litellm.proxy._experimental.mcp_server import server as srv + from litellm.proxy.mcp_server import server as srv uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) with patch( # test-quality-ok: dispatch routing is the subject; the handler is faked like its siblings here - "litellm.proxy._experimental.mcp_server.tool_search.handle_agent_search", + "litellm.proxy.mcp_server.tool_search.handle_agent_search", new_callable=AsyncMock, return_value="AGENT_RESULT", ) as mock_agent_search: @@ -1103,7 +1104,7 @@ class TestDispatchVirtualMcpTool: @pytest.mark.asyncio async def test_agent_search_rejected_when_flag_disabled(self) -> None: - from litellm.proxy._experimental.mcp_server.server import _dispatch_virtual_mcp_tool + from litellm.proxy.mcp_server.server import _dispatch_virtual_mcp_tool uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=False)) result = await _dispatch_virtual_mcp_tool( @@ -1114,11 +1115,11 @@ class TestDispatchVirtualMcpTool: @pytest.mark.asyncio async def test_routes_call_with_client_ip(self) -> None: - from litellm.proxy._experimental.mcp_server import server as srv + from litellm.proxy.mcp_server import server as srv uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) with patch( - "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", + "litellm.proxy.mcp_server.tool_search.handle_mcp_tool_call", new_callable=AsyncMock, return_value="CALL_RESULT", ) as mock_call: @@ -1147,7 +1148,7 @@ class TestDispatchVirtualMcpTool: """Regression: the SSE dispatch must run the pre-call pipeline and forward the resulting logging object to handle_mcp_tool_call, otherwise mcp_tool_call over /mcp/ skips spend logging and guardrails (unlike the REST path).""" - from litellm.proxy._experimental.mcp_server import server as srv + from litellm.proxy.mcp_server import server as srv uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) sentinel_logging_obj = object() @@ -1158,7 +1159,7 @@ class TestDispatchVirtualMcpTool: return_value=sentinel_logging_obj, ) as mock_build, patch( - "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_call", + "litellm.proxy.mcp_server.tool_search.handle_mcp_tool_call", new_callable=AsyncMock, return_value="CALL_RESULT", ) as mock_call, @@ -1177,11 +1178,11 @@ class TestDispatchVirtualMcpTool: async def test_search_coerces_non_int_top_k(self) -> None: """Regression: a non-integer top_k from an MCP client must not raise; it falls back to the default instead of ValueError propagating out.""" - from litellm.proxy._experimental.mcp_server import server as srv + from litellm.proxy.mcp_server import server as srv uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) with patch( - "litellm.proxy._experimental.mcp_server.tool_search.handle_mcp_tool_search", + "litellm.proxy.mcp_server.tool_search.handle_mcp_tool_search", new_callable=AsyncMock, return_value="SEARCH_RESULT", ) as mock_search: @@ -1200,7 +1201,7 @@ class TestDispatchVirtualMcpTool: upstream MCP servers needing pass-through auth can be called.""" from mcp.types import CallToolResult, TextContent - from litellm.proxy._experimental.mcp_server.tool_search import ( + from litellm.proxy.mcp_server.tool_search import ( handle_mcp_tool_call, ) @@ -1208,12 +1209,12 @@ class TestDispatchVirtualMcpTool: fake = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False) with ( patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[MagicMock()], ) as mock_allowed, patch( - "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool", + "litellm.proxy.mcp_server.operations.execute_mcp_tool", new_callable=AsyncMock, return_value=fake, ) as mock_exec, @@ -1248,19 +1249,19 @@ class TestDispatchVirtualMcpTool: run via the local registry without a server permission check.""" from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server.tool_search import ( + from litellm.proxy.mcp_server.tool_search import ( handle_mcp_tool_call, ) uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) with ( patch( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[], ), patch( - "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool", + "litellm.proxy.mcp_server.operations.execute_mcp_tool", new_callable=AsyncMock, ) as mock_exec, ): @@ -1278,7 +1279,7 @@ class TestDispatchVirtualMcpTool: class TestCaptureHostProgressCallback: @pytest.mark.parametrize("meta", [None, {}, {"traceparent": "trace"}]) def test_returns_none_without_progress(self, _mcp_request_ctx, meta) -> None: - from litellm.proxy._experimental.mcp_server.server import _capture_host_progress_callback + from litellm.proxy.mcp_server.server import _capture_host_progress_callback assert _capture_host_progress_callback(_mcp_request_ctx(meta=meta)) is None @@ -1292,7 +1293,7 @@ class TestCaptureHostProgressCallback: from mcp.shared.dispatcher import DispatchContext from mcp.types import CallToolRequestParams - from litellm.proxy._experimental.mcp_server.server import _capture_host_progress_callback + from litellm.proxy.mcp_server.server import _capture_host_progress_callback params = CallToolRequestParams.model_validate( {"name": "tool", "_meta": {"progressToken": token}}, by_name=False @@ -1312,11 +1313,11 @@ class TestHandleListToolsVirtual: @pytest.mark.asyncio async def test_returns_virtual_tools_when_flag_enabled(self, _mcp_request_ctx) -> None: - from litellm.proxy._experimental.mcp_server import server as srv + from litellm.proxy.mcp_server import server as srv uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) with patch( - "litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context", + "litellm.proxy.mcp_server.server.get_or_extract_auth_context", new_callable=AsyncMock, return_value=(uak, None, None, None, None, None, None), ): @@ -1338,17 +1339,17 @@ class TestMcpServerToolCallErrorHandling: async def test_virtual_tool_error_returns_iserror_not_raised(self, _mcp_request_ctx) -> None: from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server import server as srv + from litellm.proxy.mcp_server import server as srv uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) with ( patch( - "litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context", + "litellm.proxy.mcp_server.server.get_or_extract_auth_context", new_callable=AsyncMock, return_value=(uak, None, None, None, None, None, None), ), patch( - "litellm.proxy._experimental.mcp_server.operations._dispatch_virtual_mcp_tool", + "litellm.proxy.mcp_server.operations._dispatch_virtual_mcp_tool", new_callable=AsyncMock, side_effect=HTTPException(status_code=403, detail="User not allowed to call this tool"), ), @@ -1371,7 +1372,7 @@ class TestMcpServerToolCallErrorHandling: async def test_handle_mcp_tool_call_scoped_denial_names_the_binding_agent() -> None: from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server.tool_search import handle_mcp_tool_call + from litellm.proxy.mcp_server.tool_search import handle_mcp_tool_call agent_bound_key = UserAPIKeyAuth(api_key="test_key", agent_id="agent-123") @@ -1385,7 +1386,7 @@ async def test_handle_mcp_tool_call_scoped_denial_names_the_binding_agent() -> N ] with patch( # test-quality-ok: the permission resolver is a module-level function; the suite's only seam - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(side_effect=resolve), ): with pytest.raises(HTTPException) as exc_info: @@ -1435,7 +1436,7 @@ async def test_mcp_tool_search_leaves_the_listed_tools_slot_empty(monkeypatch: p async def test_cold_concurrent_schema_validation_accepts_valid_arguments() -> None: import psutil - from litellm.proxy._experimental.mcp_server.tool_search import ( + from litellm.proxy.mcp_server.tool_search import ( _MAX_VALIDATION_WORKERS, _tool_argument_validation_error, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py b/tests/unit/proxy/mcp_server/test_mcp_toolset_scope.py similarity index 88% rename from tests/unit/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py rename to tests/unit/proxy/mcp_server/test_mcp_toolset_scope.py index b9eebca3baa..f3cc38eacf9 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py +++ b/tests/unit/proxy/mcp_server/test_mcp_toolset_scope.py @@ -34,7 +34,7 @@ def _make_auth( def _granted_through_team(*team_toolset_ids: str) -> Callable[[UserAPIKeyAuth], Awaitable[frozenset[str]]]: """The real grant resolver over a team that holds ``team_toolset_ids``, with no key access rule.""" - from litellm.proxy._experimental.mcp_server.ui_session_utils import granted_toolset_ids + from litellm.proxy.mcp_server.ui_session_utils import granted_toolset_ids async def team_permission(context: UserAPIKeyAuth) -> LiteLLM_ObjectPermissionTable: return LiteLLM_ObjectPermissionTable(object_permission_id="team-op", mcp_toolsets=list(team_toolset_ids)) @@ -50,14 +50,14 @@ class TestApplyToolsetScope: @pytest.mark.asyncio async def test_restricts_to_toolset_servers_and_tools(self): - from litellm.proxy._experimental.mcp_server.server import apply_toolset_scope + from litellm.proxy.mcp_server.server import apply_toolset_scope toolset_perms = { "server-a": ["tool1", "tool2"], "server-b": ["tool3"], } with patch( - "litellm.proxy._experimental.mcp_server.server." + "litellm.proxy.mcp_server.server." "global_mcp_server_manager.resolve_toolset_tool_permissions", new=AsyncMock(return_value=toolset_perms), ): @@ -73,8 +73,8 @@ class TestApplyToolsetScope: assert set(op.mcp_servers or []) == {"server-a", "server-b"} assert op.mcp_tool_permissions == toolset_perms - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager - from litellm.proxy._experimental.mcp_server.operations import prepare_context + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.operations import prepare_context manager = MCPServerManager() unscoped_open = await manager.operator_open_server_ids( @@ -92,11 +92,11 @@ class TestApplyToolsetScope: @pytest.mark.asyncio async def test_admin_creates_object_permission_when_none(self): """Admin key with object_permission=None can access any toolset.""" - from litellm.proxy._experimental.mcp_server.server import apply_toolset_scope + from litellm.proxy.mcp_server.server import apply_toolset_scope toolset_perms = {"server-a": ["tool1"]} with patch( - "litellm.proxy._experimental.mcp_server.server." + "litellm.proxy.mcp_server.server." "global_mcp_server_manager.resolve_toolset_tool_permissions", new=AsyncMock(return_value=toolset_perms), ): @@ -116,12 +116,12 @@ class TestApplyToolsetScope: async def test_team_granted_toolset_is_served_to_a_key_without_its_own_grant(self): """A team key whose own row carries no toolset grant is admitted to the toolset its team holds (LIT-6029), scoped to that toolset's servers and tools.""" - from litellm.proxy._experimental.mcp_server.server import apply_toolset_scope + from litellm.proxy.mcp_server.server import apply_toolset_scope toolset_perms = {"server-a": ["tool1"]} auth = UserAPIKeyAuth(api_key="sk-test", team_id="team-a", object_permission=None) with patch( - "litellm.proxy._experimental.mcp_server.server." + "litellm.proxy.mcp_server.server." "global_mcp_server_manager.resolve_toolset_tool_permissions", new=AsyncMock(return_value=toolset_perms), ): @@ -138,7 +138,7 @@ class TestApplyToolsetScope: team-granted toolset is not capped by the user's own row: the row stays intact and the toolset rides along as mcp_toolset_id (LIT-6029).""" from litellm.constants import UI_SESSION_TOKEN_TEAM_ID - from litellm.proxy._experimental.mcp_server.server import apply_toolset_scope + from litellm.proxy.mcp_server.server import apply_toolset_scope session = UserAPIKeyAuth(team_id=UI_SESSION_TOKEN_TEAM_ID, user_id="user-1") own_row = LiteLLM_ObjectPermissionTable(object_permission_id="user-op", mcp_servers=["server-own"]) @@ -147,7 +147,7 @@ class TestApplyToolsetScope: granted = AsyncMock(return_value=frozenset({"toolset-123"})) resolve = AsyncMock(return_value={"server-team": ["tool1"]}) with patch( - "litellm.proxy._experimental.mcp_server.server." + "litellm.proxy.mcp_server.server." "global_mcp_server_manager.resolve_toolset_tool_permissions", new=resolve, ): @@ -163,7 +163,7 @@ class TestApplyToolsetScope: @pytest.mark.asyncio async def test_a_gateway_admitted_user_without_the_toolset_in_any_source_is_denied(self): - from litellm.proxy._experimental.mcp_server.server import apply_toolset_scope + from litellm.proxy.mcp_server.server import apply_toolset_scope admitted = UserAPIKeyAuth(user_id="user-1", object_permission=None) admitted.mcp_admitted_user_subject = True @@ -178,7 +178,7 @@ class TestApplyToolsetScope: async def test_a_resource_scoped_admitted_user_is_denied_a_team_toolset_on_another_server(self): """A gateway bearer scoped to server-own (RFC 8707 resource) cannot open a team toolset whose servers lie outside that resource, even though the team grants it (Devin Review 4150024267).""" - from litellm.proxy._experimental.mcp_server.server import apply_toolset_scope + from litellm.proxy.mcp_server.server import apply_toolset_scope admitted = UserAPIKeyAuth(user_id="user-1", object_permission=None) admitted.mcp_admitted_user_subject = True @@ -187,7 +187,7 @@ class TestApplyToolsetScope: granted = AsyncMock(return_value=frozenset({"toolset-123"})) resolve = AsyncMock(return_value={"server-team": ["tool1"]}) with patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.resolve_toolset_tool_permissions", + "litellm.proxy.mcp_server.server.global_mcp_server_manager.resolve_toolset_tool_permissions", new=resolve, ): with pytest.raises(HTTPException) as exc_info: @@ -198,7 +198,7 @@ class TestApplyToolsetScope: @pytest.mark.asyncio async def test_a_resource_scoped_admitted_user_opens_a_toolset_inside_its_resource(self): - from litellm.proxy._experimental.mcp_server.server import apply_toolset_scope + from litellm.proxy.mcp_server.server import apply_toolset_scope admitted = UserAPIKeyAuth(user_id="user-1", object_permission=None) admitted.mcp_admitted_user_subject = True @@ -206,7 +206,7 @@ class TestApplyToolsetScope: granted = AsyncMock(return_value=frozenset({"toolset-123"})) resolve = AsyncMock(return_value={"server-team": ["tool1"], "server-other": ["tool2"]}) with patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.resolve_toolset_tool_permissions", + "litellm.proxy.mcp_server.server.global_mcp_server_manager.resolve_toolset_tool_permissions", new=resolve, ): result = await apply_toolset_scope(admitted, "toolset-123", granted=granted) @@ -217,7 +217,7 @@ class TestApplyToolsetScope: @pytest.mark.asyncio async def test_team_grant_for_another_toolset_does_not_admit_a_key_to_this_one(self): - from litellm.proxy._experimental.mcp_server.server import apply_toolset_scope + from litellm.proxy.mcp_server.server import apply_toolset_scope auth = _make_auth(mcp_toolsets=[]) auth.team_id = "team-a" @@ -231,7 +231,7 @@ class TestApplyToolsetScope: """Non-admin key with object_permission=None is denied (no grants configured).""" from starlette.exceptions import HTTPException - from litellm.proxy._experimental.mcp_server.server import apply_toolset_scope + from litellm.proxy.mcp_server.server import apply_toolset_scope auth = UserAPIKeyAuth(api_key="sk-test", object_permission=None) with pytest.raises(HTTPException) as exc_info: @@ -246,7 +246,7 @@ class TestApplyToolsetScope: toolset path, which replaces mcp_servers and would drop the sentinel.""" from starlette.exceptions import HTTPException - from litellm.proxy._experimental.mcp_server.server import apply_toolset_scope + from litellm.proxy.mcp_server.server import apply_toolset_scope op = LiteLLM_ObjectPermissionTable( object_permission_id="test", @@ -259,7 +259,7 @@ class TestApplyToolsetScope: resolve = AsyncMock(return_value={"server-a": ["tool1"]}) with patch( - "litellm.proxy._experimental.mcp_server.server." + "litellm.proxy.mcp_server.server." "global_mcp_server_manager.resolve_toolset_tool_permissions", new=resolve, ): @@ -382,12 +382,12 @@ class TestFetchMCPToolsetsAccess: @pytest.mark.asyncio async def test_team_granted_toolsets_are_listed_for_a_key_without_its_own_grant(self): """GET /v1/mcp/toolset for a team key lists the team's toolsets (LIT-6029).""" - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( - MCPRequestHandler, - ) from litellm.proxy.management_endpoints.mcp_management_endpoints import ( fetch_mcp_toolsets, ) + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( + MCPRequestHandler, + ) auth = UserAPIKeyAuth(api_key="sk-test", team_id="team-a", object_permission=None) team_permission = LiteLLM_ObjectPermissionTable(object_permission_id="team-op", mcp_toolsets=["ts-team"]) @@ -417,12 +417,12 @@ class TestFetchMCPToolsetsAccess: @pytest.mark.asyncio async def test_admin_with_own_grants_is_not_narrowed_by_a_team_lookup(self): """An admin's own grant list is the only filter; no team lookup runs for admins.""" - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( - MCPRequestHandler, - ) from litellm.proxy.management_endpoints.mcp_management_endpoints import ( fetch_mcp_toolsets, ) + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( + MCPRequestHandler, + ) auth = _make_auth(mcp_toolsets=["ts-1"]) auth.user_role = LitellmUserRoles.PROXY_ADMIN @@ -454,12 +454,12 @@ class TestFetchMCPToolsetAccess: @staticmethod async def _fetch(auth: UserAPIKeyAuth, toolset_id: str, team_toolsets: list[str] | None): - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( - MCPRequestHandler, - ) from litellm.proxy.management_endpoints.mcp_management_endpoints import ( fetch_mcp_toolset, ) + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( + MCPRequestHandler, + ) team_permission = ( LiteLLM_ObjectPermissionTable(object_permission_id="team-op", mcp_toolsets=team_toolsets) @@ -542,10 +542,10 @@ class TestToolsetPrefixResolution: ): from mcp.types import Tool as MCPTool - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( filter_tools_by_key_team_permissions, ) - from litellm.proxy._experimental.mcp_server.utils import ( + from litellm.proxy.mcp_server.utils import ( add_server_prefix_to_name, get_server_prefix, ) @@ -564,12 +564,12 @@ class TestToolsetPrefixResolution: with ( patch( - "litellm.proxy._experimental.mcp_server.server." + "litellm.proxy.mcp_server.server." "MCPRequestHandler.get_allowed_tools_for_server", new=AsyncMock(return_value=allowed), ), patch( - "litellm.proxy._experimental.mcp_server.server." + "litellm.proxy.mcp_server.server." "global_mcp_server_manager.get_mcp_server_by_id", return_value=server, ), @@ -606,7 +606,7 @@ class TestToolsetPrefixResolution: async def test_resolve_keeps_a_name_that_looks_like_its_own_server_prefix( self, alias, server_name, server_id ): - from litellm.proxy._experimental.mcp_server.utils import ( + from litellm.proxy.mcp_server.utils import ( add_server_prefix_to_name, get_server_prefix, ) @@ -624,7 +624,7 @@ class TestToolsetPrefixResolution: async def _resolve(server, server_id, stored): from types import SimpleNamespace - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -635,14 +635,14 @@ class TestToolsetPrefixResolution: ) with ( patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager." + "litellm.proxy.mcp_server.mcp_server_manager." "global_mcp_server_manager.get_mcp_server_by_id", return_value=server, ), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), patch("litellm.proxy.proxy_server.user_api_key_cache", cache), patch( - "litellm.proxy._experimental.mcp_server.toolset_db.list_mcp_toolsets", + "litellm.proxy.mcp_server.toolset_db.list_mcp_toolsets", new=AsyncMock(return_value=[toolset]), ), ): @@ -661,10 +661,10 @@ class TestToolsetPrefixResolution: """ from mcp.types import Tool as MCPTool - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( filter_tools_by_key_team_permissions, ) - from litellm.proxy._experimental.mcp_server.utils import ( + from litellm.proxy.mcp_server.utils import ( add_server_prefix_to_name, get_server_prefix, strip_known_server_prefix, @@ -691,12 +691,12 @@ class TestToolsetPrefixResolution: with ( patch( - "litellm.proxy._experimental.mcp_server.server." + "litellm.proxy.mcp_server.server." "MCPRequestHandler.get_allowed_tools_for_server", new=AsyncMock(return_value=resolved["srv-collide"]), ), patch( - "litellm.proxy._experimental.mcp_server.server." + "litellm.proxy.mcp_server.server." "global_mcp_server_manager.get_mcp_server_by_id", return_value=server, ), @@ -725,10 +725,10 @@ class TestToolsetPrefixResolution: """ from mcp.types import Tool as MCPTool - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( filter_tools_by_key_team_permissions, ) - from litellm.proxy._experimental.mcp_server.utils import ( + from litellm.proxy.mcp_server.utils import ( add_server_prefix_to_name, get_server_prefix, ) @@ -748,12 +748,12 @@ class TestToolsetPrefixResolution: with ( patch( - "litellm.proxy._experimental.mcp_server.server." + "litellm.proxy.mcp_server.server." "MCPRequestHandler.get_allowed_tools_for_server", new=AsyncMock(return_value=resolved["srv-lonely"]), ), patch( - "litellm.proxy._experimental.mcp_server.server." + "litellm.proxy.mcp_server.server." "global_mcp_server_manager.get_mcp_server_by_id", return_value=server, ), @@ -770,7 +770,7 @@ class TestToolsetPrefixResolution: async def test_bare_stored_name_without_collision_grants_only_that_tool(self): """The ordinary row must stay exact; accepting both readings of an ambiguous row must not widen an unambiguous one.""" - from litellm.proxy._experimental.mcp_server.utils import get_server_prefix + from litellm.proxy.mcp_server.utils import get_server_prefix server = self._server("deepwiki", None, "srv-clean") assert get_server_prefix(server) == "deepwiki" @@ -784,12 +784,12 @@ class TestMCPActiveToolsetContextVar: """Tests for _mcp_active_toolset_id ContextVar — clients cannot inject it.""" def test_contextvar_default_is_none(self): - from litellm.proxy._experimental.mcp_server.server import mcp_active_toolset_id + from litellm.proxy.mcp_server.server import mcp_active_toolset_id assert mcp_active_toolset_id.get() is None def test_contextvar_set_and_reset(self): - from litellm.proxy._experimental.mcp_server.server import mcp_active_toolset_id + from litellm.proxy.mcp_server.server import mcp_active_toolset_id token = mcp_active_toolset_id.set("toolset-abc") assert mcp_active_toolset_id.get() == "toolset-abc" @@ -799,7 +799,7 @@ class TestMCPActiveToolsetContextVar: @pytest.mark.asyncio async def test_client_header_is_stripped_in_scope(self): """handle_streamable_http_mcp strips x-mcp-toolset-id from scope before passing to session manager.""" - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( handle_streamable_http_mcp, ) @@ -824,35 +824,35 @@ class TestMCPActiveToolsetContextVar: with ( patch( - "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + "litellm.proxy.mcp_server.server.extract_mcp_auth_context", new=AsyncMock( return_value=(mock_auth, None, [], {}, {}, scope["headers"]) ), ), patch( - "litellm.proxy._experimental.mcp_server.server.IPAddressUtils", + "litellm.proxy.mcp_server.server.IPAddressUtils", MagicMock(get_mcp_client_ip=MagicMock(return_value="127.0.0.1")), ), patch( - "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", + "litellm.proxy.mcp_server.operations.global_mcp_server_manager", MagicMock(get_mcp_server_by_name=MagicMock(return_value=None)), ), patch( - "litellm.proxy._experimental.mcp_server.server.MCPDebug", + "litellm.proxy.mcp_server.server.MCPDebug", MagicMock( maybe_build_debug_headers=MagicMock(return_value=None), ), ), patch( - "litellm.proxy._experimental.mcp_server.server.set_auth_context", + "litellm.proxy.mcp_server.server.set_auth_context", MagicMock(), ), patch( - "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + "litellm.proxy.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + "litellm.proxy.mcp_server.server._handle_stale_mcp_session", new=AsyncMock(return_value=True), ), ): diff --git a/tests/unit/proxy/_experimental/mcp_server/test_oauth2_flow_backfill.py b/tests/unit/proxy/mcp_server/test_oauth2_flow_backfill.py similarity index 99% rename from tests/unit/proxy/_experimental/mcp_server/test_oauth2_flow_backfill.py rename to tests/unit/proxy/mcp_server/test_oauth2_flow_backfill.py index ce079878328..7a7e1cf390a 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_oauth2_flow_backfill.py +++ b/tests/unit/proxy/mcp_server/test_oauth2_flow_backfill.py @@ -17,7 +17,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest -from litellm.proxy._experimental.mcp_server.oauth2_flow_backfill import ( +from litellm.proxy.mcp_server.oauth2_flow_backfill import ( backfill_null_oauth2_flows, classify_null_flow_row, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_oauth2_mcp_config.yaml b/tests/unit/proxy/mcp_server/test_oauth2_mcp_config.yaml similarity index 100% rename from tests/unit/proxy/_experimental/mcp_server/test_oauth2_mcp_config.yaml rename to tests/unit/proxy/mcp_server/test_oauth2_mcp_config.yaml diff --git a/tests/unit/proxy/_experimental/mcp_server/test_oauth2_token_cache.py b/tests/unit/proxy/mcp_server/test_oauth2_token_cache.py similarity index 90% rename from tests/unit/proxy/_experimental/mcp_server/test_oauth2_token_cache.py rename to tests/unit/proxy/mcp_server/test_oauth2_token_cache.py index 6eb93b090e9..f455096d390 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_oauth2_token_cache.py +++ b/tests/unit/proxy/mcp_server/test_oauth2_token_cache.py @@ -10,12 +10,12 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( +from litellm.proxy._types import MCPTransport +from litellm.proxy.mcp_server.oauth2_token_cache import ( MCPOAuth2TokenCache, resolve_mcp_auth, resolved_token_header, ) -from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -55,7 +55,7 @@ async def test_resolve_mcp_auth_fetches_oauth2_token(): mock_client.post.return_value = _token_response("m2m-token-1") with patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.get_async_httpx_client", + "litellm.proxy.mcp_server.oauth2_token_cache.get_async_httpx_client", return_value=mock_client, ): result = await resolve_mcp_auth(server) @@ -78,11 +78,11 @@ async def test_token_cached_across_calls(): with ( patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.get_async_httpx_client", + "litellm.proxy.mcp_server.oauth2_token_cache.get_async_httpx_client", return_value=mock_client, ), patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.mcp_oauth2_token_cache", + "litellm.proxy.mcp_server.oauth2_token_cache.mcp_oauth2_token_cache", cache, ), ): @@ -107,11 +107,11 @@ async def test_m2m_token_not_shared_across_server_ids_with_identical_config(): with ( patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.get_async_httpx_client", + "litellm.proxy.mcp_server.oauth2_token_cache.get_async_httpx_client", return_value=mock_client, ), patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.mcp_oauth2_token_cache", + "litellm.proxy.mcp_server.oauth2_token_cache.mcp_oauth2_token_cache", cache, ), ): @@ -178,7 +178,7 @@ async def test_http_error_raises_value_error(): with ( patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.get_async_httpx_client", + "litellm.proxy.mcp_server.oauth2_token_cache.get_async_httpx_client", return_value=mock_client, ), pytest.raises(ValueError, match="failed with status 401"), @@ -198,7 +198,7 @@ async def test_non_dict_response_raises_value_error(): with ( patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.get_async_httpx_client", + "litellm.proxy.mcp_server.oauth2_token_cache.get_async_httpx_client", return_value=mock_client, ), pytest.raises(ValueError, match="non-object JSON"), @@ -217,7 +217,7 @@ async def test_client_credentials_uses_client_secret_basic_when_configured(): mock_client.post.return_value = _token_response("m2m-basic") with patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.get_async_httpx_client", + "litellm.proxy.mcp_server.oauth2_token_cache.get_async_httpx_client", return_value=mock_client, ): result = await resolve_mcp_auth(server) @@ -236,7 +236,7 @@ def test_storage_ttl_capped_at_token_lifetime(): Redis fast path (which never re-checks expires_at) kept serving the dead token until eviction, while the stored refresh_token sat unused because refresh only runs on the DB read-through.""" from litellm.constants import MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS - from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( + from litellm.proxy.mcp_server.oauth2_token_cache import ( compute_per_user_token_ttl, ) @@ -247,7 +247,7 @@ def test_storage_ttl_capped_at_token_lifetime(): def test_storage_ttl_shorter_than_token_lifetime_wins(): """A configured TTL below the token lifetime is the operative value: the knob's purpose is to force earlier DB re-checks (staleness backstop), so the shorter side must win the min().""" - from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( + from litellm.proxy.mcp_server.oauth2_token_cache import ( compute_per_user_token_ttl, ) @@ -258,7 +258,7 @@ def test_storage_ttl_shorter_than_token_lifetime_wins(): def test_storage_ttl_verbatim_when_token_lifetime_unknown(): """With no expires_in from the upstream there is nothing to cap against, so the configured TTL applies as-is (matching the pre-cap behavior for lifetime-less tokens).""" - from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( + from litellm.proxy.mcp_server.oauth2_token_cache import ( compute_per_user_token_ttl, ) @@ -269,7 +269,7 @@ def test_storage_ttl_verbatim_when_token_lifetime_unknown(): def test_storage_ttl_floors_at_one_second_for_nearly_dead_token(): """A token already inside the expiry buffer yields the 1-second floor, not zero or a negative TTL, mirroring the floor the default (unconfigured) path has always had.""" - from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( + from litellm.proxy.mcp_server.oauth2_token_cache import ( compute_per_user_token_ttl, ) @@ -284,7 +284,7 @@ def test_default_ttl_paths_unchanged_without_storage_ttl(): MCP_PER_USER_TOKEN_DEFAULT_TTL, MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS, ) - from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( + from litellm.proxy.mcp_server.oauth2_token_cache import ( compute_per_user_token_ttl, ) @@ -311,7 +311,7 @@ async def test_client_credentials_sends_rfc8707_resource(configured, expected): mock_client.post.return_value = _token_response("m2m-tok") with patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.get_async_httpx_client", + "litellm.proxy.mcp_server.oauth2_token_cache.get_async_httpx_client", return_value=mock_client, ): await resolve_mcp_auth(server) @@ -340,7 +340,7 @@ async def test_token_cache_mints_afresh_when_the_token_request_changes(changed): mock_client.post.side_effect = [_token_response("tok-before"), _token_response("tok-after")] with patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.get_async_httpx_client", + "litellm.proxy.mcp_server.oauth2_token_cache.get_async_httpx_client", return_value=mock_client, ): before = await cache.async_get_token(_server()) @@ -360,7 +360,7 @@ async def test_token_cache_still_reuses_a_token_when_nothing_changed(): mock_client.post.return_value = _token_response("tok-reused") with patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.get_async_httpx_client", + "litellm.proxy.mcp_server.oauth2_token_cache.get_async_httpx_client", return_value=mock_client, ): first = await cache.async_get_token(_server(upstream_resource="auto")) @@ -383,7 +383,7 @@ async def test_invalidate_clears_every_identity_for_a_server(): ] with patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.get_async_httpx_client", + "litellm.proxy.mcp_server.oauth2_token_cache.get_async_httpx_client", return_value=mock_client, ): await cache.async_get_token(_server()) @@ -399,8 +399,8 @@ async def test_invalidate_clears_every_identity_for_a_server(): async def test_per_user_token_delete_evicts_locally_and_broadcasts_to_peer_workers(): """Revoking a user's OAuth token must not leave peer workers serving it from their in-memory layer.""" from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.oauth2_token_cache import MCPPerUserTokenCache from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.oauth2_token_cache import MCPPerUserTokenCache local_cache = UserApiKeyCache() publish = AsyncMock() @@ -433,7 +433,7 @@ async def test_m2m_mint_uses_admin_entered_token_url_when_issuer_yield_empties_r mock_client.post.return_value = _token_response("m2m-token-configured") with patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.get_async_httpx_client", + "litellm.proxy.mcp_server.oauth2_token_cache.get_async_httpx_client", return_value=mock_client, ): result = await cache.async_get_token(server) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_oauth_identity_binding.py b/tests/unit/proxy/mcp_server/test_oauth_identity_binding.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/test_oauth_identity_binding.py rename to tests/unit/proxy/mcp_server/test_oauth_identity_binding.py index 0036035f448..15c80abf576 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_oauth_identity_binding.py +++ b/tests/unit/proxy/mcp_server/test_oauth_identity_binding.py @@ -11,7 +11,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa from fastapi import HTTPException from pydantic import ValidationError -from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( +from litellm.proxy.mcp_server.oauth_identity_binding import ( RefreshOwnershipProven, RefreshTokenPresented, VerifiedRefreshToken, @@ -101,7 +101,7 @@ async def test_fetch_issuer_jwks_fetches_and_caches_keys(): client.get = AsyncMock(return_value=response) with patch( # test-quality-ok: no HTTP dependency injection seam exists for JWKS fetching - "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", + "litellm.proxy.mcp_server.oauth_identity_binding.get_async_httpx_client", return_value=client, ): first: Final = await _fetch_issuer_jwks(binding) @@ -121,7 +121,7 @@ async def test_fetch_issuer_jwks_rejects_malformed_document(): client.get = AsyncMock(return_value=response) with patch( # test-quality-ok: no HTTP dependency injection seam exists for JWKS fetching - "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", + "litellm.proxy.mcp_server.oauth_identity_binding.get_async_httpx_client", return_value=client, ): with pytest.raises(TypeError, match="has no 'keys' array"): @@ -136,7 +136,7 @@ async def test_discover_jwks_url_returns_provider_uri(): client.get = AsyncMock(return_value=response) with patch( # test-quality-ok: no HTTP dependency injection seam exists for OIDC discovery - "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", + "litellm.proxy.mcp_server.oauth_identity_binding.get_async_httpx_client", return_value=client, ): result: Final = await _discover_jwks_url("https://idp.example.com/") @@ -153,7 +153,7 @@ async def test_discover_jwks_url_rejects_missing_provider_uri(): client.get = AsyncMock(return_value=response) with patch( # test-quality-ok: no HTTP dependency injection seam exists for OIDC discovery - "litellm.proxy._experimental.mcp_server.oauth_identity_binding.get_async_httpx_client", + "litellm.proxy.mcp_server.oauth_identity_binding.get_async_httpx_client", return_value=client, ): with pytest.raises(ValueError, match="returned no jwks_uri"): @@ -177,7 +177,7 @@ async def test_load_caller_principal_supports_user_id_and_database_email(): assert await _load_caller_principal("user-a", user_id_binding) == "user-a" with patch( # test-quality-ok: caller loading is a lazy database boundary without injection - "litellm.proxy._experimental.mcp_server.bridge_token_flow.load_active_user_by_id", + "litellm.proxy.mcp_server.bridge_token_flow.load_active_user_by_id", new=AsyncMock(side_effect=["no_active_key", SimpleNamespace(user_email="alice@example.com")]), ): assert await _load_caller_principal("user-a", _server().oauth_identity_binding) is None @@ -191,7 +191,7 @@ async def test_load_stored_refresh_token_returns_credential_and_fails_closed(): get_credential: Final = AsyncMock(return_value={"refresh_token": "rt-1", "identity_binding_proof": proof}) with ( patch( # test-quality-ok: stored-token loading is a lazy database boundary without injection - "litellm.proxy._experimental.mcp_server.db.get_user_oauth_credential", + "litellm.proxy.mcp_server.db.get_user_oauth_credential", new=get_credential, ), patch( # test-quality-ok: stored-token loading is a lazy database boundary without injection @@ -627,7 +627,7 @@ async def test_authorization_code_rejects_missing_or_foreign_nonce(nonce): @pytest.mark.asyncio async def test_binding_proof_rejects_changed_user_or_policy(): - from litellm.proxy._experimental.mcp_server.oauth_identity_binding import credential_binding_matches + from litellm.proxy.mcp_server.oauth_identity_binding import credential_binding_matches binding = _server(caller_field="user_id", principal_claim="sub").oauth_identity_binding proof = await current_binding_proof(binding, "alice", "srv-1") diff --git a/tests/unit/proxy/_experimental/mcp_server/test_oauth_issuer_stamp_backfill.py b/tests/unit/proxy/mcp_server/test_oauth_issuer_stamp_backfill.py similarity index 98% rename from tests/unit/proxy/_experimental/mcp_server/test_oauth_issuer_stamp_backfill.py rename to tests/unit/proxy/mcp_server/test_oauth_issuer_stamp_backfill.py index a576bcbb3e2..4f0e91a2c1f 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_oauth_issuer_stamp_backfill.py +++ b/tests/unit/proxy/mcp_server/test_oauth_issuer_stamp_backfill.py @@ -9,10 +9,10 @@ import pytest from litellm._service_logger import ServiceTypes from litellm.proxy import proxy_server -from tests.unit.proxy.db.fake_prisma_engine import engine_call -from litellm.proxy._experimental.mcp_server.oauth_issuer_stamp_backfill import ( +from litellm.proxy.mcp_server.oauth_issuer_stamp_backfill import ( backfill_discovery_stamped_issuers, ) +from tests.unit.proxy.db.fake_prisma_engine import engine_call def _row(**overrides): diff --git a/tests/unit/proxy/_experimental/mcp_server/test_openapi_spec_path_url.py b/tests/unit/proxy/mcp_server/test_openapi_spec_path_url.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/test_openapi_spec_path_url.py rename to tests/unit/proxy/mcp_server/test_openapi_spec_path_url.py index 17a0022046e..aac85160eba 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_openapi_spec_path_url.py +++ b/tests/unit/proxy/mcp_server/test_openapi_spec_path_url.py @@ -5,7 +5,7 @@ from typing import Any, Dict import httpx import pytest -from litellm.proxy._experimental.mcp_server import openapi_to_mcp_generator as gen +from litellm.proxy.mcp_server import openapi_to_mcp_generator as gen class _FakeAsyncHTTPHandler: diff --git a/tests/unit/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py b/tests/unit/proxy/mcp_server/test_openapi_to_mcp_generator.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py rename to tests/unit/proxy/mcp_server/test_openapi_to_mcp_generator.py index 9f3db632db1..adb185e4242 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py +++ b/tests/unit/proxy/mcp_server/test_openapi_to_mcp_generator.py @@ -17,12 +17,11 @@ import pytest from fastapi import HTTPException from respx import MockRouter -from litellm.types.mcp import MCPAuth, MCPAuthType - -from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( - request_auth_header, - request_extra_headers, - request_resolved_auth_headers, +from litellm.proxy.mcp_server.exceptions import ( + MCPOpenApiUpstreamError, + MCPUpstreamAuthError, +) +from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( _request_upstream_url, _resolve_param_list, _resolve_ref, @@ -30,16 +29,15 @@ from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( create_tool_function, extract_parameters, get_base_url, + request_auth_header, + request_extra_headers, + request_resolved_auth_headers, resolve_operation_params, ) -from litellm.proxy._experimental.mcp_server.tool_outcome import JsonResult, TextResult +from litellm.proxy.mcp_server.tool_outcome import JsonResult, TextResult +from litellm.types.mcp import MCPAuth, MCPAuthType -from litellm.proxy._experimental.mcp_server.exceptions import ( - MCPOpenApiUpstreamError, - MCPUpstreamAuthError, -) - -GET_ASYNC_CLIENT_TARGET = "litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator.get_async_httpx_client" +GET_ASYNC_CLIENT_TARGET = "litellm.proxy.mcp_server.openapi_to_mcp_generator.get_async_httpx_client" @pytest.mark.asyncio @@ -1065,7 +1063,7 @@ class TestResolveOperationParams: class TestSanitizeOpenAPIToolName: def test_replaces_slashes(self): - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( sanitize_openapi_tool_name, ) @@ -1075,35 +1073,35 @@ class TestSanitizeOpenAPIToolName: ) def test_replaces_other_punctuation(self): - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( sanitize_openapi_tool_name, ) assert sanitize_openapi_tool_name("foo.bar:baz qux") == "foo_bar_baz_qux" def test_lowercases(self): - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( sanitize_openapi_tool_name, ) assert sanitize_openapi_tool_name("Pulls/List-Files") == "pulls_list-files" def test_already_valid_passes_through(self): - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( sanitize_openapi_tool_name, ) assert sanitize_openapi_tool_name("plain-tool_name") == "plain-tool_name" def test_empty_string(self): - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( sanitize_openapi_tool_name, ) assert sanitize_openapi_tool_name("") == "" def test_caps_at_128_chars(self): - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( sanitize_openapi_tool_name, ) @@ -1117,7 +1115,7 @@ class TestRegisterToolsFromOpenAPI: def test_github_style_operation_ids_are_sanitized(self, monkeypatch): import re - from litellm.proxy._experimental.mcp_server import openapi_to_mcp_generator + from litellm.proxy.mcp_server import openapi_to_mcp_generator registered: list = [] @@ -1165,7 +1163,7 @@ class TestRegisterToolsFromOpenAPI: ): import re - from litellm.proxy._experimental.mcp_server import openapi_to_mcp_generator + from litellm.proxy.mcp_server import openapi_to_mcp_generator registered: list = [] @@ -1561,7 +1559,7 @@ class TestBoundedOpenAPISpecLoading: @pytest.mark.asyncio @pytest.mark.parametrize("max_bytes", [12, 13]) async def test_exact_size_and_smaller_specs_load(self, respx_mock, monkeypatch, max_bytes): - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import load_openapi_spec_async + from litellm.proxy.mcp_server.openapi_to_mcp_generator import load_openapi_spec_async monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") route = respx_mock.get("https://93.184.216.34/spec.json").respond(200, content=b'{"paths":{}}') @@ -1572,8 +1570,9 @@ class TestBoundedOpenAPISpecLoading: @pytest.mark.parametrize("headers", [{"content-length": "1000000"}, {"content-encoding": "gzip"}]) async def test_unsafe_response_headers_reject_before_reading(self, respx_mock, monkeypatch, headers): import httpx + from litellm.llms.custom_httpx.http_handler import HTTPResponseLimitError - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( load_openapi_spec_async, ) @@ -1596,8 +1595,9 @@ class TestBoundedOpenAPISpecLoading: @pytest.mark.asyncio async def test_chunked_response_is_bounded_and_closed(self, respx_mock, monkeypatch): import httpx + from litellm.llms.custom_httpx.http_handler import HTTPResponseLimitError - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( load_openapi_spec_async, ) @@ -1624,7 +1624,7 @@ class TestBoundedOpenAPISpecLoading: @pytest.mark.parametrize("target", ["https://93.184.216.35/final.json", "http://127.0.0.1/private.json"]) async def test_bounded_spec_redirects_preserve_ssrf_protection(self, respx_mock, monkeypatch, target): from litellm.litellm_core_utils.url_utils import SSRFError - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import load_openapi_spec_async + from litellm.proxy.mcp_server.openapi_to_mcp_generator import load_openapi_spec_async monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") respx_mock.get("https://93.184.216.34/spec.json").respond(302, headers={"location": target}) @@ -1650,7 +1650,7 @@ def without_mcp(name, *args, **kwargs): raise ModuleNotFoundError('MCP SDK unavailable') return original_import(name, *args, **kwargs) builtins.__import__ = without_mcp -import litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator +import litellm.proxy.mcp_server.openapi_to_mcp_generator """ result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) assert result.returncode == 0, result.stderr diff --git a/tests/unit/proxy/_experimental/mcp_server/test_openapi_tool_auth.py b/tests/unit/proxy/mcp_server/test_openapi_tool_auth.py similarity index 91% rename from tests/unit/proxy/_experimental/mcp_server/test_openapi_tool_auth.py rename to tests/unit/proxy/mcp_server/test_openapi_tool_auth.py index ffa25d63391..3ff925c486b 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_openapi_tool_auth.py +++ b/tests/unit/proxy/mcp_server/test_openapi_tool_auth.py @@ -1,4 +1,5 @@ -from litellm.proxy._experimental.mcp_server import operations as mcp_operations +from litellm.proxy.mcp_server import operations as mcp_operations + """ VERIA-7 regression: OpenAPI-backed (local-registry) MCP tools must run through `pre_call_tool_check` before dispatch, the same as managed @@ -25,7 +26,7 @@ async def test_openapi_local_tool_runs_pre_call_tool_check(): """When `execute_mcp_tool` resolves a local-registry (OpenAPI) tool AND a server, the pre-call hook must fire before the local handler runs. Pre-fix this path skipped the hook entirely.""" - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module user = UserAPIKeyAuth( api_key="sk-user", @@ -70,11 +71,11 @@ async def test_openapi_local_tool_runs_pre_call_tool_check(): return_value=fake_tool, ), patch( - "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool", + "litellm.proxy.mcp_server.operations._handle_local_mcp_tool", new=handle_local, ), patch( - "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed", + "litellm.proxy.mcp_server.server.MCPRequestHandler.is_tool_allowed", return_value=True, ), ): @@ -112,7 +113,7 @@ async def test_openapi_local_tool_blocked_when_pre_call_check_raises(): tool), the local handler must NOT be invoked.""" from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module user = UserAPIKeyAuth( api_key="sk-user", @@ -159,11 +160,11 @@ async def test_openapi_local_tool_blocked_when_pre_call_check_raises(): return_value=fake_tool, ), patch( - "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool", + "litellm.proxy.mcp_server.operations._handle_local_mcp_tool", new=handle_local, ), patch( - "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed", + "litellm.proxy.mcp_server.server.MCPRequestHandler.is_tool_allowed", return_value=True, ), ): @@ -188,7 +189,7 @@ async def test_openapi_local_tool_denied_when_server_not_resolvable(): rather than dispatched without `pre_call_tool_check`.""" from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module user = UserAPIKeyAuth( api_key="sk-user", @@ -224,11 +225,11 @@ async def test_openapi_local_tool_denied_when_server_not_resolvable(): return_value=fake_tool, ), patch( - "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool", + "litellm.proxy.mcp_server.operations._handle_local_mcp_tool", new=handle_local, ), patch( - "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed", + "litellm.proxy.mcp_server.server.MCPRequestHandler.is_tool_allowed", return_value=True, ), ): @@ -256,14 +257,14 @@ async def test_openapi_local_tool_injects_resolved_oauth_token(): user's completed OAuth flow stored a token that never reached the upstream API. The resolved credential must land in the `_request_resolved_auth_headers` ContextVar the tool closure reads. Kills the mutant that deletes the resolve_openapi_upstream_auth call in server.py.""" - from litellm.proxy._experimental.mcp_server import server as mcp_module - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server import server as mcp_module + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( request_resolved_auth_headers, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + from litellm.proxy.mcp_server.outbound_credentials.httpx_auth import ( StaticHeaderAuth, ) - from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + from litellm.proxy.mcp_server.outbound_credentials.result import Ok from litellm.types.mcp import MCPAuth, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -315,11 +316,11 @@ async def test_openapi_local_tool_injects_resolved_oauth_token(): new=AsyncMock(return_value=Ok(StaticHeaderAuth("Bearer stored-user-token"))), ), patch( - "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool", + "litellm.proxy.mcp_server.operations._handle_local_mcp_tool", new=handle_local, ), patch( - "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed", + "litellm.proxy.mcp_server.server.MCPRequestHandler.is_tool_allowed", return_value=True, ), ): @@ -350,10 +351,10 @@ def legacy_local_tool(): Yields the server and the list the handler appends to, so a test can tell "refused" from "dispatched" by whether the handler actually ran. """ - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + from litellm.proxy.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) - from litellm.proxy._experimental.mcp_server.tool_registry import ( + from litellm.proxy.mcp_server.tool_registry import ( global_mcp_tool_registry, ) @@ -412,8 +413,8 @@ async def test_legacy_local_tool_fallback_refuses_unentitled_caller(legacy_local """ from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server import server as mcp_module - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server import server as mcp_module + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) @@ -461,7 +462,7 @@ async def test_legacy_local_tool_fallback_still_dispatches_entitled_caller( This is the backwards-compatibility half. Refusing this call would trade an authorization hole for an outage on a configuration that worked before. """ - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module server, executed = legacy_local_tool user = _caller_entitled_to([LEGACY_TOOL]) @@ -491,7 +492,7 @@ async def test_legacy_local_tool_fallback_fails_closed_on_empty_prefix( """ from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module _server, executed = legacy_local_tool @@ -520,7 +521,7 @@ async def test_legacy_local_tool_fallback_fails_closed_when_prefix_names_no_serv """ from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module _server, executed = legacy_local_tool other_server = MCPServer( @@ -534,7 +535,7 @@ async def test_legacy_local_tool_fallback_fails_closed_when_prefix_names_no_serv ) with patch( - "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed", + "litellm.proxy.mcp_server.server.MCPRequestHandler.is_tool_allowed", return_value=True, ): with pytest.raises(HTTPException) as exc: @@ -558,7 +559,7 @@ async def test_unknown_tool_name_still_reports_not_found(): """ from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy.mcp_server import server as mcp_module with pytest.raises(HTTPException) as exc: await mcp_operations.execute_mcp_tool( @@ -603,8 +604,8 @@ async def test_per_server_auth_header_reaches_both_openapi_dispatch_arms(dispatc modes the credential has to reach `resolve_openapi_upstream_auth`, whose passthrough arm outranks the ContextVar when it materializes a header. """ - from litellm.proxy._experimental.mcp_server import server as mcp_module - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server import server as mcp_module + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( request_auth_header, ) @@ -640,11 +641,11 @@ async def test_per_server_auth_header_reaches_both_openapi_dispatch_arms(dispatc patch.object(manager, "get_mcp_server_from_tool_name", return_value=server), patch.object(mcp_operations.global_mcp_tool_registry, "get_tool", return_value=fake_tool), patch( - "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool", + "litellm.proxy.mcp_server.operations._handle_local_mcp_tool", new=capture_local, ), patch( - "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed", + "litellm.proxy.mcp_server.server.MCPRequestHandler.is_tool_allowed", return_value=True, ), ): @@ -690,8 +691,8 @@ async def test_local_dispatch_reports_the_outcome_instead_of_success(failure: st `call_tool_rest_api` turns an unrecognized exception into HTTP 500 and an upstream 403 or 429 is not a gateway crash. """ - from litellm.proxy._experimental.mcp_server import server as mcp_module - from litellm.proxy._experimental.mcp_server.exceptions import ( + from litellm.proxy.mcp_server import server as mcp_module + from litellm.proxy.mcp_server.exceptions import ( MCPOpenApiUpstreamError, MCPUpstreamAuthError, ) @@ -732,7 +733,7 @@ async def test_local_dispatch_reports_the_outcome_instead_of_success(failure: st new=AsyncMock(return_value=(None, None)), ), patch( - "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed", + "litellm.proxy.mcp_server.server.MCPRequestHandler.is_tool_allowed", return_value=True, ), ): @@ -797,9 +798,9 @@ def test_the_openapi_arm_installs_the_guard_when_a_credential_rides_a_custom_slo """Pins the wiring, not just the hook: the arm must actually build a guarded client. Testing the hook alone passes even if this arm never installs it. """ - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( - request_resolved_auth_headers, + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( _upstream_client, + request_resolved_auth_headers, ) token = request_resolved_auth_headers.set({"esb-oauth": "Bearer minted"}) @@ -815,9 +816,9 @@ def test_the_guarded_client_is_reused_rather_than_built_per_call(): sets upstream_token_header would leak an httpx client and its connection pool. Both variants have to come from the shared cache. """ - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( - request_resolved_auth_headers, + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( _upstream_client, + request_resolved_auth_headers, ) token = request_resolved_auth_headers.set({"esb-oauth": "Bearer minted"}) @@ -834,10 +835,10 @@ async def test_the_shared_guard_reads_the_url_from_the_request_context(): """ import httpx - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( _drop_credential_across_origin, - request_resolved_auth_headers, _request_upstream_url, + request_resolved_auth_headers, ) creds = request_resolved_auth_headers.set({"esb-oauth": "Bearer minted"}) @@ -859,9 +860,9 @@ async def test_the_shared_guard_reads_the_url_from_the_request_context(): def test_the_openapi_arm_keeps_the_shared_client_when_no_guard_is_needed(resolved): # Authorization is already stripped across origins by the HTTP client, so taking the guarded # path for it would give up the shared connection pool for nothing. - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( - request_resolved_auth_headers, + from litellm.proxy.mcp_server.openapi_to_mcp_generator import ( _upstream_client, + request_resolved_auth_headers, ) token = request_resolved_auth_headers.set(resolved) @@ -880,7 +881,7 @@ async def test_openapi_passthrough_preparation_preserves_credential_precedence( ) -> None: from typing import Final - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager server: Final = MCPServer( server_id="openapi-passthrough", name="openapi-passthrough", transport=MCPTransport.http, diff --git a/tests/unit/proxy/_experimental/mcp_server/test_operations.py b/tests/unit/proxy/mcp_server/test_operations.py similarity index 94% rename from tests/unit/proxy/_experimental/mcp_server/test_operations.py rename to tests/unit/proxy/mcp_server/test_operations.py index 6f7437c90dc..497c4482eba 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_operations.py +++ b/tests/unit/proxy/mcp_server/test_operations.py @@ -11,20 +11,21 @@ from mcp.types import Tool as MCPTool import litellm from litellm.caching.dual_cache import DualCache from litellm.integrations.custom_logger import CustomLogger -from litellm.proxy._experimental.mcp_server import operations -from litellm.proxy._experimental.mcp_server import rest_endpoints -from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - HTTPException as MCPServerManagerHTTPException, - ListedToolsCaller, -) -from litellm.proxy._experimental.mcp_server.operations import GatewayOperations, prepare_context -from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PROXY_MaxParallelRequestsHandler_v3, ) +from litellm.proxy.mcp_server import operations, rest_endpoints +from litellm.proxy.mcp_server.mcp_server_manager import ( + HTTPException as MCPServerManagerHTTPException, +) +from litellm.proxy.mcp_server.mcp_server_manager import ( + ListedToolsCaller, +) +from litellm.proxy.mcp_server.operations import GatewayOperations, prepare_context +from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry from litellm.proxy.utils import InternalUsageCache, ProxyLogging, hash_token from litellm.types.mcp import MCPAuth, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -106,11 +107,11 @@ async def test_mapped_caller_bearer_reaches_agent_evaluation( from starlette.requests import Request from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler - from litellm.proxy._experimental.mcp_server import server as gateway - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler - from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_proxy_mode - from litellm.proxy._experimental.mcp_server.tool_search import mcp_proxy_tool_id, with_mcp_proxy_identity from litellm.proxy.guardrails.guardrail_hooks.agent_365.agent_365 import Agent365Guardrail + from litellm.proxy.mcp_server import server as gateway + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + from litellm.proxy.mcp_server.mcp_context import _mcp_proxy_mode + from litellm.proxy.mcp_server.tool_search import mcp_proxy_tool_id, with_mcp_proxy_identity bearer: Final = "signed.caller.assertion" request: Final = Request( @@ -163,7 +164,7 @@ async def test_mapped_caller_bearer_reaches_agent_evaluation( executions: Final[asyncio.Queue[str]] = asyncio.Queue() async def tool() -> str: - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import request_extra_headers + from litellm.proxy.mcp_server.openapi_to_mcp_generator import request_extra_headers assert bearer not in str(request_extra_headers.get()) executions.put_nowait("executed") @@ -313,14 +314,14 @@ async def test_listing_records_only_tools_the_caller_received( @pytest.mark.asyncio async def test_oauth_prefetch_failure_does_not_log_caller_or_exception_text(caplog): - from litellm.proxy._experimental.mcp_server.operations import _prefetch_oauth_creds_for_user + from litellm.proxy.mcp_server.operations import _prefetch_oauth_creds_for_user user_id = "caller\nFORGED-USER-LINE" fetch = AsyncMock(side_effect=RuntimeError("database\nFORGED-ERROR-LINE")) database = object() with ( patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=database), - patch("litellm.proxy._experimental.mcp_server.db.list_user_oauth_credentials", fetch), + patch("litellm.proxy.mcp_server.db.list_user_oauth_credentials", fetch), caplog.at_level("WARNING", logger="LiteLLM"), ): result = await _prefetch_oauth_creds_for_user(UserAPIKeyAuth(user_id=user_id)) @@ -337,7 +338,7 @@ async def test_oauth_prefetch_failure_does_not_log_caller_or_exception_text(capl async def test_dispatch_uses_explicit_context_when_ambient_caller_differs(): from mcp.server.auth.middleware.auth_context import auth_context_var - from litellm.proxy._experimental.mcp_server.server import set_auth_context + from litellm.proxy.mcp_server.server import set_auth_context context = prepare_context( UserAPIKeyAuth(user_id="alpha"), @@ -349,7 +350,7 @@ async def test_dispatch_uses_explicit_context_when_ambient_caller_differs(): handler = AsyncMock(return_value=GetPromptResult(messages=[])) try: set_auth_context(UserAPIKeyAuth(user_id="bravo"), raw_headers={"x-caller": "bravo"}) - with patch("litellm.proxy._experimental.mcp_server.operations.mcp_get_prompt", handler): + with patch("litellm.proxy.mcp_server.operations.mcp_get_prompt", handler): result = await GatewayOperations().execute( GetPromptRequest(params=GetPromptRequestParams(name="alpha-prompt")), context ) @@ -367,8 +368,8 @@ async def test_dispatch_uses_explicit_context_when_ambient_caller_differs(): async def test_legacy_adapter_cleans_context_after_cancelled_operation(): from types import SimpleNamespace - from litellm.proxy._experimental.mcp_server import server - from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var + from litellm.proxy.mcp_server import server + from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var previous_session = server.active_mcp_session_var.get() previous_request = active_mcp_request_ctx_var.get() @@ -382,7 +383,7 @@ async def test_legacy_adapter_cleans_context_after_cancelled_operation(): raise asyncio.CancelledError with patch( - "litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context", AsyncMock(return_value=auth) + "litellm.proxy.mcp_server.server.get_or_extract_auth_context", AsyncMock(return_value=auth) ): with pytest.raises(asyncio.CancelledError): await cancelled_operation() @@ -394,8 +395,8 @@ async def test_legacy_adapter_cleans_context_after_cancelled_operation(): async def test_legacy_adapter_cleans_context_when_trace_setup_fails(): from types import SimpleNamespace - from litellm.proxy._experimental.mcp_server import server - from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var + from litellm.proxy.mcp_server import server + from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var previous_session = server.active_mcp_session_var.get() previous_request = active_mcp_request_ctx_var.get() @@ -416,7 +417,7 @@ async def test_legacy_adapter_cleans_context_when_trace_setup_fails(): async def test_prompt_sampling_receives_explicit_operation_caller_headers_and_ip(): from unittest.mock import MagicMock - from litellm.proxy._experimental.mcp_server import operations + from litellm.proxy.mcp_server import operations from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -437,8 +438,8 @@ async def test_prompt_sampling_receives_explicit_operation_caller_headers_and_ip sampling = AsyncMock() with ( patch.object(operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[upstream])), - patch("litellm.proxy._experimental.mcp_server.upstream.MCPClient", return_value=client) as factory, - patch("litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", sampling), + patch("litellm.proxy.mcp_server.upstream.MCPClient", return_value=client) as factory, + patch("litellm.proxy.mcp_server.sampling_handler.handle_sampling_create_message", sampling), ): result = await GatewayOperations().execute( GetPromptRequest(params=GetPromptRequestParams(name="explicit_prompt-prompt")), context @@ -519,7 +520,7 @@ async def test_disabled_capability_never_reaches_upstream( from mcp.types import METHOD_NOT_FOUND from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import catalog + from litellm.proxy.mcp_server import catalog monkeypatch.setitem(proxy_server.general_settings, "mcp_disabled_capabilities", [disabled]) request, _client_method, manager_method, upstream_value, result_field = _catalog_case(operation) @@ -567,8 +568,8 @@ async def test_mcp_server_rpm_limits_every_catalog_operation(operation: str) -> from mcp.shared.exceptions import MCPError from mcp.types import INVALID_REQUEST - from litellm.proxy._experimental.mcp_server import catalog - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ServerListOk + from litellm.proxy.mcp_server import catalog + from litellm.proxy.mcp_server.faults.list_outcomes import ServerListOk server: Final = MCPServer( server_id="catalog-rpm", @@ -712,7 +713,7 @@ def _scoped_rig( from mcp.types import CallToolResult, TextContent from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager manager: Final = MCPServerManager() memory: Final = MCPServer( @@ -787,7 +788,7 @@ async def test_bare_tool_name_without_a_single_server_scope_is_still_unresolved_ @pytest.mark.asyncio async def test_tools_call_pre_call_check_enforces_mcp_server_rpm() -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager server: Final = MCPServer( server_id="catalog-call", @@ -822,7 +823,7 @@ async def test_tools_call_pre_call_check_enforces_mcp_server_rpm() -> None: async def test_tools_call_pre_call_hook_rejection_does_not_enforce_mcp_server_rpm() -> None: from unittest.mock import MagicMock - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager server: Final = MCPServer( server_id="catalog-call-pre-hook-rejected", @@ -854,7 +855,7 @@ async def test_tools_call_pre_call_hook_rejection_does_not_enforce_mcp_server_rp @pytest.mark.asyncio async def test_disallowed_tool_does_not_consume_mcp_server_rpm() -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager server: Final = MCPServer( server_id="catalog-call-authorization", @@ -900,7 +901,7 @@ async def test_catalog_helpers_preserve_context_results_and_failure_policy(metho from mcp.server.context import ServerRequestContext from mcp.types import PaginatedRequestParams - from litellm.proxy._experimental.mcp_server import operations, server + from litellm.proxy.mcp_server import operations, server from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -967,7 +968,7 @@ async def test_explicit_proxy_context_rejects_catalog_operations_before_upstream from mcp.shared.exceptions import MCPError from mcp.types import METHOD_NOT_FOUND - from litellm.proxy._experimental.mcp_server import operations + from litellm.proxy.mcp_server import operations operation, _, manager_method, _, _ = _catalog_case(method) upstream = AsyncMock() @@ -985,8 +986,8 @@ async def test_tool_operation_preserves_failure_messages_and_request_trace(failu from mcp.types import CallToolRequest, CallToolRequestParams from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException - from litellm.proxy._experimental.mcp_server import operations - from litellm.proxy._experimental.mcp_server.utils import MCPMissingUserEnvVarsError + from litellm.proxy.mcp_server import operations + from litellm.proxy.mcp_server.utils import MCPMissingUserEnvVarsError failures = { "missing_env": ( @@ -1029,7 +1030,7 @@ async def test_tool_operation_preserves_failure_messages_and_request_trace(failu ], ) async def test_catalog_operation_preserves_empty_result_for_malformed_upstream_items(method, helper): - from litellm.proxy._experimental.mcp_server import operations + from litellm.proxy.mcp_server import operations operation, _, _, _, collection = _catalog_case(method) with patch.object(operations, helper, AsyncMock(return_value=[{"unexpected": "item"}])): @@ -1042,7 +1043,7 @@ async def test_catalog_operation_preserves_empty_result_for_malformed_upstream_i async def test_tool_listing_returns_empty_result_without_dispatch_for_unavailable_catalog(catalog_unavailable): from mcp.types import ListToolsRequest - from litellm.proxy._experimental.mcp_server import operations + from litellm.proxy.mcp_server import operations allowed = AsyncMock( return_value=[], side_effect=RuntimeError("catalog unavailable") if catalog_unavailable else None @@ -1062,7 +1063,7 @@ async def test_tool_listing_returns_empty_result_without_dispatch_for_unavailabl async def test_explicit_proxy_context_lists_builtin_tools_and_blocks_direct_tool_dispatch(): from mcp.types import CallToolRequest, CallToolRequestParams, ListToolsRequest - from litellm.proxy._experimental.mcp_server import operations + from litellm.proxy.mcp_server import operations context = prepare_context(mcp_proxy_mode=True) allowed = AsyncMock() @@ -1104,7 +1105,7 @@ class TestChallengeMissingTokenExchangeSubject: raw_headers: dict[str, str] | None = None, requested_server: MCPServer | None = None, ) -> None: - from litellm.proxy._experimental.mcp_server.operations import _challenge_missing_token_exchange_subject + from litellm.proxy.mcp_server.operations import _challenge_missing_token_exchange_subject return _challenge_missing_token_exchange_subject( server=server, @@ -1200,7 +1201,7 @@ async def test_execute_mcp_tool_challenges_missing_subject_before_cold_listing() from fastapi import HTTPException - from litellm.proxy._experimental.mcp_server import operations + from litellm.proxy.mcp_server import operations server = _server("te-exec", MCPAuth.oauth2_token_exchange) listing = AsyncMock() @@ -1230,9 +1231,9 @@ async def test_local_tool_json_array_is_converted_once_for_the_caller_revision(c text block; converting at the caller's revision keeps the upstream body exactly once.""" from unittest.mock import MagicMock - from litellm.proxy._experimental.mcp_server import operations - from litellm.proxy._experimental.mcp_server.tool_outcome import WireCompat, parse_http_body - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server import operations + from litellm.proxy.mcp_server.tool_outcome import WireCompat, parse_http_body + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry body = '["a","b"]' tool = MagicMock() @@ -1253,7 +1254,7 @@ async def test_discovery_preserves_caller_scope_and_proxy_restrictions(): context = prepare_context( UserAPIKeyAuth(user_id="scoped"), mcp_servers=["only-this"], mcp_proxy_mode=True, protocol_version="2025-06-18" ) - with patch("litellm.proxy._experimental.mcp_server.operations._execute_handle_list_tools", listed): + with patch("litellm.proxy.mcp_server.operations._execute_handle_list_tools", listed): result = await GatewayOperations().execute(DiscoverRequest(), context) assert result.capabilities.tools is not None assert result.capabilities.resources is None @@ -1267,8 +1268,8 @@ async def test_discovery_preserves_caller_scope_and_proxy_restrictions(): async def test_discovery_outright_tools_failure_still_advertises_other_capabilities() -> None: from mcp.types import DiscoverRequest, ListPromptsResult, ListToolsResult, Prompt - from litellm.proxy._experimental.mcp_server import operations - from litellm.proxy._experimental.mcp_server.catalog import ToolListingFailedOutright + from litellm.proxy.mcp_server import operations + from litellm.proxy.mcp_server.catalog import ToolListingFailedOutright tools: Final = AsyncMock(side_effect=ToolListingFailedOutright(code=-32603, message="every server failed")) prompts: Final = AsyncMock(return_value=ListPromptsResult(prompts=[Prompt(name="p")])) @@ -1289,7 +1290,7 @@ async def test_discovery_propagates_non_outright_mcp_errors_from_tools_listing() from mcp import MCPError from mcp.types import INVALID_PARAMS, DiscoverRequest - from litellm.proxy._experimental.mcp_server import operations + from litellm.proxy.mcp_server import operations tools: Final = AsyncMock(side_effect=MCPError(code=INVALID_PARAMS, message="bad cursor")) context: Final = prepare_context( @@ -1304,11 +1305,11 @@ async def test_discovery_propagates_non_outright_mcp_errors_from_tools_listing() @pytest.mark.asyncio async def test_discovery_denial_cannot_advertise_tools(): - from mcp.types import DiscoverRequest from fastapi import HTTPException + from mcp.types import DiscoverRequest denied = AsyncMock(side_effect=HTTPException(status_code=403, detail="Forbidden")) - with patch("litellm.proxy._experimental.mcp_server.operations._execute_handle_list_tools", denied): + with patch("litellm.proxy.mcp_server.operations._execute_handle_list_tools", denied): with pytest.raises(HTTPException) as error: await GatewayOperations().execute(DiscoverRequest(), prepare_context(UserAPIKeyAuth(user_id="denied"))) assert error.value.status_code == 403 @@ -1319,15 +1320,16 @@ async def test_discovery_denial_cannot_advertise_tools(): async def test_discovery_lists_each_capability_with_the_same_caller(available): from mcp.types import ( DiscoverRequest, - ListToolsResult, ListPromptsResult, ListResourcesResult, ListResourceTemplatesResult, + ListToolsResult, Prompt, Resource, ResourceTemplate, ) - from litellm.proxy._experimental.mcp_server import operations + + from litellm.proxy.mcp_server import operations context = prepare_context(UserAPIKeyAuth(user_id="scoped"), mcp_servers=["authorized"]) tools = AsyncMock(return_value=ListToolsResult(tools=[])) @@ -1408,9 +1410,9 @@ async def test_discovery_skips_disabled_capability_listings(disabled: str, monke async def test_discovery_shares_one_server_admission_across_catalog_listings() -> None: from mcp import types - from litellm.proxy._experimental.mcp_server import catalog - from litellm.proxy._experimental.mcp_server.contracts import OperationContext - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ServerListOk + from litellm.proxy.mcp_server import catalog + from litellm.proxy.mcp_server.contracts import OperationContext + from litellm.proxy.mcp_server.faults.list_outcomes import ServerListOk admitted: Final = MCPServer( server_id="discover-admitted", @@ -1506,12 +1508,13 @@ async def test_discovery_shares_one_server_admission_across_catalog_listings() - async def test_discovery_concurrent_listings_drain_on_failure_and_cancellation(outcome): from mcp.types import ( DiscoverRequest, - ListToolsResult, ListPromptsResult, ListResourcesResult, ListResourceTemplatesResult, + ListToolsResult, ) - from litellm.proxy._experimental.mcp_server import operations + + from litellm.proxy.mcp_server import operations ready = [asyncio.Event() for _ in range(4)] closed = [asyncio.Event() for _ in range(4)] @@ -1570,7 +1573,8 @@ async def test_discovery_concurrent_listings_drain_on_failure_and_cancellation(o @pytest.mark.parametrize("log_enabled", [False, True]) async def test_tools_listing_preserves_explicit_spend_log_policy(log_enabled): from mcp.types import PaginatedRequestParams - from litellm.proxy._experimental.mcp_server import operations + + from litellm.proxy.mcp_server import operations listing = AsyncMock(return_value=operations.AggregateToolListing(tools=[], outcomes={})) with patch.object(operations, "_list_mcp_tools", listing): @@ -1614,14 +1618,15 @@ async def test_list_mcp_tools_records_the_catalog_only_when_asked( async def test_discovery_keeps_one_catalog_revision_across_concurrent_listings(monkeypatch): from mcp.types import ( DiscoverRequest, - ListToolsResult, ListPromptsResult, ListResourcesResult, ListResourceTemplatesResult, + ListToolsResult, ) + from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import operations - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server import operations + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager manager = MCPServerManager() original = MCPServer(server_id="catalog-server", name="before", transport=MCPTransport.http) @@ -1667,7 +1672,7 @@ async def test_local_handler_freshness_tracks_registered_owner_with_overlapping_ from fastapi import HTTPException from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager monkeypatch.setattr(proxy_server, "prisma_client", None) manager = MCPServerManager() @@ -1710,7 +1715,7 @@ async def test_local_call_cannot_borrow_another_servers_authority( from fastapi import HTTPException from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager manager: Final = MCPServerManager() allowed: Final = MCPServer(server_id="allowed", name="allowed", transport=MCPTransport.http) @@ -1747,7 +1752,7 @@ async def test_local_call_preserves_matching_and_legacy_handlers( from datetime import datetime from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager manager: Final = MCPServerManager() server: Final = MCPServer(server_id="allowed", name="allowed", transport=MCPTransport.http) @@ -1777,7 +1782,7 @@ async def test_local_handler_rejects_an_owner_absent_from_the_catalog(monkeypatc from fastapi import HTTPException from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager manager: Final = MCPServerManager() handler: Final = AsyncMock(return_value="private result") @@ -1854,6 +1859,7 @@ async def test_optional_catalog_preserves_revoked_user_error(monkeypatch, reques from types import SimpleNamespace from mcp import MCPError, types + from litellm.caching.dual_cache import DualCache from litellm.proxy import proxy_server @@ -1943,7 +1949,8 @@ async def test_guardrail_context_never_recovers_a_previous_request_bearer( ) -> None: from mcp.server.auth.middleware.auth_context import auth_context_var from starlette.requests import Request - from litellm.proxy._experimental.mcp_server import server as gateway + + from litellm.proxy.mcp_server import server as gateway request: Final = ( None @@ -1988,12 +1995,19 @@ async def test_guardrail_context_never_recovers_a_previous_request_bearer( @pytest.mark.parametrize("delay", ["target", "probe", "exhausted"]) async def test_discovery_optional_probes_use_remaining_listing_budget(delay, monkeypatch): from types import SimpleNamespace + from mcp.types import ( - DiscoverRequest, ListToolsResult, ListPromptsResult, ListResourcesResult, - ListResourceTemplatesResult, ToolsCapability, Tool, + DiscoverRequest, + ListPromptsResult, + ListResourcesResult, + ListResourceTemplatesResult, + ListToolsResult, + Tool, + ToolsCapability, ) + from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import subscriptions + from litellm.proxy.mcp_server import subscriptions entered = asyncio.Event() cleaned = asyncio.Event() diff --git a/tests/unit/proxy/_experimental/mcp_server/test_per_user_oauth_cache.py b/tests/unit/proxy/mcp_server/test_per_user_oauth_cache.py similarity index 91% rename from tests/unit/proxy/_experimental/mcp_server/test_per_user_oauth_cache.py rename to tests/unit/proxy/mcp_server/test_per_user_oauth_cache.py index efddd1beef3..c0e23319566 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_per_user_oauth_cache.py +++ b/tests/unit/proxy/mcp_server/test_per_user_oauth_cache.py @@ -19,7 +19,7 @@ for _mod in ("orjson",): if _mod not in sys.modules: sys.modules[_mod] = MagicMock() -from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( # noqa: E402 +from litellm.proxy.mcp_server.oauth2_token_cache import ( # noqa: E402 MCPPerUserTokenCache, compute_per_user_token_ttl, mcp_per_user_token_cache, @@ -30,7 +30,7 @@ from litellm.types.mcp_server.mcp_server_manager import MCPServer # noqa: E402 def _import_validate(): """Lazy import to avoid pulling orjson at collection time.""" - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + from litellm.proxy.mcp_server.discoverable_endpoints import ( _validate_token_response, ) @@ -263,7 +263,7 @@ class TestMCPPerUserTokenCache: async def test_get_returns_none_on_miss(self, cache, mock_dual_cache): with ( patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.decrypt_value_helper" + "litellm.proxy.mcp_server.oauth2_token_cache.decrypt_value_helper" ) as mock_decrypt, patch("litellm.proxy.proxy_server.user_api_key_cache", mock_dual_cache), ): @@ -278,7 +278,7 @@ class TestMCPPerUserTokenCache: fake_plaintext = "xoxb-slack-token" with ( patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.decrypt_value_helper", + "litellm.proxy.mcp_server.oauth2_token_cache.decrypt_value_helper", return_value=fake_plaintext, ) as mock_decrypt, patch("litellm.proxy.proxy_server.user_api_key_cache", mock_dual_cache), @@ -298,7 +298,7 @@ class TestMCPPerUserTokenCache: fake_encrypted = "encrypted_blob_xyz" with ( patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.encrypt_value_helper", + "litellm.proxy.mcp_server.oauth2_token_cache.encrypt_value_helper", return_value=fake_encrypted, ) as mock_encrypt, patch("litellm.proxy.proxy_server.user_api_key_cache", mock_dual_cache), @@ -315,7 +315,7 @@ class TestMCPPerUserTokenCache: async def test_set_uses_correct_cache_key(self, cache, mock_dual_cache): with ( patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.encrypt_value_helper", + "litellm.proxy.mcp_server.oauth2_token_cache.encrypt_value_helper", return_value="enc", ), patch("litellm.proxy.proxy_server.user_api_key_cache", mock_dual_cache), @@ -339,7 +339,7 @@ class TestMCPPerUserTokenCache: """Cache misses and decrypt errors should both return None without raising.""" with ( patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.decrypt_value_helper", + "litellm.proxy.mcp_server.oauth2_token_cache.decrypt_value_helper", return_value=None, # decrypt returns None on failure ), patch("litellm.proxy.proxy_server.user_api_key_cache", mock_dual_cache), @@ -355,7 +355,7 @@ class TestMCPPerUserTokenCache: mock_dual_cache.async_set_cache.side_effect = RuntimeError("Redis down") with ( patch( - "litellm.proxy._experimental.mcp_server.oauth2_token_cache.encrypt_value_helper", + "litellm.proxy.mcp_server.oauth2_token_cache.encrypt_value_helper", return_value="enc", ), patch("litellm.proxy.proxy_server.user_api_key_cache", mock_dual_cache), @@ -385,7 +385,7 @@ class TestRefreshUserOauthToken: @pytest.mark.asyncio async def test_returns_none_when_no_refresh_token(self, server): - from litellm.proxy._experimental.mcp_server.db import refresh_user_oauth_token + from litellm.proxy.mcp_server.db import refresh_user_oauth_token cred = {"type": "oauth2", "access_token": "OLD"} # no refresh_token result = await refresh_user_oauth_token( @@ -398,7 +398,7 @@ class TestRefreshUserOauthToken: @pytest.mark.asyncio async def test_returns_none_when_no_token_url(self, cred): - from litellm.proxy._experimental.mcp_server.db import refresh_user_oauth_token + from litellm.proxy.mcp_server.db import refresh_user_oauth_token server = _make_server(token_url=None) result = await refresh_user_oauth_token( @@ -411,13 +411,13 @@ class TestRefreshUserOauthToken: @pytest.mark.asyncio async def test_returns_none_on_http_error(self, server, cred): - from litellm.proxy._experimental.mcp_server.db import refresh_user_oauth_token + from litellm.proxy.mcp_server.db import refresh_user_oauth_token mock_client = AsyncMock() mock_client.post.side_effect = Exception("Connection refused") with patch( - "litellm.proxy._experimental.mcp_server.db.get_async_httpx_client", + "litellm.proxy.mcp_server.db.get_async_httpx_client", return_value=mock_client, ): result = await refresh_user_oauth_token( @@ -430,7 +430,7 @@ class TestRefreshUserOauthToken: @pytest.mark.asyncio async def test_stores_and_returns_new_credential(self, server, cred): - from litellm.proxy._experimental.mcp_server.db import refresh_user_oauth_token + from litellm.proxy.mcp_server.db import refresh_user_oauth_token new_token_response = MagicMock() new_token_response.json.return_value = { @@ -453,15 +453,15 @@ class TestRefreshUserOauthToken: with ( patch( - "litellm.proxy._experimental.mcp_server.db.get_async_httpx_client", + "litellm.proxy.mcp_server.db.get_async_httpx_client", return_value=mock_client, ), patch( - "litellm.proxy._experimental.mcp_server.db.store_user_oauth_credential", + "litellm.proxy.mcp_server.db.store_user_oauth_credential", new_callable=AsyncMock, ) as mock_store, patch( - "litellm.proxy._experimental.mcp_server.db.get_user_oauth_credential", + "litellm.proxy.mcp_server.db.get_user_oauth_credential", new_callable=AsyncMock, return_value=stored_cred, ), @@ -486,7 +486,7 @@ class TestRefreshUserOauthToken: @pytest.mark.asyncio async def test_falls_back_to_old_refresh_token_when_not_rotated(self, server, cred): """When provider doesn't return a new refresh_token, keep the old one.""" - from litellm.proxy._experimental.mcp_server.db import refresh_user_oauth_token + from litellm.proxy.mcp_server.db import refresh_user_oauth_token new_token_response = MagicMock() new_token_response.json.return_value = { @@ -501,15 +501,15 @@ class TestRefreshUserOauthToken: with ( patch( - "litellm.proxy._experimental.mcp_server.db.get_async_httpx_client", + "litellm.proxy.mcp_server.db.get_async_httpx_client", return_value=mock_client, ), patch( - "litellm.proxy._experimental.mcp_server.db.store_user_oauth_credential", + "litellm.proxy.mcp_server.db.store_user_oauth_credential", new_callable=AsyncMock, ) as mock_store, patch( - "litellm.proxy._experimental.mcp_server.db.get_user_oauth_credential", + "litellm.proxy.mcp_server.db.get_user_oauth_credential", new_callable=AsyncMock, return_value={"type": "oauth2", "access_token": "NEW_TOKEN"}, ), diff --git a/tests/unit/proxy/_experimental/mcp_server/test_proxy_api_credentials.py b/tests/unit/proxy/mcp_server/test_proxy_api_credentials.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/test_proxy_api_credentials.py rename to tests/unit/proxy/mcp_server/test_proxy_api_credentials.py index f39aec82c3e..f3d0fed0222 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_proxy_api_credentials.py +++ b/tests/unit/proxy/mcp_server/test_proxy_api_credentials.py @@ -7,20 +7,20 @@ from fastapi import HTTPException from litellm.constants import CLI_JWT_EXPIRATION_HOURS from litellm.models.user import LiteLLM_UserTable -from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ConsentTeam, MintedProxyCredential -from litellm.proxy._experimental.mcp_server.proxy_api_credentials import ( - lookup_consent_teams, - lookup_consent_teams_from_database, - mint_proxy_credential, -) from litellm.proxy._types import LitellmUserRoles from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.management_endpoints.ui_sso import CliSsoTeamDetail +from litellm.proxy.mcp_server.gateway_dcr_flow import ConsentTeam, MintedProxyCredential +from litellm.proxy.mcp_server.proxy_api_credentials import ( + lookup_consent_teams, + lookup_consent_teams_from_database, + mint_proxy_credential, +) -_LOAD_USER = "litellm.proxy._experimental.mcp_server.proxy_api_credentials.load_active_user_by_id" -_FETCH_TEAMS = "litellm.proxy._experimental.mcp_server.proxy_api_credentials.fetch_cli_sso_team_details" +_LOAD_USER = "litellm.proxy.mcp_server.proxy_api_credentials.load_active_user_by_id" +_FETCH_TEAMS = "litellm.proxy.mcp_server.proxy_api_credentials.fetch_cli_sso_team_details" _PRISMA = "litellm.proxy.proxy_server.prisma_client" TEAM_DETAILS = ( CliSsoTeamDetail(team_id="team-a", team_alias="Team A", team_models=("gpt-5.4-mini",)), diff --git a/tests/unit/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/unit/proxy/mcp_server/test_rest_endpoints.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/test_rest_endpoints.py rename to tests/unit/proxy/mcp_server/test_rest_endpoints.py index 73e248ff52e..84d32934208 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/unit/proxy/mcp_server/test_rest_endpoints.py @@ -6,7 +6,7 @@ from datetime import datetime from typing import Any, Dict, Final, Optional from unittest.mock import AsyncMock, MagicMock -from litellm.proxy._experimental.mcp_server import operations as mcp_operations +from litellm.proxy.mcp_server import operations as mcp_operations if sys.version_info < (3, 11): # BaseExceptionGroup is a builtin only from 3.11 from exceptiongroup import BaseExceptionGroup @@ -18,16 +18,16 @@ from mcp.types import CallToolResult, TextContent, Tool from starlette.requests import Request from litellm.constants import MCP_TOOL_LISTING_TIMEOUT -from litellm.proxy._experimental.mcp_server import rest_endpoints -from litellm.proxy._experimental.mcp_server.auth import ( - user_api_key_auth_mcp as auth_mcp, -) from litellm.proxy._types import ( NewMCPServerRequest, UpdateMCPServerRequest, UserAPIKeyAuth, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.mcp_server import rest_endpoints +from litellm.proxy.mcp_server.auth import ( + user_api_key_auth_mcp as auth_mcp, +) from litellm.types.mcp import MCPAuth, MCPTransport, MCPUpstreamProtocol from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -114,9 +114,9 @@ class TestExecuteWithMcpClient: ) -> None: from starlette.datastructures import Headers - from litellm.experimental_mcp_client.client import MCPClient - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.mcp_client.client import MCPClient from litellm.proxy.management_endpoints import mcp_management_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager saved: Final = MCPServer( server_id="saved-preview-server", @@ -184,8 +184,8 @@ class TestExecuteWithMcpClient: ) -> None: from starlette.datastructures import Headers - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.proxy.management_endpoints import mcp_management_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager saved: Final = MCPServer( server_id="saved-preview-server", @@ -808,10 +808,9 @@ class TestTestConnection: """The edit form resends a saved server without its masked credential; the stored one must be used, as /test/tools/list already does.""" from litellm.proxy._types import LitellmUserRoles - from litellm.types.mcp_server.mcp_server_manager import MCPServer - - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.proxy.management_endpoints import mcp_management_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager + from litellm.types.mcp_server.mcp_server_manager import MCPServer manager = MCPServerManager() monkeypatch.setattr(rest_endpoints, "global_mcp_server_manager", manager) @@ -1209,8 +1208,8 @@ class TestListToolsRestAPI: async def test_single_server_rate_limit_returns_429_without_fetching_tools( self, monkeypatch: pytest.MonkeyPatch ) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager server: Final = MCPServer( server_id="rate-limited-server", @@ -1249,8 +1248,8 @@ class TestListToolsRestAPI: async def test_admin_unfiltered_tools_list_does_not_enforce_server_rpm( self, monkeypatch: pytest.MonkeyPatch ) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager server: Final = MCPServer( server_id="admin-unfiltered-server", @@ -1404,7 +1403,7 @@ class TestListToolsRestAPI: return admitted_auth monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", fake_reload, ) @@ -1519,7 +1518,7 @@ class TestListToolsRestAPI: return ["toolset-tool-1"] monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", record_reload, ) monkeypatch.setattr( @@ -1573,9 +1572,9 @@ class TestListToolsRestAPI: ) from mcp.types import Tool as MCPTool - import litellm.experimental_mcp_client.client as mcp_client_module - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager - from litellm.proxy._experimental.mcp_server.server import MCPServer + import litellm.mcp_client.client as mcp_client_module + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport monkeypatch.setattr(rest_endpoints, "global_mcp_server_manager", MCPServerManager()) @@ -1766,7 +1765,7 @@ class TestListToolsRestAPI: """A single-server pass-through request whose upstream rejects the token must surface the upstream status (401 or 403) plus its WWW-Authenticate challenge, not collapse into a 200 ``unexpected_error`` body.""" - from litellm.proxy._experimental.mcp_server.exceptions import ( + from litellm.proxy.mcp_server.exceptions import ( MCPUpstreamAuthError, ) @@ -1835,10 +1834,10 @@ class TestListToolsRestAPI: """A single-server listing whose upstream breaks (5xx, timeout, unreachable) must answer with the truthful gateway status instead of masking the failure as an empty-success {"tools": [], "error": null} body a caller cannot distinguish from a toolless server.""" - from litellm.proxy._experimental.mcp_server.exceptions import ( + from litellm.proxy.mcp_server.exceptions import ( MCPServerListError, ) - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( + from litellm.proxy.mcp_server.faults.list_outcomes import ( ServerListFault, ) @@ -1897,7 +1896,7 @@ class TestListToolsRestAPI: caller can tell "needs upstream auth" apart from "has no tools".""" from pydantic import TypeAdapter - from litellm.proxy._experimental.mcp_server.exceptions import ( + from litellm.proxy.mcp_server.exceptions import ( MCPUpstreamAuthError, ) @@ -1971,7 +1970,7 @@ class TestListToolsRestAPI: async def test_name_resolution_finds_server_by_uuid(self, monkeypatch): """When server_id is a name string, it should be resolved to its UUID and used for the tools lookup when the UUID is in allowed_server_ids.""" - from litellm.proxy._experimental.mcp_server.server import MCPServer + from litellm.proxy.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport stub_server = MCPServer( @@ -2053,7 +2052,7 @@ class TestListToolsRestAPI: async def test_name_not_in_allowed_returns_access_denied(self, monkeypatch): """When name resolves to a server whose UUID is NOT in allowed_server_ids, the result should be an access_denied error (not a crash or silent pass).""" - from litellm.proxy._experimental.mcp_server.server import MCPServer + from litellm.proxy.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport stub_server = MCPServer( @@ -2109,7 +2108,7 @@ class TestListToolsRestAPI: async def test_mcp_server_name_query_param_resolves_to_server(self, monkeypatch): """mcp_server_name is a name-based alias for server_id: it should resolve to the matching server and scope the response to it.""" - from litellm.proxy._experimental.mcp_server.server import MCPServer + from litellm.proxy.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport stub_server = MCPServer( @@ -2189,8 +2188,8 @@ class TestListToolsRestAPI: assert result["error"] is None async def test_mcp_server_name_filter_uses_real_catalog_with_tool_search(self, monkeypatch): - from litellm.proxy._experimental.mcp_server.server import MCPServer from litellm.proxy._types import LiteLLM_ObjectPermissionTable + from litellm.proxy.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport stub_server = MCPServer( @@ -2397,7 +2396,7 @@ class TestListToolsRestAPI: async def test_oauth2_user_token_injected_for_single_server(self, monkeypatch): """For a single-server OAuth2 request, _get_user_oauth_extra_headers is called and the returned headers are forwarded to _get_tools_for_single_server.""" - from litellm.proxy._experimental.mcp_server.server import MCPServer + from litellm.proxy.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport stub_server = MCPServer( @@ -2578,8 +2577,8 @@ class TestCallToolRestAPI: ) async def test_executes_tool_when_allowed(self, monkeypatch, tool_name, native_tool_name): import litellm - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler, MCPServerAccess - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler, MCPServerAccess + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager async def fake_contexts(user_api_key_auth): return [user_api_key_auth] @@ -2713,9 +2712,9 @@ class TestCallToolRestAPI: ) -> None: import litellm from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, LiteLLM_UserTable from litellm.proxy.auth import auth_checks + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager target_server: Final = MCPServer( server_id="server-b", @@ -3141,7 +3140,7 @@ class TestCallToolRestAPI: MCPUpstreamAuthError) must reach the REST caller as that status with the upstream WWW-Authenticate preserved, so an MCP client can run the upstream OAuth flow, instead of the generic 500 the catch-all would otherwise produce.""" - from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError + from litellm.proxy.mcp_server.exceptions import MCPUpstreamAuthError async def fake_contexts(user_api_key_auth): return [user_api_key_auth] @@ -3473,9 +3472,9 @@ class TestCallToolRestAPI: without the direct branch's relay wrapper, so an MCPUpstreamAuthError from it must be relayed by the endpoint-level handler (a real 401/403 + WWW-Authenticate) rather than falling through the catch-all into a generic 500.""" - import litellm.proxy._experimental.mcp_server.tool_search as tool_search_mod - from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError + import litellm.proxy.mcp_server.tool_search as tool_search_mod from litellm.proxy._types import LiteLLM_ObjectPermissionTable + from litellm.proxy.mcp_server.exceptions import MCPUpstreamAuthError challenge = 'Bearer resource_metadata="https://gw.example.com/.well-known/oauth-protected-resource"' @@ -3587,9 +3586,9 @@ async def test_request_selected_tool_specific_guardrail_applies_to_virtual_execu import litellm from litellm.caching.caching import DualCache from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import mcp_server_manager, server, tool_registry from litellm.proxy._types import LiteLLM_ObjectPermissionTable from litellm.proxy.guardrails.guardrail_hooks.custom_code.custom_code_guardrail import CustomCodeGuardrail + from litellm.proxy.mcp_server import mcp_server_manager, server, tool_registry from litellm.proxy.utils import ProxyLogging guardrail: Final = CustomCodeGuardrail( @@ -3672,8 +3671,8 @@ class TestGetToolsForSingleServer: async def test_filters_tools_by_object_permission_mcp_tool_permissions(self, monkeypatch): """Test that tools are filtered by user_api_key_auth.object_permission.mcp_tool_permissions""" - from litellm.proxy._experimental.mcp_server.server import MCPServer from litellm.proxy._types import LiteLLM_ObjectPermissionTable + from litellm.proxy.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport mock_tools = [ @@ -3728,7 +3727,7 @@ class TestGetToolsForSingleServer: async def test_no_filtering_when_object_permission_is_none(self, monkeypatch): """Test that all tools are returned when object_permission is None""" - from litellm.proxy._experimental.mcp_server.server import MCPServer + from litellm.proxy.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport mock_tools = [ @@ -3769,8 +3768,8 @@ class TestGetToolsForSingleServer: async def test_no_filtering_when_mcp_tool_permissions_is_none(self, monkeypatch): """Test that all tools are returned when mcp_tool_permissions is None""" - from litellm.proxy._experimental.mcp_server.server import MCPServer from litellm.proxy._types import LiteLLM_ObjectPermissionTable + from litellm.proxy.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport mock_tools = [ @@ -3816,8 +3815,8 @@ class TestGetToolsForSingleServer: async def test_no_filtering_when_server_not_in_mcp_tool_permissions(self, monkeypatch): """Test that all tools are returned when server is not in mcp_tool_permissions""" - from litellm.proxy._experimental.mcp_server.server import MCPServer from litellm.proxy._types import LiteLLM_ObjectPermissionTable + from litellm.proxy.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport mock_tools = [ @@ -3863,8 +3862,8 @@ class TestGetToolsForSingleServer: async def test_combines_server_allowed_tools_and_object_permission_filters(self, monkeypatch): """Test that both server.allowed_tools and object_permission.mcp_tool_permissions filters are applied""" - from litellm.proxy._experimental.mcp_server.server import MCPServer from litellm.proxy._types import LiteLLM_ObjectPermissionTable + from litellm.proxy.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport mock_tools = [ @@ -3920,7 +3919,7 @@ class TestGetToolsForSingleServer: async def test_apply_tool_filters_false_returns_full_catalog(self, monkeypatch): """apply_tool_filters=False returns the raw catalog without the server allowed_tools gate, so the config UI can render disabled tools as off.""" - from litellm.proxy._experimental.mcp_server.server import MCPServer + from litellm.proxy.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport mock_tools = [ @@ -4188,7 +4187,7 @@ class TestPreviewOpenAPITools: } } - from litellm.proxy._experimental.mcp_server import ( + from litellm.proxy.mcp_server import ( openapi_to_mcp_generator, ) @@ -4228,7 +4227,7 @@ class TestPreviewOpenAPITools: suffixes (_2, _3, ...) get assigned to different operations and the dashboard shows names that differ from what's actually registered. """ - from litellm.proxy._experimental.mcp_server import ( + from litellm.proxy.mcp_server import ( openapi_to_mcp_generator, ) @@ -4577,7 +4576,7 @@ class TestToolResponseMcpInfoEnrichment: def test_enriches_mcp_info_with_alias_and_server_id(self): from mcp.types import Tool as MCPTool - from litellm.proxy._experimental.mcp_server.server import MCPServer + from litellm.proxy.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport server = MCPServer( @@ -4613,7 +4612,7 @@ class TestToolResponseMcpInfoEnrichment: def test_alias_none_is_explicit_in_mcp_info(self): from mcp.types import Tool as MCPTool - from litellm.proxy._experimental.mcp_server.server import MCPServer + from litellm.proxy.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport server = MCPServer( @@ -4678,10 +4677,10 @@ class TestRestListToolsetFiltering: from mcp.types import Tool as MCPTool - from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) - from litellm.proxy._experimental.mcp_server.server import MCPServer + from litellm.proxy.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport stub_server = MCPServer( @@ -4722,7 +4721,7 @@ class TestRestListToolsetFiltering: patch.object(MCPRequestHandler, "_get_key_object_permission", return_value=key_object_permission), patch.object(MCPRequestHandler, "_get_team_object_permission", AsyncMock(return_value=None)), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), ): @@ -4747,7 +4746,7 @@ class TestV1ResolvedOauth2Gate: @staticmethod def _oauth2_server(*, delegate_auth_to_upstream: bool) -> Any: - from litellm.proxy._experimental.mcp_server.server import MCPServer + from litellm.proxy.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport return MCPServer( @@ -4770,7 +4769,7 @@ class TestV1ResolvedOauth2Gate: async def test_user_oauth_headers_skip_v2_owned_servers( self, delegate_auth_to_upstream, expected_headers, expected_lookups, monkeypatch ): - from litellm.proxy._experimental.mcp_server import db as mcp_db + from litellm.proxy.mcp_server import db as mcp_db server = self._oauth2_server(delegate_auth_to_upstream=delegate_auth_to_upstream) resolve_token = AsyncMock(return_value={"access_token": "stored-token"}) @@ -4932,7 +4931,7 @@ class TestClientAllowlistOnRestRoutes: @pytest.mark.asyncio @pytest.mark.parametrize("revision", ("auto", "2024-11-05", "2025-06-18")) async def test_preview_client_honors_protocol_metadata(revision: MCPUpstreamProtocol) -> None: - from litellm.experimental_mcp_client.client import MCPClient + from litellm.mcp_client.client import MCPClient payload: Final = NewMCPServerRequest( server_name="preview", @@ -4970,9 +4969,9 @@ async def test_saved_preview_protocol_omission_and_explicit_edits( ) -> None: from starlette.datastructures import Headers - from litellm.experimental_mcp_client.client import MCPClient - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.mcp_client.client import MCPClient from litellm.proxy.management_endpoints import mcp_management_endpoints + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager saved: Final = MCPServer( server_id="saved-preview", @@ -5037,7 +5036,7 @@ async def test_rest_guardrail_bearer_excludes_admission_key( import litellm from litellm.caching.dual_cache import DualCache from litellm.integrations.custom_logger import CustomLogger - from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy.mcp_server.tool_registry import global_mcp_tool_registry observed: Final[asyncio.Queue[object]] = asyncio.Queue() diff --git a/tests/unit/proxy/_experimental/mcp_server/test_result_conversion.py b/tests/unit/proxy/mcp_server/test_result_conversion.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/test_result_conversion.py rename to tests/unit/proxy/mcp_server/test_result_conversion.py index 9926fa131ab..bbc35f11f32 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_result_conversion.py +++ b/tests/unit/proxy/mcp_server/test_result_conversion.py @@ -7,7 +7,7 @@ from mcp_types.methods import serialize_server_result from mcp_types.version import KNOWN_PROTOCOL_VERSIONS, MODERN_PROTOCOL_VERSIONS from pydantic import JsonValue, ValidationError -from litellm.proxy._experimental.mcp_server.result_conversion import ( +from litellm.proxy.mcp_server.result_conversion import ( INPUT_REQUIRED_UNSUPPORTED_MESSAGE, JsonResult, TextResult, @@ -246,7 +246,8 @@ class TestToGatewayTool: @pytest.mark.parametrize("elapsed,expected", [(0, 1000), (0.0001, 999), (0.5, 500), (1, 0), (2, 0), (-1, 1000)]) def test_freshness_aging_preserves_content_and_scope(elapsed: float, expected: int) -> None: from mcp.types import ListPromptsResult, Prompt - from litellm.proxy._experimental.mcp_server.result_conversion import age_freshness + + from litellm.proxy.mcp_server.result_conversion import age_freshness result: Final = ListPromptsResult(prompts=[Prompt(name="kept")], ttl_ms=1000, cache_scope="public") aged: Final = age_freshness(result, elapsed) @@ -263,7 +264,8 @@ def test_aggregate_freshness_never_broadens_sharing( scopes: tuple[Literal["private", "public"], ...], expected: str ) -> None: from mcp.types import CacheableResult - from litellm.proxy._experimental.mcp_server.result_conversion import aggregate_freshness + + from litellm.proxy.mcp_server.result_conversion import aggregate_freshness result: Final = aggregate_freshness(tuple(CacheableResult(ttl_ms=1000, cache_scope=scope) for scope in scopes)) assert result.cache_scope == expected diff --git a/tests/unit/proxy/_experimental/mcp_server/test_semantic_tool_filter.py b/tests/unit/proxy/mcp_server/test_semantic_tool_filter.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/test_semantic_tool_filter.py rename to tests/unit/proxy/mcp_server/test_semantic_tool_filter.py index f0b4e94f72f..c04323d7f7c 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_semantic_tool_filter.py +++ b/tests/unit/proxy/mcp_server/test_semantic_tool_filter.py @@ -17,7 +17,6 @@ if sys.version_info < (3, 11): # BaseExceptionGroup is a builtin only from 3.11 from mcp.types import Tool as MCPTool - requires_semantic_router = pytest.mark.skipif( sys.version_info >= (3, 14), reason="The semantic-router extra excludes Python 3.14" ) @@ -33,7 +32,7 @@ async def test_semantic_filter_basic_filtering(): When: Query is "send an email" Then: Email tools should rank higher than calendar tools """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) @@ -161,7 +160,7 @@ async def test_semantic_filter_top_k_limiting(): When: top_k=5 Then: Should return at most 5 tools """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) @@ -222,7 +221,7 @@ async def test_semantic_filter_disabled(): """ Test that when filter is disabled, all tools are returned. """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) @@ -261,7 +260,7 @@ async def test_semantic_filter_empty_tools(): """ Test that filter handles empty tool list gracefully. """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) @@ -289,7 +288,7 @@ async def test_semantic_filter_extract_user_query(): """ Test that user query extraction works correctly from messages. """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) @@ -341,10 +340,10 @@ async def test_semantic_filter_hook_triggers_on_completion(): """ Test that the hook triggers for completion requests with tools. """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) - from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook from litellm.types.utils import Embedding, EmbeddingResponse # Create mock filter @@ -420,10 +419,10 @@ async def test_semantic_filter_hook_skips_no_tools(): """ Test that the hook does NOT trigger when there are no tools. """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) - from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook # Create mock filter mock_router = Mock() @@ -473,10 +472,10 @@ async def test_semantic_filter_hook_preserves_native_tools(): Then: The native tools must survive unconditionally, and only MCP tools go through the semantic filter. """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) - from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook from litellm.types.utils import Embedding, EmbeddingResponse mock_router = Mock() @@ -605,10 +604,10 @@ async def test_semantic_filter_hook_all_native_tools(): Then: All tools pass through, and NO spurious semantic filter response headers are emitted (no litellm_semantic_filter_stats in metadata). """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) - from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook mock_router = Mock() filter_instance = SemanticMCPToolFilter( @@ -706,10 +705,10 @@ async def test_semantic_filter_hook_responses_api_name_collision(): Then: The native tool must NOT be sent to the semantic filter, even though its name matches an MCP canonical. """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) - from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook from litellm.types.utils import Embedding, EmbeddingResponse mock_router = Mock() @@ -802,10 +801,10 @@ async def test_semantic_filter_hook_filters_expanded_litellm_proxy_tools(): were filtered out instead of silently forwarding all tools with no stats. """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) - from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook from litellm.types.utils import Embedding, EmbeddingResponse mock_router = Mock() @@ -924,10 +923,10 @@ async def test_semantic_filter_hook_narrows_mcp_reference_for_chat_completions() reference also removed the marker the MCP gateway matches on, which disabled tool auto-execution for require_approval="never". """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) - from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook from litellm.types.utils import Embedding, EmbeddingResponse mock_router = Mock() @@ -1039,10 +1038,10 @@ async def test_semantic_filter_hook_zero_matches_exposes_all_tools_on_both_paths filter_tools to fail closed must fail this test on both paths at once, instead of silently hard-limiting one surface and not the other. """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) - from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook from litellm.types.utils import Embedding, EmbeddingResponse mock_router = Mock() @@ -1148,10 +1147,10 @@ async def test_semantic_filter_hook_filters_expanded_tools_with_string_input(): expanded-tool filtering must treat it as the user query instead of crashing (which would silently disable MCP expansion). """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) - from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook from litellm.types.utils import Embedding, EmbeddingResponse mock_router = Mock() @@ -1218,10 +1217,10 @@ async def test_semantic_filter_hook_expansion_skips_filter_when_disabled(): stats, mirroring the generic path's enabled guard. The MCP gateway then expands the reference itself, so no tool is narrowed away. """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) - from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook filter_instance = SemanticMCPToolFilter( embedding_model="text-embedding-3-small", @@ -1292,10 +1291,10 @@ async def test_semantic_filter_hook_preserves_tool_order(): Then: The output order must match the original request order, NOT native-first. """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) - from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook from litellm.types.utils import Embedding, EmbeddingResponse mock_router = Mock() @@ -1402,7 +1401,7 @@ class TestGetToolsByNames: """ def _make_filter(self): - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) @@ -1572,10 +1571,10 @@ async def test_semantic_filter_headers_hook_emits_only_complete_tool_names(): contain complete tool names, in their original order, within the cap. """ from litellm.constants import MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) - from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook filter_instance = SemanticMCPToolFilter( embedding_model="text-embedding-3-small", @@ -1654,7 +1653,7 @@ def _make_context_window_raising_router(state): def _make_context_window_filter(state, top_k: int = 3): - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) @@ -1675,7 +1674,7 @@ async def test_semantic_filter_fails_closed_on_query_time_context_window_error() user query must fail closed with a typed error instead of silently returning all tools (previously reported as N->N "success"). """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticToolFilterContextWindowError, ) @@ -1708,7 +1707,7 @@ async def test_semantic_filter_records_build_time_context_window_error(): the build, which previously left the hook unregistered and filtering silently disabled) and must fail subsequent filtering closed. """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticToolFilterContextWindowError, ) @@ -1904,7 +1903,7 @@ def test_is_context_window_error_detection_variants(): generic error must not match. """ import litellm - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.mcp_server.semantic_tool_filter import ( _is_context_window_error, ) @@ -1941,7 +1940,7 @@ def test_is_context_window_error_sees_through_trees_the_chain_walk_missed(): ``__context__`` behind a non-matching ``__cause__``, buried inside an anyio-style ``ExceptionGroup``, and chained deeper than five links.""" import litellm - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.mcp_server.semantic_tool_filter import ( _is_context_window_error, ) @@ -2002,7 +2001,7 @@ def _make_keyword_embedding_router(recorded_inputs): def _make_keyword_filter(recorded_inputs, top_k: int = 3): - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticMCPToolFilter, ) @@ -2128,7 +2127,7 @@ async def test_request_time_context_window_error_is_request_scoped(): which EVERY user's MCP requests on the worker were blocked with a 400 until restart (index poisoning via a single request). """ - from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + from litellm.proxy.mcp_server.semantic_tool_filter import ( SemanticToolFilterContextWindowError, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_server_resolution.py b/tests/unit/proxy/mcp_server/test_server_resolution.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/test_server_resolution.py rename to tests/unit/proxy/mcp_server/test_server_resolution.py index 4b9214916a8..4a5185a9410 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_server_resolution.py +++ b/tests/unit/proxy/mcp_server/test_server_resolution.py @@ -1,7 +1,6 @@ from __future__ import annotations import asyncio - from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from typing import Final, Literal @@ -10,16 +9,16 @@ from unittest.mock import Mock import pytest from fastapi import HTTPException -from litellm.proxy._experimental.mcp_server.contracts import TargetCatalog -from litellm.proxy._experimental.mcp_server.server_resolution import ( +from litellm.proxy._types import LiteLLM_MCPServerTable, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.mcp_server.contracts import TargetCatalog +from litellm.proxy.mcp_server.server_resolution import ( MCPServerTargetCatalog, ResolutionSource, ResolvedMCPServer, authorize_mcp_server, resolve_mcp_server, ) -from litellm.proxy._experimental.mcp_server.ui_session_utils import can_access_mcp_server -from litellm.proxy._types import LiteLLM_MCPServerTable, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.mcp_server.ui_session_utils import can_access_mcp_server from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -342,7 +341,7 @@ async def test_authorize_uses_real_access_helper_for_canonical_id( return await can_access_mcp_server(user_api_key_auth, requested_server_id, allowed_servers) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.server_resolution.can_access_mcp_server", + "litellm.proxy.mcp_server.server_resolution.can_access_mcp_server", spy_access, ) if expected_status is None: @@ -559,9 +558,9 @@ async def test_target_catalog_does_not_reuse_admin_authorization_for_another_cal async def test_catalog_without_listing_dependency_fails_explicitly(): from mcp.types import ListToolsRequest - from litellm.proxy._experimental.mcp_server.contracts import OperationContext - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager - from litellm.proxy._experimental.mcp_server.server_resolution import MCPServerTargetCatalog + from litellm.proxy.mcp_server.contracts import OperationContext + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.mcp_server.server_resolution import MCPServerTargetCatalog catalog = MCPServerTargetCatalog(MCPServerManager()) with pytest.raises(RuntimeError, match="listing dependency"): diff --git a/tests/unit/proxy/_experimental/mcp_server/test_short_mcp_tool_prefix.py b/tests/unit/proxy/mcp_server/test_short_mcp_tool_prefix.py similarity index 97% rename from tests/unit/proxy/_experimental/mcp_server/test_short_mcp_tool_prefix.py rename to tests/unit/proxy/mcp_server/test_short_mcp_tool_prefix.py index 7b6007c405e..89342d5a0f7 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_short_mcp_tool_prefix.py +++ b/tests/unit/proxy/mcp_server/test_short_mcp_tool_prefix.py @@ -13,8 +13,8 @@ from typing import List import pytest from mcp.types import Tool as MCPTool -from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager -from litellm.proxy._experimental.mcp_server.utils import ( +from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager +from litellm.proxy.mcp_server.utils import ( SHORT_MCP_TOOL_PREFIX_LENGTH, add_server_prefix_to_name, compute_short_server_prefix, @@ -359,7 +359,7 @@ class TestShortPrefixCollisionResolution: assert server.short_prefix is None def test_assigns_natural_hash_when_no_collision(self, monkeypatch): - from litellm.proxy._experimental.mcp_server import utils as mcp_utils + from litellm.proxy.mcp_server import utils as mcp_utils monkeypatch.setenv("LITELLM_USE_SHORT_MCP_TOOL_PREFIX", "true") manager = MCPServerManager() @@ -375,14 +375,14 @@ class TestShortPrefixCollisionResolution: # Force every attempt=0 hash to "AAA" and attempt=1 to "AAB". # That way the second server registered must rehash to "AAB". - from litellm.proxy._experimental.mcp_server import utils as mcp_utils + from litellm.proxy.mcp_server import utils as mcp_utils def _fake_hash(server_id: str, attempt: int = 0) -> str: return "AAA" if attempt == 0 else f"AA{chr(ord('A') + attempt)}" monkeypatch.setattr(mcp_utils, "compute_short_server_prefix", _fake_hash) # Also patch the symbol that the manager imported at module load. - from litellm.proxy._experimental.mcp_server import ( + from litellm.proxy.mcp_server import ( mcp_server_manager as mgr_module, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_state_tokens.py b/tests/unit/proxy/mcp_server/test_state_tokens.py similarity index 95% rename from tests/unit/proxy/_experimental/mcp_server/test_state_tokens.py rename to tests/unit/proxy/mcp_server/test_state_tokens.py index b3fc5a89526..fd121e79209 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_state_tokens.py +++ b/tests/unit/proxy/mcp_server/test_state_tokens.py @@ -3,8 +3,8 @@ from typing import Final import pytest -from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Ok -from litellm.proxy._experimental.mcp_server.state_tokens import StateTokenError, open_state, seal_state +from litellm.proxy.mcp_server.outbound_credentials.result import Error, Ok +from litellm.proxy.mcp_server.state_tokens import StateTokenError, open_state, seal_state @pytest.fixture(autouse=True) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_subscriptions.py b/tests/unit/proxy/mcp_server/test_subscriptions.py similarity index 95% rename from tests/unit/proxy/_experimental/mcp_server/test_subscriptions.py rename to tests/unit/proxy/mcp_server/test_subscriptions.py index fadb709e00f..721955a51b8 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_subscriptions.py +++ b/tests/unit/proxy/mcp_server/test_subscriptions.py @@ -12,10 +12,10 @@ from mcp.server.subscriptions import ListenHandler from mcp.shared.subscriptions import PromptsListChanged, ToolsListChanged from mcp.types import ListToolsRequest, ListToolsResult, SubscriptionFilter, SubscriptionsListenRequestParams -from litellm.experimental_mcp_client.client import MCPClient -from litellm.proxy._experimental.mcp_server import subscriptions -from litellm.proxy._experimental.mcp_server.contracts import OperationContext +from litellm.mcp_client.client import MCPClient from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.mcp_server import subscriptions +from litellm.proxy.mcp_server.contracts import OperationContext from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -181,7 +181,7 @@ async def test_listener_limits_release_capacity_after_cancellation(second_owner: @pytest.mark.asyncio @pytest.mark.parametrize("proxy_mode", [False, True]) async def test_subscription_requires_a_supported_http_gateway_route(proxy_mode: bool) -> None: - from litellm.proxy._experimental.mcp_server import server + from litellm.proxy.mcp_server import server token = server._mcp_proxy_mode.set(proxy_mode) try: @@ -206,7 +206,7 @@ async def test_subscription_requires_a_supported_http_gateway_route(proxy_mode: async def test_subscription_authorization_preserves_original_caller(changed: str | None) -> None: from dataclasses import replace - from litellm.proxy._experimental.mcp_server.server import _owner_fingerprint_for + from litellm.proxy.mcp_server.server import _owner_fingerprint_for initial = OperationContext(_caller=UserAPIKeyAuth(api_key="first", user_id="alice", team_id="team")) current = replace( @@ -341,7 +341,7 @@ async def test_legacy_upstream_without_capabilities_does_not_receive_subscribe() @pytest.mark.parametrize("proxy_mode", [False, True]) def test_legacy_notification_capabilities_require_a_notification_route(proxy_mode: bool) -> None: - from litellm.proxy._experimental.mcp_server import server + from litellm.proxy.mcp_server import server token = server._mcp_proxy_mode.set(proxy_mode) try: @@ -375,7 +375,7 @@ async def test_idle_legacy_listener_detects_closed_upstream_without_restarting_w async def test_proxy_route_rejects_legacy_resource_subscription_before_setup(method: str) -> None: from mcp.types import SubscribeRequestParams, UnsubscribeRequestParams - from litellm.proxy._experimental.mcp_server import server + from litellm.proxy.mcp_server import server params = (SubscribeRequestParams if method == "subscribe_resource" else UnsubscribeRequestParams)(uri="test://value") token = server._mcp_proxy_mode.set(True) @@ -394,7 +394,7 @@ async def test_proxy_route_rejects_legacy_resource_subscription_before_setup(met async def test_only_client_initiated_listeners_consume_server_rate_limit( monkeypatch: pytest.MonkeyPatch, client_requests: tuple[str, ...], tools_list_admitted: bool ) -> None: - from litellm.proxy._experimental.mcp_server import operations + from litellm.proxy.mcp_server import operations allow_one_request(monkeypatch, "alice", "upstream") opened: Final = asyncio.Queue[None]() @@ -447,8 +447,8 @@ async def test_legacy_listener_catalog_refreshes_do_not_consume_server_rate_limi from mcp.types import ListToolsResult, ServerCapabilities, Tool, ToolsCapability from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import catalog, operations - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ServerListOk + from litellm.proxy.mcp_server import catalog, operations + from litellm.proxy.mcp_server.faults.list_outcomes import ServerListOk server: Final = MCPServer(server_id="list-changed", name="list-changed", transport=MCPTransport.http) caller: Final = UserAPIKeyAuth(api_key="list-changed-key", user_id="alice") @@ -502,12 +502,12 @@ async def test_legacy_listener_catalog_refreshes_do_not_consume_server_rate_limi @pytest.mark.asyncio async def test_all_failed_tools_listing_keeps_empty_catalog_digest(monkeypatch: pytest.MonkeyPatch) -> None: - from litellm.proxy._experimental.mcp_server.catalog import ToolListingFailedOutright + from litellm.proxy.mcp_server.catalog import ToolListingFailedOutright listener_context: Final = OperationContext(_caller=UserAPIKeyAuth(user_id="listener")) empty_gateway: Final = _ListingGateway(result=ListToolsResult(tools=[])) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.operations.GatewayOperations", lambda: empty_gateway + "litellm.proxy.mcp_server.operations.GatewayOperations", lambda: empty_gateway ) empty_digest: Final = await subscriptions._listing(ToolsListChanged(), listener_context) @@ -515,7 +515,7 @@ async def test_all_failed_tools_listing_keeps_empty_catalog_digest(monkeypatch: error=ToolListingFailedOutright(code=-32603, message="Every MCP server in scope failed") ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.operations.GatewayOperations", lambda: failed_gateway + "litellm.proxy.mcp_server.operations.GatewayOperations", lambda: failed_gateway ) failed_digest: Final = await subscriptions._listing(ToolsListChanged(), listener_context) @@ -526,7 +526,7 @@ async def test_all_failed_tools_listing_keeps_empty_catalog_digest(monkeypatch: async def test_tools_listing_propagates_other_mcp_errors(monkeypatch: pytest.MonkeyPatch) -> None: failure: Final = MCPError(code=-32602, message="Bad cursor") gateway: Final = _ListingGateway(error=failure) - monkeypatch.setattr("litellm.proxy._experimental.mcp_server.operations.GatewayOperations", lambda: gateway) + monkeypatch.setattr("litellm.proxy.mcp_server.operations.GatewayOperations", lambda: gateway) listener_context: Final = OperationContext(_caller=UserAPIKeyAuth(user_id="listener")) expected_error: Final = pytest.raises(MCPError) diff --git a/tests/unit/proxy/experimental/mcp_server/test_tool_registry.py b/tests/unit/proxy/mcp_server/test_tool_registry.py similarity index 92% rename from tests/unit/proxy/experimental/mcp_server/test_tool_registry.py rename to tests/unit/proxy/mcp_server/test_tool_registry.py index c7df359aae2..d05204b8e64 100644 --- a/tests/unit/proxy/experimental/mcp_server/test_tool_registry.py +++ b/tests/unit/proxy/mcp_server/test_tool_registry.py @@ -2,8 +2,7 @@ import json import pytest - -from litellm.proxy._experimental.mcp_server.tool_registry import MCPToolRegistry +from litellm.proxy.mcp_server.tool_registry import MCPToolRegistry # Test handler function @@ -59,7 +58,7 @@ def test_load_tools_from_config(): "name": "config_tool", "description": "A tool from config", "input_schema": {"type": "object"}, - "handler": "tests.unit.proxy.experimental.mcp_server.test_tool_registry.example_handler", + "handler": "tests.unit.proxy.mcp_server.test_tool_registry.example_handler", } ] diff --git a/tests/unit/proxy/_experimental/mcp_server/test_ui_session_utils.py b/tests/unit/proxy/mcp_server/test_ui_session_utils.py similarity index 91% rename from tests/unit/proxy/_experimental/mcp_server/test_ui_session_utils.py rename to tests/unit/proxy/mcp_server/test_ui_session_utils.py index 293d9443ced..55adbf19ffe 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_ui_session_utils.py +++ b/tests/unit/proxy/mcp_server/test_ui_session_utils.py @@ -6,14 +6,14 @@ import pytest from fastapi import HTTPException from litellm.constants import UI_SESSION_TOKEN_TEAM_ID -from litellm.proxy._experimental.mcp_server.ui_session_utils import ( +from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth +from litellm.proxy.mcp_server.ui_session_utils import ( build_effective_auth_contexts, clone_user_api_key_auth_with_team, granted_toolset_ids, - toolset_grant_contexts, resolve_ui_session_team_ids, + toolset_grant_contexts, ) -from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth def test_clone_user_api_key_auth_with_team_creates_independent_copy(): @@ -68,7 +68,7 @@ async def test_build_effective_auth_contexts_returns_cloned_contexts(monkeypatch mock_resolve = AsyncMock(return_value=["team-one", "team-two"]) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.ui_session_utils.resolve_ui_session_team_ids", + "litellm.proxy.mcp_server.ui_session_utils.resolve_ui_session_team_ids", mock_resolve, ) @@ -87,7 +87,7 @@ async def test_build_effective_auth_contexts_returns_original_when_no_resolution mock_resolve = AsyncMock(return_value=[]) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.ui_session_utils.resolve_ui_session_team_ids", + "litellm.proxy.mcp_server.ui_session_utils.resolve_ui_session_team_ids", mock_resolve, ) @@ -114,7 +114,7 @@ async def test_build_effective_auth_contexts_handles_unpicklable_parent_span( mock_resolve = AsyncMock(return_value=["team-span"]) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.ui_session_utils.resolve_ui_session_team_ids", + "litellm.proxy.mcp_server.ui_session_utils.resolve_ui_session_team_ids", mock_resolve, ) @@ -133,12 +133,12 @@ async def test_build_effective_auth_contexts_appends_admitted_user_context(monke admitted_auth = UserAPIKeyAuth(user_id="user-42") monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.ui_session_utils.resolve_ui_session_team_ids", + "litellm.proxy.mcp_server.ui_session_utils.resolve_ui_session_team_ids", AsyncMock(return_value=["team-one"]), ) reload_mock = AsyncMock(return_value=admitted_auth) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", reload_mock, ) @@ -154,7 +154,7 @@ async def test_build_effective_auth_contexts_never_widens_caller_passed_keys(mon normal_user = UserAPIKeyAuth(team_id="regular-team", user_id="user-1") reload_mock = AsyncMock() monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", reload_mock, ) @@ -169,11 +169,11 @@ async def test_build_effective_auth_contexts_survives_admitted_reload_failure(mo user_auth = UserAPIKeyAuth(team_id=UI_SESSION_TOKEN_TEAM_ID, user_id="user-9") monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.ui_session_utils.resolve_ui_session_team_ids", + "litellm.proxy.mcp_server.ui_session_utils.resolve_ui_session_team_ids", AsyncMock(return_value=["team-a"]), ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", AsyncMock(side_effect=HTTPException(status_code=503, detail="db down")), ) @@ -186,13 +186,13 @@ async def test_build_effective_auth_contexts_survives_admitted_reload_failure(mo async def test_acting_user_auth_returns_admitted_subject_for_non_admin_sessions(monkeypatch): """LIT-4861: acting-as-user MCP routes must resolve a non-admin dashboard session as the admitted subject so tool ceilings, reachability, and limits bind exactly as on /mcp.""" - from litellm.proxy._experimental.mcp_server.ui_session_utils import acting_user_auth + from litellm.proxy.mcp_server.ui_session_utils import acting_user_auth user_auth = UserAPIKeyAuth(team_id=UI_SESSION_TOKEN_TEAM_ID, user_id="user-42", user_role="internal_user") admitted_auth = UserAPIKeyAuth(user_id="user-42") reload_mock = AsyncMock(return_value=admitted_auth) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", reload_mock, ) @@ -204,11 +204,11 @@ async def test_acting_user_auth_returns_admitted_subject_for_non_admin_sessions( @pytest.mark.asyncio async def test_acting_user_auth_keeps_admin_sessions_and_passed_keys_unchanged(monkeypatch): - from litellm.proxy._experimental.mcp_server.ui_session_utils import acting_user_auth + from litellm.proxy.mcp_server.ui_session_utils import acting_user_auth reload_mock = AsyncMock() monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", reload_mock, ) @@ -223,11 +223,11 @@ async def test_acting_user_auth_keeps_admin_sessions_and_passed_keys_unchanged(m @pytest.mark.asyncio async def test_acting_user_auth_falls_back_to_session_auth_on_reload_failure(monkeypatch): - from litellm.proxy._experimental.mcp_server.ui_session_utils import acting_user_auth + from litellm.proxy.mcp_server.ui_session_utils import acting_user_auth user_auth = UserAPIKeyAuth(team_id=UI_SESSION_TOKEN_TEAM_ID, user_id="user-9", user_role="internal_user") monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", AsyncMock(side_effect=HTTPException(status_code=503, detail="db down")), ) @@ -239,7 +239,7 @@ async def test_admitted_user_context_carries_the_request_span(monkeypatch): """Swapping the principal must not drop the request: the admitted subject is rebuilt from the user row and carries no span of its own, so every consumer would otherwise lose trace linkage for the resolution and logging it drives.""" - from litellm.proxy._experimental.mcp_server.ui_session_utils import acting_user_auth + from litellm.proxy.mcp_server.ui_session_utils import acting_user_auth class DummySpan: def __init__(self) -> None: @@ -253,7 +253,7 @@ async def test_admitted_user_context_carries_the_request_span(monkeypatch): parent_otel_span=parent_span, ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", AsyncMock(return_value=UserAPIKeyAuth(user_id="user-42")), ) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_utils.py b/tests/unit/proxy/mcp_server/test_utils.py similarity index 98% rename from tests/unit/proxy/_experimental/mcp_server/test_utils.py rename to tests/unit/proxy/mcp_server/test_utils.py index 8d10219796b..00af79cd860 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_utils.py +++ b/tests/unit/proxy/mcp_server/test_utils.py @@ -3,14 +3,14 @@ from unittest.mock import patch import pytest from fastapi import HTTPException -from litellm.proxy._experimental.mcp_server.utils import ( - upstream_credential_headers, +from litellm.proxy._types import NewMCPServerRequest +from litellm.proxy.mcp_server.utils import ( build_synthetic_mcp_request, logging_safe_mcp_headers, + upstream_credential_headers, validate_and_normalize_mcp_server_payload, validate_tool_display_names, ) -from litellm.proxy._types import NewMCPServerRequest from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -26,7 +26,7 @@ def _server_forwarding(*header_names: str) -> MCPServer: def _configured_servers(*servers: MCPServer): return patch.dict( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager.config_mcp_servers", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager.config_mcp_servers", {server.server_id: server for server in servers}, clear=False, ) @@ -273,7 +273,7 @@ class TestBuildSyntheticMcpRequest: @pytest.mark.parametrize("field", ["structuredContent", "structured_content"]) def test_structured_content_redaction_updates_shared_dictionary(field): - from litellm.proxy._experimental.mcp_server.utils import ( + from litellm.proxy.mcp_server.utils import ( mcp_tool_result_structured_content, set_mcp_tool_result_structured_content, ) diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index cc61b6f2fcf..f60c2254b41 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -1235,8 +1235,8 @@ def test_public_mcp_hub_derives_publication_metadata_without_mutating_registry( expected_listed: bool, stored_public: bool | None, ) -> None: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.proxy._types import MCPTransport + from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager from litellm.types.mcp_server.mcp_server_manager import MCPServer app: Final = FastAPI() @@ -1275,7 +1275,7 @@ def test_public_mcp_hub_derives_publication_metadata_without_mutating_registry( patch("litellm.public_mcp_servers", [server.server_id] if explicit else []), patch("litellm.public_mcp_hub_strict_whitelist", strict), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", manager, ), ): @@ -1307,7 +1307,7 @@ def test_public_mcp_hub_returns_empty_when_whitelist_unset(): with ( patch("litellm.public_mcp_servers", None), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), ): @@ -1345,7 +1345,7 @@ def test_public_mcp_hub_does_not_expose_upstream_url(): with ( patch("litellm.public_mcp_servers", ["listed"]), patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", mock_manager, ), ): diff --git a/tests/unit/proxy/test_dynamic_mcp_route.py b/tests/unit/proxy/test_dynamic_mcp_route.py index 4a8dddb1bf4..dc5c1ba1b79 100644 --- a/tests/unit/proxy/test_dynamic_mcp_route.py +++ b/tests/unit/proxy/test_dynamic_mcp_route.py @@ -18,21 +18,20 @@ from unittest.mock import ANY, AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException - # --------------------------------------------------------------------------- # helpers # --------------------------------------------------------------------------- -_MCP_MANAGER = "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" +_MCP_MANAGER = "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" _HANDLE_HTTP = ( - "litellm.proxy._experimental.mcp_server.server.handle_streamable_http_mcp" + "litellm.proxy.mcp_server.server.handle_streamable_http_mcp" ) _STREAM_ASGI = "litellm.proxy.proxy_server._stream_mcp_asgi_response" _PRISMA = "litellm.proxy.proxy_server.prisma_client" _IS_ACCESS_GROUP = "litellm.proxy.proxy_server._is_mcp_access_group_cached" _USER_API_KEY_CACHE = "litellm.proxy.proxy_server.user_api_key_cache" _GET_ACCESS_GROUP_SERVERS = ( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp." + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp." "MCPRequestHandler.get_mcp_servers_from_access_groups" ) _FORWARD = "litellm.proxy.proxy_server._mcp_forward_as_path" @@ -285,7 +284,7 @@ async def test_dynamic_mcp_route_resolves_toolset(): async def fake_stream(fn, scope, receive): nonlocal captured_toolset_id - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy.mcp_server.server import ( mcp_active_toolset_id, ) @@ -545,7 +544,7 @@ async def test_toolset_mcp_route_unexpected_exception_returns_500_without_traceb # 7. Aggregate /mcp without a trailing slash (bare mount prefix) # --------------------------------------------------------------------------- -_IS_MCP_AVAILABLE = "litellm.proxy._experimental.mcp_server.utils.is_mcp_available" +_IS_MCP_AVAILABLE = "litellm.proxy.mcp_server.utils.is_mcp_available" def _test_client(): @@ -623,8 +622,8 @@ async def test_dynamic_route_observes_committed_peer_catalog_changes(monkeypatch from litellm.caching.caching import DualCache from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server import mcp_server_manager from litellm.proxy._types import LiteLLM_MCPServerTable + from litellm.proxy.mcp_server import mcp_server_manager old_row = LiteLLM_MCPServerTable( server_id="catalog-route", server_name="previous", alias="previous", transport="http", diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index e03fd9850c1..a44373b87c8 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -37,7 +37,6 @@ from litellm.litellm_core_utils.redact_messages import _get_turn_off_message_log from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id from litellm.proxy._types import AddTeamCallback, ProxyException, TeamCallbackMetadata, UserAPIKeyAuth -from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES from litellm.proxy.litellm_pre_call_utils import ( KeyAndTeamLoggingSettings, LiteLLMProxyRequestSetup, @@ -59,6 +58,7 @@ from litellm.proxy.litellm_pre_call_utils import ( move_guardrails_to_metadata, ) from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload +from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES from litellm.types.utils import CredentialItem @@ -9576,7 +9576,7 @@ async def test_mcp_credentials_only_removed_from_logging_copies(path: str, custo with ( patch("litellm.proxy.proxy_server.general_settings", settings), patch.dict( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager.config_mcp_servers", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager.config_mcp_servers", {"header-test": server}, clear=True, ), @@ -10371,8 +10371,8 @@ class TestResolveUserProviderCredentials: _get_hierarchical_router_settings) must be part of discovery.""" from types import SimpleNamespace - from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request import litellm.proxy.auth.auth_checks as auth_checks + from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request table = self._env(monkeypatch) monkeypatch.setattr( diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 745488bf93f..8c62b225992 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -29,9 +29,9 @@ import litellm import litellm.proxy.proxy_server as proxy_server_module from litellm._internal_context import current_service_target from litellm.caching.caching import RedisCache +from litellm.caching.dual_cache import DualCache from litellm.caching.redis_cluster_cache import RedisClusterCache from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded -from litellm.caching.dual_cache import DualCache from litellm.proxy._types import ( AwsErrorTypeProxyException, LitellmUserRoles, @@ -9335,8 +9335,8 @@ async def test_increment_spend_counters_reseeds_from_db_on_bad_reserved_counter( from the authoritative DB and adds this request's settled cost, which the async spend flush has not written yet, so budget gating continues.""" from litellm.caching.dual_cache import DualCache - from litellm.proxy.proxy_server import increment_spend_counters from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed + from litellm.proxy.proxy_server import increment_spend_counters counter_cache = DualCache() budget_reservation = { @@ -12350,8 +12350,8 @@ def _config_field_info_client(monkeypatch, user_role): from fastapi.testclient import TestClient import litellm.proxy.proxy_server as ps - from litellm.proxy.config_resolvers import SettingsStore from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.config_resolvers import SettingsStore from litellm.proxy.proxy_server import app db_record = types.SimpleNamespace( @@ -13742,11 +13742,11 @@ def test_startup_is_silent_when_mock_testing_params_disabled(caplog): def _window_spend_enqueue_env(cached_objects: dict): """Point increment_spend_counters at throwaway caches and a real WindowSpendUpdateQueue, and hand back the queue to inspect.""" + import litellm.proxy.proxy_server as ps from litellm.caching.dual_cache import DualCache from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( WindowSpendUpdateQueue, ) - import litellm.proxy.proxy_server as ps user_api_key_cache = MagicMock() user_api_key_cache.async_get_cache = AsyncMock(side_effect=lambda key, **_: cached_objects.get(key)) @@ -13825,8 +13825,8 @@ async def test_team_window_spend_row_is_enqueued(): async def test_window_spend_row_is_enqueued_even_when_the_counter_was_reserved(): """A reservation only pre-charged the cache counter with an estimate; the row still owes the actual cost, so the enqueue must not be skipped.""" - from litellm.proxy.proxy_server import increment_spend_counters import litellm.proxy.spend_tracking.budget_reservation as br + from litellm.proxy.proxy_server import increment_spend_counters reset_at = datetime.now(timezone.utc) + timedelta(days=10) key_obj = MagicMock() @@ -15166,7 +15166,7 @@ async def test_update_general_settings_keeps_yaml_openai_websocket_passthrough() def test_settings_store_exposes_dashboard_saved_mcp_client_allowlist_to_the_mcp_gateway() -> None: - from litellm.proxy._experimental.mcp_server.client_allowlist import MCPClientAllowlist, load_mcp_client_allowlist + from litellm.proxy.mcp_server.client_allowlist import MCPClientAllowlist, load_mcp_client_allowlist from litellm.proxy.proxy_server import ProxyConfig settings: Final = ProxyConfig().settings @@ -15252,9 +15252,8 @@ async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeyp async def test_token_counter_loads_a_custom_tokenizer_once_per_identifier_revision_and_token(monkeypatch): - from litellm.rust_bridge._native import Tokenizer - from litellm import Router + from litellm.rust_bridge._native import Tokenizer from litellm.types.router import DeploymentTypedDict claude_tokenizer: Final[Tokenizer] = litellm.utils.select_tokenizer("claude-fable-5")["tokenizer"] @@ -15307,13 +15306,13 @@ async def test_auth_cache_invalidation_subscriber_evicts_byok_credentials_cached """A peer worker's BYOK revocation broadcast must reach this worker's BYOK credential cache.""" from redis.asyncio import Redis - from litellm.proxy._experimental.mcp_server.byok_credential_cache import ( + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.mcp_server.byok_credential_cache import ( byok_credential_cache, byok_credential_cache_key, cache_byok_credential, get_cached_byok_credential, ) - from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache class _QueuePubSub: def __init__(self, messages: list[object]) -> None: diff --git a/tests/unit/proxy/types_utils/test_get_instance_fn_runtime_gate.py b/tests/unit/proxy/types_utils/test_get_instance_fn_runtime_gate.py index 3d76ad54a9c..3b0672b306e 100644 --- a/tests/unit/proxy/types_utils/test_get_instance_fn_runtime_gate.py +++ b/tests/unit/proxy/types_utils/test_get_instance_fn_runtime_gate.py @@ -151,7 +151,7 @@ def test_mcp_tool_registry_threads_config_file_path(): # MCP tool handlers declared in ``config.yaml`` mcp_tools[].handler # may legitimately be ``s3://...``; the YAML-load path must thread # ``config_file_path`` so they resolve. - from litellm.proxy._experimental.mcp_server import tool_registry as tr + from litellm.proxy.mcp_server import tool_registry as tr fake_handler = lambda **kwargs: None # noqa: E731 — registry requires callable with patch.object(tr, "get_instance_fn", return_value=fake_handler) as mock_get: diff --git a/tests/unit/responses/mcp/conftest.py b/tests/unit/responses/mcp/conftest.py index c41ba624ac8..ac661b9a9fb 100644 --- a/tests/unit/responses/mcp/conftest.py +++ b/tests/unit/responses/mcp/conftest.py @@ -5,4 +5,4 @@ import pytest @pytest.fixture(scope="module", autouse=True) def import_mcp_server_before_manager_doubles() -> None: - importlib.import_module("litellm.proxy._experimental.mcp_server.server") + importlib.import_module("litellm.proxy.mcp_server.server") diff --git a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py index 285d90ef8de..4cbcc6c1519 100644 --- a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py +++ b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py @@ -17,7 +17,6 @@ import respx from mcp.types import CallToolResult, ListToolsResult, TextContent from mcp.types import Tool as MCPTool - # Import required modules import litellm from litellm.exceptions import GuardrailRaisedException, MidStreamFallbackError @@ -25,15 +24,15 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.langfuse.langfuse import langfuse_trace_id, langfuse_trace_metadata from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.xai.common_utils import xai_reported_cost_in_usd -from litellm.proxy._experimental.mcp_server import mcp_server_manager -from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ServerListOk +from litellm.proxy.mcp_server import mcp_server_manager +from litellm.proxy.mcp_server.faults.list_outcomes import ServerListOk from litellm.proxy.utils import _estimate_dispatched_failure_usage, _failure_fields_to_lift from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler from litellm.responses.mcp.mcp_streaming_iterator import MCPEnhancedStreamingIterator from litellm.types.llms.openai import ( + OpenAIMcpServerTool, ResponsesAPIResponse, ResponsesAPIStreamingResponse, - OpenAIMcpServerTool, ToolParam, ) from litellm.types.utils import ModelResponse, StandardLoggingPayload, Usage @@ -1079,10 +1078,11 @@ async def test_aresponses_mcp_call_does_not_forward_admission_authorization( import types from unittest.mock import AsyncMock - from mcp.types import CallToolResult, TextContent, Tool as MCPTool + from mcp.types import CallToolResult, TextContent + from mcp.types import Tool as MCPTool - from litellm.proxy._experimental.mcp_server import mcp_server_manager from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server import mcp_server_manager from litellm.types.mcp import MCPAuth, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -1185,6 +1185,7 @@ async def test_no_duplicate_mcp_tools_in_streaming_e2e(): sent to the LLM to ensure no duplication occurs. """ from unittest.mock import AsyncMock, patch + from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) @@ -1715,7 +1716,7 @@ async def _auto_execute_through_mcp_server( sys.modules, "litellm.proxy.proxy_server", SimpleNamespace(proxy_logging_obj=None, prisma_client=None) ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", SimpleNamespace( catalog=SimpleNamespace(operation=nullcontext), get_registry=MagicMock(return_value={}), @@ -2042,7 +2043,7 @@ def _serve_the_demo_mcp_listing(monkeypatch: pytest.MonkeyPatch) -> _ListingReco sys.modules, "litellm.proxy.proxy_server", SimpleNamespace(proxy_logging_obj=None, prisma_client=None) ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", SimpleNamespace( catalog=SimpleNamespace(operation=nullcontext), get_registry=MagicMock(return_value={}), @@ -2054,11 +2055,11 @@ def _serve_the_demo_mcp_listing(monkeypatch: pytest.MonkeyPatch) -> _ListingReco ), ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", AsyncMock(return_value=[demo_server]), ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.catalog.get_filtered_server_tools", + "litellm.proxy.mcp_server.catalog.get_filtered_server_tools", AsyncMock(return_value=(ListToolsResult(tools=[echo_tool]), ServerListOk(tool_count=1))), ) listing_recorder: Final = _ListingRecorder() diff --git a/tests/unit/responses/mcp/test_chat_completions_handler.py b/tests/unit/responses/mcp/test_chat_completions_handler.py index 9064eba161d..ec5968c2a1a 100644 --- a/tests/unit/responses/mcp/test_chat_completions_handler.py +++ b/tests/unit/responses/mcp/test_chat_completions_handler.py @@ -6,19 +6,17 @@ from collections.abc import Mapping, Sequence from datetime import datetime from types import MappingProxyType from typing import Final, cast +from unittest.mock import AsyncMock, patch import pytest import respx from httpx import Response from mcp.types import Tool as MCPTool -from unittest.mock import AsyncMock, patch import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.types.utils import ModelResponse, StandardLoggingPayload - from litellm.responses.mcp import chat_completions_handler from litellm.responses.mcp.chat_completions_handler import ( acompletion_with_mcp, @@ -27,6 +25,7 @@ from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.utils import ModelResponse, StandardLoggingPayload @pytest.mark.asyncio @@ -181,15 +180,16 @@ async def test_acompletion_with_mcp_passes_mcp_server_auth_headers_to_process_to @pytest.mark.asyncio async def test_acompletion_with_mcp_auto_exec_performs_follow_up(monkeypatch): - from litellm.utils import CustomStreamWrapper + from unittest.mock import MagicMock + from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + Delta, + Function, ModelResponseStream, StreamingChoices, - Delta, - ChatCompletionDeltaToolCall, - Function, ) - from unittest.mock import MagicMock + from litellm.utils import CustomStreamWrapper tools = [{"type": "function", "function": {"name": "tool"}}] @@ -432,9 +432,9 @@ async def test_acompletion_with_mcp_adds_metadata_to_streaming(monkeypatch): Test that acompletion_with_mcp adds MCP metadata to CustomStreamWrapper and it appears in the final chunk's delta.provider_specific_fields. """ - from litellm.utils import CustomStreamWrapper - from litellm.types.utils import ModelResponseStream, StreamingChoices, Delta from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + from litellm.utils import CustomStreamWrapper tools = [{"type": "mcp", "server_url": "litellm_proxy/mcp/local"}] openai_tools = [{"type": "function", "function": {"name": "local_search"}}] @@ -588,8 +588,8 @@ async def test_acompletion_with_mcp_streaming_initial_call_is_streaming(monkeypa Test that acompletion_with_mcp makes the initial LLM call with streaming=True when stream=True is requested, instead of making a non-streaming call first. """ + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices from litellm.utils import CustomStreamWrapper - from litellm.types.utils import ModelResponseStream, StreamingChoices, Delta tools = [{"type": "mcp", "server_url": "litellm_proxy/mcp/local"}] openai_tools = [{"type": "function", "function": {"name": "local_search"}}] @@ -762,14 +762,14 @@ async def test_acompletion_with_mcp_streaming_metadata_in_correct_chunks(monkeyp - mcp_list_tools should be in the first chunk - mcp_tool_calls and mcp_call_results should be in the final chunk of initial response """ - from litellm.utils import CustomStreamWrapper from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + Delta, + Function, ModelResponseStream, StreamingChoices, - Delta, - ChatCompletionDeltaToolCall, - Function, ) + from litellm.utils import CustomStreamWrapper tools = [{"type": "mcp", "server_url": "litellm_proxy/mcp/local"}] openai_tools = [{"type": "function", "function": {"name": "local_search"}}] @@ -1083,7 +1083,7 @@ async def test_execute_tool_calls_sets_proxy_server_request_arguments(monkeypatc ) monkeypatch.setattr(mcp_handler_module, "function_setup", mock_function_setup) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager.call_tool", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager.call_tool", mock_call_tool, ) @@ -1366,7 +1366,7 @@ async def test_acompletion_with_mcp_streaming_drains_inner_stream_after_exhausti @pytest.mark.asyncio @respx.mock async def test_acompletion_with_mcp_forwards_unserved_external_mcp_tool_to_the_provider(monkeypatch): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager zapier_tool = {"type": "mcp", "server_label": "zapier", "server_url": "https://mcp.zapier.com/api/mcp/mcp"} monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", types.SimpleNamespace(prisma_client=None)) @@ -1405,14 +1405,15 @@ async def test_acompletion_with_mcp_forwards_unserved_external_mcp_tool_to_the_p @pytest.mark.parametrize("selection_source", ["metadata", "litellm_metadata", "body"]) @pytest.mark.parametrize("logging_failure", [False, True]) async def test_request_selected_mcp_guardrail_blocks_before_upstream(monkeypatch, selected, stream, selection_source, logging_failure): - from litellm.exceptions import GuardrailRaisedException from mcp.types import Tool + from litellm.caching.caching import DualCache + from litellm.exceptions import GuardrailRaisedException from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy import proxy_server - from litellm.proxy._types import UserAPIKeyAuth, LiteLLM_ObjectPermissionTable - from litellm.proxy._experimental.mcp_server import mcp_server_manager, server, tool_registry - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth + from litellm.proxy.mcp_server import mcp_server_manager, server, tool_registry + from litellm.proxy.mcp_server.faults.list_outcomes import AggregateToolListing from litellm.proxy.utils import ProxyLogging from litellm.types.guardrails import GuardrailEventHooks from litellm.types.mcp_server.mcp_server_manager import MCPServer diff --git a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py index 26553558298..4da1bd97c3a 100644 --- a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -18,10 +18,10 @@ import litellm from litellm.caching.caching import DualCache from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.proxy._experimental.mcp_server import operations as mcp_operations -from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing -from litellm.proxy._experimental.mcp_server.mcp_server_manager import ListedToolsCaller, MCPServerManager from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.mcp_server import operations as mcp_operations +from litellm.proxy.mcp_server.faults.list_outcomes import AggregateToolListing +from litellm.proxy.mcp_server.mcp_server_manager import ListedToolsCaller, MCPServerManager from litellm.proxy.utils import ProxyLogging from litellm.responses import main as responses_main from litellm.responses.mcp import litellm_proxy_mcp_handler as mcp_handler_module @@ -55,7 +55,7 @@ def _setup_mcp_call_environment(monkeypatch: pytest.MonkeyPatch) -> AsyncMock: get_mcp_server_by_name=MagicMock(return_value=None), ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", fake_manager, ) return fake_manager.call_tool @@ -187,11 +187,11 @@ def test_transform_mcp_tools_to_openai_uses_chat_format(monkeypatch): return {"responses": True} monkeypatch.setattr( - "litellm.experimental_mcp_client.tools.transform_mcp_tool_to_openai_tool", + "litellm.mcp_client.tools.transform_mcp_tool_to_openai_tool", fake_transform_chat, ) monkeypatch.setattr( - "litellm.experimental_mcp_client.tools.transform_mcp_tool_to_openai_responses_api_tool", + "litellm.mcp_client.tools.transform_mcp_tool_to_openai_responses_api_tool", fake_transform_responses, ) @@ -313,7 +313,7 @@ async def test_execute_tool_calls_strips_prefix_when_alias_differs_from_server_n mcp_info=None, tool_name_to_display_name=None, ) - from litellm.proxy._experimental.mcp_server import mcp_server_manager as _msm + from litellm.proxy.mcp_server import mcp_server_manager as _msm _msm.global_mcp_server_manager.get_mcp_server_from_tool_name = MagicMock(return_value=fake_server) @@ -355,7 +355,7 @@ async def test_execute_tool_calls_reverse_maps_display_name(monkeypatch): mcp_info=None, tool_name_to_display_name={"read_wiki_structure": "browse_repo_docs"}, ) - from litellm.proxy._experimental.mcp_server import mcp_server_manager as _msm + from litellm.proxy.mcp_server import mcp_server_manager as _msm _msm.global_mcp_server_manager.get_mcp_server_from_tool_name = MagicMock(return_value=colliding_server) _msm.global_mcp_server_manager.get_mcp_server_by_name = MagicMock(return_value=fake_server) @@ -393,7 +393,7 @@ async def test_execute_tool_calls_logs_failure_via_post_call_failure_hook(monkey call_tool=AsyncMock(side_effect=HTTPException(status_code=500, detail="boom")), ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", fake_manager, ) @@ -515,7 +515,7 @@ async def test_execute_tool_calls_applies_post_call_hook_content(monkeypatch): get_mcp_server_by_name=MagicMock(return_value=None), ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", fake_manager, ) @@ -557,7 +557,7 @@ async def test_execute_tool_calls_returns_proxy_result_without_logging(monkeypat get_mcp_server_by_name=MagicMock(return_value=None), ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", fake_manager, ) handler_module = importlib.import_module("litellm.responses.mcp.litellm_proxy_mcp_handler") @@ -591,7 +591,7 @@ async def test_execute_tool_calls_passes_logging_details_to_proxy_hook(monkeypat get_mcp_server_by_name=MagicMock(return_value=None), ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", fake_manager, ) logging_obj = MagicMock() @@ -627,7 +627,7 @@ async def test_execute_tool_calls_continues_when_post_call_logging_fails(monkeyp get_mcp_server_by_name=MagicMock(return_value=None), ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", fake_manager, ) @@ -670,7 +670,7 @@ async def test_get_mcp_tools_from_manager_enables_list_tools_logging(monkeypatch ] mock_get_tools = AsyncMock(return_value=AggregateToolListing(tools=served_tools, outcomes={})) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.server._get_tools_from_mcp_servers", + "litellm.proxy.mcp_server.server._get_tools_from_mcp_servers", mock_get_tools, ) @@ -683,7 +683,7 @@ async def test_get_mcp_tools_from_manager_enables_list_tools_logging(monkeypatch get_mcp_server_by_name=MagicMock(return_value=None), ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", fake_manager, ) @@ -730,7 +730,7 @@ def test_get_parent_request_tags_from_nested_litellm_params(): async def test_get_mcp_tools_from_manager_forwards_request_tags(monkeypatch): mock_get_tools = AsyncMock(return_value=AggregateToolListing(tools=[], outcomes={})) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.server._get_tools_from_mcp_servers", + "litellm.proxy.mcp_server.server._get_tools_from_mcp_servers", mock_get_tools, ) fake_manager = types.SimpleNamespace( @@ -741,7 +741,7 @@ async def test_get_mcp_tools_from_manager_forwards_request_tags(monkeypatch): get_mcp_server_by_name=MagicMock(return_value=None), ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", fake_manager, ) @@ -952,7 +952,7 @@ async def test_gateway_served_names_matches_alias_server_name_name_access_group_ @pytest.mark.asyncio async def test_gateway_served_names_matches_server_id_short_prefix_and_alias_case_like_the_gateway(): - from litellm.proxy._experimental.mcp_server.utils import compute_short_server_prefix + from litellm.proxy.mcp_server.utils import compute_short_server_prefix from litellm.responses.mcp.litellm_proxy_mcp_handler import _gateway_served_names server_id = "0b9ae4ca-1bd2-4faa-b183-7dd812597e3b" @@ -1040,7 +1040,7 @@ def test_should_use_gateway_still_triggers_on_http_mcp_path(): @pytest.mark.asyncio async def test_aresponses_api_with_mcp_forwards_unserved_external_mcp_tool_to_the_provider(monkeypatch): - from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.responses import main as responses_main from litellm.types.llms.openai import ResponsesAPIResponse @@ -1287,7 +1287,7 @@ async def test_mcp_follow_up_call_is_stateless_when_store_is_false( @pytest.mark.asyncio async def test_responses_discovery_logs_sanitized_caller_headers(monkeypatch: pytest.MonkeyPatch): - from litellm.proxy._experimental.mcp_server import mcp_server_manager, operations + from litellm.proxy.mcp_server import mcp_server_manager, operations headers: Final = { "x-app-id": "app-a", @@ -1365,7 +1365,7 @@ async def test_bridge_listing_leaves_the_callers_catalog_unchanged( get_mcp_server_by_name=MagicMock(return_value=None), ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", fake_manager, ) with ( @@ -1453,7 +1453,7 @@ async def test_concurrent_bridge_calls_use_their_own_served_metadata(monkeypatch monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", logger) monkeypatch.setattr(mcp_operations, "global_mcp_server_manager", manager) monkeypatch.setattr(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[server])) - monkeypatch.setattr("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", manager) + monkeypatch.setattr("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", manager) first_listed: Final = asyncio.Event() second_listed: Final = asyncio.Event() @@ -1508,13 +1508,13 @@ def _toolset_gateway_manager(toolset_id: str, server_id: str) -> types.SimpleNam async def _tools_listing_kwargs_for_toolset_url(monkeypatch, team_toolset_id: str) -> dict[str, object]: - from litellm.proxy._experimental.mcp_server.ui_session_utils import granted_toolset_ids from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.mcp_server.ui_session_utils import granted_toolset_ids mock_get_tools = AsyncMock(return_value=AggregateToolListing(tools=[], outcomes={})) - monkeypatch.setattr("litellm.proxy._experimental.mcp_server.server._get_tools_from_mcp_servers", mock_get_tools) + monkeypatch.setattr("litellm.proxy.mcp_server.server._get_tools_from_mcp_servers", mock_get_tools) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", _toolset_gateway_manager("ts-granted", "srv-1"), ) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MagicMock()) @@ -1561,7 +1561,7 @@ async def test_apply_toolset_permissions_pins_the_auth_to_explicit_grants_only(m resolve_toolset_tool_permissions=AsyncMock(return_value={"srv-1": ["add"]}), ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", fake_manager, ) diff --git a/tests/unit/responses/mcp/test_mcp_streaming_iterator.py b/tests/unit/responses/mcp/test_mcp_streaming_iterator.py index ed39da1b7a9..78b0e3d4e5b 100644 --- a/tests/unit/responses/mcp/test_mcp_streaming_iterator.py +++ b/tests/unit/responses/mcp/test_mcp_streaming_iterator.py @@ -85,7 +85,7 @@ def _mock_mcp_environment(monkeypatch: pytest.MonkeyPatch) -> AsyncMock: get_mcp_server_by_name=MagicMock(return_value=None), ) monkeypatch.setattr( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager", fake_manager, ) monkeypatch.setitem( diff --git a/tests/unit/responses/mcp/test_tool_precheck.py b/tests/unit/responses/mcp/test_tool_precheck.py index a3b2c3a0efb..bd37114ccea 100644 --- a/tests/unit/responses/mcp/test_tool_precheck.py +++ b/tests/unit/responses/mcp/test_tool_precheck.py @@ -14,7 +14,7 @@ import respx import litellm from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway -GATEWAY_REGISTRY_MODULE: Final = "litellm.proxy._experimental.mcp_server.mcp_server_manager" +GATEWAY_REGISTRY_MODULE: Final = "litellm.proxy.mcp_server.mcp_server_manager" GATEWAY_HANDLER_MODULES: Final = ( "litellm.responses.mcp.litellm_proxy_mcp_handler", "litellm.responses.mcp.chat_completions_handler", diff --git a/tests/unit/test_check_mcp_operation_boundary.py b/tests/unit/test_check_mcp_operation_boundary.py index d7ac72de9f0..0969c461ad2 100644 --- a/tests/unit/test_check_mcp_operation_boundary.py +++ b/tests/unit/test_check_mcp_operation_boundary.py @@ -9,11 +9,11 @@ from scripts.check_mcp_operation_boundary import main, violations "source", ( "from mcp.server.auth.middleware.auth_context import auth_context_var as hidden", - "from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_proxy_mode as mode", + "from litellm.proxy.mcp_server.mcp_context import _mcp_proxy_mode as mode", "caller = legacy.get_active_auth_context()", "owners = transport._stateful_session_owners", "from weakref import WeakKeyDictionary", - "from litellm.proxy._experimental.mcp_server.server import get_auth_context", + "from litellm.proxy.mcp_server.server import get_auth_context", ), ) def test_shared_operation_boundary_rejects_ambient_state(source): @@ -21,7 +21,7 @@ def test_shared_operation_boundary_rejects_ambient_state(source): def test_legacy_adapter_may_resolve_context_but_policy_must_receive_it(): - source = "from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_proxy_mode" + source = "from litellm.proxy.mcp_server.mcp_context import _mcp_proxy_mode" assert violations(Path("server.py"), source) == () assert violations(Path("legacy_callbacks.py"), source) == () assert violations(Path("operations.py"), "def execute(context):\n return context.client_ip") == () @@ -32,7 +32,7 @@ def test_boundary_command_rejects_shared_state_and_accepts_explicit_context(tmp_ import subprocess import sys - package = tmp_path / "litellm/proxy/_experimental/mcp_server" + package = tmp_path / "litellm/proxy/mcp_server" package.mkdir(parents=True) module = package / "operations.py" module.write_text("from mcp.server.auth.middleware.auth_context import auth_context_var as hidden\n") diff --git a/tests/unit/test_circleci_path_filter.py b/tests/unit/test_circleci_path_filter.py index c44560b96f1..9f695960517 100644 --- a/tests/unit/test_circleci_path_filter.py +++ b/tests/unit/test_circleci_path_filter.py @@ -51,7 +51,7 @@ CI = [".github/workflows/test-unit.yml"] [ ("mcp-dependencies", ["pyproject.toml"], "run"), ("mcp-dependencies", ["uv.lock"], "run"), - ("mcp-dependencies", ["litellm/experimental_mcp_client/client.py"], "run"), + ("mcp-dependencies", ["litellm/mcp_client/client.py"], "run"), ("mcp-dependencies", ["tests/e2e/mcp/datadog_mcp.py"], "run"), ("mcp-dependencies", ["litellm-proxy-extras/pyproject.toml"], "run"), ("mcp-dependencies", ["scripts/check_mcp_sdk_install.py"], "run"), diff --git a/tests/unit/test_internal_context.py b/tests/unit/test_internal_context.py index e104cf16b04..f78c6064f76 100644 --- a/tests/unit/test_internal_context.py +++ b/tests/unit/test_internal_context.py @@ -62,13 +62,13 @@ _IN_MEMORY_ONLY_CALLERS: Final = frozenset( "litellm/llms/openai_like/model_info.py", "litellm/llms/vertex_ai/vertex_ai_non_gemini.py", "litellm/llms/watsonx/common_utils.py", - "litellm/proxy/_experimental/mcp_server/byok_credential_cache.py", - "litellm/proxy/_experimental/mcp_server/catalog.py", - "litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py", - "litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py", - "litellm/proxy/_experimental/mcp_server/operations.py", - "litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py", - "litellm/proxy/_experimental/mcp_server/outbound_credentials/token_endpoint.py", + "litellm/proxy/mcp_server/byok_credential_cache.py", + "litellm/proxy/mcp_server/catalog.py", + "litellm/proxy/mcp_server/discoverable_endpoints.py", + "litellm/proxy/mcp_server/oauth_identity_binding.py", + "litellm/proxy/mcp_server/operations.py", + "litellm/proxy/mcp_server/outbound_credentials/sso_assertion_store.py", + "litellm/proxy/mcp_server/outbound_credentials/token_endpoint.py", "litellm/proxy/agent_endpoints/databricks_oauth.py", "litellm/proxy/common_utils/registry_read_through.py", "litellm/proxy/container_endpoints/ownership.py", diff --git a/tests/unit/test_private_usage_aliases.py b/tests/unit/test_private_usage_aliases.py index 88ae584f34f..1d9c2e87100 100644 --- a/tests/unit/test_private_usage_aliases.py +++ b/tests/unit/test_private_usage_aliases.py @@ -177,157 +177,157 @@ MODULE_IMPORT_ALIAS_CASES: Final = ( "PROXY_DynamicRateLimitHandlerV3", ), ( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", "_read_request_body", "read_request_body", ), ( - "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp", "_run_centralized_common_checks", "run_centralized_common_checks", ), ( - "litellm.proxy._experimental.mcp_server.bridge_token_flow", + "litellm.proxy.mcp_server.bridge_token_flow", "_V2_GCM_PREFIX", "V2_GCM_PREFIX", ), ( - "litellm.proxy._experimental.mcp_server.db", + "litellm.proxy.mcp_server.db", "_get_salt_key", "get_salt_key", ), ( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints", + "litellm.proxy.mcp_server.discoverable_endpoints", "_bridge_mint_error_response", "bridge_mint_error_response", ), ( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints", + "litellm.proxy.mcp_server.discoverable_endpoints", "_extract_user_id_from_request", "extract_user_id_from_request", ), ( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints", + "litellm.proxy.mcp_server.discoverable_endpoints", "_finish_bridge_mint", "finish_bridge_mint", ), ( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints", + "litellm.proxy.mcp_server.discoverable_endpoints", "_prepare_bridge_mint", "prepare_bridge_mint", ), ( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints", + "litellm.proxy.mcp_server.discoverable_endpoints", "_prepare_bridge_refresh", "prepare_bridge_refresh", ), ( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints", + "litellm.proxy.mcp_server.discoverable_endpoints", "_read_request_body", "read_request_body", ), ( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints", + "litellm.proxy.mcp_server.discoverable_endpoints", "_reload_active_user_by_id", "reload_active_user_by_id", ), ( - "litellm.proxy._experimental.mcp_server.mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager", "_is_mcp_admitted_user_subject", "is_mcp_admitted_user_subject", ), ( - "litellm.proxy._experimental.mcp_server.mcp_server_manager", + "litellm.proxy.mcp_server.mcp_server_manager", "_redact_mcp_resource_url", "redact_mcp_resource_url", ), ( - "litellm.proxy._experimental.mcp_server.oauth2_flow_backfill", + "litellm.proxy.mcp_server.oauth2_flow_backfill", "_decode_oauth_payload", "decode_oauth_payload", ), ( - "litellm.proxy._experimental.mcp_server.operations", + "litellm.proxy.mcp_server.operations", "_caller_authorization_fans_out", "caller_authorization_fans_out", ), ( - "litellm.proxy._experimental.mcp_server.operations", + "litellm.proxy.mcp_server.operations", "_client_forwarded_authorization_headers", "client_forwarded_authorization_headers", ), ( - "litellm.proxy._experimental.mcp_server.operations", + "litellm.proxy.mcp_server.operations", "_redact_mcp_resource_url", "redact_mcp_resource_url", ), ( - "litellm.proxy._experimental.mcp_server.operations", + "litellm.proxy.mcp_server.operations", "_request_auth_header", "request_auth_header", ), ( - "litellm.proxy._experimental.mcp_server.operations", + "litellm.proxy.mcp_server.operations", "_request_extra_headers", "request_extra_headers", ), ( - "litellm.proxy._experimental.mcp_server.operations", + "litellm.proxy.mcp_server.operations", "_request_resolved_auth_headers", "request_resolved_auth_headers", ), ( - "litellm.proxy._experimental.mcp_server.operations", + "litellm.proxy.mcp_server.operations", "_resolve_openapi_tool_auth", "resolve_openapi_tool_auth", ), ( - "litellm.proxy._experimental.mcp_server.operations", + "litellm.proxy.mcp_server.operations", "_should_strip_caller_authorization", "should_strip_caller_authorization", ), ( - "litellm.proxy._experimental.mcp_server.rest_endpoints", + "litellm.proxy.mcp_server.rest_endpoints", "_UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES", "UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES", ), ( - "litellm.proxy._experimental.mcp_server.rest_endpoints", + "litellm.proxy.mcp_server.rest_endpoints", "_apply_toolset_scope", "apply_toolset_scope", ), ( - "litellm.proxy._experimental.mcp_server.rest_endpoints", + "litellm.proxy.mcp_server.rest_endpoints", "_inherit_credentials_from_existing_server", "inherit_credentials_from_existing_server", ), ( - "litellm.proxy._experimental.mcp_server.rest_endpoints", + "litellm.proxy.mcp_server.rest_endpoints", "_redact_mcp_resource_url", "redact_mcp_resource_url", ), ( - "litellm.proxy._experimental.mcp_server.rest_endpoints", + "litellm.proxy.mcp_server.rest_endpoints", "_safe_get_request_headers", "safe_get_request_headers", ), ( - "litellm.proxy._experimental.mcp_server.server", + "litellm.proxy.mcp_server.server", "_is_mcp_admitted_user_subject", "is_mcp_admitted_user_subject", ), ( - "litellm.proxy._experimental.mcp_server.server", + "litellm.proxy.mcp_server.server", "_mcp_active_toolset_id", "mcp_active_toolset_id", ), ( - "litellm.proxy._experimental.mcp_server.server", + "litellm.proxy.mcp_server.server", "_mcp_gateway_initialize_instructions", "mcp_gateway_initialize_instructions", ), ( - "litellm.proxy._experimental.mcp_server.server", + "litellm.proxy.mcp_server.server", "_mcp_gateway_server_name", "mcp_gateway_server_name", ), @@ -2310,294 +2310,294 @@ LLMS_ALIAS_CASES: Final = ( ) PROXY_ALIAS_CASES: Final = ( ( - 'litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp', + 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', '', '_is_mcp_admitted_user_subject', 'is_mcp_admitted_user_subject', False, ), ( - 'litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp', + 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', 'MCPRequestHandler', '_get_mcp_auth_header_from_headers', 'get_mcp_auth_header_from_headers', False, ), ( - 'litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp', + 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', 'MCPRequestHandler', '_get_mcp_server_auth_headers_from_headers', 'get_mcp_server_auth_headers_from_headers', False, ), ( - 'litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp', + 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', 'MCPRequestHandler', '_get_mcp_servers_from_access_groups', 'get_mcp_servers_from_access_groups', False, ), ( - 'litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp', + 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', 'MCPRequestHandler', '_get_oauth2_headers_from_headers', 'get_oauth2_headers_from_headers', False, ), ( - 'litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp', + 'litellm.proxy.mcp_server.auth.user_api_key_auth_mcp', 'MCPRequestHandler', '_safe_get_headers_from_scope', 'safe_get_headers_from_scope', False, ), ( - 'litellm.proxy._experimental.mcp_server.bridge_token_flow', + 'litellm.proxy.mcp_server.bridge_token_flow', '', '_bridge_mint_error_response', 'bridge_mint_error_response', False, ), ( - 'litellm.proxy._experimental.mcp_server.bridge_token_flow', + 'litellm.proxy.mcp_server.bridge_token_flow', '', '_extract_user_id_from_request', 'extract_user_id_from_request', False, ), ( - 'litellm.proxy._experimental.mcp_server.bridge_token_flow', + 'litellm.proxy.mcp_server.bridge_token_flow', '', '_finish_bridge_mint', 'finish_bridge_mint', False, ), ( - 'litellm.proxy._experimental.mcp_server.bridge_token_flow', + 'litellm.proxy.mcp_server.bridge_token_flow', '', '_prepare_bridge_mint', 'prepare_bridge_mint', False, ), ( - 'litellm.proxy._experimental.mcp_server.bridge_token_flow', + 'litellm.proxy.mcp_server.bridge_token_flow', '', '_prepare_bridge_refresh', 'prepare_bridge_refresh', False, ), ( - 'litellm.proxy._experimental.mcp_server.bridge_token_flow', + 'litellm.proxy.mcp_server.bridge_token_flow', '', '_reload_active_user_by_id', 'reload_active_user_by_id', False, ), ( - 'litellm.proxy._experimental.mcp_server.byok_oauth_endpoints', + 'litellm.proxy.mcp_server.byok_oauth_endpoints', '', '_user_id_from_session_cookie', 'user_id_from_session_cookie', False, ), ( - 'litellm.proxy._experimental.mcp_server.db', + 'litellm.proxy.mcp_server.db', '', '_decode_oauth_payload', 'decode_oauth_payload', False, ), ( - 'litellm.proxy._experimental.mcp_server.discoverable_endpoints', + 'litellm.proxy.mcp_server.discoverable_endpoints', '', '_raise_if_not_oauth2', 'raise_if_not_oauth2', False, ), ( - 'litellm.proxy._experimental.mcp_server.mcp_context', + 'litellm.proxy.mcp_server.mcp_context', '', '_mcp_active_toolset_id', 'mcp_active_toolset_id', False, ), ( - 'litellm.proxy._experimental.mcp_server.mcp_context', + 'litellm.proxy.mcp_server.mcp_context', '', '_mcp_gateway_initialize_instructions', 'mcp_gateway_initialize_instructions', False, ), ( - 'litellm.proxy._experimental.mcp_server.mcp_context', + 'litellm.proxy.mcp_server.mcp_context', '', '_mcp_gateway_server_name', 'mcp_gateway_server_name', False, ), ( - 'litellm.proxy._experimental.mcp_server.mcp_server_manager', + 'litellm.proxy.mcp_server.mcp_server_manager', '', '_UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES', 'UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES', False, ), ( - 'litellm.proxy._experimental.mcp_server.mcp_server_manager', + 'litellm.proxy.mcp_server.mcp_server_manager', '', '_caller_authorization_fans_out', 'caller_authorization_fans_out', False, ), ( - 'litellm.proxy._experimental.mcp_server.mcp_server_manager', + 'litellm.proxy.mcp_server.mcp_server_manager', '', '_client_forwarded_authorization_headers', 'client_forwarded_authorization_headers', False, ), ( - 'litellm.proxy._experimental.mcp_server.mcp_server_manager', + 'litellm.proxy.mcp_server.mcp_server_manager', '', '_resolve_openapi_tool_auth', 'resolve_openapi_tool_auth', False, ), ( - 'litellm.proxy._experimental.mcp_server.mcp_server_manager', + 'litellm.proxy.mcp_server.mcp_server_manager', '', '_should_strip_caller_authorization', 'should_strip_caller_authorization', False, ), ( - 'litellm.proxy._experimental.mcp_server.mcp_server_manager', + 'litellm.proxy.mcp_server.mcp_server_manager', 'MCPServerManager', '_build_mcp_server_table', 'build_mcp_server_table', False, ), ( - 'litellm.proxy._experimental.mcp_server.mcp_server_manager', + 'litellm.proxy.mcp_server.mcp_server_manager', 'MCPServerManager', '_build_stdio_env', 'build_stdio_env', False, ), ( - 'litellm.proxy._experimental.mcp_server.mcp_server_manager', + 'litellm.proxy.mcp_server.mcp_server_manager', 'MCPServerManager', '_create_mcp_client', 'create_mcp_client', False, ), ( - 'litellm.proxy._experimental.mcp_server.mcp_server_manager', + 'litellm.proxy.mcp_server.mcp_server_manager', 'MCPServerManager', '_ensure_upstream_initialize_instructions_cached', 'ensure_upstream_initialize_instructions_cached', False, ), ( - 'litellm.proxy._experimental.mcp_server.mcp_server_manager', + 'litellm.proxy.mcp_server.mcp_server_manager', 'MCPServerManager', '_extract_subject_token', 'extract_subject_token', False, ), ( - 'litellm.proxy._experimental.mcp_server.mcp_server_manager', + 'litellm.proxy.mcp_server.mcp_server_manager', 'MCPServerManager', '_get_mcp_server_from_tool_name', 'get_mcp_server_from_tool_name', False, ), ( - 'litellm.proxy._experimental.mcp_server.mcp_server_manager', + 'litellm.proxy.mcp_server.mcp_server_manager', 'MCPServerManager', '_get_tools_from_server', 'get_tools_from_server', False, ), ( - 'litellm.proxy._experimental.mcp_server.mcp_server_manager', + 'litellm.proxy.mcp_server.mcp_server_manager', 'MCPServerManager', '_is_server_accessible_from_ip', 'is_server_accessible_from_ip', False, ), ( - 'litellm.proxy._experimental.mcp_server.oauth2_token_cache', + 'litellm.proxy.mcp_server.oauth2_token_cache', '', '_compute_per_user_token_ttl', 'compute_per_user_token_ttl', False, ), ( - 'litellm.proxy._experimental.mcp_server.oauth_utils', + 'litellm.proxy.mcp_server.oauth_utils', '', '_redact_mcp_resource_url', 'redact_mcp_resource_url', False, ), ( - 'litellm.proxy._experimental.mcp_server.server', + 'litellm.proxy.mcp_server.server', '', '_redact_mcp_resource_url', 'redact_mcp_resource_url', False, ), ( - 'litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator', + 'litellm.proxy.mcp_server.openapi_to_mcp_generator', '', '_OPENAPI_TOOL_NAME_MAX_LEN', 'OPENAPI_TOOL_NAME_MAX_LEN', False, ), ( - 'litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator', + 'litellm.proxy.mcp_server.openapi_to_mcp_generator', '', '_request_auth_header', 'request_auth_header', False, ), ( - 'litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator', + 'litellm.proxy.mcp_server.openapi_to_mcp_generator', '', '_request_extra_headers', 'request_extra_headers', False, ), ( - 'litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator', + 'litellm.proxy.mcp_server.openapi_to_mcp_generator', '', '_request_resolved_auth_headers', 'request_resolved_auth_headers', False, ), ( - 'litellm.proxy._experimental.mcp_server.semantic_tool_filter', + 'litellm.proxy.mcp_server.semantic_tool_filter', 'SemanticMCPToolFilter', '_extract_tool_info', 'extract_tool_info', False, ), ( - 'litellm.proxy._experimental.mcp_server.server', + 'litellm.proxy.mcp_server.server', '', '_apply_toolset_scope', 'apply_toolset_scope', False, ), ( - 'litellm.proxy._experimental.mcp_server.server_resolution', + 'litellm.proxy.mcp_server.server_resolution', 'MCPServerRegistry', '_build_mcp_server_table', 'build_mcp_server_table', False, ), ( - 'litellm.proxy._experimental.mcp_server.server_resolution', + 'litellm.proxy.mcp_server.server_resolution', 'MCPServerRegistry', '_is_server_accessible_from_ip', 'is_server_accessible_from_ip', diff --git a/ui/litellm-dashboard/src/utils/mcpHeaderUtils.ts b/ui/litellm-dashboard/src/utils/mcpHeaderUtils.ts index ebe7116c18f..f1123dd70f7 100644 --- a/ui/litellm-dashboard/src/utils/mcpHeaderUtils.ts +++ b/ui/litellm-dashboard/src/utils/mcpHeaderUtils.ts @@ -2,7 +2,7 @@ * Sanitize an MCP server alias for use in HTTP header names (x-mcp-{alias}-...). * RFC 7230 tchar allows token chars; aliases with spaces or hyphens break parsing * because the backend splits on the first dash after the x-mcp- prefix. - * Keep in sync with litellm.proxy._experimental.mcp_server.utils.sanitize_mcp_alias_for_header. + * Keep in sync with litellm.proxy.mcp_server.utils.sanitize_mcp_alias_for_header. */ export function sanitizeMcpAliasForHeader(alias: string): string { return alias From 71a95262549661ae2e8534a2976f2870abbf50e9 Mon Sep 17 00:00:00 2001 From: NingLin Liu Date: Wed, 28 Oct 2026 02:55:16 +0800 Subject: [PATCH 004/102] fix: correct Palm 'decommissioned' typo in error message (#46171) * fix: correct Palm decommissioned typo in error message Fixes #46170 * ci: retrigger checks after unrelated mcp-server flake --- litellm/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index 23eac8ce0aa..7775922ea44 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6326,7 +6326,7 @@ def completion( response = _complete_vercel_ai_gateway(_dispatch_ctx) elif custom_llm_provider == "palm": raise ValueError( - "Palm was decommisioned on October 2024. Please use the `gemini/` route for Gemini Google AI Studio Models. Announcement: https://ai.google.dev/palm_docs/palm?hl=en" + "Palm was decommissioned on October 2024. Please use the `gemini/` route for Gemini Google AI Studio Models. Announcement: https://ai.google.dev/palm_docs/palm?hl=en" ) elif custom_llm_provider == "vertex_ai_beta" or custom_llm_provider == "gemini": response = _complete_vertex_ai_beta(_dispatch_ctx) From a2c23d95ae4561b071546bc5de872f07fcf1f514 Mon Sep 17 00:00:00 2001 From: prassu017 <72042793+prassu017@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:58:01 -0400 Subject: [PATCH 005/102] fix(responses): replay web_search_call input items that have no action (#48417) * fix(responses): replay web_search_call input items that have no action * Inline web_search_call replay fallback * fix(responses): validate replay fields when action is missing --------- Co-authored-by: moe-berri --- .../transformation.py | 14 +++++- .../test_litellm_completion_responses.py | 46 ++++++++++++++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index a9c7818b577..3850654daf9 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -2062,7 +2062,19 @@ class LiteLLMCompletionResponsesConfig: - ItemReference """ if input_item.get("type") == "web_search_call": - search: Final = ResponseFunctionWebSearch.model_validate(input_item) + try: + search: Final = ResponseFunctionWebSearch.model_validate(input_item) + except ValidationError as exc: + errors: Final = exc.errors(include_url=False, include_input=False) + if len(errors) != 1 or errors[0]["type"] != "missing" or errors[0]["loc"] != ("action",): + raise + replay: Final = ResponseFunctionWebSearch.model_construct(None, **input_item) + return [ + GenericChatCompletionMessage( + role="assistant", + content="Hosted web search: " + replay.model_dump_json(exclude_none=True), + ) + ] return [ GenericChatCompletionMessage( role="assistant", diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 023d0ea26fb..63c98b571cb 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -14,7 +14,7 @@ from openai.types.responses.response_function_web_search import ( ActionSearchSource, ResponseFunctionWebSearch, ) -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError import litellm from litellm.constants import CALLER_SESSION_ID_METADATA_KEY @@ -5062,6 +5062,50 @@ class TestHostedWebSearchReplay: assert replayed == search assert input_item == original + def test_replays_search_without_action(self) -> None: + input_item: Final = {"type": "web_search_call", "id": "ws_no_action", "status": "completed"} + + messages: Final = ( + LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=input_item + ) + ) + + assert len(messages) == 1 + assert messages[0]["role"] == "assistant" + content: Final = messages[0].get("content") + assert isinstance(content, str) + assert json.loads(content[content.index("{") :]) == input_item + + @pytest.mark.parametrize( + "fields", + ( + {"status": "completed"}, + {"id": "ws_invalid"}, + {"id": 42, "status": "completed"}, + {"id": "ws_invalid", "status": "invalid"}, + {"id": "ws_invalid", "status": "completed", "action": None}, + {"id": "ws_invalid", "status": "completed", "action": 42}, + {"id": "ws_invalid", "status": "completed", "action": {"type": "invalid"}}, + {"id": "ws_invalid", "status": "completed", "action": {"type": "find_in_page"}}, + ), + ids=( + "missing-id", + "missing-status", + "invalid-id", + "invalid-status", + "null-action", + "invalid-action-type", + "unknown-action", + "incomplete-action", + ), + ) + def test_rejects_search_with_other_invalid_fields(self, fields: Mapping[str, object]) -> None: + with pytest.raises(ValidationError): + LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=[{"type": "web_search_call", **fields}], responses_api_request={} + ) + @pytest.mark.parametrize("order", ((0, 1, 2, 3), (1, 0, 3, 2), (1, 3, 0, 2))) @pytest.mark.parametrize("modify_params", (False, True)) @pytest.mark.parametrize("structured_content", (False, True)) From c39b5a3efdcb794aab1f8103faccdda207013204 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 27 Oct 2026 12:06:55 -0700 Subject: [PATCH 006/102] feat: generate provider_endpoints_support.json from the cost map (#48619) * feat: generate provider endpoint support flags * test: remove derived endpoint flag assertions * test: run provider endpoint generator in CI --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-linting.yml | 6 + .../provider_endpoints_support_backup.json | 4191 +++++++++++++---- provider_endpoints_support.json | 3897 +++++++++++---- .../generate_provider_endpoints_support.py | 248 + ...est_generate_provider_endpoints_support.py | 179 + .../chat/test_edenai_chat_transformation.py | 11 - .../llms/openai_like/test_cortecs_provider.py | 20 - .../openai_like/test_libertai_provider.py | 7 - .../llms/openai_like/test_prism_provider.py | 19 - .../llms/openai_like/test_reka_provider.py | 22 - 10 files changed, 6705 insertions(+), 1895 deletions(-) create mode 100644 scripts/generate_provider_endpoints_support.py create mode 100644 tests/code_coverage_tests/test_generate_provider_endpoints_support.py diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 3b9a2614725..b9f8d7e7dfb 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -524,6 +524,12 @@ jobs: - name: check_provider_folders_documented run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py + - name: check_provider_endpoints_support_generated + run: uv run --no-sync python scripts/generate_provider_endpoints_support.py --check + + - name: test_generate_provider_endpoints_support + run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_generate_provider_endpoints_support.py + - name: check_prisma_binary_cache run: uv run --no-sync python ./tests/code_coverage_tests/check_prisma_binary_cache.py diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 49b667af819..9ba6ed342ea 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -6,6 +6,7 @@ "url": "Link to provider documentation", "endpoints": { "chat_completions": "Supports /chat/completions endpoint", + "systemone": "Supports native System One typed decisions", "messages": "Supports /messages endpoint (Anthropic format)", "responses": "Supports /responses endpoint (OpenAI/Anthropic unified)", "embeddings": "Supports /embeddings endpoint", @@ -19,9 +20,9 @@ "search": "Supports /search endpoint", "skills": "Supports /skills endpoint", "interactions": "Supports /interactions endpoint (Google AI Interactions API)", - "a2a": "Supports /a2a/{agent}/message/send endpoint (A2A Protocol)", + "a2a_(Agent Gateway)": "Supports /a2a/{agent}/message/send endpoint (A2A Protocol)", "container": "Supports OpenAI's /containers endpoint", - "container_files": "Supports OpenAI's /containers/{id}/files endpoint", + "container_file": "Supports OpenAI's /containers/{id}/files endpoint", "compact": "Supports /responses/compact endpoint", "files": "Supports /files endpoint for file operations", "image_edits": "Supports /images/edits endpoint for image editing", @@ -37,33 +38,51 @@ "url": "https://docs.litellm.ai/docs/providers/refuseless", "endpoints": { "chat_completions": true, - "messages": false, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "a2a": { "display_name": "A2A (Agent-to-Agent) (`a2a`)", "url": "https://docs.litellm.ai/docs/providers/a2a", "endpoints": { - "chat_completions": true, - "messages": false, + "chat_completions": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "abliteration": { @@ -71,34 +90,51 @@ "url": "https://docs.litellm.ai/docs/providers/abliteration", "endpoints": { "chat_completions": true, - "messages": false, - "responses": false, + "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "aiml": { "display_name": "AI/ML API (`aiml`)", "url": "https://docs.litellm.ai/docs/providers/aiml", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, "image_generations": true, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "ai21": { @@ -106,35 +142,51 @@ "url": "https://docs.litellm.ai/docs/providers/ai21", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "ai21_chat": { "display_name": "AI21 Chat (`ai21_chat`)", "url": "https://docs.litellm.ai/docs/providers/ai21", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "amazon_nova": { @@ -142,17 +194,25 @@ "url": "https://docs.litellm.ai/docs/providers/amazon_nova", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "anthropic": { @@ -160,18 +220,26 @@ "url": "https://docs.litellm.ai/docs/providers/anthropic", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": true, "rerank": false, - "skills": true, - "a2a": true, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true, + "skills": true, "count_tokens": true } }, @@ -179,71 +247,130 @@ "display_name": "Anthropic Text (`anthropic_text`)", "url": "https://docs.litellm.ai/docs/providers/anthropic", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": true, "rerank": false, - "skills": true, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "skills": true } }, "apertis": { "display_name": "Apertis (`apertis`)", "endpoints": { - "chat_completions": true, - "messages": false, + "chat_completions": false, "responses": false, - "embeddings": true, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "aihubmix": { + "display_name": "AIHubMix (`aihubmix`)", + "url": "https://docs.litellm.ai/docs/providers/aihubmix", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "assemblyai": { "display_name": "AssemblyAI (`assemblyai`)", "url": "https://docs.litellm.ai/docs/pass_through/assembly_ai", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": true, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "auto_router": { "display_name": "Auto Router (`auto_router`)", "url": "https://docs.litellm.ai/docs/proxy/auto_routing", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "bedrock": { @@ -251,17 +378,25 @@ "url": "https://docs.litellm.ai/docs/providers/bedrock", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": true, - "image_generations": false, + "image_generations": true, + "image_edits": true, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": true, - "a2a": true, + "ocr": false, + "search": false, + "moderations": false, + "realtime": true, + "video_generations": false, + "batches": false, + "messages": true, "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true, "bedrock_invoke": true, "bedrock_converse": true, "vector_stores_search": true, @@ -275,17 +410,25 @@ "url": "https://docs.litellm.ai/docs/providers/s3_vectors", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, "vector_stores_create": true, "vector_stores_search": true } @@ -295,17 +438,25 @@ "url": "https://docs.litellm.ai/docs/providers/aws_sagemaker", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": true, + "text_completion": true, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "aws_polly": { @@ -313,15 +464,25 @@ "url": "https://docs.litellm.ai/docs/providers/aws_polly", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": true, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "azure": { @@ -329,21 +490,28 @@ "url": "https://docs.litellm.ai/docs/providers/azure", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": true, "image_generations": true, + "image_edits": true, "audio_transcriptions": true, "audio_speech": true, - "moderations": true, - "batches": true, "rerank": false, - "a2a": true, + "ocr": false, + "search": false, + "moderations": false, + "realtime": true, + "video_generations": true, + "batches": true, + "messages": true, "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true, "vector_stores_search": true, "assistants": true, - "fine_tuning": true, - "text_completion": true + "fine_tuning": true } }, "azure_ai": { @@ -351,19 +519,25 @@ "url": "https://docs.litellm.ai/docs/providers/azure_ai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": true, "image_generations": true, "image_edits": true, "audio_transcriptions": true, - "audio_speech": true, - "moderations": true, - "batches": true, - "rerank": false, + "audio_speech": false, + "rerank": true, "ocr": true, - "a2a": true, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": true, + "messages": true, "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true, "vector_stores_create": true, "vector_stores_search": true } @@ -373,52 +547,77 @@ "url": "https://docs.litellm.ai/docs/providers/azure_document_intelligence", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "ocr": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "azure_ai/agents": { "display_name": "Azure AI Foundry Agents (`azure_ai/agents`)", "url": "https://docs.litellm.ai/docs/providers/azure_ai_agents", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "azure_text": { "display_name": "Azure Text (`azure_text`)", "url": "https://docs.litellm.ai/docs/providers/azure", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": true, "embeddings": false, "image_generations": false, - "audio_transcriptions": true, - "audio_speech": true, - "moderations": true, - "batches": true, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "baseten": { @@ -426,35 +625,77 @@ "url": "https://docs.litellm.ai/docs/providers/baseten", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "bytez": { "display_name": "Bytez (`bytez`)", "url": "https://docs.litellm.ai/docs/providers/bytez", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "nadir": { + "display_name": "Nadir (`nadir`)", + "url": "https://docs.litellm.ai/docs/providers/nadir", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "cerebras": { @@ -462,51 +703,102 @@ "url": "https://docs.litellm.ai/docs/providers/cerebras", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true + } + }, + "charity_engine": { + "display_name": "Charity Engine (`charity_engine`)", + "url": "https://docs.litellm.ai/docs/providers/charity_engine", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "chutes": { "display_name": "Chutes (`chutes`)", "endpoints": { - "chat_completions": true, - "messages": false, + "chat_completions": false, "responses": false, - "embeddings": true, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "clarifai": { "display_name": "Clarifai (`clarifai`)", "url": "https://docs.litellm.ai/docs/providers/clarifai", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "cloudflare": { @@ -514,17 +806,25 @@ "url": "https://docs.litellm.ai/docs/providers/cloudflare_workers", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, - "audio_transcriptions": false, + "image_edits": false, + "audio_transcriptions": true, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "codestral": { @@ -532,17 +832,25 @@ "url": "https://docs.litellm.ai/docs/providers/codestral", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "cognition": { @@ -550,35 +858,51 @@ "url": "https://docs.litellm.ai/docs/providers/cognition", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "cohere": { "display_name": "Cohere (`cohere`)", "url": "https://docs.litellm.ai/docs/providers/cohere", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": true, "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": true, "ocr": true, - "a2a": true, - "interactions": true + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "cohere_chat": { @@ -586,53 +910,77 @@ "url": "https://docs.litellm.ai/docs/providers/cohere", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "cometapi": { "display_name": "CometAPI (`cometapi`)", "url": "https://docs.litellm.ai/docs/providers/cometapi", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "compactifai": { "display_name": "CompactifAI (`compactifai`)", "url": "https://docs.litellm.ai/docs/providers/compactifai", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "coralbricks": { @@ -640,71 +988,129 @@ "url": "https://docs.litellm.ai/docs/providers/coralbricks", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false, - "interactions": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "cortecs": { "display_name": "Cortecs (`cortecs`)", "url": "https://docs.litellm.ai/docs/providers/cortecs", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, "a2a": false, - "interactions": false + "generateContent": false + } + }, + "crusoe": { + "display_name": "Crusoe (`crusoe`)", + "url": "https://docs.litellm.ai/docs/providers/crusoe", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "custom": { "display_name": "Custom (`custom`)", "url": "https://docs.litellm.ai/docs/providers/custom_llm_server", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "custom_openai": { "display_name": "Custom OpenAI (`custom_openai`)", "url": "https://docs.litellm.ai/docs/providers/openai_compatible", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, - "audio_transcriptions": true, - "audio_speech": true, - "moderations": true, - "batches": true, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "dashscope": { @@ -712,17 +1118,25 @@ "url": "https://docs.litellm.ai/docs/providers/dashscope", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, - "image_generations": false, + "image_generations": true, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": true, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "qwencloud": { @@ -730,17 +1144,25 @@ "url": "https://docs.litellm.ai/docs/providers/qwencloud", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": true, + "text_completion": true, + "embeddings": false, "image_generations": true, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": true, "batches": false, - "rerank": true, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "qwen_ai_platform": { @@ -748,17 +1170,25 @@ "url": "https://docs.litellm.ai/docs/providers/qwencloud", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": true, + "text_completion": true, + "embeddings": false, "image_generations": true, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": true, "batches": false, - "rerank": true, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "databricks": { @@ -766,17 +1196,25 @@ "url": "https://docs.litellm.ai/docs/providers/databricks", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "dataforseo": { @@ -784,52 +1222,77 @@ "url": "https://docs.litellm.ai/docs/search/dataforseo", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "datarobot": { "display_name": "DataRobot (`datarobot`)", "url": "https://docs.litellm.ai/docs/providers/datarobot", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "deepgram": { "display_name": "Deepgram (`deepgram`)", "url": "https://docs.litellm.ai/docs/providers/deepgram", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": true, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "deepinfra": { @@ -837,17 +1300,25 @@ "url": "https://docs.litellm.ai/docs/providers/deepinfra", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "deepseek": { @@ -855,35 +1326,51 @@ "url": "https://docs.litellm.ai/docs/providers/deepseek", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "edenai": { "display_name": "Eden AI (`edenai`)", "url": "https://docs.litellm.ai/docs/providers/edenai", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, - "image_generations": true, - "audio_transcriptions": true, - "audio_speech": true, - "moderations": false, - "batches": false, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, "interactions": false, - "video_generations": true + "systemone": false, + "a2a": false, + "generateContent": false } }, "duckduckgo": { @@ -891,34 +1378,78 @@ "url": "https://docs.litellm.ai/docs/search/duckduckgo", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "e2b": { + "display_name": "E2B (`e2b`)", + "url": "https://docs.litellm.ai/docs/providers/e2b", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "sandbox": true } }, "elevenlabs": { "display_name": "ElevenLabs (`elevenlabs`)", "url": "https://docs.litellm.ai/docs/providers/elevenlabs", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, - "audio_transcriptions": false, + "image_edits": false, + "audio_transcriptions": true, "audio_speech": true, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "exa_ai": { @@ -926,16 +1457,25 @@ "url": "https://docs.litellm.ai/docs/search/exa_ai", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "brave": { @@ -943,34 +1483,51 @@ "url": "https://docs.litellm.ai/docs/search/brave", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "empower": { "display_name": "Empower (`empower`)", "url": "https://docs.litellm.ai/docs/providers/empower", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "fal_ai": { @@ -978,17 +1535,25 @@ "url": "https://docs.litellm.ai/docs/providers/fal_ai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": true, + "image_edits": true, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": true, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "featherless_ai": { @@ -996,17 +1561,25 @@ "url": "https://docs.litellm.ai/docs/providers/featherless_ai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "fireworks_ai": { @@ -1014,17 +1587,25 @@ "url": "https://docs.litellm.ai/docs/providers/fireworks_ai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, + "text_completion": true, + "embeddings": true, + "image_generations": true, + "image_edits": false, + "audio_transcriptions": true, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": true, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "firecrawl": { @@ -1032,16 +1613,51 @@ "url": "https://docs.litellm.ai/docs/search/firecrawl", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "fastcrw": { + "display_name": "fastCRW (`fastcrw`)", + "url": "https://docs.litellm.ai/docs/search/fastcrw", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "linkup": { @@ -1049,16 +1665,25 @@ "url": "https://docs.litellm.ai/docs/search/linkup", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "friendliai": { @@ -1066,35 +1691,77 @@ "url": "https://docs.litellm.ai/docs/providers/friendliai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "galadriel": { "display_name": "Galadriel (`galadriel`)", "url": "https://docs.litellm.ai/docs/providers/galadriel", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "gdc": { + "display_name": "Google Distributed Cloud (GDC)", + "url": "https://docs.litellm.ai/docs/providers/gdc", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "github_copilot": { @@ -1102,17 +1769,25 @@ "url": "https://docs.litellm.ai/docs/providers/github_copilot", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "chatgpt": { @@ -1120,35 +1795,51 @@ "url": "https://docs.litellm.ai/docs/providers/chatgpt", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false, - "interactions": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "github": { "display_name": "GitHub Models (`github`)", "url": "https://docs.litellm.ai/docs/providers/github", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "gmi": { @@ -1156,17 +1847,25 @@ "url": "https://docs.litellm.ai/docs/providers/gmi_cloud", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "vertex_ai": { @@ -1174,25 +1873,30 @@ "url": "https://docs.litellm.ai/docs/providers/vertex", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": true, "image_generations": true, - "audio_transcriptions": false, + "image_edits": true, + "audio_transcriptions": true, "audio_speech": true, - "moderations": false, - "batches": false, - "rerank": false, + "rerank": true, "ocr": true, - "a2a": true, + "search": false, + "moderations": false, + "realtime": true, + "video_generations": true, + "batches": true, + "messages": true, "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true, "vector_stores_search": true, "count_tokens": true, "fine_tuning": true, "rag_ingest": true, - "rag_query": true, - "generateContent": true, - "realtime": true + "rag_query": true } }, "gemini": { @@ -1200,22 +1904,28 @@ "url": "https://docs.litellm.ai/docs/providers/gemini", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, - "moderations": false, - "batches": false, + "text_completion": true, + "embeddings": true, + "image_generations": true, + "image_edits": true, + "audio_transcriptions": true, + "audio_speech": true, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": true, + "video_generations": true, + "batches": true, + "messages": true, "interactions": true, + "systemone": false, "a2a": true, + "generateContent": true, "vector_stores_search": true, "count_tokens": true, - "rag_ingest": true, - "realtime": true, - "generateContent": true + "rag_ingest": true } }, "gradient_ai": { @@ -1223,17 +1933,25 @@ "url": "https://docs.litellm.ai/docs/providers/gradient_ai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "groq": { @@ -1241,17 +1959,25 @@ "url": "https://docs.litellm.ai/docs/providers/groq", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, - "moderations": false, - "batches": false, + "image_edits": false, + "audio_transcriptions": true, + "audio_speech": true, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "heroku": { @@ -1259,55 +1985,78 @@ "url": "https://docs.litellm.ai/docs/providers/heroku", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "hosted_vllm": { "display_name": "Hosted VLLM (`hosted_vllm`)", "url": "https://docs.litellm.ai/docs/providers/vllm", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, - "batches": true, - "files": true, - "rerank": true, - "a2a": true, - "interactions": true, - "video_generations": true + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "files": true } }, "huggingface": { "display_name": "Huggingface (`huggingface`)", "url": "https://docs.litellm.ai/docs/providers/huggingface", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": true, - "a2a": true, - "interactions": true + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "hyperbolic": { @@ -1315,17 +2064,25 @@ "url": "https://docs.litellm.ai/docs/providers/hyperbolic", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "watsonx": { @@ -1333,17 +2090,51 @@ "url": "https://docs.litellm.ai/docs/providers/watsonx", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": true, + "text_completion": true, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": true, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true + } + }, + "inception": { + "display_name": "Inception (`inception`)", + "url": "https://docs.litellm.ai/docs/providers/inception", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "infinity": { @@ -1351,15 +2142,25 @@ "url": "https://docs.litellm.ai/docs/providers/infinity", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, - "embeddings": true, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "jina_ai": { @@ -1367,15 +2168,77 @@ "url": "https://docs.litellm.ai/docs/providers/jina_ai", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, - "embeddings": true, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": true, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "bespoke": { + "display_name": "Bespoke Nimble (`bespoke`)", + "url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": true, + "a2a": false, + "generateContent": false + } + }, + "laya": { + "display_name": "Laya (`laya`)", + "url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": true, + "a2a": false, + "generateContent": false } }, "lambda_ai": { @@ -1383,17 +2246,25 @@ "url": "https://docs.litellm.ai/docs/providers/lambda_ai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "lemonade": { @@ -1401,17 +2272,25 @@ "url": "https://docs.litellm.ai/docs/providers/lemonade", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "libertai": { @@ -1419,88 +2298,129 @@ "url": "https://docs.litellm.ai/docs/providers/libertai", "endpoints": { "chat_completions": true, - "messages": true, - "responses": false, - "embeddings": false, + "responses": true, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "litellm_proxy": { "display_name": "LiteLLM Proxy (`litellm_proxy`)", "url": "https://docs.litellm.ai/docs/providers/litellm_proxy", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, - "image_generations": true, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "llamafile": { "display_name": "Llamafile (`llamafile`)", "url": "https://docs.litellm.ai/docs/providers/llamafile", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "lm_studio": { "display_name": "LM Studio (`lm_studio`)", "url": "https://docs.litellm.ai/docs/providers/lm_studio", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "maritalk": { "display_name": "Maritalk (`maritalk`)", "url": "https://docs.litellm.ai/docs/providers/maritalk", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "meta_llama": { @@ -1508,17 +2428,25 @@ "url": "https://docs.litellm.ai/docs/providers/meta_llama", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "mistral": { @@ -1526,18 +2454,51 @@ "url": "https://docs.litellm.ai/docs/providers/mistral", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": true, "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, - "moderations": false, - "batches": true, + "image_edits": false, + "audio_transcriptions": true, + "audio_speech": true, "rerank": false, "ocr": true, + "search": false, + "moderations": true, + "realtime": false, + "video_generations": false, + "batches": true, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true + } + }, + "modelscope": { + "display_name": "ModelScope (`modelscope`)", + "url": "https://docs.litellm.ai/docs/providers/modelscope", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "moonshot": { @@ -1545,35 +2506,51 @@ "url": "https://docs.litellm.ai/docs/providers/moonshot", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "docker_model_runner": { "display_name": "Docker Model Runner (`docker_model_runner`)", "url": "https://docs.litellm.ai/docs/providers/docker_model_runner", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "morph": { @@ -1581,33 +2558,50 @@ "url": "https://docs.litellm.ai/docs/providers/morph", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "nanogpt": { "display_name": "NanoGPT (`nanogpt`)", "endpoints": { - "chat_completions": true, - "messages": false, + "chat_completions": false, "responses": false, - "embeddings": true, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "nebius": { @@ -1615,17 +2609,51 @@ "url": "https://docs.litellm.ai/docs/providers/nebius", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true + } + }, + "neosantara": { + "display_name": "Neosantara (`neosantara`)", + "url": "https://docs.litellm.ai/docs/providers/neosantara", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "nlp_cloud": { @@ -1633,17 +2661,25 @@ "url": "https://docs.litellm.ai/docs/providers/nlp_cloud", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "novita": { @@ -1651,17 +2687,25 @@ "url": "https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": true, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "nscale": { @@ -1669,52 +2713,77 @@ "url": "https://docs.litellm.ai/docs/providers/nscale", "endpoints": { "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": true, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, "messages": true, - "responses": true, - "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, - "moderations": false, - "batches": false, - "rerank": false, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true - } - }, - "neosantara": { - "display_name": "Neosantara (`neosantara`)", - "url": "https://docs.litellm.ai/docs/providers/neosantara", - "endpoints": { - "chat_completions": true, - "messages": false, - "responses": true, - "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, - "moderations": false, - "batches": false, - "rerank": false, - "a2a": false + "generateContent": true } }, "nvidia_nim": { "display_name": "Nvidia NIM (`nvidia_nim`)", "url": "https://docs.litellm.ai/docs/providers/nvidia_nim", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": true, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "nvidia_riva": { + "display_name": "Nvidia Riva (`nvidia_riva`)", + "url": "https://docs.litellm.ai/docs/providers/nvidia_riva", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "oci": { @@ -1722,17 +2791,25 @@ "url": "https://docs.litellm.ai/docs/providers/oci", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "ollama": { @@ -1740,53 +2817,77 @@ "url": "https://docs.litellm.ai/docs/providers/ollama", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": true, + "text_completion": true, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "ollama_chat": { "display_name": "Ollama Chat (`ollama_chat`)", "url": "https://docs.litellm.ai/docs/providers/ollama", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "oobabooga": { "display_name": "Oobabooga (`oobabooga`)", "url": "https://docs.litellm.ai/docs/providers/openai_compatible", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, - "audio_transcriptions": true, - "audio_speech": true, - "moderations": true, - "batches": true, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "openai": { @@ -1794,19 +2895,27 @@ "url": "https://docs.litellm.ai/docs/providers/openai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": true, "image_generations": true, + "image_edits": true, "audio_transcriptions": true, "audio_speech": true, - "moderations": true, - "batches": true, "rerank": false, + "ocr": false, + "search": false, + "moderations": true, + "realtime": true, + "video_generations": true, + "batches": true, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true, "container": true, "compact": true, - "a2a": true, - "interactions": true, "vector_store_files": true, "vector_stores_create": true, "vector_stores_search": true, @@ -1815,9 +2924,34 @@ "fine_tuning": true, "image_variations": true, "rag_ingest": true, - "rag_query": true, - "realtime": true, - "text_completion": true + "rag_query": true + } + }, + "opensandbox": { + "display_name": "OpenSandbox (`opensandbox`)", + "url": "https://open-sandbox.ai/api/", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "sandbox": true } }, "openai_like": { @@ -1825,15 +2959,25 @@ "url": "https://docs.litellm.ai/docs/providers/openai_compatible", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, - "embeddings": true, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, "assistants": true } }, @@ -1841,18 +2985,26 @@ "display_name": "OpenRouter (`openrouter`)", "url": "https://docs.litellm.ai/docs/providers/openrouter", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "ovhcloud": { @@ -1860,17 +3012,25 @@ "url": "https://docs.litellm.ai/docs/providers/ovhcloud", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, - "audio_transcriptions": true, + "image_edits": false, + "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "parallel_ai": { @@ -1878,16 +3038,51 @@ "url": "https://docs.litellm.ai/docs/search/parallel_ai", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "parasail": { + "display_name": "Parasail (`parasail`)", + "url": "https://docs.litellm.ai/docs/providers/parasail", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "perplexity": { @@ -1895,52 +3090,154 @@ "url": "https://docs.litellm.ai/docs/providers/perplexity", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "petals": { "display_name": "Petals (`petals`)", "url": "https://docs.litellm.ai/docs/providers/petals", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "sail": { + "display_name": "Sail (`sail`)", + "url": "https://docs.litellm.ai/docs/providers/sail", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true + } + }, + "meta": { + "display_name": "Meta Model API (`meta`)", + "url": "https://docs.litellm.ai/docs/providers/meta", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": true, + "image_edits": true, + "audio_transcriptions": true, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": true, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, + "pinstripes": { + "display_name": "Pinstripes (`pinstripes`)", + "url": "https://docs.litellm.ai/docs/providers/pinstripes", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "poe": { "display_name": "Poe (`poe`)", "endpoints": { - "chat_completions": true, - "messages": false, + "chat_completions": false, "responses": false, - "embeddings": true, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "publicai": { @@ -1948,17 +3245,25 @@ "url": "https://docs.litellm.ai/docs/providers/publicai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "darkbloom": { @@ -1966,34 +3271,51 @@ "url": "https://docs.litellm.ai/docs/providers/darkbloom", "endpoints": { "chat_completions": true, - "messages": false, - "responses": false, + "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "predibase": { "display_name": "Predibase (`predibase`)", "url": "https://docs.litellm.ai/docs/providers/predibase", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "prism": { @@ -2001,16 +3323,25 @@ "url": "https://docs.litellm.ai/docs/providers/prism", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "recraft": { @@ -2018,15 +3349,51 @@ "url": "https://docs.litellm.ai/docs/providers/recraft", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": true, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "reducto": { + "display_name": "Reducto (`reducto`)", + "url": "https://docs.litellm.ai/docs/providers/reducto", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": true, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "replicate": { @@ -2034,17 +3401,25 @@ "url": "https://docs.litellm.ai/docs/providers/replicate", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "runwayml": { @@ -2052,34 +3427,51 @@ "url": "https://docs.litellm.ai/docs/providers/runwayml/videos", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": true, + "image_edits": false, "audio_transcriptions": false, "audio_speech": true, - "moderations": false, - "batches": false, "rerank": false, - "video_generations": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": true, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "sagemaker_chat": { "display_name": "Sagemaker Chat (`sagemaker_chat`)", "url": "https://docs.litellm.ai/docs/providers/aws_sagemaker", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "searxng": { @@ -2087,16 +3479,25 @@ "url": "https://docs.litellm.ai/docs/search/searxng", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "scaledown": { @@ -2104,17 +3505,25 @@ "url": "https://docs.litellm.ai/docs/providers/scaledown", "endpoints": { "chat_completions": true, - "messages": false, - "responses": false, + "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false, - "interactions": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "sambanova": { @@ -2122,53 +3531,51 @@ "url": "https://docs.litellm.ai/docs/providers/sambanova", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "sap": { "display_name": "SAP Generative AI Hub (`sap`)", "url": "https://docs.litellm.ai/docs/providers/sap", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true - } - }, - "reka": { - "display_name": "Reka (`reka`)", - "url": "https://docs.litellm.ai/docs/providers/reka", - "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false, + "messages": false, + "interactions": false, + "systemone": false, "a2a": false, - "interactions": false + "generateContent": false } }, "scaleway": { @@ -2176,17 +3583,25 @@ "url": "https://docs.litellm.ai/docs/providers/scaleway", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": true, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "scx-ai": { @@ -2194,16 +3609,25 @@ "url": "https://docs.litellm.ai/docs/providers/scx_ai", "endpoints": { "chat_completions": true, - "messages": false, - "responses": false, + "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "inferenceapis": { @@ -2211,17 +3635,25 @@ "url": "https://docs.litellm.ai/docs/providers/inferenceapis", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false, - "interactions": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "snowflake": { @@ -2229,69 +3661,180 @@ "url": "https://docs.litellm.ai/docs/providers/snowflake", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true + } + }, + "soniox": { + "display_name": "Soniox (`soniox`)", + "url": "https://docs.litellm.ai/docs/providers/soniox", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": true, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "synthetic": { "display_name": "Synthetic (`synthetic`)", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "tensormesh": { + "display_name": "Tensormesh (`tensormesh`)", + "url": "https://docs.litellm.ai/docs/providers/tensormesh", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, + "tencent": { + "display_name": "Tencent TokenHub (`tencent`)", + "url": "https://docs.litellm.ai/docs/providers/tencent", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "text-completion-codestral": { "display_name": "Text Completion Codestral (`text-completion-codestral`)", "url": "https://docs.litellm.ai/docs/providers/codestral", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "text-completion-openai": { "display_name": "Text Completion OpenAI (`text-completion-openai`)", "url": "https://docs.litellm.ai/docs/providers/text_completion_openai", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": true, "embeddings": false, "image_generations": false, - "audio_transcriptions": true, - "audio_speech": true, - "moderations": true, - "batches": true, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "together_ai": { @@ -2299,56 +3842,312 @@ "url": "https://docs.litellm.ai/docs/providers/togetherai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": true, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "topaz": { "display_name": "Topaz (`topaz`)", "url": "https://docs.litellm.ai/docs/providers/topaz", "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, "image_variations": true } }, + "nace": { + "display_name": "Nace (`nace`)", + "url": "https://console.nace.ai/docs/drex", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "typesafe": { + "display_name": "TypeSafe (`typesafe`)", + "url": "https://docs.typesafe.ai/models", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "strands_decider": { + "display_name": "Strands Decider (`strands_decider`)", + "url": "https://docs.litellm.ai/docs/providers", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, "tavily": { "display_name": "Tavily (`tavily`)", "url": "https://docs.litellm.ai/docs/search/tavily", "endpoints": { - "search": true + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "serper": { "display_name": "Serper (`serper`)", "url": "https://docs.litellm.ai/docs/search/serper", "endpoints": { - "search": true + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "you_com": { + "display_name": "You.com (`you_com`)", + "url": "https://docs.litellm.ai/docs/search/you_com", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "apiserpent": { + "display_name": "APISerpent (`apiserpent`)", + "url": "https://docs.litellm.ai/docs/search/apiserpent", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "tinyfish": { + "display_name": "TinyFish (`tinyfish`)", + "url": "https://docs.tinyfish.ai/search-api", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "nimble": { + "display_name": "Nimble (`nimble`)", + "url": "https://docs.nimbleway.com/api-reference/search/search", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "triton": { "display_name": "Triton (`triton`)", "url": "https://docs.litellm.ai/docs/providers/triton-inference-server", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "v0": { @@ -2356,17 +4155,25 @@ "url": "https://docs.litellm.ai/docs/providers/v0", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "vercel_ai_gateway": { @@ -2374,36 +4181,52 @@ "url": "https://docs.litellm.ai/docs/providers/vercel_ai_gateway", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "vllm": { "display_name": "VLLM (`vllm`)", "url": "https://docs.litellm.ai/docs/providers/vllm", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, - "batches": true, - "files": true, - "rerank": true, - "a2a": true, - "interactions": true + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "files": true } }, "volcengine": { @@ -2411,17 +4234,25 @@ "url": "https://docs.litellm.ai/docs/providers/volcano", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": true, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "voyage": { @@ -2429,15 +4260,25 @@ "url": "https://docs.litellm.ai/docs/providers/voyage", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": true, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": true + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "wandb": { @@ -2445,35 +4286,51 @@ "url": "https://docs.litellm.ai/docs/providers/wandb_inference", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "watsonx_text": { "display_name": "Watsonx Text (`watsonx_text`)", "url": "https://docs.litellm.ai/docs/providers/watsonx", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "xai": { @@ -2481,18 +4338,25 @@ "url": "https://docs.litellm.ai/docs/providers/xai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, + "image_generations": true, + "image_edits": false, + "audio_transcriptions": true, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, + "ocr": false, + "search": false, + "moderations": false, + "realtime": true, + "video_generations": true, + "batches": false, + "messages": true, "interactions": true, - "realtime": true + "systemone": false, + "a2a": true, + "generateContent": true } }, "xinference": { @@ -2500,15 +4364,25 @@ "url": "https://docs.litellm.ai/docs/providers/xinference", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, - "embeddings": true, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "zai": { @@ -2516,90 +4390,182 @@ "url": "https://docs.litellm.ai/docs/providers/zai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "ragflow": { "display_name": "RAGFlow (`ragflow`)", "url": "https://docs.litellm.ai/docs/providers/ragflow", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "vector_stores_create": true, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "vector_stores_create": true } }, "cursor": { "display_name": "Cursor BYOK (`cursor`)", "url": "https://docs.litellm.ai/docs/providers/cursor", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "langgraph": { "display_name": "LangGraph (`langgraph`)", "url": "https://docs.litellm.ai/docs/providers/langgraph", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "microsoft_365_copilot": { + "display_name": "Microsoft 365 Copilot (`microsoft_365_copilot`)", + "url": "https://docs.litellm.ai/docs/providers/microsoft_365_copilot", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true + } + }, + "langflow": { + "display_name": "LangFlow (`langflow`)", + "url": "https://docs.litellm.ai/docs/providers/langflow", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "vertex_ai/agent_engine": { "display_name": "Vertex AI Agent Engine (`vertex_ai/agent_engine`)", "url": "https://docs.litellm.ai/docs/providers/vertex_ai_agent_engine", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "pydantic_ai_agents": { @@ -2607,16 +4573,25 @@ "url": "https://docs.litellm.ai/docs/providers/pydantic_ai_agent", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "stability": { @@ -2624,34 +4599,51 @@ "url": "https://docs.litellm.ai/docs/providers/stability", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": true, "image_edits": true, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "venice": { "display_name": "Venice.ai (`venice`)", "url": "https://docs.litellm.ai/docs/providers/venice", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "gigachat": { @@ -2659,22 +4651,77 @@ "url": "https://docs.litellm.ai/docs/providers/gigachat", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": true + "text_completion": true, + "embeddings": true, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "google_pse": { "display_name": "Google PSE (`google_pse`)", "url": "https://docs.litellm.ai/docs/search/google_pse", "endpoints": { - "search": true + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "milvus": { "display_name": "Milvus (`milvus`)", "url": "https://docs.litellm.ai/docs/providers/milvus_vector_stores", "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, "vector_stores_search": true } }, @@ -2683,14 +4730,105 @@ "url": "https://docs.litellm.ai/docs/providers/minimax", "endpoints": { "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": true, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": true, + "batches": false, "messages": true, - "responses": true + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "pg_vector": { "display_name": "PG Vector (`pg_vector`)", "url": "https://docs.litellm.ai/docs/providers/pg_vector", "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "vector_stores_search": true + } + }, + "mongodb": { + "display_name": "MongoDB Atlas (`mongodb`)", + "url": "https://docs.litellm.ai/docs/providers/mongodb_vector_stores", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "vector_stores_search": true + } + }, + "valkey": { + "display_name": "Valkey (`valkey`)", + "url": "https://docs.litellm.ai/docs/providers/valkey_vector_stores", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, "vector_stores_search": true } }, @@ -2698,9 +4836,26 @@ "display_name": "Helicone (`helicone`)", "url": "https://docs.litellm.ai/docs/providers/helicone", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "llamagate": { @@ -2708,8 +4863,25 @@ "url": "https://docs.litellm.ai/docs/providers/llamagate", "endpoints": { "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": true, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, "messages": true, - "responses": true + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "xiaomi_mimo": { @@ -2717,27 +4889,52 @@ "url": "https://docs.litellm.ai/docs/providers/xiaomi_mimo", "endpoints": { "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, "messages": true, - "responses": true + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "manus": { "display_name": "Manus (`manus`)", "url": "https://docs.litellm.ai/docs/providers/manus", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, - "files": true, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "files": true } }, "sarvam": { @@ -2745,8 +4942,155 @@ "url": "https://docs.litellm.ai/docs/providers/sarvam", "endpoints": { "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, "messages": true, - "responses": true + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, + "bedrock_mantle": { + "display_name": "Bedrock Mantle (`bedrock_mantle`)", + "url": "https://docs.litellm.ai/docs/providers/bedrock", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, + "searchapi": { + "display_name": "SearchAPI (`searchapi`)", + "url": "https://docs.litellm.ai/docs/providers/searchapi", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "black_forest_labs": { + "display_name": "Black Forest Labs (`black_forest_labs`)", + "url": "https://docs.litellm.ai/docs/providers/black_forest_labs", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": true, + "image_edits": true, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "reka": { + "display_name": "Reka (`reka`)", + "url": "https://docs.litellm.ai/docs/providers/reka", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "empiriolabs": { + "display_name": "EmpirioLabs (`empiriolabs`)", + "url": "https://docs.litellm.ai/docs/providers/empiriolabs", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "openinfer": { @@ -2754,16 +5098,155 @@ "url": "https://docs.litellm.ai/docs/providers/openinfer", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, + "agentcore": { + "display_name": "AgentCore (`agentcore`)", + "url": "https://docs.litellm.ai/docs/providers", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "anyscale": { + "display_name": "Anyscale (`anyscale`)", + "url": "https://docs.litellm.ai/docs/providers/anyscale", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, + "bing_grounding": { + "display_name": "Bing Grounding (`bing_grounding`)", + "url": "https://docs.litellm.ai/docs/providers", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "opencode_go": { + "display_name": "OpenCode Go (`opencode_go`)", + "url": "https://docs.litellm.ai/docs/providers/opencode_go", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, + "transcribe": { + "display_name": "Transcribe (`transcribe`)", + "url": "https://docs.litellm.ai/docs/providers", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": true, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } } }, @@ -2976,6 +5459,13 @@ "provider_json_field": "compact", "url": "https://docs.litellm.ai/docs/response_api" }, + "sandbox": { + "docs_label": "sandbox", + "display_name": "Code Execution Sandbox API", + "leftnav_label": "/sandbox", + "provider_json_field": "sandbox", + "url": "https://docs.litellm.ai/docs/providers/e2b" + }, "search": { "docs_label": "search", "display_name": "Search API", @@ -2990,6 +5480,13 @@ "provider_json_field": "skills", "url": "https://docs.litellm.ai/docs/skills" }, + "systemone": { + "docs_label": "systemone", + "display_name": "System One Decision API", + "leftnav_label": "/laya/v1/systemone", + "provider_json_field": "systemone", + "url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers" + }, "text_completion": { "docs_label": "text_completion", "display_name": "OpenAI Completions API", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index e62b74aa87d..9ba6ed342ea 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -38,33 +38,51 @@ "url": "https://docs.litellm.ai/docs/providers/refuseless", "endpoints": { "chat_completions": true, - "messages": false, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "a2a": { "display_name": "A2A (Agent-to-Agent) (`a2a`)", "url": "https://docs.litellm.ai/docs/providers/a2a", "endpoints": { - "chat_completions": true, - "messages": false, + "chat_completions": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "abliteration": { @@ -72,34 +90,51 @@ "url": "https://docs.litellm.ai/docs/providers/abliteration", "endpoints": { "chat_completions": true, - "messages": false, - "responses": false, + "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "aiml": { "display_name": "AI/ML API (`aiml`)", "url": "https://docs.litellm.ai/docs/providers/aiml", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, "image_generations": true, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "ai21": { @@ -107,35 +142,51 @@ "url": "https://docs.litellm.ai/docs/providers/ai21", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "ai21_chat": { "display_name": "AI21 Chat (`ai21_chat`)", "url": "https://docs.litellm.ai/docs/providers/ai21", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "amazon_nova": { @@ -143,17 +194,25 @@ "url": "https://docs.litellm.ai/docs/providers/amazon_nova", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "anthropic": { @@ -161,18 +220,26 @@ "url": "https://docs.litellm.ai/docs/providers/anthropic", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": true, "rerank": false, - "skills": true, - "a2a": true, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true, + "skills": true, "count_tokens": true } }, @@ -180,35 +247,52 @@ "display_name": "Anthropic Text (`anthropic_text`)", "url": "https://docs.litellm.ai/docs/providers/anthropic", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": true, "rerank": false, - "skills": true, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "skills": true } }, "apertis": { "display_name": "Apertis (`apertis`)", "endpoints": { - "chat_completions": true, - "messages": false, + "chat_completions": false, "responses": false, - "embeddings": true, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "aihubmix": { @@ -216,52 +300,77 @@ "url": "https://docs.litellm.ai/docs/providers/aihubmix", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": true, - "image_generations": true, - "audio_transcriptions": true, - "audio_speech": true, - "moderations": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": true, - "a2a": false + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "assemblyai": { "display_name": "AssemblyAI (`assemblyai`)", "url": "https://docs.litellm.ai/docs/pass_through/assembly_ai", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": true, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "auto_router": { "display_name": "Auto Router (`auto_router`)", "url": "https://docs.litellm.ai/docs/proxy/auto_routing", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "bedrock": { @@ -269,17 +378,25 @@ "url": "https://docs.litellm.ai/docs/providers/bedrock", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": true, - "image_generations": false, + "image_generations": true, + "image_edits": true, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": true, - "a2a": true, + "ocr": false, + "search": false, + "moderations": false, + "realtime": true, + "video_generations": false, + "batches": false, + "messages": true, "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true, "bedrock_invoke": true, "bedrock_converse": true, "vector_stores_search": true, @@ -293,17 +410,25 @@ "url": "https://docs.litellm.ai/docs/providers/s3_vectors", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, "vector_stores_create": true, "vector_stores_search": true } @@ -313,17 +438,25 @@ "url": "https://docs.litellm.ai/docs/providers/aws_sagemaker", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": true, + "text_completion": true, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "aws_polly": { @@ -331,15 +464,25 @@ "url": "https://docs.litellm.ai/docs/providers/aws_polly", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": true, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "azure": { @@ -347,21 +490,28 @@ "url": "https://docs.litellm.ai/docs/providers/azure", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": true, "image_generations": true, + "image_edits": true, "audio_transcriptions": true, "audio_speech": true, - "moderations": true, - "batches": true, "rerank": false, - "a2a": true, + "ocr": false, + "search": false, + "moderations": false, + "realtime": true, + "video_generations": true, + "batches": true, + "messages": true, "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true, "vector_stores_search": true, "assistants": true, - "fine_tuning": true, - "text_completion": true + "fine_tuning": true } }, "azure_ai": { @@ -369,19 +519,25 @@ "url": "https://docs.litellm.ai/docs/providers/azure_ai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": true, "image_generations": true, "image_edits": true, "audio_transcriptions": true, - "audio_speech": true, - "moderations": true, - "batches": true, - "rerank": false, + "audio_speech": false, + "rerank": true, "ocr": true, - "a2a": true, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": true, + "messages": true, "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true, "vector_stores_create": true, "vector_stores_search": true } @@ -391,52 +547,77 @@ "url": "https://docs.litellm.ai/docs/providers/azure_document_intelligence", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "ocr": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "azure_ai/agents": { "display_name": "Azure AI Foundry Agents (`azure_ai/agents`)", "url": "https://docs.litellm.ai/docs/providers/azure_ai_agents", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "azure_text": { "display_name": "Azure Text (`azure_text`)", "url": "https://docs.litellm.ai/docs/providers/azure", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": true, "embeddings": false, "image_generations": false, - "audio_transcriptions": true, - "audio_speech": true, - "moderations": true, - "batches": true, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "baseten": { @@ -444,53 +625,77 @@ "url": "https://docs.litellm.ai/docs/providers/baseten", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "bytez": { "display_name": "Bytez (`bytez`)", "url": "https://docs.litellm.ai/docs/providers/bytez", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "nadir": { "display_name": "Nadir (`nadir`)", "url": "https://docs.litellm.ai/docs/providers/nadir", "endpoints": { - "chat_completions": true, - "messages": false, + "chat_completions": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, "a2a": false, - "interactions": false + "generateContent": false } }, "cerebras": { @@ -498,69 +703,102 @@ "url": "https://docs.litellm.ai/docs/providers/cerebras", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "charity_engine": { "display_name": "Charity Engine (`charity_engine`)", "url": "https://docs.litellm.ai/docs/providers/charity_engine", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, "a2a": false, - "interactions": false + "generateContent": false } }, "chutes": { "display_name": "Chutes (`chutes`)", "endpoints": { - "chat_completions": true, - "messages": false, + "chat_completions": false, "responses": false, - "embeddings": true, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "clarifai": { "display_name": "Clarifai (`clarifai`)", "url": "https://docs.litellm.ai/docs/providers/clarifai", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "cloudflare": { @@ -568,17 +806,25 @@ "url": "https://docs.litellm.ai/docs/providers/cloudflare_workers", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, - "audio_transcriptions": false, + "image_edits": false, + "audio_transcriptions": true, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "codestral": { @@ -586,17 +832,25 @@ "url": "https://docs.litellm.ai/docs/providers/codestral", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "cognition": { @@ -604,35 +858,51 @@ "url": "https://docs.litellm.ai/docs/providers/cognition", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "cohere": { "display_name": "Cohere (`cohere`)", "url": "https://docs.litellm.ai/docs/providers/cohere", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": true, "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": true, "ocr": true, - "a2a": true, - "interactions": true + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "cohere_chat": { @@ -640,53 +910,77 @@ "url": "https://docs.litellm.ai/docs/providers/cohere", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "cometapi": { "display_name": "CometAPI (`cometapi`)", "url": "https://docs.litellm.ai/docs/providers/cometapi", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "compactifai": { "display_name": "CompactifAI (`compactifai`)", "url": "https://docs.litellm.ai/docs/providers/compactifai", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "coralbricks": { @@ -694,35 +988,51 @@ "url": "https://docs.litellm.ai/docs/providers/coralbricks", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false, - "interactions": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "cortecs": { "display_name": "Cortecs (`cortecs`)", "url": "https://docs.litellm.ai/docs/providers/cortecs", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, "a2a": false, - "interactions": false + "generateContent": false } }, "crusoe": { @@ -730,53 +1040,77 @@ "url": "https://docs.litellm.ai/docs/providers/crusoe", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "custom": { "display_name": "Custom (`custom`)", "url": "https://docs.litellm.ai/docs/providers/custom_llm_server", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "custom_openai": { "display_name": "Custom OpenAI (`custom_openai`)", "url": "https://docs.litellm.ai/docs/providers/openai_compatible", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, - "audio_transcriptions": true, - "audio_speech": true, - "moderations": true, - "batches": true, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "dashscope": { @@ -784,18 +1118,25 @@ "url": "https://docs.litellm.ai/docs/providers/dashscope", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, - "image_generations": false, + "image_generations": true, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": true, + "batches": false, + "messages": true, "interactions": true, - "video_generations": true + "systemone": false, + "a2a": true, + "generateContent": true } }, "qwencloud": { @@ -803,18 +1144,25 @@ "url": "https://docs.litellm.ai/docs/providers/qwencloud", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": true, + "text_completion": true, + "embeddings": false, "image_generations": true, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": true, "batches": false, - "rerank": true, - "a2a": true, + "messages": true, "interactions": true, - "video_generations": true + "systemone": false, + "a2a": true, + "generateContent": true } }, "qwen_ai_platform": { @@ -822,18 +1170,25 @@ "url": "https://docs.litellm.ai/docs/providers/qwencloud", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": true, + "text_completion": true, + "embeddings": false, "image_generations": true, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": true, "batches": false, - "rerank": true, - "a2a": true, + "messages": true, "interactions": true, - "video_generations": true + "systemone": false, + "a2a": true, + "generateContent": true } }, "databricks": { @@ -841,17 +1196,25 @@ "url": "https://docs.litellm.ai/docs/providers/databricks", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "dataforseo": { @@ -859,52 +1222,77 @@ "url": "https://docs.litellm.ai/docs/search/dataforseo", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "datarobot": { "display_name": "DataRobot (`datarobot`)", "url": "https://docs.litellm.ai/docs/providers/datarobot", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "deepgram": { "display_name": "Deepgram (`deepgram`)", "url": "https://docs.litellm.ai/docs/providers/deepgram", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": true, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "deepinfra": { @@ -912,17 +1300,25 @@ "url": "https://docs.litellm.ai/docs/providers/deepinfra", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "deepseek": { @@ -930,36 +1326,51 @@ "url": "https://docs.litellm.ai/docs/providers/deepseek", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, "interactions": true, - "search": true + "systemone": false, + "a2a": true, + "generateContent": true } }, "edenai": { "display_name": "Eden AI (`edenai`)", "url": "https://docs.litellm.ai/docs/providers/edenai", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, - "image_generations": true, - "audio_transcriptions": true, - "audio_speech": true, - "moderations": false, - "batches": false, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, "interactions": false, - "video_generations": true + "systemone": false, + "a2a": false, + "generateContent": false } }, "duckduckgo": { @@ -967,16 +1378,25 @@ "url": "https://docs.litellm.ai/docs/search/duckduckgo", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "e2b": { @@ -984,15 +1404,25 @@ "url": "https://docs.litellm.ai/docs/providers/e2b", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, "sandbox": true } }, @@ -1000,18 +1430,26 @@ "display_name": "ElevenLabs (`elevenlabs`)", "url": "https://docs.litellm.ai/docs/providers/elevenlabs", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, - "audio_transcriptions": false, + "image_edits": false, + "audio_transcriptions": true, "audio_speech": true, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "exa_ai": { @@ -1019,16 +1457,25 @@ "url": "https://docs.litellm.ai/docs/search/exa_ai", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "brave": { @@ -1036,34 +1483,51 @@ "url": "https://docs.litellm.ai/docs/search/brave", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "empower": { "display_name": "Empower (`empower`)", "url": "https://docs.litellm.ai/docs/providers/empower", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "fal_ai": { @@ -1071,17 +1535,25 @@ "url": "https://docs.litellm.ai/docs/providers/fal_ai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": true, + "image_edits": true, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": true, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "featherless_ai": { @@ -1089,17 +1561,25 @@ "url": "https://docs.litellm.ai/docs/providers/featherless_ai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "fireworks_ai": { @@ -1107,18 +1587,25 @@ "url": "https://docs.litellm.ai/docs/providers/fireworks_ai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, + "text_completion": true, + "embeddings": true, + "image_generations": true, + "image_edits": false, + "audio_transcriptions": true, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": true, - "a2a": true, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, "interactions": true, - "systemone": true + "systemone": false, + "a2a": true, + "generateContent": true } }, "firecrawl": { @@ -1126,16 +1613,25 @@ "url": "https://docs.litellm.ai/docs/search/firecrawl", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "fastcrw": { @@ -1143,16 +1639,25 @@ "url": "https://docs.litellm.ai/docs/search/fastcrw", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "linkup": { @@ -1160,16 +1665,25 @@ "url": "https://docs.litellm.ai/docs/search/linkup", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "friendliai": { @@ -1177,45 +1691,77 @@ "url": "https://docs.litellm.ai/docs/providers/friendliai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "galadriel": { "display_name": "Galadriel (`galadriel`)", "url": "https://docs.litellm.ai/docs/providers/galadriel", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "gdc": { "display_name": "Google Distributed Cloud (GDC)", "url": "https://docs.litellm.ai/docs/providers/gdc", "endpoints": { - "chat_completions": true, - "messages": false, + "chat_completions": false, "responses": false, - "embeddings": false + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "github_copilot": { @@ -1223,17 +1769,25 @@ "url": "https://docs.litellm.ai/docs/providers/github_copilot", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "chatgpt": { @@ -1241,35 +1795,51 @@ "url": "https://docs.litellm.ai/docs/providers/chatgpt", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false, - "interactions": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "github": { "display_name": "GitHub Models (`github`)", "url": "https://docs.litellm.ai/docs/providers/github", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "gmi": { @@ -1277,17 +1847,25 @@ "url": "https://docs.litellm.ai/docs/providers/gmi_cloud", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "vertex_ai": { @@ -1295,25 +1873,30 @@ "url": "https://docs.litellm.ai/docs/providers/vertex", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": true, "image_generations": true, - "audio_transcriptions": false, + "image_edits": true, + "audio_transcriptions": true, "audio_speech": true, - "moderations": false, - "batches": false, - "rerank": false, + "rerank": true, "ocr": true, - "a2a": true, + "search": false, + "moderations": false, + "realtime": true, + "video_generations": true, + "batches": true, + "messages": true, "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true, "vector_stores_search": true, "count_tokens": true, "fine_tuning": true, "rag_ingest": true, - "rag_query": true, - "generateContent": true, - "realtime": true + "rag_query": true } }, "gemini": { @@ -1321,22 +1904,28 @@ "url": "https://docs.litellm.ai/docs/providers/gemini", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, - "moderations": false, - "batches": false, + "text_completion": true, + "embeddings": true, + "image_generations": true, + "image_edits": true, + "audio_transcriptions": true, + "audio_speech": true, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": true, + "video_generations": true, + "batches": true, + "messages": true, "interactions": true, + "systemone": false, "a2a": true, + "generateContent": true, "vector_stores_search": true, "count_tokens": true, - "rag_ingest": true, - "realtime": true, - "generateContent": true + "rag_ingest": true } }, "gradient_ai": { @@ -1344,17 +1933,25 @@ "url": "https://docs.litellm.ai/docs/providers/gradient_ai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "groq": { @@ -1362,17 +1959,25 @@ "url": "https://docs.litellm.ai/docs/providers/groq", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, - "moderations": false, - "batches": false, + "image_edits": false, + "audio_transcriptions": true, + "audio_speech": true, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "heroku": { @@ -1380,56 +1985,78 @@ "url": "https://docs.litellm.ai/docs/providers/heroku", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "hosted_vllm": { "display_name": "Hosted VLLM (`hosted_vllm`)", "url": "https://docs.litellm.ai/docs/providers/vllm", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, - "batches": true, - "files": true, - "rerank": true, - "a2a": true, - "interactions": true, - "video_generations": true, - "systemone": true + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "files": true } }, "huggingface": { "display_name": "Huggingface (`huggingface`)", "url": "https://docs.litellm.ai/docs/providers/huggingface", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": true, - "a2a": true, - "interactions": true + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "hyperbolic": { @@ -1437,17 +2064,25 @@ "url": "https://docs.litellm.ai/docs/providers/hyperbolic", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "watsonx": { @@ -1455,17 +2090,25 @@ "url": "https://docs.litellm.ai/docs/providers/watsonx", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": true, + "text_completion": true, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": true, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "inception": { @@ -1473,17 +2116,25 @@ "url": "https://docs.litellm.ai/docs/providers/inception", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "infinity": { @@ -1491,15 +2142,25 @@ "url": "https://docs.litellm.ai/docs/providers/infinity", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, - "embeddings": true, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "jina_ai": { @@ -1507,29 +2168,77 @@ "url": "https://docs.litellm.ai/docs/providers/jina_ai", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, - "embeddings": true, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": true, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "bespoke": { "display_name": "Bespoke Nimble (`bespoke`)", "url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers", "endpoints": { - "systemone": true + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": true, + "a2a": false, + "generateContent": false } }, "laya": { "display_name": "Laya (`laya`)", "url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers", "endpoints": { - "systemone": true + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": true, + "a2a": false, + "generateContent": false } }, "lambda_ai": { @@ -1537,17 +2246,25 @@ "url": "https://docs.litellm.ai/docs/providers/lambda_ai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "lemonade": { @@ -1555,17 +2272,25 @@ "url": "https://docs.litellm.ai/docs/providers/lemonade", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "libertai": { @@ -1573,88 +2298,129 @@ "url": "https://docs.litellm.ai/docs/providers/libertai", "endpoints": { "chat_completions": true, - "messages": true, - "responses": false, - "embeddings": false, + "responses": true, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "litellm_proxy": { "display_name": "LiteLLM Proxy (`litellm_proxy`)", "url": "https://docs.litellm.ai/docs/providers/litellm_proxy", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, - "image_generations": true, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "llamafile": { "display_name": "Llamafile (`llamafile`)", "url": "https://docs.litellm.ai/docs/providers/llamafile", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "lm_studio": { "display_name": "LM Studio (`lm_studio`)", "url": "https://docs.litellm.ai/docs/providers/lm_studio", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "maritalk": { "display_name": "Maritalk (`maritalk`)", "url": "https://docs.litellm.ai/docs/providers/maritalk", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "meta_llama": { @@ -1662,17 +2428,25 @@ "url": "https://docs.litellm.ai/docs/providers/meta_llama", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "mistral": { @@ -1680,36 +2454,51 @@ "url": "https://docs.litellm.ai/docs/providers/mistral", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": true, "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, - "moderations": false, - "batches": true, + "image_edits": false, + "audio_transcriptions": true, + "audio_speech": true, "rerank": false, "ocr": true, + "search": false, + "moderations": true, + "realtime": false, + "video_generations": false, + "batches": true, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "modelscope": { "display_name": "ModelScope (`modelscope`)", "url": "https://docs.litellm.ai/docs/providers/modelscope", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, - "image_generations": true, + "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, "a2a": false, - "interactions": false + "generateContent": false } }, "moonshot": { @@ -1717,35 +2506,51 @@ "url": "https://docs.litellm.ai/docs/providers/moonshot", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "docker_model_runner": { "display_name": "Docker Model Runner (`docker_model_runner`)", "url": "https://docs.litellm.ai/docs/providers/docker_model_runner", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "morph": { @@ -1753,33 +2558,50 @@ "url": "https://docs.litellm.ai/docs/providers/morph", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "nanogpt": { "display_name": "NanoGPT (`nanogpt`)", "endpoints": { - "chat_completions": true, - "messages": false, + "chat_completions": false, "responses": false, - "embeddings": true, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "nebius": { @@ -1787,34 +2609,51 @@ "url": "https://docs.litellm.ai/docs/providers/nebius", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "neosantara": { "display_name": "Neosantara (`neosantara`)", "url": "https://docs.litellm.ai/docs/providers/neosantara", "endpoints": { - "chat_completions": true, - "messages": false, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "nlp_cloud": { @@ -1822,17 +2661,25 @@ "url": "https://docs.litellm.ai/docs/providers/nlp_cloud", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "novita": { @@ -1840,17 +2687,25 @@ "url": "https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": true, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "nscale": { @@ -1858,35 +2713,51 @@ "url": "https://docs.litellm.ai/docs/providers/nscale", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, - "image_generations": false, + "image_generations": true, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "nvidia_nim": { "display_name": "Nvidia NIM (`nvidia_nim`)", "url": "https://docs.litellm.ai/docs/providers/nvidia_nim", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": true, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false, - "a2a": true, - "interactions": true + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "nvidia_riva": { @@ -1894,15 +2765,25 @@ "url": "https://docs.litellm.ai/docs/providers/nvidia_riva", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, - "audio_transcriptions": true, + "image_edits": false, + "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "oci": { @@ -1910,17 +2791,25 @@ "url": "https://docs.litellm.ai/docs/providers/oci", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "ollama": { @@ -1928,53 +2817,77 @@ "url": "https://docs.litellm.ai/docs/providers/ollama", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": true, + "text_completion": true, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "ollama_chat": { "display_name": "Ollama Chat (`ollama_chat`)", "url": "https://docs.litellm.ai/docs/providers/ollama", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "oobabooga": { "display_name": "Oobabooga (`oobabooga`)", "url": "https://docs.litellm.ai/docs/providers/openai_compatible", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, - "audio_transcriptions": true, - "audio_speech": true, - "moderations": true, - "batches": true, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "openai": { @@ -1982,19 +2895,27 @@ "url": "https://docs.litellm.ai/docs/providers/openai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": true, "image_generations": true, + "image_edits": true, "audio_transcriptions": true, "audio_speech": true, - "moderations": true, - "batches": true, "rerank": false, + "ocr": false, + "search": false, + "moderations": true, + "realtime": true, + "video_generations": true, + "batches": true, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true, "container": true, "compact": true, - "a2a": true, - "interactions": true, "vector_store_files": true, "vector_stores_create": true, "vector_stores_search": true, @@ -2003,9 +2924,7 @@ "fine_tuning": true, "image_variations": true, "rag_ingest": true, - "rag_query": true, - "realtime": true, - "text_completion": true + "rag_query": true } }, "opensandbox": { @@ -2013,15 +2932,25 @@ "url": "https://open-sandbox.ai/api/", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, "sandbox": true } }, @@ -2030,15 +2959,25 @@ "url": "https://docs.litellm.ai/docs/providers/openai_compatible", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, - "embeddings": true, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, "assistants": true } }, @@ -2046,18 +2985,26 @@ "display_name": "OpenRouter (`openrouter`)", "url": "https://docs.litellm.ai/docs/providers/openrouter", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "ovhcloud": { @@ -2065,17 +3012,25 @@ "url": "https://docs.litellm.ai/docs/providers/ovhcloud", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, - "audio_transcriptions": true, + "image_edits": false, + "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "parallel_ai": { @@ -2083,33 +3038,51 @@ "url": "https://docs.litellm.ai/docs/search/parallel_ai", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "parasail": { "display_name": "Parasail (`parasail`)", "url": "https://docs.litellm.ai/docs/providers/parasail", "endpoints": { - "chat_completions": true, - "messages": false, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "perplexity": { @@ -2117,36 +3090,51 @@ "url": "https://docs.litellm.ai/docs/providers/perplexity", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "petals": { "display_name": "Petals (`petals`)", "url": "https://docs.litellm.ai/docs/providers/petals", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "sail": { @@ -2154,16 +3142,25 @@ "url": "https://docs.litellm.ai/docs/providers/sail", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "meta": { @@ -2171,16 +3168,25 @@ "url": "https://docs.litellm.ai/docs/providers/meta", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, + "image_generations": true, + "image_edits": true, + "audio_transcriptions": true, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": true, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "pinstripes": { @@ -2188,32 +3194,50 @@ "url": "https://docs.litellm.ai/docs/providers/pinstripes", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": true, + "text_completion": true, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "poe": { "display_name": "Poe (`poe`)", "endpoints": { - "chat_completions": true, - "messages": false, + "chat_completions": false, "responses": false, - "embeddings": true, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "publicai": { @@ -2221,17 +3245,25 @@ "url": "https://docs.litellm.ai/docs/providers/publicai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "darkbloom": { @@ -2239,34 +3271,51 @@ "url": "https://docs.litellm.ai/docs/providers/darkbloom", "endpoints": { "chat_completions": true, - "messages": false, - "responses": false, + "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "predibase": { "display_name": "Predibase (`predibase`)", "url": "https://docs.litellm.ai/docs/providers/predibase", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "prism": { @@ -2274,16 +3323,25 @@ "url": "https://docs.litellm.ai/docs/providers/prism", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "recraft": { @@ -2291,15 +3349,25 @@ "url": "https://docs.litellm.ai/docs/providers/recraft", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": true, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "reducto": { @@ -2307,16 +3375,25 @@ "url": "https://docs.litellm.ai/docs/providers/reducto", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "ocr": true + "ocr": true, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "replicate": { @@ -2324,17 +3401,25 @@ "url": "https://docs.litellm.ai/docs/providers/replicate", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "runwayml": { @@ -2342,34 +3427,51 @@ "url": "https://docs.litellm.ai/docs/providers/runwayml/videos", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": true, + "image_edits": false, "audio_transcriptions": false, "audio_speech": true, - "moderations": false, - "batches": false, "rerank": false, - "video_generations": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": true, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "sagemaker_chat": { "display_name": "Sagemaker Chat (`sagemaker_chat`)", "url": "https://docs.litellm.ai/docs/providers/aws_sagemaker", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "searxng": { @@ -2377,16 +3479,25 @@ "url": "https://docs.litellm.ai/docs/search/searxng", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "scaledown": { @@ -2394,17 +3505,25 @@ "url": "https://docs.litellm.ai/docs/providers/scaledown", "endpoints": { "chat_completions": true, - "messages": false, - "responses": false, + "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false, - "interactions": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "sambanova": { @@ -2412,35 +3531,51 @@ "url": "https://docs.litellm.ai/docs/providers/sambanova", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "sap": { "display_name": "SAP Generative AI Hub (`sap`)", "url": "https://docs.litellm.ai/docs/providers/sap", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "scaleway": { @@ -2448,17 +3583,25 @@ "url": "https://docs.litellm.ai/docs/providers/scaleway", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": true, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": true, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "scx-ai": { @@ -2466,16 +3609,25 @@ "url": "https://docs.litellm.ai/docs/providers/scx_ai", "endpoints": { "chat_completions": true, - "messages": false, - "responses": false, + "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "inferenceapis": { @@ -2483,17 +3635,25 @@ "url": "https://docs.litellm.ai/docs/providers/inferenceapis", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false, - "interactions": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "snowflake": { @@ -2501,17 +3661,25 @@ "url": "https://docs.litellm.ai/docs/providers/snowflake", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "soniox": { @@ -2519,31 +3687,50 @@ "url": "https://docs.litellm.ai/docs/providers/soniox", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": true, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "synthetic": { "display_name": "Synthetic (`synthetic`)", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "tensormesh": { @@ -2551,17 +3738,25 @@ "url": "https://docs.litellm.ai/docs/providers/tensormesh", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false, - "text_completion": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "tencent": { @@ -2569,53 +3764,77 @@ "url": "https://docs.litellm.ai/docs/providers/tencent", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false, - "text_completion": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "text-completion-codestral": { "display_name": "Text Completion Codestral (`text-completion-codestral`)", "url": "https://docs.litellm.ai/docs/providers/codestral", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "text-completion-openai": { "display_name": "Text Completion OpenAI (`text-completion-openai`)", "url": "https://docs.litellm.ai/docs/providers/text_completion_openai", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": true, "embeddings": false, "image_generations": false, - "audio_transcriptions": true, - "audio_speech": true, - "moderations": true, - "batches": true, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "together_ai": { @@ -2623,23 +3842,51 @@ "url": "https://docs.litellm.ai/docs/providers/togetherai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": true, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "topaz": { "display_name": "Topaz (`topaz`)", "url": "https://docs.litellm.ai/docs/providers/topaz", "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, "image_variations": true } }, @@ -2647,78 +3894,260 @@ "display_name": "Nace (`nace`)", "url": "https://console.nace.ai/docs/drex", "endpoints": { - "systemone": true + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "typesafe": { "display_name": "TypeSafe (`typesafe`)", "url": "https://docs.typesafe.ai/models", "endpoints": { - "systemone": true + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "strands_decider": { "display_name": "Strands Decider (`strands_decider`)", "url": "https://docs.litellm.ai/docs/providers", "endpoints": { - "systemone": true + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "tavily": { "display_name": "Tavily (`tavily`)", "url": "https://docs.litellm.ai/docs/search/tavily", "endpoints": { - "search": true + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "serper": { "display_name": "Serper (`serper`)", "url": "https://docs.litellm.ai/docs/search/serper", "endpoints": { - "search": true + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "you_com": { "display_name": "You.com (`you_com`)", - "url": "https://docs.litellm.ai/docs/search/you_com" + "url": "https://docs.litellm.ai/docs/search/you_com", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } }, "apiserpent": { "display_name": "APISerpent (`apiserpent`)", "url": "https://docs.litellm.ai/docs/search/apiserpent", "endpoints": { - "search": true + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "tinyfish": { "display_name": "TinyFish (`tinyfish`)", "url": "https://docs.tinyfish.ai/search-api", "endpoints": { - "search": true + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "nimble": { "display_name": "Nimble (`nimble`)", "url": "https://docs.nimbleway.com/api-reference/search/search", "endpoints": { - "search": true + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "triton": { "display_name": "Triton (`triton`)", "url": "https://docs.litellm.ai/docs/providers/triton-inference-server", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "v0": { @@ -2726,17 +4155,25 @@ "url": "https://docs.litellm.ai/docs/providers/v0", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "vercel_ai_gateway": { @@ -2744,36 +4181,52 @@ "url": "https://docs.litellm.ai/docs/providers/vercel_ai_gateway", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "vllm": { "display_name": "VLLM (`vllm`)", "url": "https://docs.litellm.ai/docs/providers/vllm", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": true, + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, - "batches": true, - "files": true, - "rerank": true, - "a2a": true, - "interactions": true + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "files": true } }, "volcengine": { @@ -2781,18 +4234,25 @@ "url": "https://docs.litellm.ai/docs/providers/volcano", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": false, + "text_completion": true, + "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": true, + "batches": false, + "messages": true, "interactions": true, - "video_generations": true + "systemone": false, + "a2a": true, + "generateContent": true } }, "voyage": { @@ -2800,15 +4260,25 @@ "url": "https://docs.litellm.ai/docs/providers/voyage", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": true, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": true, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": true + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "wandb": { @@ -2816,35 +4286,51 @@ "url": "https://docs.litellm.ai/docs/providers/wandb_inference", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "watsonx_text": { "display_name": "Watsonx Text (`watsonx_text`)", "url": "https://docs.litellm.ai/docs/providers/watsonx", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "xai": { @@ -2852,18 +4338,25 @@ "url": "https://docs.litellm.ai/docs/providers/xai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, + "image_generations": true, + "image_edits": false, + "audio_transcriptions": true, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, + "ocr": false, + "search": false, + "moderations": false, + "realtime": true, + "video_generations": true, + "batches": false, + "messages": true, "interactions": true, - "realtime": true + "systemone": false, + "a2a": true, + "generateContent": true } }, "xinference": { @@ -2871,15 +4364,25 @@ "url": "https://docs.litellm.ai/docs/providers/xinference", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, - "embeddings": true, + "text_completion": false, + "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "zai": { @@ -2887,72 +4390,104 @@ "url": "https://docs.litellm.ai/docs/providers/zai", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, "a2a": true, - "interactions": true + "generateContent": true } }, "ragflow": { "display_name": "RAGFlow (`ragflow`)", "url": "https://docs.litellm.ai/docs/providers/ragflow", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "vector_stores_create": true, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "vector_stores_create": true } }, "cursor": { "display_name": "Cursor BYOK (`cursor`)", "url": "https://docs.litellm.ai/docs/providers/cursor", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "langgraph": { "display_name": "LangGraph (`langgraph`)", "url": "https://docs.litellm.ai/docs/providers/langgraph", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "microsoft_365_copilot": { @@ -2960,53 +4495,77 @@ "url": "https://docs.litellm.ai/docs/providers/microsoft_365_copilot", "endpoints": { "chat_completions": true, - "messages": false, - "responses": false, + "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false, - "interactions": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "langflow": { "display_name": "LangFlow (`langflow`)", "url": "https://docs.litellm.ai/docs/providers/langflow", "endpoints": { - "chat_completions": true, - "messages": false, + "chat_completions": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "vertex_ai/agent_engine": { "display_name": "Vertex AI Agent Engine (`vertex_ai/agent_engine`)", "url": "https://docs.litellm.ai/docs/providers/vertex_ai_agent_engine", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "pydantic_ai_agents": { @@ -3014,16 +4573,25 @@ "url": "https://docs.litellm.ai/docs/providers/pydantic_ai_agent", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "stability": { @@ -3031,34 +4599,51 @@ "url": "https://docs.litellm.ai/docs/providers/stability", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": true, "image_edits": true, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "venice": { "display_name": "Venice.ai (`venice`)", "url": "https://docs.litellm.ai/docs/providers/venice", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "gigachat": { @@ -3066,22 +4651,77 @@ "url": "https://docs.litellm.ai/docs/providers/gigachat", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, - "embeddings": true + "text_completion": true, + "embeddings": true, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "google_pse": { "display_name": "Google PSE (`google_pse`)", "url": "https://docs.litellm.ai/docs/search/google_pse", "endpoints": { - "search": true + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "milvus": { "display_name": "Milvus (`milvus`)", "url": "https://docs.litellm.ai/docs/providers/milvus_vector_stores", "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, "vector_stores_search": true } }, @@ -3090,14 +4730,51 @@ "url": "https://docs.litellm.ai/docs/providers/minimax", "endpoints": { "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": true, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": true, + "batches": false, "messages": true, - "responses": true + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "pg_vector": { "display_name": "PG Vector (`pg_vector`)", "url": "https://docs.litellm.ai/docs/providers/pg_vector", "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, "vector_stores_search": true } }, @@ -3105,6 +4782,26 @@ "display_name": "MongoDB Atlas (`mongodb`)", "url": "https://docs.litellm.ai/docs/providers/mongodb_vector_stores", "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, "vector_stores_search": true } }, @@ -3112,6 +4809,26 @@ "display_name": "Valkey (`valkey`)", "url": "https://docs.litellm.ai/docs/providers/valkey_vector_stores", "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, "vector_stores_search": true } }, @@ -3119,9 +4836,26 @@ "display_name": "Helicone (`helicone`)", "url": "https://docs.litellm.ai/docs/providers/helicone", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "llamagate": { @@ -3129,8 +4863,25 @@ "url": "https://docs.litellm.ai/docs/providers/llamagate", "endpoints": { "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": true, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, "messages": true, - "responses": true + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "xiaomi_mimo": { @@ -3138,27 +4889,52 @@ "url": "https://docs.litellm.ai/docs/providers/xiaomi_mimo", "endpoints": { "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, "messages": true, - "responses": true + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "manus": { "display_name": "Manus (`manus`)", "url": "https://docs.litellm.ai/docs/providers/manus", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, - "files": true, "rerank": false, - "a2a": true, - "interactions": true + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false, + "files": true } }, "sarvam": { @@ -3166,8 +4942,25 @@ "url": "https://docs.litellm.ai/docs/providers/sarvam", "endpoints": { "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, "messages": true, - "responses": true + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "bedrock_mantle": { @@ -3175,16 +4968,25 @@ "url": "https://docs.litellm.ai/docs/providers/bedrock", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "searchapi": { @@ -3192,17 +4994,25 @@ "url": "https://docs.litellm.ai/docs/providers/searchapi", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "search": true, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "black_forest_labs": { @@ -3210,67 +5020,77 @@ "url": "https://docs.litellm.ai/docs/providers/black_forest_labs", "endpoints": { "chat_completions": false, - "messages": false, "responses": false, + "text_completion": false, "embeddings": false, "image_generations": true, "image_edits": true, "audio_transcriptions": false, "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "reka": { "display_name": "Reka (`reka`)", "url": "https://docs.litellm.ai/docs/providers/reka", "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false, - "interactions": false - } - }, - "charity_engine": { - "display_name": "Charity Engine (`charity_engine`)", - "url": "https://docs.litellm.ai/docs/providers/charity_engine", - "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, + "ocr": false, + "search": false, "moderations": false, + "realtime": false, + "video_generations": false, "batches": false, - "rerank": false + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "empiriolabs": { "display_name": "EmpirioLabs (`empiriolabs`)", "url": "https://docs.litellm.ai/docs/providers/empiriolabs", "endpoints": { - "chat_completions": true, - "messages": false, - "responses": true, + "chat_completions": false, + "responses": false, + "text_completion": false, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } }, "openinfer": { @@ -3278,16 +5098,155 @@ "url": "https://docs.litellm.ai/docs/providers/openinfer", "endpoints": { "chat_completions": true, - "messages": true, "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, + "agentcore": { + "display_name": "AgentCore (`agentcore`)", + "url": "https://docs.litellm.ai/docs/providers", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "anyscale": { + "display_name": "Anyscale (`anyscale`)", + "url": "https://docs.litellm.ai/docs/providers/anyscale", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, + "bing_grounding": { + "display_name": "Bing Grounding (`bing_grounding`)", + "url": "https://docs.litellm.ai/docs/providers", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": true, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false + } + }, + "opencode_go": { + "display_name": "OpenCode Go (`opencode_go`)", + "url": "https://docs.litellm.ai/docs/providers/opencode_go", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, + "transcribe": { + "display_name": "Transcribe (`transcribe`)", + "url": "https://docs.litellm.ai/docs/providers", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": true, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": false, + "a2a": false, + "generateContent": false } } }, diff --git a/scripts/generate_provider_endpoints_support.py b/scripts/generate_provider_endpoints_support.py new file mode 100644 index 00000000000..a2329ccded3 --- /dev/null +++ b/scripts/generate_provider_endpoints_support.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Iterator, Mapping, Sequence +from itertools import chain +from pathlib import Path +from types import MappingProxyType +from typing import Final + +from pydantic import JsonValue, TypeAdapter + +ROOT: Final = Path(__file__).resolve().parents[1] +COST_MAP_PATH: Final = ROOT / "model_prices_and_context_window.json" +SOURCE_PATH: Final = ROOT / "provider_endpoints_support.json" +OUTPUT_PATHS: Final = ( + SOURCE_PATH, + ROOT / "litellm/provider_endpoints_support_backup.json", +) +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + +MODE_TO_FLAG: Final[Mapping[str, str]] = MappingProxyType( + { + "chat": "chat_completions", + "responses": "responses", + "completion": "text_completion", + "embedding": "embeddings", + "image_generation": "image_generations", + "image_edit": "image_edits", + "audio_transcription": "audio_transcriptions", + "audio_speech": "audio_speech", + "rerank": "rerank", + "ocr": "ocr", + "search": "search", + "moderation": "moderations", + "realtime": "realtime", + "video_generation": "video_generations", + } +) +SUPPORTED_ENDPOINT_TO_FLAG: Final[Mapping[str, str]] = MappingProxyType( + { + "/v1/chat/completions": "chat_completions", + "/v1/responses": "responses", + "/v1/completions": "text_completion", + "/v1/embeddings": "embeddings", + "/v1/images/generations": "image_generations", + "/v1/images/edits": "image_edits", + "/v1/audio/transcriptions": "audio_transcriptions", + "/v1/audio/speech": "audio_speech", + "/v1/ocr": "ocr", + "/v1/realtime": "realtime", + "/v1/videos": "video_generations", + "/v1/batch": "batches", + "/v1/messages": "messages", + "/v1beta/interactions": "interactions", + "/v1/systemone": "systemone", + } +) + + +def _json_object(value: JsonValue, name: str) -> Mapping[str, JsonValue]: + if not isinstance(value, dict): + raise ValueError(f"{name} must be a JSON object") + return value + + +def _provider_slug(provider: str) -> str: + if provider.startswith("vertex_ai-"): + return "vertex_ai" + if provider == "fireworks_ai-embedding-models": + return "fireworks_ai" + if provider == "bedrock_converse": + return "bedrock" + if provider == "text-completion-inception": + return "inception" + return provider + + +def _model_provider_slug(model: JsonValue) -> str | None: + if not isinstance(model, dict): + return None + provider: Final = model.get("litellm_provider") + if not isinstance(provider, str) or not provider: + return None + return _provider_slug(provider) + + +def _model_flags(model: Mapping[str, JsonValue]) -> tuple[str, ...]: + mode: Final = model.get("mode") + mode_flag: Final = MODE_TO_FLAG.get(mode) if isinstance(mode, str) else None + supported_endpoints: Final = model.get("supported_endpoints") + endpoint_flags: Final = ( + tuple( + SUPPORTED_ENDPOINT_TO_FLAG[endpoint] + for endpoint in supported_endpoints + if isinstance(endpoint, str) and endpoint in SUPPORTED_ENDPOINT_TO_FLAG + ) + if isinstance(supported_endpoints, list) + else () + ) + return ((mode_flag,) if mode_flag is not None else ()) + endpoint_flags + + +def _bridge_field(endpoint: JsonValue) -> str | None: + if not isinstance(endpoint, dict) or endpoint.get("bridges_to_chat_completion") is not True: + return None + provider_json_field: Final = endpoint.get("provider_json_field") + return provider_json_field if isinstance(provider_json_field, str) else None + + +def _bridged_fields(endpoints: Mapping[str, JsonValue]) -> tuple[str, ...]: + return tuple( + dict.fromkeys( + provider_json_field + for endpoint in endpoints.values() + if (provider_json_field := _bridge_field(endpoint)) is not None + ) + ) + + +def _derived_keys(bridged_fields: tuple[str, ...]) -> tuple[str, ...]: + return tuple(dict.fromkeys((*MODE_TO_FLAG.values(), *SUPPORTED_ENDPOINT_TO_FLAG.values(), *bridged_fields))) + + +def _model_evidence( + model: JsonValue, bridged_fields: tuple[str, ...] +) -> Iterator[tuple[str, str]]: + provider: Final = _model_provider_slug(model) + if provider is None or not isinstance(model, dict): + return + flags: Final = _model_flags(model) + yield from ((provider, flag) for flag in flags) + if "chat_completions" not in flags: + return + yield from ((provider, flag) for flag in bridged_fields) + + +def _provider_evidence( + cost_map: Mapping[str, JsonValue], bridged_fields: tuple[str, ...] +) -> Iterator[tuple[str, str]]: + return chain.from_iterable( + _model_evidence(model, bridged_fields) + for model_name, model in cost_map.items() + if model_name != "sample_spec" + ) + + +def _provider_slugs(cost_map: Mapping[str, JsonValue]) -> frozenset[str]: + return frozenset( + provider_slug + for model_name, model in cost_map.items() + if model_name != "sample_spec" and (provider_slug := _model_provider_slug(model)) is not None + ) + + +def _generated_provider( + slug: str, + value: JsonValue, + derived_keys: tuple[str, ...], + derived_key_set: frozenset[str], + evidence: frozenset[tuple[str, str]], +) -> dict[str, JsonValue]: + provider: Final = _json_object(value, f"provider {slug}") + current_flags_value: Final = provider.get("endpoints") + current_flags: Final = ( + _json_object(current_flags_value, f"provider {slug} endpoints") + if current_flags_value is not None + else {} + ) + derived_flags: Final = {flag: (slug, flag) in evidence for flag in derived_keys} + handwritten_flags: Final = { + flag: value for flag, value in current_flags.items() if flag not in derived_key_set + } + return { + **provider, + "endpoints": {**derived_flags, **handwritten_flags}, + } + + +def generate_provider_endpoints_support( + cost_map: Mapping[str, JsonValue], + current_json: Mapping[str, JsonValue], +) -> dict[str, JsonValue]: + current_providers: Final = _json_object( + current_json.get("providers"), "provider_endpoints_support providers" + ) + endpoints: Final = _json_object(current_json.get("endpoints"), "provider_endpoints_support endpoints") + bridged_fields: Final = _bridged_fields(endpoints) + derived_keys: Final = _derived_keys(bridged_fields) + derived_key_set: Final = frozenset(derived_keys) + provider_slugs: Final = _provider_slugs(cost_map) + missing_slugs: Final = tuple(sorted(provider_slugs - frozenset(current_providers))) + if missing_slugs: + raise ValueError(f"Cost map providers missing from provider_endpoints_support.json: {', '.join(missing_slugs)}") + evidence: Final = frozenset(_provider_evidence(cost_map, bridged_fields)) + providers: Final = { + slug: _generated_provider(slug, value, derived_keys, derived_key_set, evidence) + for slug, value in current_providers.items() + } + return {**current_json, "providers": providers} + + +def generate_files( + cost_map: Mapping[str, JsonValue], + current_json: Mapping[str, JsonValue], + output_paths: tuple[Path, ...], + check: bool, +) -> int: + generated: Final = generate_provider_endpoints_support(cost_map, current_json) + content: Final = json.dumps(generated, indent=2, ensure_ascii=False) + "\n" + stale_paths: Final = tuple( + path for path in output_paths if not path.exists() or path.read_text(encoding="utf-8") != content + ) + if check: + if not stale_paths: + return 0 + sys.stderr.write( + "provider endpoint support is stale in " + + ", ".join(str(path) for path in stale_paths) + + "\nRun `uv run --no-sync python scripts/generate_provider_endpoints_support.py` to regenerate it\n" + ) + return 1 + for path in output_paths: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return 0 + + +def main(argv: Sequence[str] | None = None, *, repo_root: Path = ROOT) -> int: + parser: Final = argparse.ArgumentParser(description="Generate provider endpoint support from the model cost map") + parser.add_argument("--check", action="store_true", help="fail if either generated file is stale") + args: Final = parser.parse_args(argv) + cost_map: Final = JSON_OBJECT.validate_json((repo_root / COST_MAP_PATH.name).read_bytes()) + current_json: Final = JSON_OBJECT.validate_json((repo_root / SOURCE_PATH.name).read_bytes()) + return generate_files( + cost_map, + current_json, + ( + repo_root / SOURCE_PATH.name, + repo_root / OUTPUT_PATHS[1].relative_to(ROOT), + ), + args.check, + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/code_coverage_tests/test_generate_provider_endpoints_support.py b/tests/code_coverage_tests/test_generate_provider_endpoints_support.py new file mode 100644 index 00000000000..c2ec6f56c3a --- /dev/null +++ b/tests/code_coverage_tests/test_generate_provider_endpoints_support.py @@ -0,0 +1,179 @@ +import json +from collections.abc import Mapping +from pathlib import Path +from typing import Final + +import pytest +from pydantic import JsonValue + +from scripts.generate_provider_endpoints_support import ( + generate_provider_endpoints_support, + main, +) + +ENDPOINTS: Final = { + "a2a": {"provider_json_field": "a2a", "bridges_to_chat_completion": True}, + "messages": {"provider_json_field": "messages", "bridges_to_chat_completion": True}, + "responses": {"provider_json_field": "responses", "bridges_to_chat_completion": True}, + "generateContent": {"provider_json_field": "generateContent", "bridges_to_chat_completion": True}, + "text_completion": {"provider_json_field": "text_completion", "bridges_to_chat_completion": True}, + "count_tokens": {"provider_json_field": "count_tokens"}, +} + + +def _current_json(providers: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return { + "_comment": "retained", + "_schema": {"version": 1}, + "providers": dict(providers), + "endpoints": ENDPOINTS, + } + + +def _provider(flags: Mapping[str, JsonValue] | None = None) -> dict[str, JsonValue]: + return { + "display_name": "Example (`example`)", + "url": "https://docs.litellm.ai/docs/providers/example", + "endpoints": dict(flags or {}), + } + + +def _flags(generated: Mapping[str, JsonValue], slug: str) -> Mapping[str, JsonValue]: + providers: Final = generated["providers"] + assert isinstance(providers, dict) + provider: Final = providers[slug] + assert isinstance(provider, dict) + flags: Final = provider["endpoints"] + assert isinstance(flags, dict) + return flags + + +def test_mode_derives_chat_flag() -> None: + cost_map: Final = {"example/model": {"litellm_provider": "example", "mode": "chat"}} + current_json: Final = _current_json({"example": _provider()}) + + generated: Final = generate_provider_endpoints_support(cost_map, current_json) + + assert _flags(generated, "example")["chat_completions"] is True + + +def test_supported_endpoint_derives_embedding_flag() -> None: + cost_map: Final = { + "example/model": { + "litellm_provider": "example", + "mode": "completion", + "supported_endpoints": ["/v1/embeddings"], + } + } + current_json: Final = _current_json({"example": _provider()}) + + generated: Final = generate_provider_endpoints_support(cost_map, current_json) + + assert _flags(generated, "example")["embeddings"] is True + + +def test_chat_models_enable_bridge_flags_but_other_models_do_not() -> None: + cost_map: Final = { + "example/chat": {"litellm_provider": "has_chat", "mode": "chat"}, + "example/endpoint-chat": { + "litellm_provider": "endpoint_chat", + "mode": "image_generation", + "supported_endpoints": ["/v1/chat/completions"], + }, + "example/image": {"litellm_provider": "no_chat", "mode": "image_generation"}, + } + current_json: Final = _current_json( + { + "has_chat": _provider(), + "endpoint_chat": _provider(), + "no_chat": _provider(), + } + ) + + generated: Final = generate_provider_endpoints_support(cost_map, current_json) + + assert _flags(generated, "has_chat")["a2a"] is True + assert _flags(generated, "has_chat")["generateContent"] is True + assert _flags(generated, "endpoint_chat")["a2a"] is True + assert _flags(generated, "no_chat")["a2a"] is False + assert _flags(generated, "no_chat")["generateContent"] is False + + +def test_unbacked_derived_flags_reset_and_handwritten_flags_remain() -> None: + cost_map: Final = {"example/model": {"litellm_provider": "example", "mode": "image_generation"}} + current_json: Final = _current_json( + { + "example": _provider({"responses": True, "count_tokens": True, "files": False}), + "without_models": _provider({"responses": True}), + } + ) + + generated: Final = generate_provider_endpoints_support(cost_map, current_json) + flags: Final = _flags(generated, "example") + + assert flags["responses"] is False + assert flags["count_tokens"] is True + assert flags["files"] is False + assert _flags(generated, "without_models")["chat_completions"] is False + + +def test_provider_aliases_share_their_mapped_slugs() -> None: + cost_map: Final = { + "vertex/model-a": {"litellm_provider": "vertex_ai-chat", "mode": "chat"}, + "vertex/model-b": {"litellm_provider": "vertex_ai-embedding", "mode": "embedding"}, + "bedrock/model": {"litellm_provider": "bedrock_converse", "mode": "responses"}, + } + current_json: Final = _current_json({"vertex_ai": _provider(), "bedrock": _provider()}) + + generated: Final = generate_provider_endpoints_support(cost_map, current_json) + + assert _flags(generated, "vertex_ai")["chat_completions"] is True + assert _flags(generated, "vertex_ai")["embeddings"] is True + assert _flags(generated, "bedrock")["responses"] is True + + +def test_cost_map_provider_without_file_entry_is_an_error() -> None: + cost_map: Final = {"example/model": {"litellm_provider": "missing_provider", "mode": "chat"}} + current_json: Final = _current_json({"example": _provider()}) + + with pytest.raises(ValueError, match="missing_provider"): + generate_provider_endpoints_support(cost_map, current_json) + + +def test_check_reports_stale_output_without_writing(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + cost_map: Final = {"example/model": {"litellm_provider": "example", "mode": "chat"}} + current_json: Final = _current_json({"example": _provider()}) + (tmp_path / "model_prices_and_context_window.json").write_text(json.dumps(cost_map), encoding="utf-8") + source_path: Final = tmp_path / "provider_endpoints_support.json" + source_content: Final = json.dumps(current_json) + source_path.write_text(source_content, encoding="utf-8") + + result: Final = main(["--check"], repo_root=tmp_path) + + assert result == 1 + assert source_path.read_text(encoding="utf-8") == source_content + assert not (tmp_path / "litellm/provider_endpoints_support_backup.json").exists() + error_output: Final = capsys.readouterr().err + assert "provider endpoint support is stale" in error_output + assert "scripts/generate_provider_endpoints_support.py" in error_output + + +def test_check_reports_stale_backup_without_writing(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + cost_map: Final = {"example/model": {"litellm_provider": "example", "mode": "chat"}} + current_json: Final = _current_json({"example": _provider()}) + generated: Final = generate_provider_endpoints_support(cost_map, current_json) + generated_content: Final = json.dumps(generated, indent=2, ensure_ascii=False) + "\n" + source_path: Final = tmp_path / "provider_endpoints_support.json" + backup_path: Final = tmp_path / "litellm/provider_endpoints_support_backup.json" + backup_content: Final = "stale\n" + (tmp_path / "model_prices_and_context_window.json").write_text(json.dumps(cost_map), encoding="utf-8") + source_path.write_text(generated_content, encoding="utf-8") + backup_path.parent.mkdir() + backup_path.write_text(backup_content, encoding="utf-8") + + result: Final = main(["--check"], repo_root=tmp_path) + + assert result == 1 + assert source_path.read_text(encoding="utf-8") == generated_content + assert backup_path.read_text(encoding="utf-8") == backup_content + assert str(backup_path) in capsys.readouterr().err diff --git a/tests/unit/llms/edenai/chat/test_edenai_chat_transformation.py b/tests/unit/llms/edenai/chat/test_edenai_chat_transformation.py index 4f1e2c11a51..739ad144277 100644 --- a/tests/unit/llms/edenai/chat/test_edenai_chat_transformation.py +++ b/tests/unit/llms/edenai/chat/test_edenai_chat_transformation.py @@ -440,14 +440,3 @@ class TestDashboardRegistration: entry = json.loads(matrix_path.read_text())["providers"]["edenai"] assert entry["url"] == "https://docs.litellm.ai/docs/providers/edenai" - served = {name for name, flag in entry["endpoints"].items() if flag} - assert served == { - "chat_completions", - "messages", - "responses", - "embeddings", - "image_generations", - "audio_transcriptions", - "audio_speech", - "video_generations", - } diff --git a/tests/unit/llms/openai_like/test_cortecs_provider.py b/tests/unit/llms/openai_like/test_cortecs_provider.py index 142bb1b7588..c5a6160784a 100644 --- a/tests/unit/llms/openai_like/test_cortecs_provider.py +++ b/tests/unit/llms/openai_like/test_cortecs_provider.py @@ -58,26 +58,6 @@ def test_cortecs_is_available_in_add_model_form(): } -def test_cortecs_supported_endpoints(): - matrix_path = Path(litellm.__file__).parent / "provider_endpoints_support_backup.json" - providers = json.loads(matrix_path.read_text())["providers"] - - assert providers["cortecs"]["endpoints"] == { - "chat_completions": True, - "messages": True, - "responses": True, - "embeddings": False, - "image_generations": False, - "audio_transcriptions": False, - "audio_speech": False, - "moderations": False, - "batches": False, - "rerank": False, - "a2a": False, - "interactions": False, - } - - def test_cortecs_chat_completion_request(): with respx.mock() as upstream: route: Final = upstream.post("https://api.cortecs.ai/v1/chat/completions").respond( diff --git a/tests/unit/llms/openai_like/test_libertai_provider.py b/tests/unit/llms/openai_like/test_libertai_provider.py index c17eaf7c87f..7553efb1341 100644 --- a/tests/unit/llms/openai_like/test_libertai_provider.py +++ b/tests/unit/llms/openai_like/test_libertai_provider.py @@ -91,10 +91,3 @@ class TestLibertAIProviderConfig: matrix = json.loads(backup_path.read_text()) assert "libertai" in matrix["providers"] - endpoints = matrix["providers"]["libertai"]["endpoints"] - assert endpoints["chat_completions"] is True - # embeddings is advertised false: the JSON-configured-provider path only - # wires chat routing (the OpenAILike embedding handler is reached solely - # for the literal openai_like/llamafile/lm_studio providers), matching - # the llamagate precedent. bge-m3 stays in the cost map for metadata. - assert endpoints["embeddings"] is False diff --git a/tests/unit/llms/openai_like/test_prism_provider.py b/tests/unit/llms/openai_like/test_prism_provider.py index c1775c63dc9..9bf5cdbbbb4 100644 --- a/tests/unit/llms/openai_like/test_prism_provider.py +++ b/tests/unit/llms/openai_like/test_prism_provider.py @@ -99,25 +99,6 @@ def test_prism_is_available_in_add_model_form(): } -def test_prism_supported_endpoints(): - matrix_path = Path(litellm.__file__).parent / "provider_endpoints_support_backup.json" - providers = json.loads(matrix_path.read_text())["providers"] - - assert providers["prism"]["endpoints"] == { - "chat_completions": True, - "messages": True, - "responses": True, - "embeddings": False, - "image_generations": False, - "audio_transcriptions": False, - "audio_speech": False, - "moderations": False, - "batches": False, - "rerank": False, - "a2a": False, - } - - def test_prism_responses_request(): with respx.mock() as upstream: route: Final = upstream.post("https://api.prisminference.com/v1/responses").respond( diff --git a/tests/unit/llms/openai_like/test_reka_provider.py b/tests/unit/llms/openai_like/test_reka_provider.py index 3b482f23641..5ba7a2f6680 100644 --- a/tests/unit/llms/openai_like/test_reka_provider.py +++ b/tests/unit/llms/openai_like/test_reka_provider.py @@ -115,28 +115,6 @@ def test_reka_is_available_in_add_model_form(): } -def test_reka_supported_endpoints(): - expected: Final = { - "chat_completions": True, - "messages": True, - "responses": True, - "embeddings": False, - "image_generations": False, - "audio_transcriptions": False, - "audio_speech": False, - "moderations": False, - "batches": False, - "rerank": False, - "a2a": False, - "interactions": False, - } - backup_path = Path(litellm.__file__).parent / "provider_endpoints_support_backup.json" - root_path = Path(litellm.__file__).parent.parent / "provider_endpoints_support.json" - - assert json.loads(backup_path.read_text())["providers"]["reka"]["endpoints"] == expected - assert json.loads(root_path.read_text())["providers"]["reka"]["endpoints"] == expected - - def test_reka_chat_completion_request(): with respx.mock() as upstream: route: Final = upstream.post("https://api.reka.ai/v1/chat/completions").respond(200, json=_CHAT_COMPLETION) From 0b66a27528354d9a85edb6bd1136935d1a94dc6a Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 27 Oct 2026 12:10:45 -0700 Subject: [PATCH 007/102] feat(bedrock): price native guardrailConfig trace usage into spend and x-litellm-response-cost (#48278) * feat(bedrock): price native guardrailConfig trace usage into spend and x-litellm-response-cost Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): bill native Bedrock guardrail trace usage on chat, responses and pass-through Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(streaming): record the Bedrock guardrail trace of a streamed metadata chunk on the logging object Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(bedrock): apply guardrailConfig on /v1/messages and price its trace, record the stream trace on both builder paths, redact the native guardrail_response Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(bedrock): validate the guardrail trace with one pydantic model Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): record the native guardrail entry when litellm_params metadata is None Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(bedrock): import the renamed messages invoke transformation module Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): skip native guardrail billing on cache hits and redact the trace with message logging off Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): walk the guardrail trace iteratively when redacting it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(bedrock): record the native guardrail trace where the Bedrock response is parsed Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(bedrock): type the native guardrail trace fixtures and locals Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 30 ++ .../llm_cost_calc/guardrail_cost.py | 170 ++++++- litellm/litellm_core_utils/redact_messages.py | 49 +- .../streaming_chunk_builder_utils.py | 40 ++ .../litellm_core_utils/streaming_handler.py | 4 + .../bedrock/chat/converse_transformation.py | 2 + litellm/llms/bedrock/chat/invoke_handler.py | 17 +- .../anthropic_claude_transformation.py | 10 +- litellm/llms/bedrock/common_utils.py | 2 + .../anthropic_claude_transformation.py | 9 +- litellm/llms/custom_httpx/llm_http_handler.py | 1 + litellm/proxy/common_request_processing.py | 28 +- .../anthropic_passthrough_logging_handler.py | 14 + .../handler.py | 1 - ...st_bedrock_native_guardrail_trace_spend.py | 423 ++++++++++++++++++ .../llm_cost_calc/test_guardrail_cost.py | 119 +++++ .../test_litellm_logging.py | 51 +++ .../test_redact_messages.py | 38 +- .../test_streaming_chunk_builder_utils.py | 206 ++++----- .../llms/bedrock/chat/test_invoke_handler.py | 35 +- .../test_anthropic_claude_transformation.py | 29 ++ ...t_anthropic_passthrough_logging_handler.py | 83 +++- 22 files changed, 1227 insertions(+), 134 deletions(-) create mode 100644 tests/integration/providers/test_bedrock_native_guardrail_trace_spend.py diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9e0461e951d..55f0f5d49f6 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -97,8 +97,13 @@ from litellm.litellm_core_utils.internal_call_metadata import ( is_unbilled_non_inference_call, ) from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import ( + bedrock_guardrail_trace, cost_breakdown_with_guardrail, guardrail_information_cost, + is_str_keyed_dict, + native_bedrock_guardrail_information, + object_mapping, + with_guardrail_entry, ) from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, @@ -2725,6 +2730,31 @@ class Logging(LiteLLMLoggingBaseClass): return self.model_call_details["response_headers"] = headers + def record_native_bedrock_guardrail(self, result: object) -> None: + """Bill the guardrail trace a Bedrock response carries when the deployment's ``guardrailConfig`` ran it, by + recording it as guardrail information on this call, the same entry the proxy-side Bedrock guardrail writes. + A cache hit carries the original response's trace but made no Bedrock call, so it bills nothing.""" + trace: Final = bedrock_guardrail_trace(result) + if trace is None or served_from_cache(result) or self.model_call_details.get("cache_hit") is True: + return + optional_params: Final = object_mapping(getattr(self, "optional_params", None)) or {} + guardrail_config: Final = object_mapping(optional_params.get("guardrailConfig")) or {} + identifier: Final = guardrail_config.get("guardrailIdentifier") + litellm_params_in: Final = object_mapping(getattr(self, "litellm_params", None)) or {} + region: Final = optional_params.get("aws_region_name") or litellm_params_in.get("aws_region_name") + entry: Final = native_bedrock_guardrail_information( + trace, + aws_region_name=region if isinstance(region, str) else None, + guardrail_identifier=identifier if isinstance(identifier, str) else None, + ) + litellm_params: Final[object] = self.model_call_details.get("litellm_params") + if entry is None or not is_str_keyed_dict(litellm_params): + return + metadata: Final[object] = litellm_params.get("metadata") + bucket: Final = metadata if is_str_keyed_dict(metadata) else {} + litellm_params["metadata"] = bucket + with_guardrail_entry(bucket, entry) + def _merge_hidden_params_from_response_into_metadata(self, logging_result: object) -> None: """ Copy response._hidden_params into litellm_params.metadata['hidden_params']. diff --git a/litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py b/litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py index 61ec2fc39c1..e4ad36f50bc 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py +++ b/litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py @@ -1,15 +1,24 @@ import math -from collections.abc import Mapping +from collections.abc import Iterator, Mapping, Sequence +from functools import partial +from itertools import chain from typing import Annotated, Final -from pydantic import ConfigDict, Field, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError +from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13 import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import ( + redact_nested_match_and_regex_keys, # pyright: ignore[reportUnknownVariableType] # signature uses bare dict +) +from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.utils import CostBreakdown +from litellm.types.utils import CostBreakdown, GuardrailStatus, StandardLoggingGuardrailInformation BEDROCK_GUARDRAIL_PRICING_KEY: Final = "bedrock/guardrails" +BEDROCK_TRACE_FIELDS: Final = ("trace", "amazon-bedrock-trace") +_INTERVENING_ACTIONS: Final = frozenset({"BLOCKED", "ANONYMIZED"}) class GuardrailPricing(LiteLLMBaseModel): @@ -96,6 +105,161 @@ def bedrock_guardrail_cost(usage_units: Mapping[str, int], aws_region_name: str return guardrail_cost_total(bedrock_guardrail_cost_by_unit(usage_units, aws_region_name)) +_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_OBJECT_ITEMS: Final = TypeAdapter(tuple[object, ...]) + + +def object_mapping(value: object) -> Mapping[str, object] | None: + """``value`` as a str-keyed mapping, None for anything else. Vendor JSON arrives untyped, so this is the one place + that turns it into something the rest of the module can read without guessing.""" + try: + return _OBJECT_MAPPING.validate_python(value) + except ValidationError: + return None + + +def object_items(value: object) -> tuple[object, ...]: + if isinstance(value, (str, bytes, Mapping)): + return () + try: + return _OBJECT_ITEMS.validate_python(value) + except ValidationError: + return () + + +def is_str_keyed_dict( + value: object, +) -> TypeIs[dict[str, object]]: # guard-ok: JSON object keys are str # mutable-ok: the caller edits this dict in place + return isinstance(value, dict) + + +def _trace_field(response: object, field: str) -> Mapping[str, object] | None: + source: Final = object_mapping(response) + trace: Final = object_mapping(source.get(field) if source is not None else getattr(response, field, None)) + if trace is None or object_mapping(trace.get("guardrail")) is None: + return None + return trace + + +def bedrock_guardrail_trace(response: object) -> Mapping[str, object] | None: + """The guardrail trace a Bedrock response carries: Converse puts it under ``trace``, Invoke under + ``amazon-bedrock-trace``. Returned as is, or None when the response has no guardrail section.""" + traces: Final = map(partial(_trace_field, response), BEDROCK_TRACE_FIELDS) + return next((trace for trace in traces if trace is not None), None) + + +class _Assessment(BaseModel): + model_config = ConfigDict(frozen=True, extra="allow") + invocation_metrics: Mapping[str, JsonValue] = Field(default_factory=dict, alias="invocationMetrics") + + @property + def billable_usage(self) -> Mapping[str, int]: + usage: Final = self.invocation_metrics.get("usage") + if not isinstance(usage, dict): + return {} + return { + counter: units + for counter, units in usage.items() + if counter.endswith("Units") and isinstance(units, int) and not isinstance(units, bool) + } + + +class _GuardrailTrace(BaseModel): + """Both trace shapes: Converse ``inputAssessment`` {id: a} and ``outputAssessments`` {id: [a, ...]}, Invoke + ``input`` {id: a} and ``outputs`` [{id: a}, ...].""" + + model_config = ConfigDict(frozen=True, extra="allow") + input_assessment: Mapping[str, _Assessment] = Field(default_factory=dict, alias="inputAssessment") + input: Mapping[str, _Assessment] = Field(default_factory=dict) + output_assessments: Mapping[str, tuple[_Assessment, ...]] = Field(default_factory=dict, alias="outputAssessments") + outputs: tuple[Mapping[str, _Assessment], ...] = () + + @property + def assessments(self) -> tuple[tuple[str, _Assessment], ...]: + listed: Final = ( + (gid, a) for gid, items in self.output_assessments.items() for a in items + ) # comprehension-ok: id then its list + return ( + *self.input_assessment.items(), + *self.input.items(), + *listed, + *chain.from_iterable(by_id.items() for by_id in self.outputs), + ) + + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def _summed_usage(usages: Sequence[Mapping[str, int]]) -> Mapping[str, int]: + counters: Final = tuple(dict.fromkeys(chain.from_iterable(usages))) + return {counter: sum(usage.get(counter, 0) for usage in usages) for counter in counters} + + +def _actions(node: JsonValue) -> Iterator[str]: + if isinstance(node, list): + yield from chain.from_iterable(map(_actions, node)) + return + if not isinstance(node, dict): + return + action: Final = node.get("action") + if isinstance(action, str): + yield action + yield from chain.from_iterable(map(_actions, node.values())) + + +def native_bedrock_guardrail_information( + trace: Mapping[str, object], aws_region_name: str | None, guardrail_identifier: str | None +) -> StandardLoggingGuardrailInformation | None: + """The guardrail entry for a trace Bedrock returned because the deployment's ``guardrailConfig`` ran with + ``trace: enabled``: every assessment's ``invocationMetrics.usage`` summed and priced like the proxy-side Bedrock + guardrail. None when the trace carries no usage, which is what ``trace: disabled`` leaves.""" + try: + guardrail: Final = _JSON_OBJECT.validate_python(trace.get("guardrail")) + pairs: Final = _GuardrailTrace.model_validate(guardrail).assessments + except ValidationError: + return None + usage: Final = _summed_usage(tuple(assessment.billable_usage for _, assessment in pairs)) + if not usage: + return None + cost_by_unit: Final = bedrock_guardrail_cost_by_unit(usage, aws_region_name) + name: Final = guardrail_identifier or pairs[0][0] + status: Final[GuardrailStatus] = ( + "guardrail_intervened" if any(action in _INTERVENING_ACTIONS for action in _actions(guardrail)) else "success" + ) + priced: Final[StandardLoggingGuardrailInformation] = ( + {"guardrail_cost_by_unit": cost_by_unit} if cost_by_unit is not None else {} + ) + return { + "guardrail_name": name, + "guardrail_id": name, + "guardrail_provider": "bedrock", + "guardrail_mode": GuardrailEventHooks.during_call, + "guardrail_response": redact_nested_match_and_regex_keys(guardrail), + "guardrail_status": status, + "guardrail_usage": usage, + "guardrail_cost": guardrail_cost_total(cost_by_unit), + **priced, + } + + +def guardrail_entries(raw: object) -> tuple[object, ...]: + """The entries under ``standard_logging_guardrail_information``, which older writers stored as one dict.""" + single: Final = object_mapping(raw) + return (single,) if single is not None else object_items(raw) + + +def with_guardrail_entry( + metadata: dict[str, object], # mutable-ok: the proxy's own request metadata, read later for headers and spend + entry: StandardLoggingGuardrailInformation, +) -> None: + """Append ``entry`` to the metadata's ``standard_logging_guardrail_information`` once, so the same trace recorded + from the response headers path and again from the success handler bills a single time.""" + existing: Final = guardrail_entries(metadata.get("standard_logging_guardrail_information")) + if entry in existing: + return + metadata["standard_logging_guardrail_information"] = [*existing, entry] # rebind-ok: appending here is the job + + AZURE_PROMPT_SHIELD_TEXT_RECORD_UNIT: Final = "text_records" diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 7f8a38cc8ec..548d088d833 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -14,7 +14,7 @@ from collections.abc import Mapping, Sequence from dataclasses import replace from typing import TYPE_CHECKING, Any, Final -from pydantic import TypeAdapter +from pydantic import JsonValue, TypeAdapter, ValidationError import litellm from litellm.constants import REDACTED_BY_LITELLM @@ -120,11 +120,55 @@ def _redacted_streaming_response_copy(streaming_response: object) -> object: return _redact_streaming_response(redacted_response) +BEDROCK_GUARDRAIL_TRACE_FIELDS: Final = ("trace", "amazon-bedrock-trace") +_MATCHED_TEXT_KEYS: Final = frozenset({"match", "regex"}) +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def _without_matched_text(trace: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]: + redacted: Final = dict(copy.deepcopy(trace)) + stack: Final[list[JsonValue]] = [redacted] # mutable-ok: iterative walk, recursive_detector bans recursion here + while stack: + node = stack.pop() + if isinstance(node, dict): + for key in _MATCHED_TEXT_KEYS & node.keys(): + node[key] = "[REDACTED]" + stack.extend(node.values()) + elif isinstance(node, list): + stack.extend(node) + return redacted + + +def _redacted_bedrock_trace(trace: object) -> object: + try: + return _without_matched_text(_JSON_OBJECT.validate_python(trace)) + except ValidationError: + return trace + + +def _redact_bedrock_guardrail_trace(response: object) -> None: + """A Bedrock guardrail trace echoes the matched text of its sensitive-information and word findings back.""" + mapping: Final = _as_mapping(response) + if isinstance(response, dict): + traces: Final = { + field: _redacted_bedrock_trace(value) + for field, value in (mapping or {}).items() + if field in BEDROCK_GUARDRAIL_TRACE_FIELDS + } + response.update(traces) + return + for field, value in ((name, getattr(response, name, None)) for name in BEDROCK_GUARDRAIL_TRACE_FIELDS): + if value is not None: + setattr(response, field, _redacted_bedrock_trace(value)) + + def _redact_streaming_response(streaming_response): if hasattr(streaming_response, "choices"): for choice in streaming_response.choices: _redact_choice_content(choice) redact_vertex_ai_metadata_from_logged_object(streaming_response) + if isinstance(streaming_response, litellm.ModelResponse): + _redact_bedrock_guardrail_trace(streaming_response) elif isinstance(streaming_response, litellm.ResponsesAPIResponse): return perform_redaction({}, streaming_response) elif hasattr(streaming_response, "output"): @@ -351,6 +395,7 @@ def _redact_standard_logging_object( if isinstance(response.get("choices"), list): _redact_model_response_dict_choices(response["choices"], redacted_str) redact_vertex_ai_metadata_from_logged_object(response) + _redact_bedrock_guardrail_trace(response) elif isinstance(response, str): standard_logging_object["response"] = redacted_str else: @@ -523,11 +568,13 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons for choice in _result.choices: _redact_choice_content(choice) redact_vertex_ai_metadata_from_logged_object(_result) + _redact_bedrock_guardrail_trace(_result) elif isinstance(_result, dict) and "choices" in _result: # Handle dict representation of ModelResponse (e.g., from model_dump()) if _result.get("choices") is not None: _redact_model_response_dict_choices(_result["choices"], REDACTED_BY_LITELLM) redact_vertex_ai_metadata_from_logged_object(_result) + _redact_bedrock_guardrail_trace(_result) elif isinstance(_result, dict) and "output" in _result: return _redacted_responses_api_response(_result, redact_instructions=redact_streaming_responses) elif isinstance(_result, litellm.ResponsesAPIResponse): diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 68cdc204af4..8d243e572fa 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -14,6 +14,7 @@ import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params +from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict ) @@ -1484,6 +1485,35 @@ def _simple_text_part(choices: Sequence[object]) -> str | None: return content if isinstance(content, str) else "" +_NO_TRACE_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _carry_bedrock_guardrail_trace(chunks: Sequence[object], response: ModelResponse) -> None: + """Bedrock reports a guardrail trace once per stream, on the final metadata chunk, so the built response carries it + under the same field as the non-streaming response (``trace`` on Converse, ``amazon-bedrock-trace`` on Invoke).""" + traced: Final = map(_bedrock_trace_fields, reversed(chunks)) + carried: Final = next((fields for fields in traced if fields), _NO_TRACE_FIELDS) + for field, trace in carried.items(): + setattr(response, field, trace) + + +def _record_bedrock_guardrail_trace( + chunks: Sequence[object], response: ModelResponse, logging_obj: Optional["Logging"] +) -> None: + _carry_bedrock_guardrail_trace(chunks, response) + if logging_obj is not None: + logging_obj.record_native_bedrock_guardrail(response) + + +def _bedrock_trace_fields(chunk: object) -> Mapping[str, object]: + as_mapping: Final = object_mapping(chunk) + raw: Final = ( + as_mapping.get("provider_specific_fields") if as_mapping else getattr(chunk, "provider_specific_fields", None) + ) + fields: Final[Mapping[str, object]] = object_mapping(raw) or {} + return {field: fields[field] for field in BEDROCK_TRACE_FIELDS if field in fields} + + def stream_chunk_builder( chunks: list, messages: Sequence | None = None, @@ -1565,6 +1595,11 @@ def stream_chunk_builder( ) break + _record_bedrock_guardrail_trace( + object_items(chunks), # pyright: ignore[reportUnknownArgumentType] # chunks is an untyped list param + response, + logging_obj, + ) _stamp_streaming_usage_cost(usage, response, logging_obj) _set_stream_builder_response_cost( response, @@ -1760,6 +1795,11 @@ def stream_chunk_builder( chunks, # pyright: ignore[reportUnknownArgumentType] # legacy chunk list is untyped ) + _record_bedrock_guardrail_trace( + object_items(chunks), # pyright: ignore[reportUnknownArgumentType] # chunks is an untyped list param + response, + logging_obj, + ) processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj) return response except Exception as e: diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 2df470edc59..8fb602ebeb0 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1610,6 +1610,8 @@ class CustomStreamWrapper: return _ProviderChunkParsed(response_obj) def chunk_creator(self, chunk: Any): + if isinstance(chunk, ModelResponseStream) and chunk.provider_specific_fields: + self.logging_obj.record_native_bedrock_guardrail(chunk.provider_specific_fields) if self.custom_llm_provider != "cached_response" and isinstance( chunk, (ChatCompletionChunk, ModelResponseStream) ): @@ -1717,6 +1719,8 @@ class CustomStreamWrapper: model_response.usage = Usage(**chunk_usage.model_dump()) ## RETURN ARG + if isinstance(chunk, ModelResponseStream) and chunk.provider_specific_fields: + self.copy_model_response_level_provider_specific_fields(chunk, model_response) result: Final = self.return_processed_chunk_logic( completion_obj=completion_obj, model_response=model_response, diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 0bc07ac6b22..f54c352fd6e 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -2859,6 +2859,8 @@ class AmazonConverseConfig(BaseConfig): # Add "trace" from Bedrock guardrails - if user has opted in to returning it if "trace" in completion_response: setattr(model_response, "trace", completion_response["trace"]) + if logging_obj is not None: + logging_obj.record_native_bedrock_guardrail(model_response) # Add service_tier if present in Bedrock response # Map Bedrock serviceTier (object) to OpenAI service_tier (string) diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 3d4821fcaa2..a6b34010297 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -13,6 +13,7 @@ from litellm.caching.caching import InMemoryCache from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers from litellm.litellm_core_utils.optional_imports import ensure_optional_import from litellm.litellm_core_utils.prompt_templates.common_utils import native_redacted_thinking_data @@ -22,6 +23,7 @@ from litellm.llms.anthropic.chat.handler import ( from litellm.llms.anthropic.chat.transformation import JsonModeRequest from litellm.llms.base_llm import base_model_iterator from litellm.llms.base_llm.base_utils import ResponseFormatSchema +from litellm.llms.bedrock.common_utils import BEDROCK_INVOKE_TRACE_FIELD from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -962,8 +964,19 @@ class AmazonAnthropicClaudeStreamDecoder(AWSEventStreamDecoder): response_format_schema=response_format_schema, ) - def _chunk_parser(self, chunk_data: dict) -> ModelResponseStream: - return self.anthropic_model_response_iterator.chunk_parser(chunk=chunk_data) + def _chunk_parser(self, chunk_data: dict[str, object]) -> ModelResponseStream: + parsed: Final = self.anthropic_model_response_iterator.chunk_parser(chunk=chunk_data) + trace: Final = object_mapping(chunk_data.get(BEDROCK_INVOKE_TRACE_FIELD)) + if trace is None: + return parsed + return parsed.model_copy( + update={ + "provider_specific_fields": { + **(parsed.provider_specific_fields or {}), + BEDROCK_INVOKE_TRACE_FIELD: trace, + } + } + ) def _stream_end_chunk(self) -> ModelResponseStream | None: return self.anthropic_model_response_iterator.cut_stream_chunk() diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py index 240a85ce327..eb6ed5c75e3 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py @@ -5,6 +5,7 @@ import httpx from pydantic import TypeAdapter from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers +from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_anthropic_image_obj, ) @@ -22,6 +23,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation AmazonInvokeConfig, ) from litellm.llms.bedrock.common_utils import ( + BEDROCK_INVOKE_TRACE_FIELD, apply_bedrock_invoke_structured_output, bedrock_chat_uses_native_structured_output, bedrock_cleaned_output_schema, @@ -406,7 +408,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: - return AnthropicConfig.transform_response( + response: Final = AnthropicConfig.transform_response( self, model=model, raw_response=raw_response, @@ -420,3 +422,9 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): api_key=api_key, json_mode=json_mode, ) + body: Final = object_mapping(raw_response.json()) or {} + trace: Final = object_mapping(body.get(BEDROCK_INVOKE_TRACE_FIELD)) + if trace is not None: + setattr(response, BEDROCK_INVOKE_TRACE_FIELD, trace) + logging_obj.record_native_bedrock_guardrail(response) + return response diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 2b946d9664c..28888f1a23a 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -45,6 +45,8 @@ if TYPE_CHECKING: from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.types.llms.openai import AllMessageValues +BEDROCK_INVOKE_TRACE_FIELD: Final = "amazon-bedrock-trace" + _ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs" _OPENAI_FAMILY_MODEL_RE: Final = re.compile(r"(^|[./])openai\.") diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py index 008b977f485..e1e15861a90 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py @@ -33,6 +33,7 @@ from litellm.llms.base_llm.anthropic_messages.transformation import ( from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, + bedrock_invoke_guardrail_headers, ) from litellm.llms.bedrock.common_utils import ( BedrockError, @@ -151,8 +152,14 @@ class AmazonAnthropicClaudeMessagesConfig( api_key: str | None = None, api_base: str | None = None, ) -> tuple[dict, str | None]: + raw_guardrail_config: Final = litellm_params.get("guardrailConfig") # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # litellm_params is an untyped dict param + guardrail_headers: Final = ( + () + if raw_guardrail_config is None + else tuple(bedrock_invoke_guardrail_headers(raw_guardrail_config).items()) # pyright: ignore[reportUnknownArgumentType] # read from the untyped dict param + ) owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params) - return merge_bedrock_invoke_headers(headers, (), metadata_headers, owned_names), api_base + return merge_bedrock_invoke_headers(headers, guardrail_headers, metadata_headers, owned_names), api_base # pyright: ignore[reportUnknownArgumentType] # headers is an untyped dict param def sign_request( self, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 91ac5a0ca9a..ef009e70137 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2705,6 +2705,7 @@ class BaseLLMHTTPHandler: raw_response=response, logging_obj=logging_obj, ) + logging_obj.record_native_bedrock_guardrail(initial_response) return await self._finalize_anthropic_messages_response( initial_response=initial_response, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 8e681c615ec..3b9c17baa02 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -6,6 +6,7 @@ import math from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Mapping, Sequence from datetime import datetime from functools import lru_cache +from itertools import chain from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -69,7 +70,11 @@ from litellm.litellm_core_utils.get_supported_openai_params import ( from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost +from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import ( + guardrail_entries, + guardrail_information_cost, + object_mapping, +) from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, ) @@ -306,6 +311,23 @@ async def close_guarded_stream(stream: object) -> None: ) +def _guardrail_entries_for_headers( + request_metadata_bucket: object, logging_obj: LiteLLMLoggingObj +) -> tuple[object, ...]: + """Guardrail entries the proxy hooks wrote on the request plus the ones the call recorded on its own logging + object (a Bedrock deployment's native guardrail trace), each once: the hooks copy theirs into the logging object.""" + litellm_params: Final = object_mapping(logging_obj.model_call_details.get("litellm_params")) or {} + buckets: Final = (object_mapping(request_metadata_bucket), object_mapping(litellm_params.get("metadata"))) + entries: Final = tuple( + chain.from_iterable( + guardrail_entries(metadata.get("standard_logging_guardrail_information")) + for metadata in buckets + if metadata is not None + ) + ) + return tuple(entry for index, entry in enumerate(entries) if entry not in entries[:index]) + + def resolve_litellm_call_id(client_call_id: str | None) -> str: if client_call_id is not None and 0 < len(client_call_id) <= MAX_LITELLM_CALL_ID_LENGTH: return client_call_id @@ -3238,9 +3260,9 @@ class ProxyBaseLLMRequestProcessing: if is_unbilled_non_inference_call_from_params(logging_obj.call_type, logging_obj.litellm_params, response) else computed_cost_for_headers ) - _, request_metadata_bucket = get_or_create_metadata_bucket(self.data) + request_metadata_bucket: Final[object] = get_or_create_metadata_bucket(self.data)[1] # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # self.data is an untyped dict guardrail_cost_for_headers: Final = guardrail_information_cost( - request_metadata_bucket.get("standard_logging_guardrail_information") + _guardrail_entries_for_headers(request_metadata_bucket, logging_obj) # pyright: ignore[reportUnknownArgumentType] # bucket comes from the untyped request dict ) response_cost_for_headers: Final = ( (llm_cost_for_headers if isinstance(llm_cost_for_headers, (int, float)) else 0.0) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index c7aa530434a..7c7eb51bc5d 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -28,6 +28,7 @@ from litellm.llms.anthropic.messages.stream_assembly import ( collapse_pure_text_chunks, split_sse_chunk_into_events, ) +from litellm.llms.bedrock.common_utils import BEDROCK_INVOKE_TRACE_FIELD from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.proxy.auth.auth_utils import get_end_user_id_from_request_body from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import ( @@ -70,6 +71,17 @@ def _event_payload(event: str) -> Mapping[str, object] | None: return None +def _bedrock_trace_payload(all_chunks: Sequence[str | bytes]) -> Mapping[str, object] | None: + payloads: Final = ( + _event_payload(event) + for chunk in reversed(all_chunks) + for event in reversed(split_sse_chunk_into_events(chunk)) + ) # comprehension-ok: chunk then event, read newest first + return next( + (payload for payload in payloads if payload is not None and BEDROCK_INVOKE_TRACE_FIELD in payload), None + ) + + def _message_delta_usages(all_chunks: Sequence[str | bytes]) -> Iterator[object]: for chunk in reversed(all_chunks): yield from ( @@ -141,6 +153,7 @@ class AnthropicPassthroughLoggingHandler: json_mode=False, litellm_params={}, ) + logging_obj.record_native_bedrock_guardrail(response_body) # pyright: ignore[reportUnknownArgumentType] # response_body is an untyped dict param kwargs = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( litellm_model_response=litellm_model_response, @@ -546,6 +559,7 @@ class AnthropicPassthroughLoggingHandler: "result": None, "kwargs": {}, } + litellm_logging_obj.record_native_bedrock_guardrail(_bedrock_trace_payload(all_chunks)) kwargs: Final = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( litellm_model_response=complete_streaming_response, model=model, diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index c9d800421f7..ea3bda1b8b5 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -133,7 +133,6 @@ class LiteLLMCompletionTransformationHandler: litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion( **acompletion_args, ) - if isinstance(litellm_completion_response, ModelResponse): responses_api_response: Final[ResponsesAPIResponse] = ( LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( diff --git a/tests/integration/providers/test_bedrock_native_guardrail_trace_spend.py b/tests/integration/providers/test_bedrock_native_guardrail_trace_spend.py new file mode 100644 index 00000000000..b1f2d5a9181 --- /dev/null +++ b/tests/integration/providers/test_bedrock_native_guardrail_trace_spend.py @@ -0,0 +1,423 @@ +"""A Bedrock deployment whose ``guardrailConfig`` has ``trace: enabled`` bills the policy units Bedrock reports in the +trace as guardrail cost, on every surface the deployment serves: ``/v1/chat/completions`` and ``/v1/responses`` over +Converse and Invoke, streamed and not, and the four ``/bedrock/model/{model}`` pass-through actions. The cost rides +``x-litellm-response-cost`` on non-streaming responses and the spend row's ``spend`` and ``guardrail_information`` +everywhere. ``trace: disabled`` leaves no usage, so nothing is billed beyond the tokens (LIT-10101)""" + +import base64 +import json +import uuid +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path +from typing import Final + +import boto3 +import pytest +import yaml +from botocore.config import Config +from integration._support.budget import case_timeout_seconds +from integration._support.client import Gateway, Scenario, eventually +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.upstream import _aws_event_frame +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +import litellm + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 240) + +_MODEL_ID: Final = "anthropic.claude-sonnet-4-5-20250929-v1:0" +_EVENT_STREAM: Final = "application/vnd.amazon.eventstream" +_GUARDRAIL_ID: Final = "native-guardrail" +_INPUT_COST_PER_TOKEN: Final = 0.001 +_OUTPUT_COST_PER_TOKEN: Final = 0.002 +_PROMPT_TOKENS: Final = 5 +_COMPLETION_TOKENS: Final = 1 +_INPUT_USAGE: Final[Mapping[str, int]] = {"topicPolicyUnits": 1, "contentPolicyUnits": 1, "wordPolicyUnits": 1} +_OUTPUT_USAGE: Final[Mapping[str, int]] = {"topicPolicyUnits": 2, "sensitiveInformationPolicyUnits": 2} +_SUMMED_USAGE: Final[Mapping[str, int]] = { + "topicPolicyUnits": 3, + "contentPolicyUnits": 1, + "wordPolicyUnits": 1, + "sensitiveInformationPolicyUnits": 2, +} +_ACTIONS: Final = ("converse", "converse-stream", "invoke", "invoke-with-response-stream") +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_SPEND_SQL: Final = 'SELECT spend, metadata FROM "LiteLLM_SpendLogs" WHERE api_key=%s ORDER BY "startTime"' + + +def _llm_cost() -> float: + return _PROMPT_TOKENS * _INPUT_COST_PER_TOKEN + _COMPLETION_TOKENS * _OUTPUT_COST_PER_TOKEN + + +def _guardrail_cost() -> float: + """What the proxy's own cost map prices the traced units at, so the test owns the relation, not AWS's rates.""" + rates: Final = litellm.model_cost["bedrock/guardrails"]["guardrail_cost_per_unit"] + return sum(units * rates[counter] for counter, units in _SUMMED_USAGE.items()) + + +def _action(request: Request) -> str: + return request.target.partition("?")[0].rsplit("/", 1)[-1] + + +def _traced(request: Request) -> bool: + if _action(request).startswith("converse"): + config: Final = json.loads(request.body).get("guardrailConfig") or {} + return config.get("trace") == "enabled" + return request.headers.get("x-amzn-bedrock-trace", "").upper() == "ENABLED" + + +def _assessment(usage: Mapping[str, int]) -> dict[str, JsonValue]: + return {"invocationMetrics": {"usage": dict(usage), "guardrailProcessingLatency": 7}} + + +def _converse_trace() -> dict[str, JsonValue]: + return { + "guardrail": { + "inputAssessment": {_GUARDRAIL_ID: _assessment(_INPUT_USAGE)}, + "outputAssessments": {_GUARDRAIL_ID: [_assessment(_OUTPUT_USAGE)]}, + } + } + + +def _invoke_trace() -> dict[str, JsonValue]: + return { + "guardrail": { + "input": {_GUARDRAIL_ID: _assessment(_INPUT_USAGE)}, + "outputs": [{_GUARDRAIL_ID: _assessment(_OUTPUT_USAGE)}], + } + } + + +def _bedrock(request: Request) -> Reply: + """Bedrock's runtime as the caller sees it: a one-token answer, and the guardrail trace with its policy units only + when the request enabled tracing, in the shape of the action that served it.""" + traced: Final = _traced(request) + action: Final = _action(request) + usage: Final = {"inputTokens": _PROMPT_TOKENS, "outputTokens": _COMPLETION_TOKENS, "totalTokens": 6} + converse_trace: Final = {"trace": _converse_trace()} if traced else {} + if action == "converse": + body: Final = { + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": usage, + "metrics": {"latencyMs": 3}, + **converse_trace, + } + return Reply(body=json.dumps(body).encode()) + if action == "converse-stream" and "toolConfig" in json.loads(request.body): + return Reply( + chunks=( + _aws_event_frame("messageStart", {"role": "assistant"}, "sc", "u"), + _aws_event_frame( + "contentBlockStart", + {"start": {"toolUse": {"toolUseId": "tool-1", "name": "lookup"}}, "contentBlockIndex": 0}, + "sc", + "u", + ), + _aws_event_frame( + "contentBlockDelta", + {"delta": {"toolUse": {"input": '{"q": "x"}'}}, "contentBlockIndex": 0}, + "sc", + "u", + ), + _aws_event_frame("contentBlockStop", {"contentBlockIndex": 0}, "sc", "u"), + _aws_event_frame("messageStop", {"stopReason": "tool_use"}, "sc", "u"), + _aws_event_frame( + "metadata", {"usage": usage, "metrics": {"latencyMs": 3}, **converse_trace}, "sc", "u" + ), + ), + content_type=_EVENT_STREAM, + ) + if action == "converse-stream": + return Reply( + chunks=( + _aws_event_frame("messageStart", {"role": "assistant"}, "sc", "u"), + _aws_event_frame("contentBlockDelta", {"delta": {"text": "ok"}, "contentBlockIndex": 0}, "sc", "u"), + _aws_event_frame("contentBlockStop", {"contentBlockIndex": 0}, "sc", "u"), + _aws_event_frame("messageStop", {"stopReason": "end_turn"}, "sc", "u"), + _aws_event_frame( + "metadata", {"usage": usage, "metrics": {"latencyMs": 3}, **converse_trace}, "sc", "u" + ), + ), + content_type=_EVENT_STREAM, + ) + invoke_trace: Final = {"amazon-bedrock-trace": _invoke_trace()} if traced else {} + if action == "invoke": + message: Final = { + "id": f"msg_{uuid.uuid4().hex[:12]}", + "type": "message", + "role": "assistant", + "model": "claude", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": _PROMPT_TOKENS, "output_tokens": _COMPLETION_TOKENS}, + **invoke_trace, + } + return Reply(body=json.dumps(message).encode()) + events: Final = ( + { + "type": "message_start", + "message": { + "id": f"msg_{uuid.uuid4().hex[:12]}", + "type": "message", + "role": "assistant", + "model": "claude", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": _PROMPT_TOKENS, "output_tokens": 0}, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "ok"}}, + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": _COMPLETION_TOKENS}, + }, + { + "type": "message_stop", + "amazon-bedrock-invocationMetrics": { + "inputTokenCount": _PROMPT_TOKENS, + "outputTokenCount": _COMPLETION_TOKENS, + "invocationLatency": 3, + "firstByteLatency": 1, + }, + **invoke_trace, + }, + ) + return Reply( + chunks=tuple( + _aws_event_frame("chunk", {"bytes": base64.b64encode(json.dumps(event).encode()).decode()}, "sc", "u") + for event in events + ), + content_type=_EVENT_STREAM, + ) + + +def _deployment(wire: Wire, model: str, trace: str) -> Mapping[str, JsonValue]: + return { + "model": model, + "api_base": wire.url, + "aws_access_key_id": "AKIASCRIPTEDPROVIDER", + "aws_secret_access_key": "scripted-secret", + "aws_region_name": "us-east-1", + "use_in_pass_through": True, + "input_cost_per_token": _INPUT_COST_PER_TOKEN, + "output_cost_per_token": _OUTPUT_COST_PER_TOKEN, + "guardrailConfig": {"guardrailIdentifier": _GUARDRAIL_ID, "guardrailVersion": "1", "trace": trace}, + } + + +@dataclass(frozen=True, slots=True) +class Rig: + proxy: Gateway + scenario: Scenario + converse_model: str + invoke_model: str + untraced_model: str + + def key(self) -> str: + return self.scenario.key(key_alias=f"native-trace-{uuid.uuid4().hex[:8]}") + + def runtime(self, monkeypatch: pytest.MonkeyPatch, key: str): # noqa: ANN201 # boto3 clients are untyped + monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", key) + return boto3.client( + "bedrock-runtime", + endpoint_url=f"{str(self.proxy.client.base_url).rstrip('/')}/bedrock", + region_name="us-east-1", + config=Config(read_timeout=case_timeout_seconds(), retries={"total_max_attempts": 1}), + ) + + +@pytest.fixture(scope="module") +def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + suffix: Final = uuid.uuid4().hex[:12] + converse_model: Final = f"bedrock-traced-converse-{suffix}" + invoke_model: Final = f"bedrock-traced-invoke-{suffix}" + untraced_model: Final = f"bedrock-untraced-{suffix}" + directory: Final = tmp_path_factory.mktemp("bedrock-native-guardrail-trace") + base_config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + with wire_server(_bedrock) as wire: + config: Final = { + **base_config, + "model_list": [ + *base_config.get("model_list", []), + {"model_name": converse_model, "litellm_params": _deployment(wire, f"bedrock/{_MODEL_ID}", "enabled")}, + { + "model_name": invoke_model, + "litellm_params": _deployment(wire, f"bedrock/invoke/{_MODEL_ID}", "enabled"), + }, + {"model_name": untraced_model, "litellm_params": _deployment(wire, f"bedrock/{_MODEL_ID}", "disabled")}, + ], + } + config_path: Final = directory / "bedrock-native-guardrail-trace.yaml" + config_path.write_text(yaml.safe_dump(config)) + with owned_proxy(session_gateway, directory, {}, config=config_path) as proxy, proxy.scenario() as scenario: + yield Rig( + proxy=proxy, + scenario=scenario, + converse_model=converse_model, + invoke_model=invoke_model, + untraced_model=untraced_model, + ) + + +@dataclass(frozen=True, slots=True) +class SpendRow: + spend: float + guardrail_information: tuple[dict[str, JsonValue], ...] + + +def _spend_row(proxy: Gateway, key: str) -> SpendRow: + digest: Final = sha256(key.encode()).hexdigest() + (row,) = eventually(lambda: proxy.rows(_SPEND_SQL, (digest,)), lambda rows: len(rows) == 1, seconds=70) + metadata: Final = _JSON_OBJECT.validate_python( + json.loads(row["metadata"]) if isinstance(row["metadata"], str) else row["metadata"] + ) + raw_information: Final = metadata.get("guardrail_information") + entries: Final = raw_information if isinstance(raw_information, list) else [raw_information] + return SpendRow( + spend=float(str(row["spend"])), + guardrail_information=tuple(_JSON_OBJECT.validate_python(entry) for entry in entries if entry is not None), + ) + + +def _assert_billed_as_guardrail_cost(row: SpendRow) -> None: + assert row.spend == pytest.approx(_llm_cost() + _guardrail_cost()) + (entry,) = row.guardrail_information + assert entry["guardrail_provider"] == "bedrock" + assert entry["guardrail_name"] == _GUARDRAIL_ID + assert entry["guardrail_usage"] == _SUMMED_USAGE + assert isinstance(entry["guardrail_cost"], float) and entry["guardrail_cost"] == pytest.approx(_guardrail_cost()) + + +def _chat_body(model: str, stream: bool) -> dict[str, JsonValue]: + return {"model": model, "messages": [{"role": "user", "content": "hello"}], "max_tokens": 16, "stream": stream} + + +def _responses_body(model: str, stream: bool) -> dict[str, JsonValue]: + return {"model": model, "input": "hello", "stream": stream} + + +@pytest.mark.parametrize("api", ["converse", "invoke"]) +@pytest.mark.parametrize("endpoint", ["/v1/chat/completions", "/v1/responses"]) +def test_non_streaming_response_prices_the_trace_into_the_cost_header_and_the_spend_row( + rig: Rig, endpoint: str, api: str +) -> None: + key: Final = rig.key() + model: Final = rig.converse_model if api == "converse" else rig.invoke_model + body: Final = _chat_body(model, False) if endpoint == "/v1/chat/completions" else _responses_body(model, False) + response: Final = rig.proxy.request("POST", endpoint, body, key=key) + assert response.is_success, response.text[:400] + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(_llm_cost() + _guardrail_cost()) + _assert_billed_as_guardrail_cost(_spend_row(rig.proxy, key)) + + +@pytest.mark.parametrize("api", ["converse", "invoke"]) +@pytest.mark.parametrize("endpoint", ["/v1/chat/completions", "/v1/responses"]) +def test_streamed_response_prices_the_trace_of_the_final_event_into_the_spend_row( + rig: Rig, endpoint: str, api: str +) -> None: + key: Final = rig.key() + model: Final = rig.converse_model if api == "converse" else rig.invoke_model + body: Final = _chat_body(model, True) if endpoint == "/v1/chat/completions" else _responses_body(model, True) + with rig.proxy.client.stream( + "POST", endpoint, json=body, headers={"Authorization": f"Bearer {key}"}, timeout=case_timeout_seconds() + ) as response: + frames: Final = tuple(line for line in response.iter_lines() if line.startswith("data:")) + assert response.is_success and frames + _assert_billed_as_guardrail_cost(_spend_row(rig.proxy, key)) + + +@pytest.mark.parametrize("action", _ACTIONS) +def test_passthrough_action_prices_the_trace_into_the_spend_row( + rig: Rig, monkeypatch: pytest.MonkeyPatch, action: str +) -> None: + key: Final = rig.key() + runtime: Final = rig.runtime(monkeypatch, key) + messages: Final = [{"role": "user", "content": [{"text": "hello"}]}] + invoke_body: Final = json.dumps( + { + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 16, + "messages": [{"role": "user", "content": "hello"}], + } + ) + if action == "converse": + getattr(runtime, "converse")(modelId=rig.converse_model, messages=messages) + elif action == "converse-stream": + list(getattr(runtime, "converse_stream")(modelId=rig.converse_model, messages=messages)["stream"]) + elif action == "invoke": + getattr(runtime, "invoke_model")(modelId=rig.converse_model, body=invoke_body)["body"].read() + else: + list( + getattr(runtime, "invoke_model_with_response_stream")(modelId=rig.converse_model, body=invoke_body)["body"] + ) + _assert_billed_as_guardrail_cost(_spend_row(rig.proxy, key)) + + +def test_disabled_trace_bills_only_the_tokens(rig: Rig) -> None: + key: Final = rig.key() + response: Final = rig.proxy.request("POST", "/v1/chat/completions", _chat_body(rig.untraced_model, False), key=key) + assert response.is_success, response.text[:400] + assert "trace" not in response.json() + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(_llm_cost()) + row: Final = _spend_row(rig.proxy, key) + assert row.spend == pytest.approx(_llm_cost()) + assert row.guardrail_information == () + + +def _messages_body(model: str, stream: bool) -> dict[str, JsonValue]: + return { + "model": model, + "messages": [{"role": "user", "content": f"hello from the messages api, stream={stream}"}], + "max_tokens": 16, + "stream": stream, + } + + +def test_messages_non_streaming_response_applies_the_guardrail_and_prices_its_trace(rig: Rig) -> None: + key: Final = rig.key() + response: Final = rig.proxy.request("POST", "/v1/messages", _messages_body(rig.invoke_model, False), key=key) + assert response.is_success, response.text[:400] + assert "amazon-bedrock-trace" in response.json() + _assert_billed_as_guardrail_cost(_spend_row(rig.proxy, key)) + + +def test_messages_streamed_response_prices_the_trace_of_the_message_stop_event(rig: Rig) -> None: + key: Final = rig.key() + with rig.proxy.client.stream( + "POST", + "/v1/messages", + json=_messages_body(rig.invoke_model, True), + headers={"Authorization": f"Bearer {key}"}, + timeout=case_timeout_seconds(), + ) as response: + body: Final = b"".join(response.iter_bytes()) + assert response.is_success and b"message_stop" in body, body[:400] + _assert_billed_as_guardrail_cost(_spend_row(rig.proxy, key)) + + +def test_passthrough_streamed_tool_call_prices_the_trace_of_the_final_event( + rig: Rig, monkeypatch: pytest.MonkeyPatch +) -> None: + key: Final = rig.key() + runtime: Final = rig.runtime(monkeypatch, key) + tool_config: Final = { + "tools": [{"toolSpec": {"name": "lookup", "inputSchema": {"json": {"type": "object", "properties": {}}}}}] + } + events: Final = list( + getattr(runtime, "converse_stream")( + modelId=rig.converse_model, + messages=[{"role": "user", "content": [{"text": "hello"}]}], + toolConfig=tool_config, + )["stream"] + ) + assert any("contentBlockStart" in event for event in events) + _assert_billed_as_guardrail_cost(_spend_row(rig.proxy, key)) diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py index aa2fc0b9a45..949f55b8a82 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py @@ -1,7 +1,11 @@ +import json +from typing import Final import pytest +from pydantic import JsonValue import litellm +from litellm.litellm_core_utils.llm_cost_calc import guardrail_cost from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import ( bedrock_guardrail_cost, bedrock_guardrail_cost_by_unit, @@ -230,3 +234,118 @@ def test_guardrail_information_cost_skips_malformed_entry_keeps_siblings(): ] assert guardrail_information_cost(entries) == pytest.approx(0.0003) assert guardrail_information_cost({"guardrail_cost": 0.5, "guardrail_cost_in_spend": "maybe"}) == 0.0 + + +_TRACE_USAGE_IN: Final = { + "topicPolicyUnits": 1, + "contentPolicyUnits": 1, + "wordPolicyUnits": 1, + "sensitiveInformationPolicyUnits": 0, + "contextualGroundingPolicyUnits": 0, + "automatedReasoningPolicies": 0, +} +_TRACE_USAGE_OUT: Final = { + **_TRACE_USAGE_IN, + "topicPolicyUnits": 2, + "contentPolicyUnits": 2, + "sensitiveInformationPolicyUnits": 2, +} +_TRACE_PRICING: Final = { + "bedrock/guardrails": { + "litellm_provider": "bedrock", + "mode": "guardrails", + "guardrail_cost_per_unit": { + "topicPolicyUnits": 0.00015, + "contentPolicyUnits": 0.00015, + "wordPolicyUnits": 0.0, + "sensitiveInformationPolicyUnits": 0.0001, + }, + } +} + + +def _converse_trace(action: str = "NONE") -> dict[str, JsonValue]: + return { + "guardrail": { + "inputAssessment": {"gid": {"invocationMetrics": {"usage": _TRACE_USAGE_IN}}}, + "outputAssessments": { + "gid": [ + { + "topicPolicy": {"topics": [{"name": "t", "type": "DENY", "action": action}]}, + "invocationMetrics": {"usage": _TRACE_USAGE_OUT}, + } + ] + }, + } + } + + +def _invoke_trace() -> dict[str, JsonValue]: + return { + "guardrail": { + "input": {"gid": {"invocationMetrics": {"usage": _TRACE_USAGE_IN}}}, + "outputs": [{"gid": {"invocationMetrics": {"usage": _TRACE_USAGE_OUT}}}], + } + } + + +@pytest.mark.parametrize("trace", [_converse_trace(), _invoke_trace()], ids=["converse", "invoke"]) +def test_native_trace_sums_every_assessment_and_prices_it_like_the_proxy_guardrail(monkeypatch, trace) -> None: + monkeypatch.setattr(litellm, "model_cost", _TRACE_PRICING) + entry: Final = guardrail_cost.native_bedrock_guardrail_information( + trace, aws_region_name=None, guardrail_identifier=None + ) + assert entry is not None + assert entry["guardrail_usage"] == { + "topicPolicyUnits": 3, + "contentPolicyUnits": 3, + "wordPolicyUnits": 2, + "sensitiveInformationPolicyUnits": 2, + "contextualGroundingPolicyUnits": 0, + } + assert entry["guardrail_cost"] == pytest.approx(3 * 0.00015 + 3 * 0.00015 + 2 * 0.0001) + assert entry["guardrail_cost_by_unit"]["contextualGroundingPolicyUnits"] is None + assert entry["guardrail_name"] == "gid" + assert entry["guardrail_provider"] == "bedrock" + assert entry["guardrail_status"] == "success" + assert guardrail_cost.guardrail_information_cost([entry]) == pytest.approx(entry["guardrail_cost"]) + + +def test_native_trace_marks_a_blocked_assessment_intervened_and_prefers_the_configured_identifier(monkeypatch) -> None: + monkeypatch.setattr(litellm, "model_cost", _TRACE_PRICING) + entry: Final = guardrail_cost.native_bedrock_guardrail_information( + _converse_trace("BLOCKED"), aws_region_name="us-east-1", guardrail_identifier="configured-id" + ) + assert entry is not None + assert entry["guardrail_status"] == "guardrail_intervened" + assert entry["guardrail_name"] == "configured-id" + + +def test_native_trace_without_usage_is_not_billed(monkeypatch) -> None: + monkeypatch.setattr(litellm, "model_cost", _TRACE_PRICING) + unpriced: Final = {"guardrail": {"inputAssessment": {"gid": {"topicPolicy": {"topics": []}}}}} + assert guardrail_cost.native_bedrock_guardrail_information(unpriced, None, None) is None + assert guardrail_cost.bedrock_guardrail_trace({"usage": {}}) is None + + +def test_with_guardrail_entry_records_the_same_trace_once() -> None: + entry: Final = {"guardrail_name": "gid", "guardrail_cost": 0.001} + metadata: dict = {"standard_logging_guardrail_information": {"guardrail_name": "hook"}} + guardrail_cost.with_guardrail_entry(metadata, entry) # pyright: ignore[reportArgumentType] # partial entry + guardrail_cost.with_guardrail_entry(metadata, entry) # pyright: ignore[reportArgumentType] # partial entry + assert metadata["standard_logging_guardrail_information"] == [{"guardrail_name": "hook"}, entry] + + +def test_native_trace_entry_redacts_the_pii_matches_before_they_reach_spend_logs(monkeypatch) -> None: + monkeypatch.setattr(litellm, "model_cost", _TRACE_PRICING) + trace: Final = _converse_trace() + trace["guardrail"]["inputAssessment"]["gid"]["sensitiveInformationPolicy"] = { + "piiEntities": [{"type": "US_SSN", "match": "123-45-6789", "action": "ANONYMIZED"}] + } + entry: Final = guardrail_cost.native_bedrock_guardrail_information( + trace, aws_region_name=None, guardrail_identifier=None + ) + assert entry is not None + assert entry["guardrail_status"] == "guardrail_intervened" + assert "123-45-6789" not in json.dumps(entry["guardrail_response"]) + assert entry["guardrail_usage"]["sensitiveInformationPolicyUnits"] == 2 diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index b46c65d4460..317d3c35664 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -13642,3 +13642,54 @@ def test_native_messages_body_carries_the_reported_list_cost_into_the_priced_res assert isinstance(priced, ModelResponse) headers = priced._hidden_params.get("additional_headers") or {} assert headers.get("llm_provider-x-litellm-response-cost") == expected_header + + +def test_record_native_bedrock_guardrail_bills_when_the_metadata_is_none() -> None: + import datetime + + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj = Logging( + model="bedrock/anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=[{"role": "user", "content": "hello"}], + stream=False, + call_type="completion", + start_time=datetime.datetime.now(), + litellm_call_id="call-none-metadata", + function_id="fn", + ) + logging_obj.model_call_details["litellm_params"] = {"metadata": None} + trace = {"guardrail": {"inputAssessment": {"gid": {"invocationMetrics": {"usage": {"contentPolicyUnits": 2}}}}}} + + logging_obj.record_native_bedrock_guardrail({"trace": trace}) + + (entry,) = logging_obj.model_call_details["litellm_params"]["metadata"]["standard_logging_guardrail_information"] + assert entry["guardrail_usage"] == {"contentPolicyUnits": 2} + assert entry["guardrail_cost"] > 0 + + +def test_record_native_bedrock_guardrail_bills_nothing_for_a_response_served_from_cache() -> None: + import datetime + + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj = Logging( + model="bedrock/anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=[{"role": "user", "content": "hello"}], + stream=False, + call_type="completion", + start_time=datetime.datetime.now(), + litellm_call_id="call-cache-hit", + function_id="fn", + ) + logging_obj.model_call_details["litellm_params"] = {"metadata": {}} + trace = {"guardrail": {"inputAssessment": {"gid": {"invocationMetrics": {"usage": {"contentPolicyUnits": 2}}}}}} + cached = litellm.ModelResponse(choices=[{"message": {"role": "assistant", "content": "hi"}}], model="x") + setattr(cached, "trace", trace) + cached._hidden_params = {"cache_hit": True} + + logging_obj.record_native_bedrock_guardrail(cached) + logging_obj.model_call_details["cache_hit"] = True + logging_obj.record_native_bedrock_guardrail({"trace": trace}) + + assert "standard_logging_guardrail_information" not in logging_obj.model_call_details["litellm_params"]["metadata"] diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index 6a734ca3b70..5b64e42d8a8 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -26,15 +26,16 @@ import litellm from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.litellm_core_utils.redact_messages import ( _redact_responses_api_output, _redacted_responses_api_response, perform_redaction, redacted_model_call_details_for_custom_logger, + redacted_response_obj_for_custom_logger, redacted_standard_logging_payload, should_redact_message_logging, ) +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.responses.main import mock_responses_api_response from litellm.types.router import BaselineRouteStamp from litellm.types.utils import ( @@ -176,7 +177,11 @@ def test_redaction_hides_a_compaction_summary_in_responses_api_output(shape: str object="response", status="completed", output=[ - {"type": "compaction", "id": "cmp_1", "encrypted_content": json.dumps({"type": "compaction", "content": SECRET})} + { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": json.dumps({"type": "compaction", "content": SECRET}), + } ], ) result: Final = response if shape == "object" else response.model_dump() @@ -2041,3 +2046,32 @@ async def test_redaction_with_metadata_completion_api(): response = standard_logging_payload["response"] assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" + + +def test_perform_redaction_strips_the_matched_text_from_the_bedrock_guardrail_traces() -> None: + finding = { + "guardrail": { + "inputAssessment": { + "gid": { + "sensitiveInformationPolicy": { + "piiEntities": [{"type": "US_SSN", "match": "123-45-6789", "action": "ANONYMIZED"}] + } + } + } + } + } + response = litellm.ModelResponse(choices=[{"message": {"role": "assistant", "content": "hi"}}], model="x") + setattr(response, "trace", copy.deepcopy(finding)) + setattr(response, "amazon-bedrock-trace", copy.deepcopy(finding)) + opted_out: Final = CustomLogger(message_logging=False) + + redacted: Final = redacted_response_obj_for_custom_logger(response, opted_out) + redacted_dict: Final = redacted_response_obj_for_custom_logger( + {"choices": [], "trace": copy.deepcopy(finding)}, opted_out + ) + + assert isinstance(redacted, litellm.ModelResponse) + assert "123-45-6789" not in redacted.model_dump_json() + assert "US_SSN" in redacted.model_dump_json(), "the finding stays, only its matched text goes" + assert "123-45-6789" not in str(redacted_dict) + assert "123-45-6789" in response.model_dump_json(), "the caller's own response is not redacted" diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index 5c56245cf33..59946d54dc7 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -8,17 +8,19 @@ import httpx import pytest import respx - import litellm from litellm import ChatCompletionUsageBlock, stream_chunk_builder -from litellm.types.utils import GenericStreamingChunk -from litellm.litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor, stream_chunk_builder_text_completion +from litellm.litellm_core_utils.streaming_chunk_builder_utils import ( + ChunkProcessor, + stream_chunk_builder_text_completion, +) from litellm.llms.anthropic.chat.handler import ModelResponseIterator from litellm.types.utils import ( ChatCompletionDeltaToolCall, ChatCompletionMessageToolCall, Delta, Function, + GenericStreamingChunk, ModelResponseStream, PromptTokensDetails, ServerToolUse, @@ -196,11 +198,7 @@ def test_get_combined_thinking_content_preserves_interleaved_blocks(): } ] ), - make_chunk( - thinking_blocks=[ - {"type": "thinking", "thinking": None, "signature": "sig_block1"} - ] - ), + make_chunk(thinking_blocks=[{"type": "thinking", "thinking": None, "signature": "sig_block1"}]), make_chunk( thinking_blocks=[ { @@ -218,16 +216,10 @@ def test_get_combined_thinking_content_preserves_interleaved_blocks(): } ] ), - make_chunk( - thinking_blocks=[ - {"type": "thinking", "thinking": None, "signature": "sig_block2"} - ] - ), + make_chunk(thinking_blocks=[{"type": "thinking", "thinking": None, "signature": "sig_block2"}]), ] - thinking_chunks = [ - chunk for chunk in chunks if chunk["choices"][0]["delta"].get("thinking_blocks") - ] + thinking_chunks = [chunk for chunk in chunks if chunk["choices"][0]["delta"].get("thinking_blocks")] processor = ChunkProcessor(chunks=chunks) result = processor.get_combined_thinking_content(thinking_chunks) @@ -297,9 +289,7 @@ def test_cache_read_input_tokens_retained(): prompt_tokens=11779, total_tokens=11784, completion_tokens_details=None, - prompt_tokens_details=PromptTokensDetails( - audio_tokens=None, cached_tokens=11775 - ), + prompt_tokens_details=PromptTokensDetails(audio_tokens=None, cached_tokens=11775), cache_creation_input_tokens=4, cache_read_input_tokens=11775, ), @@ -333,9 +323,7 @@ def test_cache_read_input_tokens_retained(): prompt_tokens=0, total_tokens=214, completion_tokens_details=None, - prompt_tokens_details=PromptTokensDetails( - audio_tokens=None, cached_tokens=0 - ), + prompt_tokens_details=PromptTokensDetails(audio_tokens=None, cached_tokens=0), cache_creation_input_tokens=0, cache_read_input_tokens=0, ), @@ -395,10 +383,7 @@ def test_streaming_preserves_anthropic_1hr_cache_creation_breakdown(): ) # Sanity: the delta event genuinely lacks the breakdown - this is the input # condition that used to defeat cost calc. - assert ( - getattr(message_delta_usage.prompt_tokens_details, "cache_creation_token_details", None) - is None - ) + assert getattr(message_delta_usage.prompt_tokens_details, "cache_creation_token_details", None) is None def _usage_chunk(usage, finish_reason): return ModelResponseStream( @@ -517,9 +502,7 @@ def test_cache_read_input_tokens_retained_genericstreamingchunk(): prompt_tokens=1234, total_tokens=1239, completion_tokens_details=None, - prompt_tokens_details=PromptTokensDetails( - audio_tokens=None, cached_tokens=543 - ).model_dump(), + prompt_tokens_details=PromptTokensDetails(audio_tokens=None, cached_tokens=543).model_dump(), ), index=2, ) @@ -536,6 +519,7 @@ def test_cache_read_input_tokens_retained_genericstreamingchunk(): assert usage.prompt_tokens_details.cached_tokens == 543 + def test_stream_chunk_builder_litellm_usage_chunks(): """ Validate ChunkProcessor.calculate_usage uses provided usage fields from streaming chunks @@ -609,9 +593,7 @@ def test_stream_chunk_builder_litellm_usage_chunks(): chunks = [chunk1, chunk2] processor = ChunkProcessor(chunks=chunks) - usage = processor.calculate_usage( - chunks=chunks, model="gemini/gemini-2.5-flash-lite", completion_output="" - ) + usage = processor.calculate_usage(chunks=chunks, model="gemini/gemini-2.5-flash-lite", completion_output="") assert usage.prompt_tokens == 50 assert usage.completion_tokens == 27 @@ -655,15 +637,11 @@ def test_calculate_usage_honors_openai_sdk_completion_usage_chunks(): provider_specific_fields=None, stream_options={"include_usage": True}, ) - usage_chunk.usage = CompletionUsage( - prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704 - ) + usage_chunk.usage = CompletionUsage(prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704) assert type(usage_chunk.usage) is CompletionUsage chunks = [content_chunk, usage_chunk] - usage = ChunkProcessor(chunks=chunks).calculate_usage( - chunks=chunks, model="mantle-claude", completion_output="" - ) + usage = ChunkProcessor(chunks=chunks).calculate_usage(chunks=chunks, model="mantle-claude", completion_output="") assert usage.prompt_tokens == 20 assert usage.completion_tokens == 60 @@ -686,9 +664,7 @@ def test_get_model_from_chunks_azure_model_router(): {"model": "gpt-4.1-nano-2025-04-14", "id": "chatcmpl-123", "choices": []}, ] - result = ChunkProcessor._get_model_from_chunks( - chunks=chunks, first_chunk_model="azure-model-router" - ) + result = ChunkProcessor._get_model_from_chunks(chunks=chunks, first_chunk_model="azure-model-router") # Should return the actual model, not the request model assert result == "gpt-4.1-nano-2025-04-14" @@ -699,9 +675,7 @@ def test_get_model_from_chunks_azure_model_router(): {"model": "gpt-4", "id": "chatcmpl-456", "choices": []}, ] - result_same = ChunkProcessor._get_model_from_chunks( - chunks=chunks_same_model, first_chunk_model="gpt-4" - ) + result_same = ChunkProcessor._get_model_from_chunks(chunks=chunks_same_model, first_chunk_model="gpt-4") # Should return the first chunk's model when all are the same assert result_same == "gpt-4" @@ -777,9 +751,7 @@ def test_stream_chunk_builder_anthropic_web_search(): chunks = [chunk1, chunk2] processor = ChunkProcessor(chunks=chunks) - usage = processor.calculate_usage( - chunks=chunks, model="claude-sonnet-4-5-20250929", completion_output="" - ) + usage = processor.calculate_usage(chunks=chunks, model="claude-sonnet-4-5-20250929", completion_output="") assert usage.prompt_tokens == 50 assert usage.completion_tokens == 27 @@ -931,15 +903,11 @@ def test_stream_chunk_builder_dict_snapshot_preserves_hidden_provider_fields(): ], ) chunk_dict = chunk.model_dump() - chunk_dict["_hidden_params"] = { - "provider_specific_fields": {"traffic_type": "default"} - } + chunk_dict["_hidden_params"] = {"provider_specific_fields": {"traffic_type": "default"}} response = stream_chunk_builder(chunks=[chunk_dict]) assert response is not None - assert ( - response._hidden_params["provider_specific_fields"]["traffic_type"] == "default" - ) + assert response._hidden_params["provider_specific_fields"]["traffic_type"] == "default" def test_stream_chunk_builder_propagates_vertex_ai_metadata_from_chunks(): @@ -984,10 +952,7 @@ def test_stream_chunk_builder_propagates_vertex_ai_metadata_from_chunks(): assert getattr(response, "vertex_ai_grounding_metadata") == grounding_metadata assert getattr(response, "vertex_ai_url_context_metadata") == url_context_metadata assert response._hidden_params["vertex_ai_grounding_metadata"] == grounding_metadata - assert ( - response._hidden_params["vertex_ai_url_context_metadata"] - == url_context_metadata - ) + assert response._hidden_params["vertex_ai_url_context_metadata"] == url_context_metadata dumped = response.model_dump() assert dumped["vertex_ai_grounding_metadata"] == grounding_metadata @@ -1034,9 +999,7 @@ def test_stream_chunk_builder_uses_assembled_model_for_provider_metadata(): def test_stream_chunk_builder_propagates_vertex_ai_safety_results(): """Assembled response must expose safety data under the non-streaming field name.""" - safety_ratings = [ - [{"category": "HARM_CATEGORY_HATE_SPEECH", "probability": "NEGLIGIBLE"}] - ] + safety_ratings = [[{"category": "HARM_CATEGORY_HATE_SPEECH", "probability": "NEGLIGIBLE"}]] chunk = ModelResponseStream( id="chatcmpl-vertex-safety", @@ -1078,18 +1041,12 @@ def test_stream_chunk_builder_propagates_vertex_ai_metadata_from_dict_chunks(): ) ], ).model_dump() - chunk_dict["_hidden_params"] = { - "vertex_ai_grounding_metadata": [{"webSearchQueries": ["test query"]}] - } + chunk_dict["_hidden_params"] = {"vertex_ai_grounding_metadata": [{"webSearchQueries": ["test query"]}]} response = stream_chunk_builder(chunks=[chunk_dict]) assert response is not None - assert getattr(response, "vertex_ai_grounding_metadata") == [ - {"webSearchQueries": ["test query"]} - ] - assert response.model_dump()["vertex_ai_grounding_metadata"] == [ - {"webSearchQueries": ["test query"]} - ] + assert getattr(response, "vertex_ai_grounding_metadata") == [{"webSearchQueries": ["test query"]}] + assert response.model_dump()["vertex_ai_grounding_metadata"] == [{"webSearchQueries": ["test query"]}] def test_cost_field_in_usage_chunks(): @@ -1098,29 +1055,21 @@ def test_cost_field_in_usage_chunks(): id="chatcmpl-1", created=1745513206, model="openrouter/claude", - choices=[ - StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi")) - ], + choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi"))], usage=chunk1_usage, ) - chunk2_usage = Usage( - completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.00025 - ) + chunk2_usage = Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.00025) chunk2 = ModelResponseStream( id="chatcmpl-1", created=1745513207, model="openrouter/claude", - choices=[ - StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) - ], + choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=""))], usage=chunk2_usage, ) processor = ChunkProcessor(chunks=[chunk1, chunk2]) - usage = processor.calculate_usage( - chunks=[chunk1, chunk2], model="openrouter/claude", completion_output="Hi" - ) + usage = processor.calculate_usage(chunks=[chunk1, chunk2], model="openrouter/claude", completion_output="Hi") assert hasattr(usage, "cost") assert usage.cost == 0.00025 @@ -1203,25 +1152,19 @@ def test_prompt_tokens_details_survive_later_usage_chunk_without_details(): id="chatcmpl-1", created=1745513206, model="openai/gpt-5.6-sol", - choices=[ - StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi")) - ], + choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi"))], usage=Usage( prompt_tokens=6017, completion_tokens=4, total_tokens=6021, - prompt_tokens_details=PromptTokensDetailsWrapper( - cached_tokens=6004, cache_write_tokens=10 - ), + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=6004, cache_write_tokens=10), ), ) chunk_without_details = ModelResponseStream( id="chatcmpl-1", created=1745513207, model="openai/gpt-5.6-sol", - choices=[ - StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) - ], + choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=""))], usage=Usage(prompt_tokens=6017, completion_tokens=4, total_tokens=6021), ) @@ -1559,9 +1502,7 @@ def test_calculate_usage_fills_unknown_split_from_reasoning_estimate( assert usage.completion_tokens_details.text_tokens == expected_text_tokens -def _openai_chunk( - choices: Sequence[Mapping[str, object]], usage: Mapping[str, int] | None = None -) -> dict[str, object]: +def _openai_chunk(choices: Sequence[Mapping[str, object]], usage: Mapping[str, int] | None = None) -> dict[str, object]: base: Final = { "id": "chatcmpl-lit6552", "object": "chat.completion.chunk", @@ -1996,7 +1937,11 @@ def test_stream_chunk_builder_litellm_tool_call(respx_mock: respx.MockRouter) -> ) assert [tool_call.to_dict() for tool_call in response.choices[0].message.tool_calls] == [ - {"function": {"arguments": '{"city":"Boston"}', "name": "get_weather"}, "id": "call_weather", "type": "function"} + { + "function": {"arguments": '{"city":"Boston"}', "name": "get_weather"}, + "id": "call_weather", + "type": "function", + } ] assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (5, 3, 8) @@ -2018,7 +1963,9 @@ def test_stream_chunk_builder_litellm_tool_call_regular_message(respx_mock: resp def test_stream_chunk_builder_multiple_tool_calls() -> None: - def tool_call_chunk(*tool_calls: ChatCompletionDeltaToolCall, finish_reason: str | None = None) -> ModelResponseStream: + def tool_call_chunk( + *tool_calls: ChatCompletionDeltaToolCall, finish_reason: str | None = None + ) -> ModelResponseStream: return _legacy_stream_chunk("openai/gpt-4o", tool_calls=tool_calls, finish_reason=finish_reason) chunks: Final = ( @@ -2029,7 +1976,9 @@ def test_stream_chunk_builder_multiple_tool_calls() -> None: ), tool_call_chunk(ChatCompletionDeltaToolCall(index=0, function=Function(arguments='{"base": 3, '))), tool_call_chunk( - ChatCompletionDeltaToolCall(id="call_add", index=1, type="function", function=Function(name="add", arguments="")) + ChatCompletionDeltaToolCall( + id="call_add", index=1, type="function", function=Function(name="add", arguments="") + ) ), tool_call_chunk(ChatCompletionDeltaToolCall(index=0, function=Function(arguments='"exponent": 5}'))), tool_call_chunk( @@ -2043,7 +1992,11 @@ def test_stream_chunk_builder_multiple_tool_calls() -> None: assert response is not None assert response.choices[0].finish_reason == "tool_calls" assert [tool_call.to_dict() for tool_call in response.choices[0].message.tool_calls] == [ - {"function": {"arguments": '{"base": 3, "exponent": 5}', "name": "exponentiate"}, "id": "call_exp", "type": "function"}, + { + "function": {"arguments": '{"base": 3, "exponent": 5}', "name": "exponentiate"}, + "id": "call_exp", + "type": "function", + }, {"function": {"arguments": '{"x": 1, "y": 2}', "name": "add"}, "id": "call_add", "type": "function"}, ] @@ -2077,9 +2030,7 @@ def test_grok_bug() -> None: response: Final = stream_chunk_builder(chunks=chunks) assert response is not None - assert json.loads(response.choices[0].message.tool_calls[0].function.arguments) == { - "location": "Boston" - } + assert json.loads(response.choices[0].message.tool_calls[0].function.arguments) == {"location": "Boston"} def test_stream_chunk_builder_openai_audio_output_usage() -> None: @@ -2196,7 +2147,9 @@ def test_stream_chunk_builder_keeps_usage_dict_set_on_a_typed_text_completion_ch @pytest.mark.parametrize("build", [litellm.stream_chunk_builder, litellm.stream_chunk_builder_text_completion]) -def test_stream_chunk_builder_rebuilds_dict_text_completion_chunks(build: Callable[..., TextCompletionResponse]) -> None: +def test_stream_chunk_builder_rebuilds_dict_text_completion_chunks( + build: Callable[..., TextCompletionResponse], +) -> None: def chunk(choices: list[dict[str, object]], **extra: object) -> dict[str, object]: return { "id": "cmpl-1", @@ -2450,7 +2403,9 @@ def test_stream_chunk_builder_rebuilds_text_completion_choice_indexes_that_are_n ] ) - assert tuple(choice.model_dump(include={"index", "text", "finish_reason"}) for choice in response.choices) == expected + assert ( + tuple(choice.model_dump(include={"index", "text", "finish_reason"}) for choice in response.choices) == expected + ) def test_stream_chunk_builder_text_completion_survives_uncountable_prompt() -> None: @@ -2517,3 +2472,50 @@ def test_stream_chunk_builder_keeps_reasoning_items_on_text_only_replies(): assert response is not None assert response.choices[0].message.content == "Hello world" assert response.choices[0].message.reasoning_items == reasoning_items + + +def test_stream_chunk_builder_carries_the_bedrock_guardrail_trace_and_records_it_on_the_logging_object( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A Bedrock stream reports its guardrail trace on the last chunk only; the built response carries it like the + non-streaming response and the call bills it as guardrail cost.""" + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + monkeypatch.setattr( + litellm, + "model_cost", + { + "bedrock/guardrails": { + "litellm_provider": "bedrock", + "mode": "guardrails", + "guardrail_cost_per_unit": {"topicPolicyUnits": 0.00015}, + } + }, + ) + trace = {"guardrail": {"inputAssessment": {"gid": {"invocationMetrics": {"usage": {"topicPolicyUnits": 2}}}}}} + chunks = [ + ModelResponseStream(id="s", model="m", choices=[StreamingChoices(index=0, delta=Delta(content="hi"))]), + ModelResponseStream( + id="s", + model="m", + choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")], + provider_specific_fields={"trace": trace}, + ), + ] + logging_obj = Logging( + model="bedrock/m", + messages=[{"role": "user", "content": "q"}], + stream=True, + call_type="acompletion", + start_time=None, + litellm_call_id="call", + function_id="fn", + ) + logging_obj.model_call_details["litellm_params"] = {"metadata": {}} + built = stream_chunk_builder(chunks=chunks, messages=[{"role": "user", "content": "q"}], logging_obj=logging_obj) + assert built is not None + assert getattr(built, "trace") == trace + (entry,) = logging_obj.model_call_details["litellm_params"]["metadata"]["standard_logging_guardrail_information"] + assert entry["guardrail_usage"] == {"topicPolicyUnits": 2} + assert entry["guardrail_cost"] == pytest.approx(2 * 0.00015) diff --git a/tests/unit/llms/bedrock/chat/test_invoke_handler.py b/tests/unit/llms/bedrock/chat/test_invoke_handler.py index f3ceac1d93c..996cfec6e22 100644 --- a/tests/unit/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/unit/llms/bedrock/chat/test_invoke_handler.py @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.litellm_logging import _get_provider_request_id from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.bedrock.chat.invoke_handler import ( + AmazonAnthropicClaudeStreamDecoder, AmazonOpenAICompatibleStreamDecoder, AWSEventStreamDecoder, _stream_decoder, @@ -1120,6 +1121,7 @@ async def test_converse_streaming_completion_yields_text_and_finish_reason( ), ) ) + def handle_request(request: httpx.Request) -> httpx.Response: assert request.url.path.endswith("/converse-stream") return httpx.Response(200, content=event_stream_body, headers=_event_stream_headers()) @@ -1179,9 +1181,7 @@ def test_mistral_invoke_stream_yields_text_and_finish_reason( assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == ( "A court case moves through the courts." ) - assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == [ - "stop" - ] + assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == ["stop"] def test_converse_streaming_tool_call_arguments_are_valid_json( @@ -1721,6 +1721,7 @@ def test_filter_headers_for_aws_signature(): @pytest.mark.parametrize("missing", ["botocore", "unrelated_dependency"]) def test_invoke_decoder_reports_only_missing_aws_dependency(missing): from unittest.mock import patch + from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder failure = ModuleNotFoundError("missing dependency", name=missing) @@ -1835,9 +1836,7 @@ async def test_converse_stream_thinking_merges_into_content_with_think_tags() -> "contentBlockDelta", {"contentBlockIndex": 0, "delta": {"reasoningContent": {"signature": "c2lnbmF0dXJl"}}} ), _converse_event_frame("contentBlockStop", {"contentBlockIndex": 0}), - _converse_event_frame( - "contentBlockDelta", {"contentBlockIndex": 1, "delta": {"text": "Hi I am Anthropic, "}} - ), + _converse_event_frame("contentBlockDelta", {"contentBlockIndex": 1, "delta": {"text": "Hi I am Anthropic, "}}), _converse_event_frame( "contentBlockDelta", {"contentBlockIndex": 1, "delta": {"text": "I am a helpful assistant"}} ), @@ -1919,7 +1918,11 @@ _CUT_JSON_TOOL_STREAM: Final = b"".join( "content_block": {"type": "tool_use", "id": "toolu_cut_1", "name": "json_tool_call", "input": {}}, }, {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": '{"city"'}}, - {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": ': "Paris", '}}, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": ': "Paris", '}, + }, { "type": "content_block_delta", "index": 0, @@ -1961,3 +1964,21 @@ async def test_async_anthropic_invoke_stream_cut_before_the_json_tool_block_ends chunks: Final = [chunk async for chunk in _cut_json_tool_stream_decoder().aiter_bytes(_chunks())] assert json.loads(_streamed_content(chunks)) == {"city": "Paris", "country": "France"} + + +def test_invoke_stream_decoder_carries_the_guardrail_trace_of_the_message_stop_chunk() -> None: + """Bedrock Invoke reports the native guardrail trace once, on the final ``message_stop`` chunk, under + ``amazon-bedrock-trace``. The chunk keeps it so the stream builder and the spend log can price it.""" + decoder = AmazonAnthropicClaudeStreamDecoder(model="anthropic.claude-sonnet-5-v1:0", sync_stream=True) + trace = {"guardrail": {"input": {"gid": {"invocationMetrics": {"usage": {"topicPolicyUnits": 1}}}}}} + plain = decoder._chunk_parser({"type": "message_stop", "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3}}) + traced = decoder._chunk_parser( + { + "type": "message_stop", + "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3}, + "amazon-bedrock-trace": trace, + } + ) + assert (plain.provider_specific_fields or {}).get("amazon-bedrock-trace") is None + assert traced.provider_specific_fields is not None + assert traced.provider_specific_fields["amazon-bedrock-trace"] == trace diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py index 93367fdff16..3c0bbadc133 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py @@ -4169,3 +4169,32 @@ async def test_bedrock_messages_400_is_not_retried_unless_it_rejects_opt_in_nati assert isinstance(raised, litellm.BadRequestError) assert len(bedrock.bodies) == 1 + + +@pytest.mark.parametrize("trace", ["enabled", "disabled"]) +def test_bedrock_messages_applies_the_deployment_guardrail_config_as_invoke_headers(trace: str) -> None: + from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( + AmazonAnthropicClaudeMessagesConfig, + ) + + headers, _ = AmazonAnthropicClaudeMessagesConfig().validate_anthropic_messages_environment( + headers={}, + model="anthropic.claude-sonnet-5-v1:0", + messages=[], + optional_params={}, + litellm_params={"guardrailConfig": {"guardrailIdentifier": "gid", "guardrailVersion": "2", "trace": trace}}, + ) + assert headers["X-Amzn-Bedrock-GuardrailIdentifier"] == "gid" + assert headers["X-Amzn-Bedrock-GuardrailVersion"] == "2" + assert headers["X-Amzn-Bedrock-Trace"] == trace.upper() + + +def test_bedrock_messages_without_guardrail_config_sends_no_guardrail_headers() -> None: + from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( + AmazonAnthropicClaudeMessagesConfig, + ) + + headers, _ = AmazonAnthropicClaudeMessagesConfig().validate_anthropic_messages_environment( + headers={}, model="anthropic.claude-sonnet-5-v1:0", messages=[], optional_params={}, litellm_params={} + ) + assert not any(name.startswith("X-Amzn-Bedrock-Guardrail") for name in headers) diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index 31a963d3e47..e09a179eadf 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -1,7 +1,7 @@ import asyncio import json from datetime import datetime -from typing import Any, Dict, Final, List +from typing import Final from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx @@ -284,9 +284,10 @@ class TestAzureAnthropicCostCalculation: @patch("litellm.completion_cost") def test_cost_calculation_with_azure_ai_custom_llm_provider(self, mock_completion_cost): """Test that custom_llm_provider is passed to completion_cost for Azure AI Anthropic""" - from litellm.types.utils import ModelResponse from datetime import datetime + from litellm.types.utils import ModelResponse + mock_completion_cost.return_value = 0.001 logging_obj = self._create_mock_logging_obj( @@ -319,9 +320,10 @@ class TestAzureAnthropicCostCalculation: @patch("litellm.completion_cost") def test_cost_calculation_without_custom_llm_provider(self, mock_completion_cost): """Test that cost calculation works without custom_llm_provider (standard Anthropic)""" - from litellm.types.utils import ModelResponse from datetime import datetime + from litellm.types.utils import ModelResponse + mock_completion_cost.return_value = 0.001 # No custom_llm_provider in model_call_details @@ -353,9 +355,10 @@ class TestAzureAnthropicCostCalculation: @patch("litellm.completion_cost") def test_cost_calculation_does_not_duplicate_provider_prefix(self, mock_completion_cost): """Test that provider prefix is not duplicated if already present in model name""" - from litellm.types.utils import ModelResponse from datetime import datetime + from litellm.types.utils import ModelResponse + mock_completion_cost.return_value = 0.001 logging_obj = self._create_mock_logging_obj( @@ -826,9 +829,10 @@ class TestAnthropicBatchPassthroughCostTracking: mock_request_body, ): """Test that model gets 'anthropic/' prefix when not found in router""" + import base64 + from litellm.llms.anthropic.batches.transformation import AnthropicBatchesConfig from litellm.types.utils import LiteLLMBatch - import base64 # Model not in router - returns same model name mock_get_model_id.return_value = "claude-sonnet-4-5-20250929" @@ -1089,7 +1093,7 @@ class TestAnthropicBatchPassthroughCostTracking: class TestBuildCompleteStreamingResponseRobustness: """_build_complete_streaming_response must tolerate non-standard SSE frames.""" - def _build(self, chunks: List[str]): + def _build(self, chunks: list[str]): return AnthropicPassthroughLoggingHandler._build_complete_streaming_response( all_chunks=chunks, litellm_logging_obj=MagicMock(), @@ -1840,9 +1844,6 @@ class TestStreamFalseDeduplication: This is the _is_assembled_stream_success gate: with stream=False it always returned False and the guard was permanently disabled. """ - from litellm.types.passthrough_endpoints.pass_through_endpoints import ( - EndpointType, - ) from litellm.types.utils import ModelResponse # Simulate what pass_through_endpoints.py now does after stream detection @@ -2642,8 +2643,6 @@ def test_create_anthropic_response_logging_payload(mock_logging_obj, metadata_pa def test_handle_logging_anthropic_collected_chunks(all_chunks): from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( AnthropicPassthroughLoggingHandler, - PassthroughStandardLoggingPayload, - EndpointType, ) from litellm.types.utils import ModelResponse @@ -2943,3 +2942,65 @@ def test_streamed_messages_from_other_providers_ignore_a_zero_usage_cost_field() headers = result["result"]._hidden_params.get("additional_headers") or {} assert "llm_provider-x-litellm-response-cost" not in headers assert result["kwargs"].get("response_cost") != 0.0 + + +_BEDROCK_TRACE_PRICING: Final = { + "bedrock/guardrails": { + "litellm_provider": "bedrock", + "mode": "guardrail", + "guardrail_cost_per_unit": {"topicPolicyUnits": 0.00015, "contentPolicyUnits": 0.00015}, + } +} +_BEDROCK_INVOKE_TRACE: Final = { + "guardrail": { + "input": {"gid": {"invocationMetrics": {"usage": {"topicPolicyUnits": 1, "contentPolicyUnits": 1}}}}, + "outputs": [{"gid": {"invocationMetrics": {"usage": {"topicPolicyUnits": 2, "contentPolicyUnits": 2}}}}], + } +} + + +def _recorded_guardrail_entries(logging_obj: LiteLLMLoggingObj) -> list: + return logging_obj.model_call_details["litellm_params"]["metadata"]["standard_logging_guardrail_information"] + + +def test_anthropic_passthrough_handler_bills_the_bedrock_guardrail_trace_of_a_messages_response( + monkeypatch, mock_httpx_response, mock_response, mock_logging_obj +) -> None: + monkeypatch.setattr(litellm, "model_cost", _BEDROCK_TRACE_PRICING) + mock_logging_obj.model_call_details["litellm_params"] = {"metadata": {}} + AnthropicPassthroughLoggingHandler.anthropic_passthrough_handler( + httpx_response=mock_httpx_response, + response_body={**mock_response, "amazon-bedrock-trace": _BEDROCK_INVOKE_TRACE}, + logging_obj=mock_logging_obj, + url_route="/v1/messages", + result="success", + start_time=datetime(2025, 1, 1), + end_time=datetime(2025, 1, 1), + cache_hit=False, + ) + (entry,) = _recorded_guardrail_entries(mock_logging_obj) + assert entry["guardrail_provider"] == "bedrock" + assert entry["guardrail_usage"] == {"topicPolicyUnits": 3, "contentPolicyUnits": 3} + assert entry["guardrail_cost"] == pytest.approx(6 * 0.00015) + + +def test_anthropic_passthrough_stream_bills_the_bedrock_guardrail_trace_of_the_message_stop_event( + monkeypatch, mock_logging_obj, all_chunks +) -> None: + monkeypatch.setattr(litellm, "model_cost", _BEDROCK_TRACE_PRICING) + mock_logging_obj.model_call_details["litellm_params"] = {"metadata": {}} + stop_event: Final = json.dumps({"type": "message_stop", "amazon-bedrock-trace": _BEDROCK_INVOKE_TRACE}) + chunks: Final = [*all_chunks, f"event: message_stop\ndata: {stop_event}\n\n"] + AnthropicPassthroughLoggingHandler.handle_logging_anthropic_collected_chunks( + litellm_logging_obj=mock_logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route="/v1/messages", + request_body={"model": "claude", "messages": []}, + endpoint_type=MagicMock(), + start_time=datetime(2025, 1, 1), + all_chunks=chunks, + end_time=datetime(2025, 1, 1), + ) + (entry,) = _recorded_guardrail_entries(mock_logging_obj) + assert entry["guardrail_usage"] == {"topicPolicyUnits": 3, "contentPolicyUnits": 3} + assert entry["guardrail_cost"] == pytest.approx(6 * 0.00015) From 5667ac8497240541cd81d573ed1a7637456cd1f6 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 27 Oct 2026 12:14:42 -0700 Subject: [PATCH 008/102] feat(proxy): refuse migrations that existing rows would make fail, before the first one runs (#48493) * feat(proxy): refuse migrations that existing rows would make fail, before the first one runs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(migrations): start the serving-proxy preflight case from a database without the api_key index Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): type the preflight count rows without casts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(proxy): ruff format the db preflight flag Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(proxy-extras): keep the extras files at their existing formatting Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy-extras): type the ledger query and the reader pid so the type gate holds Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): declare the preflight report type where the extras package may be absent Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): suppress the optional-extras unknown on the preflight report variables Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): suppress the optional-extras unknown on the preflight import Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): return a typed preflight report when the extras package is unknown Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(proxy): sort the preflight import block Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): mark the preflight report binding Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(proxy): size the whole btree entry in the row check and name the flag for what it does The index check now lays out each row's entry the way index_form_tuple does (tuple header, null bitmap, every indexed column at its alignment, rounded to 8) and compares it with BTMaxItemSize for the server's block_size, so a key that passes octet_length > 2704 but still fails the (api_key, startTime) index is caught. --db_preflight becomes --check_migrations_against_existing_data, db_preflight.py becomes migration_data_checks.py and the timeout env var LITELLM_MIGRATION_DATA_CHECK_TIMEOUT Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../migration_data_checks.py | 639 ++++++++++++++++++ .../request_log_indexes.py | 12 + .../litellm_proxy_extras/utils.py | 226 +++---- litellm/proxy/db/prisma_client.py | 12 + litellm/proxy/proxy_cli.py | 27 + .../migrations/test_request_log_indexes.py | 201 +++++- .../test_migration_data_checks.py | 156 +++++ 7 files changed, 1144 insertions(+), 129 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py create mode 100644 tests/unit/litellm_proxy_extras/test_migration_data_checks.py diff --git a/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py b/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py new file mode 100644 index 00000000000..a2d3728349a --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py @@ -0,0 +1,639 @@ +"""Read-only checks that the rows already in the database can take the pending schema +changes, run before `prisma migrate deploy` and before each request-log index build. + +Postgres has a short list of schema statements that fail because of existing rows: a +btree index whose entry for some row would not fit in a third of a page (2704 bytes on +the default 8 KB page, counting every indexed column), UNIQUE (duplicate values), SET NOT NULL and ADD COLUMN NOT NULL with no default (nulls, any row at all), +FOREIGN KEY (orphans) and CHECK (rows failing the expression). Each pending statement +of one of those kinds becomes one count query derived from the statement text and the +column types in the catalog. A count above zero names the table, the column, the +statement it blocks and the SQL that clears it, and the migration is refused before +anything is applied. Every other statement a migration can contain cannot fail on +existing data and gets no query. A statement the derivation cannot read gets none +either, so the migration keeps today's behavior for it. +""" + +import os +import re +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from functools import reduce +from itertools import accumulate +from pathlib import Path +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal + +from litellm_proxy_extras._logging import logger + +if TYPE_CHECKING: + import psycopg + from psycopg import sql + +DEFAULT_BLOCK_SIZE: Final = 8192 +INDEX_TUPLE_HEADER_BYTES: Final = 8 +NULL_BITMAP_BYTES: Final = 8 +VARLENA_HEADER_BYTES: Final = 4 +SHORT_VARLENA_MAX_BYTES: Final = 126 +_PAGE_HEADER_WITH_THREE_LINE_POINTERS_BYTES: Final = 40 +_BTREE_PAGE_OPAQUE_BYTES: Final = 16 +_HEAP_TID_BYTES: Final = 8 +_ALIGNMENT: Final = MappingProxyType({"c": 1, "s": 2, "i": 4, "d": 8}) +_OCTET_LENGTH_TYPES: Final = frozenset({"text", "varchar", "bpchar", "bytea", "name"}) +MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR: Final = "LITELLM_MIGRATION_DATA_CHECK_TIMEOUT" +DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS: Final = 600.0 +REFUSAL: Final = "Existing rows would make a pending schema change fail:" +LEGACY_SPEND_LOG_KEY_REMEDY: Final = ( + "Rows written before v1.99.0 hold the raw key. Hash them in place (one-way, idempotent):\n" + ' UPDATE "LiteLLM_SpendLogs" SET api_key = encode(sha256(convert_to(' + "regexp_replace(api_key, '^[Bb][Ee][Aa][Rr][Ee][Rr] ', ''), 'UTF8')), 'hex')\n" + " WHERE api_key <> '' AND api_key !~ '^[0-9a-fA-F]{64}$' AND api_key !~ '^hashed-jwt-[0-9A-Fa-f]{64}$';" +) + +Kind = Literal["too_long", "duplicates", "nulls", "any_row", "orphans", "check_violations"] + + +@dataclass(frozen=True, slots=True) +class Check: + kind: Kind + table: str + columns: tuple[str, ...] + statement: str + source: str + index: str | None = None + referenced_table: str | None = None + referenced_columns: tuple[str, ...] = () + expression: str | None = None + + +@dataclass(frozen=True, slots=True) +class BlockedChange: + check: Check + rows: int + detail: str + + +@dataclass(frozen=True, slots=True) +class CouldNotVerify: + check: Check + timeout_seconds: float + + +Finding = BlockedChange | CouldNotVerify + + +@dataclass(frozen=True, slots=True) +class ColumnType: + """One column as `pg_attribute` and `pg_type` describe it: `length` is `attlen` (-1 for a + variable-length value), `align` is `attalign` (c, s, i or d), `typmod` is `atttypmod`.""" + + type_name: str + length: int + align: str + typmod: int + + @property + def is_variable_length(self) -> bool: + return self.length == -1 + + @property + def can_overflow_btree(self) -> bool: + """Whether one value of this column alone can push a btree entry past the limit: a + variable-length type with no declared cap, or a cap of more bytes than the limit.""" + if not self.is_variable_length: + return False + return self.typmod == -1 or (self.typmod - VARLENA_HEADER_BYTES) * 4 > btree_max_entry_bytes(DEFAULT_BLOCK_SIZE) + + +ColumnTypeLookup = Callable[[str, str], ColumnType | None] + +_IDENT: Final = r'"?(\w+)"?' +_QUALIFIED: Final = r'(?:"?\w+"?\s*\.\s*)?"?(\w+)"?' +_CREATE_INDEX: Final = re.compile( + rf"^\s*CREATE\s+(UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?{_IDENT}\s+ON\s+(?:ONLY\s+)?" + rf"{_QUALIFIED}\s*(?:USING\s+(\w+)\s*)?\((.*?)\)", + re.IGNORECASE | re.DOTALL, +) +_ALTER_TABLE: Final = re.compile( + rf"^\s*ALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?(?:ONLY\s+)?{_QUALIFIED}\s+(.*)$", re.IGNORECASE | re.DOTALL +) +_SET_NOT_NULL: Final = re.compile(rf"^ALTER\s+(?:COLUMN\s+)?{_IDENT}\s+SET\s+NOT\s+NULL$", re.IGNORECASE) +_ADD_COLUMN: Final = re.compile( + rf"^ADD\s+(?:COLUMN\s+)?(?:IF\s+NOT\s+EXISTS\s+)?(?!(?:CONSTRAINT|FOREIGN|UNIQUE|CHECK|PRIMARY|EXCLUDE)\b){_IDENT}\s+(.*)$", + re.IGNORECASE | re.DOTALL, +) +_CONSTRAINT: Final = rf"^ADD\s+(?:CONSTRAINT\s+{_IDENT}\s+)?" +_ADD_FOREIGN_KEY: Final = re.compile( + rf"{_CONSTRAINT}FOREIGN\s+KEY\s*\(([^)]*)\)\s*REFERENCES\s+{_QUALIFIED}\s*\(([^)]*)\)(.*)$", + re.IGNORECASE | re.DOTALL, +) +_ADD_UNIQUE: Final = re.compile(rf"{_CONSTRAINT}UNIQUE\s*\(([^)]*)\)", re.IGNORECASE | re.DOTALL) +_ADD_CHECK: Final = re.compile(rf"{_CONSTRAINT}CHECK\s*\((.*)\)\s*(NOT\s+VALID)?\s*$", re.IGNORECASE | re.DOTALL) +_PLAIN_COLUMN: Final = re.compile(rf"^{_IDENT}(?:\s+(?:ASC|DESC|NULLS\s+(?:FIRST|LAST)))*$", re.IGNORECASE) +_COMMENT: Final = re.compile(r"--[^\n]*") +_NOT_NULL: Final = re.compile(r"\bNOT\s+NULL\b", re.IGNORECASE) +_HAS_DEFAULT: Final = re.compile(r"\bDEFAULT\b|\bGENERATED\b", re.IGNORECASE) + + +def migration_data_check_timeout_seconds() -> float: + """Seconds one count query may run before the preflight gives up on verifying it.""" + raw: Final = os.getenv(MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR) + if raw is None: + return DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS + try: + return float(raw) + except ValueError: + logger.warning( + "%s=%r is not a number, using %s", + MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR, + raw, + DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS, + ) + return DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS + + +def statements(migration_sql: str) -> tuple[str, ...]: + """The statements of one migration file, comments stripped, empty ones dropped.""" + parts: Final = _COMMENT.sub("", migration_sql).split(";") + return tuple(part.strip() for part in parts if part.strip()) + + +def _split_top_level(text: str) -> tuple[str, ...]: + """Split on the commas outside parentheses.""" + depth: Final = accumulate((1 if char == "(" else -1 if char == ")" else 0) for char in text) + cuts: Final = tuple( + position for position, (char, level) in enumerate(zip(text, depth, strict=True)) if char == "," and level == 0 + ) + starts: Final = (0, *(cut + 1 for cut in cuts)) + stops: Final = (*cuts, len(text)) + pieces: Final = tuple(text[start:stop].strip() for start, stop in zip(starts, stops, strict=True)) + return tuple(piece for piece in pieces if piece) + + +def _plain_columns(column_list: str) -> tuple[str, ...] | None: + """The column names of an index or constraint column list, None when one entry is an + expression the derivation cannot turn into a count.""" + matches: Final = tuple(_PLAIN_COLUMN.match(entry) for entry in _split_top_level(column_list)) + if any(match is None for match in matches): + return None + return tuple(match.group(1) for match in matches if match is not None) + + +def _index_checks(match: "re.Match[str]", statement_text: str, source: str) -> Iterator[Check]: + unique, name, table, method, column_list = match.groups() + columns: Final = _plain_columns(column_list) + if columns is None: + return + label: Final = f'CREATE {"UNIQUE " if unique else ""}INDEX "{name}"' + if method is None or method.lower() == "btree": + yield Check("too_long", table, columns, label, source, index=name) + if unique: + yield Check("duplicates", table, columns, label, source, index=name) + + +def _alter_table_checks(table: str, action: str, source: str) -> Iterator[Check]: + if (set_not_null := _SET_NOT_NULL.match(action)) is not None: + yield Check("nulls", table, (set_not_null.group(1),), f'ALTER TABLE "{table}" {action}', source) + return + if (add_column := _ADD_COLUMN.match(action)) is not None: + column, body = add_column.groups() + if _NOT_NULL.search(body) is not None and _HAS_DEFAULT.search(body) is None: + yield Check( + "any_row", table, (column,), f'ALTER TABLE "{table}" ADD COLUMN "{column}" ... NOT NULL', source + ) + return + if (foreign_key := _ADD_FOREIGN_KEY.match(action)) is not None: + constraint, columns, referenced_table, referenced_columns, tail = foreign_key.groups() + own: Final = _plain_columns(columns) + theirs: Final = _plain_columns(referenced_columns) + if own is None or theirs is None or len(own) != len(theirs) or "NOT VALID" in tail.upper(): + return + yield Check( + "orphans", + table, + own, + _constraint_label(table, constraint, "FOREIGN KEY"), + source, + referenced_table=referenced_table, + referenced_columns=theirs, + ) + return + if (unique := _ADD_UNIQUE.match(action)) is not None: + constraint, columns = unique.groups() + unique_columns: Final = _plain_columns(columns) + if unique_columns is not None: + yield Check("duplicates", table, unique_columns, _constraint_label(table, constraint, "UNIQUE"), source) + return + if (check := _ADD_CHECK.match(action)) is not None: + constraint, expression, not_valid = check.groups() + if not_valid is None: + yield Check( + "check_violations", + table, + (), + _constraint_label(table, constraint, "CHECK"), + source, + expression=expression, + ) + + +def _constraint_label(table: str, constraint: str | None, kind: str) -> str: + named: Final = f'CONSTRAINT "{constraint}" ' if constraint else "" + return f'ALTER TABLE "{table}" ADD {named}{kind}' + + +def checks_for_statement(statement_text: str, source: str) -> tuple[Check, ...]: + """The count queries one SQL statement needs before it can be applied, empty for a + statement that cannot fail on existing rows or that the derivation cannot read.""" + if (index := _CREATE_INDEX.match(statement_text)) is not None: + return tuple(_index_checks(index, statement_text, source)) + if (alter := _ALTER_TABLE.match(statement_text)) is not None: + table, actions = alter.groups() + return tuple( + check for action in _split_top_level(actions) for check in _alter_table_checks(table, action, source) + ) # comprehension-ok: flattening the per-action checks of one statement + return () + + +def checks_for_migration(name: str, migration_sql: str) -> tuple[Check, ...]: + return tuple( + check + for statement_text in statements(migration_sql) + for check in checks_for_statement(statement_text, f"migration {name}") + ) # comprehension-ok: flattening the per-statement checks of one migration + + +def checks_for_index(table: str, name: str, columns: tuple[str, ...]) -> tuple[Check, ...]: + """The entry-size check a request-log index needs before `CREATE INDEX CONCURRENTLY`.""" + return (Check("too_long", table, columns, f'CREATE INDEX "{name}"', "request-log index", index=name),) + + +def _applied_migrations(connection: "psycopg.Connection[tuple[object, ...]]", schema: str) -> frozenset[str]: + import psycopg + from psycopg import sql + + try: + rows: Final = connection.execute( + sql.SQL("SELECT migration_name FROM {} WHERE finished_at IS NOT NULL AND rolled_back_at IS NULL").format( + sql.Identifier(schema, "_prisma_migrations") + ) + ).fetchall() + except psycopg.errors.UndefinedTable: + return frozenset() + return frozenset(str(row[0]) for row in rows) + + +def pending_migrations( + connection: "psycopg.Connection[tuple[object, ...]]", schema: str, migrations_dir: Path +) -> tuple[tuple[str, str], ...]: + """(name, sql) of every shipped migration the ledger has not finished, oldest first. + Without a ledger every migration is pending, and the checks skip tables that do not exist yet.""" + applied: Final = _applied_migrations(connection, schema) + folders: Final = tuple(folder for folder in sorted(migrations_dir.iterdir()) if folder.is_dir()) + return tuple( + (folder.name, (folder / "migration.sql").read_text()) + for folder in folders + if folder.name not in applied and (folder / "migration.sql").is_file() + ) + + +def _int(value: object, default: int) -> int: + return value if isinstance(value, int) else default + + +def column_type_lookup(connection: "psycopg.Connection[tuple[object, ...]]", schema: str) -> ColumnTypeLookup: + def lookup(table: str, column: str) -> ColumnType | None: + row: Final = connection.execute( + "SELECT t.typname, a.attlen, a.attalign, a.atttypmod FROM pg_attribute a " + "JOIN pg_class c ON c.oid = a.attrelid JOIN pg_namespace n ON n.oid = c.relnamespace " + "JOIN pg_type t ON t.oid = a.atttypid " + "WHERE n.nspname = %s AND c.relname = %s AND a.attname = %s AND a.attnum > 0 AND NOT a.attisdropped", + (schema, table, column), + ).fetchone() + if row is None: + return None + return ColumnType(str(row[0]), _int(row[1], -1), str(row[2]), _int(row[3], -1)) + + return lookup + + +def _column_types(check: Check, column_type: ColumnTypeLookup) -> tuple[ColumnType, ...] | None: + """The catalog types of the check's columns, None when one of them is not there yet.""" + types: Final = tuple(column_type(check.table, column) for column in check.columns) + if any(column is None for column in types): + return None + return tuple(column for column in types if column is not None) + + +def _table_exists(connection: "psycopg.Connection[tuple[object, ...]]", schema: str, table: str) -> bool: + row: Final = connection.execute( + "SELECT 1 FROM information_schema.tables WHERE table_schema = %s AND table_name = %s", (schema, table) + ).fetchone() + return row is not None + + +def _index_is_valid(connection: "psycopg.Connection[tuple[object, ...]]", schema: str, index: str) -> bool: + row: Final = connection.execute( + "SELECT x.indisvalid FROM pg_index x JOIN pg_class i ON i.oid = x.indexrelid " + "JOIN pg_namespace n ON n.oid = i.relnamespace WHERE n.nspname = %s AND i.relname = %s", + (schema, index), + ).fetchone() + return row is not None and bool(row[0]) + + +def btree_max_entry_bytes(block_size: int) -> int: + """`BTMaxItemSize` from nbtree.h for btree version 4: a third of the page left after the page + header, three line pointers and the btree opaque area, rounded down to 8, minus the heap TID a + pivot tuple carries. 2704 on the default 8 KB page, the number in Postgres's error.""" + usable: Final = block_size - _PAGE_HEADER_WITH_THREE_LINE_POINTERS_BYTES - _BTREE_PAGE_OPAQUE_BYTES + return (usable // 3) // 8 * 8 - _HEAP_TID_BYTES + + +def _value_bytes(column: ColumnType, identifier: "sql.Identifier") -> "sql.Composed": + from psycopg import sql + + if column.type_name in _OCTET_LENGTH_TYPES: + return sql.SQL("octet_length({})").format(identifier) + return sql.SQL("octet_length({}::text)").format(identifier) + + +def _aligned(offset: "sql.Composable", align: str) -> "sql.Composed": + from psycopg import sql + + unit: Final = _ALIGNMENT.get(align, 8) + return sql.SQL("(({off} + {pad}) / {unit}) * {unit}").format( + off=offset, pad=sql.Literal(unit - 1), unit=sql.Literal(unit) + ) + + +def _value_end(column: ColumnType, identifier: "sql.Identifier", offset: "sql.Composable") -> "sql.Composed": + """Where this column's value ends when laid down from `offset` the way `index_form_tuple` + does it: NULL takes no room, a fixed-length value is aligned to its type, a variable-length + value of up to 126 bytes gets a 1-byte header and no alignment, a longer one a 4-byte header.""" + from psycopg import sql + + length: Final = _value_bytes(column, identifier) + placed: Final = ( + sql.SQL("{} + {}").format(_aligned(offset, column.align), sql.Literal(column.length)) + if not column.is_variable_length + else sql.SQL("CASE WHEN {len} <= {short} THEN {off} + 1 + {len} ELSE {aligned} + {hdr} + {len} END").format( + len=length, + short=sql.Literal(SHORT_VARLENA_MAX_BYTES), + off=offset, + aligned=_aligned(offset, column.align), + hdr=sql.Literal(VARLENA_HEADER_BYTES), + ) + ) + return sql.SQL("CASE WHEN {col} IS NULL THEN {off} ELSE {placed} END").format( + col=identifier, off=offset, placed=placed + ) + + +def _entry_size_query(check: Check, schema: str, types: tuple[ColumnType, ...], limit_bytes: int) -> "sql.Composed": + """count(*) and the largest entry over the rows whose btree entry would not fit, plus the + longest value of each variable-length column: an 8-byte tuple header, 8 more for the null + bitmap when a column is NULL, each value laid down in index order, the total rounded up to 8.""" + from psycopg import sql + + identifiers: Final = tuple(sql.Identifier(column) for column in check.columns) + any_null: Final = sql.SQL(" OR ").join(sql.SQL("{} IS NULL").format(identifier) for identifier in identifiers) + header: Final = sql.SQL("{} + CASE WHEN {} THEN {} ELSE 0 END").format( + sql.Literal(INDEX_TUPLE_HEADER_BYTES), any_null, sql.Literal(NULL_BITMAP_BYTES) + ) + innermost: Final = sql.SQL("SELECT {cols}, {hdr} AS off0 FROM {t}").format( + cols=sql.SQL(", ").join(identifiers), hdr=header, t=sql.Identifier(schema, check.table) + ) + + def lay_down(inner: "sql.Composed", step: tuple[int, tuple[ColumnType, "sql.Identifier"]]) -> "sql.Composed": + position, (column, identifier) = step + return sql.SQL("SELECT *, {end} AS {next} FROM ({inner}) AS {alias}").format( + end=_value_end(column, identifier, sql.Identifier(f"off{position}")), + next=sql.Identifier(f"off{position + 1}"), + inner=inner, + alias=sql.Identifier(f"s{position}"), + ) + + laid_out: Final = reduce(lay_down, enumerate(zip(types, identifiers, strict=True)), innermost) + entry: Final = _aligned(sql.Identifier(f"off{len(types)}"), "d") + longest: Final = sql.SQL("").join( + sql.SQL(", max({})").format(_value_bytes(column, identifier)) + for column, identifier in zip(types, identifiers, strict=True) + if column.is_variable_length + ) + return sql.SQL("SELECT count(*), max({entry}){longest} FROM ({rows}) AS entries WHERE {entry} > {limit}").format( + entry=entry, longest=longest, rows=laid_out, limit=sql.Literal(limit_bytes) + ) + + +def _count_query(check: Check, schema: str, types: tuple[ColumnType, ...], limit_bytes: int) -> "sql.Composed": + from psycopg import sql + + table: Final = sql.Identifier(schema, check.table) + columns: Final = sql.SQL(", ").join(sql.Identifier(column) for column in check.columns) + match check.kind: + case "too_long": + return _entry_size_query(check, schema, types, limit_bytes) + case "duplicates": + not_null: Final = sql.SQL(" AND ").join( + sql.SQL("{} IS NOT NULL").format(sql.Identifier(column)) for column in check.columns + ) + return sql.SQL( + "SELECT count(*) FROM (SELECT 1 FROM {t} WHERE {nn} GROUP BY {cols} HAVING count(*) > 1) AS d" + ).format(t=table, nn=not_null, cols=columns) + case "nulls": + return sql.SQL("SELECT count(*) FROM {t} WHERE {col} IS NULL").format( + t=table, col=sql.Identifier(check.columns[0]) + ) + case "any_row": + return sql.SQL("SELECT count(*) FROM {t}").format(t=table) + case "orphans": + pairs: Final = sql.SQL(" AND ").join( + sql.SQL("parent.{} = child.{}").format(sql.Identifier(theirs), sql.Identifier(own)) + for own, theirs in zip(check.columns, check.referenced_columns, strict=True) + ) + present: Final = sql.SQL(" AND ").join( + sql.SQL("child.{} IS NOT NULL").format(sql.Identifier(column)) for column in check.columns + ) + return sql.SQL( + "SELECT count(*) FROM {t} AS child WHERE {present} AND NOT EXISTS (SELECT 1 FROM {ref} AS parent WHERE {pairs})" + ).format(t=table, present=present, ref=sql.Identifier(schema, check.referenced_table or ""), pairs=pairs) + case "check_violations": + return sql.SQL("SELECT count(*) FROM {t} WHERE NOT ({expr})").format( + t=table, + expr=sql.SQL(check.expression or "true"), # pyright: ignore[reportArgumentType] # copied from our own migration file + ) + + +def _applies( + connection: "psycopg.Connection[tuple[object, ...]]", + schema: str, + check: Check, + types: tuple[ColumnType, ...] | None, +) -> bool: + """False when the check has nothing to count yet: the table or a column is not there + (a pending migration adds it), the index is already valid, or no column can overflow.""" + if not _table_exists(connection, schema, check.table): + return False + if check.index is not None and _index_is_valid(connection, schema, check.index): + return False + if check.kind == "any_row": + return types is None + if types is None: + return False + if check.kind == "too_long": + return any(column.can_overflow_btree for column in types) + if check.kind == "orphans": + return check.referenced_table is not None and _table_exists(connection, schema, check.referenced_table) + return True + + +def _too_long_detail(check: Check, types: tuple[ColumnType, ...], values: tuple[object, ...], limit_bytes: int) -> str: + variable: Final = tuple( + column for column, column_type in zip(check.columns, types, strict=True) if column_type.is_variable_length + ) + longest: Final = ", ".join( + f"{column} up to {length} bytes" + for column, length in zip(variable, values[2:], strict=True) + if length is not None + ) + return f"with an index entry over {limit_bytes} bytes, largest {values[1]} bytes ({longest})" + + +def _run_check( + connection: "psycopg.Connection[tuple[object, ...]]", + schema: str, + check: Check, + types: tuple[ColumnType, ...], + limit_bytes: int, + timeout_seconds: float, +) -> Finding | None: + import psycopg + + try: + row: Final = connection.execute(_count_query(check, schema, types, limit_bytes)).fetchone() + except psycopg.errors.QueryCanceled: + return CouldNotVerify(check, timeout_seconds) + values: Final[tuple[object, ...]] = row if row is not None else () + rows: Final = _int(values[0], 0) if values else 0 + if rows == 0: + return None + match check.kind: + case "too_long": + return BlockedChange(check, rows, _too_long_detail(check, types, values, limit_bytes)) + case "duplicates": + return BlockedChange(check, rows, "duplicated value groups") + case "nulls": + return BlockedChange(check, rows, "NULL") + case "any_row": + return BlockedChange( + check, rows, "rows, and a NOT NULL column with no DEFAULT cannot be added to a table with rows" + ) + case "orphans": + return BlockedChange(check, rows, f'with no matching row in "{check.referenced_table}"') + case "check_violations": + return BlockedChange(check, rows, f"failing CHECK ({check.expression})") + + +def find_blockers( + connection: "psycopg.Connection[tuple[object, ...]]", + schema: str, + checks: tuple[Check, ...], + timeout_seconds: float = DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS, +) -> tuple[Finding, ...]: + """Run every applicable check read-only, each under `timeout_seconds`. Never writes + and never takes a table lock; a check that times out is a finding, not a pass.""" + from psycopg import sql + + previous_timeout: Final = connection.execute("SHOW statement_timeout").fetchone() + previous_read_only: Final = connection.execute("SHOW default_transaction_read_only").fetchone() + block_size: Final = connection.execute("SHOW block_size").fetchone() + limit_bytes: Final = btree_max_entry_bytes(int(str(block_size[0])) if block_size else DEFAULT_BLOCK_SIZE) + connection.execute("SET default_transaction_read_only = on") + connection.execute(sql.SQL("SET statement_timeout = {}").format(sql.Literal(int(timeout_seconds * 1000)))) + column_type: Final = column_type_lookup(connection, schema) + typed: Final = tuple((check, _column_types(check, column_type)) for check in checks) + applicable: Final = tuple( + (check, types if types is not None else ()) + for check, types in typed + if _applies(connection, schema, check, types) + ) + results: Final = tuple( + _run_check(connection, schema, check, types, limit_bytes, timeout_seconds) for check, types in applicable + ) + connection.execute( + sql.SQL("SET statement_timeout = {}").format(sql.Literal(str(previous_timeout[0]) if previous_timeout else "0")) + ) + connection.execute( + sql.SQL("SET default_transaction_read_only = {}").format( + sql.Literal(str(previous_read_only[0]) if previous_read_only else "off") + ) + ) + return tuple(result for result in results if result is not None) + + +def _remedy(check: Check) -> str: + if check.kind == "too_long" and check.table == "LiteLLM_SpendLogs" and "api_key" in check.columns: + return LEGACY_SPEND_LOG_KEY_REMEDY + joined: Final = ", ".join(check.columns) + match check.kind: + case "too_long": + return f'Shorten or hash the long "{check.table}" ({joined}) values so every row\'s index entry fits.' + case "duplicates": + return f'Remove or merge the duplicate "{check.table}" ({joined}) values.' + case "nulls": + return f'Fill the NULL "{check.table}"."{joined}" values, or give the column a default first.' + case "any_row": + return "Give the new column a DEFAULT in the migration, or add it nullable and fill it afterwards." + case "orphans": + return f'Delete or repoint the "{check.table}" ({joined}) values with no "{check.referenced_table}" row.' + case "check_violations": + return f'Fix the "{check.table}" rows that fail the expression before adding the constraint.' + + +def _describe(finding: Finding) -> str: + check: Final = finding.check + where: Final = f'"{check.table}"' + (f".{', '.join(check.columns)}" if check.columns else "") + match finding: + case CouldNotVerify(timeout_seconds=seconds): + return ( + f" {where}: could not be checked within {seconds:g} s, refusing rather than guessing\n" + f" blocks: {check.statement} ({check.source})\n" + f" raise {MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR} to give the scan more time" + ) + case BlockedChange(rows=rows, detail=detail): + return ( + f" {where}: {rows} row(s) {detail}\n" + f" blocks: {check.statement} ({check.source})\n" + f" fix: {_remedy(check).replace(chr(10), chr(10) + ' ' * 8)}" + ) + + +def report(findings: tuple[Finding, ...]) -> str: + """The operator-facing refusal: one block per finding, then what to do next.""" + lines: Final = "\n".join(_describe(finding) for finding in findings) + return ( + f"{REFUSAL}\n{lines}\n" + "No schema change was made. Fix the rows and start again, or run `litellm --check_migrations_against_existing_data` with the new image to check again." + ) + + +def pending_change_checks( + connection: "psycopg.Connection[tuple[object, ...]]", + schema: str, + migrations_dir: Path | None, + indexes: tuple[tuple[str, str, tuple[str, ...]], ...], +) -> tuple[Check, ...]: + """Every check the pending migrations and the request-log indexes need. `migrations_dir` + is None when the schema is pushed rather than migrated, so only the indexes are checked.""" + migration_checks: Final = ( + tuple( + check + for name, migration_sql in pending_migrations(connection, schema, migrations_dir) + for check in checks_for_migration(name, migration_sql) + ) # comprehension-ok: flattening the per-migration checks + if migrations_dir is not None + else () + ) + index_checks: Final = tuple( + check for table, name, columns in indexes for check in checks_for_index(table, name, columns) + ) # comprehension-ok: flattening the per-index checks + return migration_checks + index_checks diff --git a/litellm-proxy-extras/litellm_proxy_extras/request_log_indexes.py b/litellm-proxy-extras/litellm_proxy_extras/request_log_indexes.py index 6c31e8364a9..4c856c741d3 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/request_log_indexes.py +++ b/litellm-proxy-extras/litellm_proxy_extras/request_log_indexes.py @@ -18,6 +18,12 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Final from litellm_proxy_extras._logging import logger +from litellm_proxy_extras.migration_data_checks import ( + checks_for_index, + find_blockers, + migration_data_check_timeout_seconds, + report, +) from litellm_proxy_extras.migration_lock import held_migration_lock if TYPE_CHECKING: @@ -355,6 +361,12 @@ def _build_leaf_index( connection.execute(sql.SQL("DROP INDEX CONCURRENTLY {}").format(sql.Identifier(schema, name))) elif _adopt_equivalent_index(connection, schema, table, name, index): return True + blockers: Final = find_blockers( + connection, schema, checks_for_index(table, name, index.columns), migration_data_check_timeout_seconds() + ) + if blockers: + logger.warning("Not building index %s on %s. %s", name, table, report(blockers)) + return False logger.info("Building index %s on %s concurrently", name, table) prefix: Final = sql.SQL("CREATE INDEX CONCURRENTLY IF NOT EXISTS {} ON {} ").format( sql.Identifier(name), sql.Identifier(schema, table) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index be785e2fd78..528bf7def20 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -16,6 +16,13 @@ from typing import TYPE_CHECKING, Final, Optional, Protocol, Union from litellm_proxy_extras import prisma_toolchain from litellm_proxy_extras._logging import logger from litellm_proxy_extras._redaction import _redact_credentials +from litellm_proxy_extras.migration_data_checks import ( + Finding, + find_blockers, + migration_data_check_timeout_seconds, + pending_change_checks, + report, +) from litellm_proxy_extras.migration_lock import held_migration_lock from litellm_proxy_extras.prisma_toolchain import ( PRISMA_COMMAND_TIMEOUT_ENV_VAR, @@ -28,7 +35,11 @@ from litellm_proxy_extras.replica_identity import ( REPLICA_IDENTITY_FULL_ENV_VAR, apply_replica_identity_full, ) -from litellm_proxy_extras.request_log_indexes import ensure_request_log_indexes, filter_request_log_index_diff +from litellm_proxy_extras.request_log_indexes import ( + REQUEST_LOG_INDEXES, + ensure_request_log_indexes, + filter_request_log_index_diff, +) if TYPE_CHECKING: import psycopg @@ -53,9 +64,7 @@ def _get_prisma_env() -> dict: if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")): # These env vars prevent Prisma from attempting downloads prisma_env["NPM_CONFIG_PREFER_OFFLINE"] = "true" - prisma_env["NPM_CONFIG_CACHE"] = os.getenv( - "NPM_CONFIG_CACHE", "/app/.cache/npm" - ) + prisma_env["NPM_CONFIG_CACHE"] = os.getenv("NPM_CONFIG_CACHE", "/app/.cache/npm") return prisma_env @@ -86,6 +95,7 @@ class _InvalidIndex: name: str table_size: str + MAX_MIGRATE_DEPLOY_ATTEMPTS = 4 LIBPQ_URL_PARAMS: Final = frozenset( { @@ -135,9 +145,7 @@ class _MigrateAttemptBudget: _SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE) -_SPEND_LOGS_ARTIFACT_DROP_RE = re.compile( - r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE -) +_SPEND_LOGS_ARTIFACT_DROP_RE = re.compile(r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE) _SPEND_LOGS_PK_CLAUSE_RE = re.compile( r'^(?:DROP\s+CONSTRAINT\s+"[^"]*_pkey"' r'|ADD\s+(?:CONSTRAINT\s+"[^"]*"\s+)?PRIMARY\s+KEY\s*\([^)]*\))$', @@ -146,9 +154,9 @@ _SPEND_LOGS_PK_CLAUSE_RE = re.compile( PARTITIONED_SPEND_LOGS_PUSH_ERROR = ( "LiteLLM_SpendLogs is a partitioned table (see db_scripts/partition_spend_logs.sql), " - "so its primary key must include the partition key (\"startTime\"). `prisma db push` " + 'so its primary key must include the partition key ("startTime"). `prisma db push` ' "reconciles the database against schema.prisma, which declares the unpartitioned " - "primary key (\"request_id\"), and Postgres rejects that rewrite with: unique " + 'primary key ("request_id"), and Postgres rejects that rewrite with: unique ' "constraint on partitioned table must include all partitioning columns. Start the " "proxy without --use_prisma_db_push so it uses `prisma migrate deploy`, which only " "applies shipped migrations and leaves the partitioned primary key alone." @@ -157,9 +165,7 @@ PARTITIONED_SPEND_LOGS_PUSH_ERROR = ( def _without_sql_comments(statement: str) -> str: return "\n".join( - line - for line in statement.splitlines() - if line.strip() and not line.strip().startswith("--") + line for line in statement.splitlines() if line.strip() and not line.strip().startswith("--") ).strip() @@ -169,7 +175,7 @@ def _without_spend_logs_pk_clauses(statement: str) -> Optional[str]: return statement kept = tuple( clause.strip() - for clause in statement[prefix_match.end():].split(",\n") + for clause in statement[prefix_match.end() :].split(",\n") if not _SPEND_LOGS_PK_CLAUSE_RE.match(clause.strip()) ) if not kept: @@ -243,8 +249,7 @@ def _get_prisma_command() -> str: # If not found, log warning and fall back logger.warning( - f"Prisma CLI not found at {default_cli_path}. " - "Falling back to Python wrapper (may attempt downloads)" + f"Prisma CLI not found at {default_cli_path}. Falling back to Python wrapper (may attempt downloads)" ) # Fall back to the Python wrapper (will work in online mode) @@ -331,9 +336,7 @@ class ProxyExtrasDBManager: return True except subprocess.TimeoutExpired: - logger.warning( - "Migration timed out - the database might be under heavy load." - ) + logger.warning("Migration timed out - the database might be under heavy load.") return False except subprocess.CalledProcessError as e: logger.warning( @@ -386,10 +389,12 @@ class ProxyExtrasDBManager: except ImportError: return None + from psycopg import sql + cleaned_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(database_url) - ledger_table: Final = psycopg.sql.SQL("{}.{}").format( - psycopg.sql.Identifier(ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"), - psycopg.sql.Identifier("_prisma_migrations"), + ledger_table: Final = sql.SQL("{}.{}").format( + sql.Identifier(ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"), + sql.Identifier("_prisma_migrations"), ) try: with psycopg.connect(cleaned_url, connect_timeout=10, autocommit=True) as conn: @@ -500,9 +505,7 @@ class ProxyExtrasDBManager: return filter_partitioned_spend_logs_diff(without_indexes) @staticmethod - def _resolve_all_migrations( - migrations_dir: str, schema_path: str, mark_all_applied: bool = True - ): + def _resolve_all_migrations(migrations_dir: str, schema_path: str, mark_all_applied: bool = True): """ 1. Compare the current database state to schema.prisma and generate a migration for the diff. 2. Run prisma migrate deploy to apply any pending migrations. @@ -571,9 +574,7 @@ class ProxyExtrasDBManager: ) logger.info(f"Applied migration: {mig_file.parent.name}") except subprocess.CalledProcessError as e: - logger.warning( - f"Failed to apply migration {mig_file.parent.name}: {e.stderr}" - ) + logger.warning(f"Failed to apply migration {mig_file.parent.name}: {e.stderr}") except subprocess.TimeoutExpired: logger.warning(f"Migration {mig_file.parent.name} timed out.") return @@ -645,9 +646,7 @@ class ProxyExtrasDBManager: logger.debug(f"Resolved migration: {migration_name}") except subprocess.CalledProcessError as e: if "is already recorded as applied in the database." not in e.stderr: - logger.warning( - f"Failed to resolve migration {migration_name}: {e.stderr}" - ) + logger.warning(f"Failed to resolve migration {migration_name}: {e.stderr}") @staticmethod def raise_if_lens_rename_pending(connect: LensCheckConnect | None = None) -> None: @@ -707,9 +706,7 @@ class ProxyExtrasDBManager: cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url) try: - with psycopg.connect( - cleaned_url, connect_timeout=10, autocommit=True - ) as conn: + with psycopg.connect(cleaned_url, connect_timeout=10, autocommit=True) as conn: row = conn.execute( "SELECT 1 " "FROM pg_partitioned_table pt " @@ -717,10 +714,7 @@ class ProxyExtrasDBManager: "JOIN pg_namespace n ON n.oid = c.relnamespace " "WHERE c.relname = 'LiteLLM_SpendLogs' " " AND n.nspname = %s", - ( - ProxyExtrasDBManager._prisma_schema_param(database_url) - or "public", - ), + (ProxyExtrasDBManager._prisma_schema_param(database_url) or "public",), ).fetchone() except (psycopg.OperationalError, psycopg.DatabaseError): return False @@ -815,9 +809,7 @@ class ProxyExtrasDBManager: # it, psycopg3's `with conn` calls COMMIT on clean exit — which # fails after `UndefinedTable` (fresh DB) leaves the transaction # in an aborted state. - with psycopg.connect( - cleaned_url, connect_timeout=10, autocommit=True - ) as conn: + with psycopg.connect(cleaned_url, connect_timeout=10, autocommit=True) as conn: try: rows = conn.execute( "SELECT migration_name FROM _prisma_migrations " @@ -838,9 +830,7 @@ class ProxyExtrasDBManager: return head_newest_ts = _max_migration_timestamp(known) - hostile = { - name for name in unknown if _migration_timestamp(name) > head_newest_ts - } + hostile = {name for name in unknown if _migration_timestamp(name) > head_newest_ts} if not hostile: return @@ -931,8 +921,7 @@ class ProxyExtrasDBManager: if not found: return True logger.warning( - "Found %d invalid index(es) left by an interrupted CREATE INDEX " - "CONCURRENTLY, rebuilding: %s", + "Found %d invalid index(es) left by an interrupted CREATE INDEX CONCURRENTLY, rebuilding: %s", len(found), ", ".join(f"{index.name} (table size {index.table_size})" for index in found), ) @@ -1403,8 +1392,7 @@ class ProxyExtrasDBManager: prisma_env = _get_prisma_env() except OSError as e: logger.error( - "Could not resolve the migrations directory for the REPLICA " - "IDENTITY FULL step, skipping it. Error: %s", + "Could not resolve the migrations directory for the REPLICA IDENTITY FULL step, skipping it. Error: %s", e, ) return False @@ -1435,18 +1423,57 @@ class ProxyExtrasDBManager: Returns: bool: True if setup was successful, False otherwise """ - ensure_prisma_toolchain( - prisma_command=_get_prisma_command(), prisma_env=_get_prisma_env() - ) - migrated = ProxyExtrasDBManager._run_migrations( - use_migrate=use_migrate, use_v2_resolver=use_v2_resolver - ) + ensure_prisma_toolchain(prisma_command=_get_prisma_command(), prisma_env=_get_prisma_env()) + ProxyExtrasDBManager.raise_if_rows_block_pending_changes(use_migrate=use_migrate) + migrated = ProxyExtrasDBManager._run_migrations(use_migrate=use_migrate, use_v2_resolver=use_v2_resolver) if not migrated: return False ProxyExtrasDBManager.repair_invalid_indexes() ProxyExtrasDBManager.apply_replica_identity_full_if_requested() return True + @staticmethod + def migration_data_check_findings(use_migrate: bool = True) -> tuple[Finding, ...]: + """What in the existing rows would make a pending migration or request-log index + fail, read with count queries only: no write, no table lock. Empty when nothing + blocks, when no database is configured, or when it cannot be reached, so the + migration step that follows reports the connection problem the way it does today.""" + import psycopg + + database_url: Final = os.environ.get("DATABASE_URL") + if not database_url: + return () + direct_url: Final = ProxyExtrasDBManager._strip_prisma_query_params( + os.environ.get("DIRECT_URL") or database_url + ) + schema: Final = ProxyExtrasDBManager._prisma_schema_param(database_url) or "public" + migrations_dir: Final = Path(ProxyExtrasDBManager._get_prisma_dir()) / "migrations" if use_migrate else None + indexes: Final = tuple((index.table, index.name, index.columns) for index in REQUEST_LOG_INDEXES) + try: + with psycopg.connect(direct_url, connect_timeout=10, autocommit=True) as connection: + checks: Final = pending_change_checks(connection, schema, migrations_dir, indexes) + return find_blockers(connection, schema, checks, migration_data_check_timeout_seconds()) + except psycopg.OperationalError as exc: + logger.warning( + "Could not reach the database for the pre-migration row check: %s", _redact_credentials(str(exc)) + ) + return () + + @staticmethod + def migration_data_check_report(use_migrate: bool = True) -> Optional[str]: + """The operator-facing refusal for `migration_data_check_findings`, None when nothing blocks.""" + findings: Final = ProxyExtrasDBManager.migration_data_check_findings(use_migrate=use_migrate) + return report(findings) if findings else None + + @staticmethod + def raise_if_rows_block_pending_changes(use_migrate: bool) -> None: + """Refuse to run migrations that the existing rows would make fail, before any is + applied. The RuntimeError carries the report and reaches the operator through the + proxy's "Database migration cannot proceed" exit.""" + refusal: Final = ProxyExtrasDBManager.migration_data_check_report(use_migrate=use_migrate) + if refusal is not None: + raise RuntimeError(refusal) + @staticmethod def build_request_log_indexes(build: Callable[[str, str], bool] = ensure_request_log_indexes) -> bool: """Build the indexes in `REQUEST_LOG_INDEXES` on the writer, in the schema the @@ -1513,9 +1540,7 @@ class ProxyExtrasDBManager: # Skip sanity check when deploy reports no pending migrations — # DB already matches schema, no drift to correct. if "No pending migrations to apply" in result.stdout: - logger.info( - "No pending migrations — skipping post-migration sanity check" - ) + logger.info("No pending migrations — skipping post-migration sanity check") return True # Run sanity check to ensure DB matches schema @@ -1530,9 +1555,7 @@ class ProxyExtrasDBManager: logger.info(f"prisma db error: {stderr}, e: {e.stdout}") if "P3009" in stderr: # Extract the failed migration name from the error message - migration_match = re.search( - r"`(\d+_.*)` migration", stderr - ) + migration_match = re.search(r"`(\d+_.*)` migration", stderr) if migration_match: failed_migration = migration_match.group(1) if ProxyExtrasDBManager._is_idempotent_error(stderr): @@ -1540,9 +1563,7 @@ class ProxyExtrasDBManager: f"Migration {failed_migration} failed due to idempotent error (e.g., column already exists), resolving as applied" ) try: - ProxyExtrasDBManager._roll_back_migration( - failed_migration - ) + ProxyExtrasDBManager._roll_back_migration(failed_migration) except ( subprocess.CalledProcessError, subprocess.TimeoutExpired, @@ -1552,9 +1573,7 @@ class ProxyExtrasDBManager: f"It may already be in a rolled-back state." ) try: - ProxyExtrasDBManager._resolve_specific_migration( - failed_migration - ) + ProxyExtrasDBManager._resolve_specific_migration(failed_migration) logger.info( f"✅ Migration {failed_migration} resolved, retrying to apply remaining migrations" ) @@ -1562,9 +1581,7 @@ class ProxyExtrasDBManager: subprocess.CalledProcessError, subprocess.TimeoutExpired, ) as resolve_err: - logger.warning( - f"Failed to resolve migration {failed_migration}: {resolve_err}" - ) + logger.warning(f"Failed to resolve migration {failed_migration}: {resolve_err}") # Apply any schema drift not covered by the marked-as-applied migration ProxyExtrasDBManager._resolve_all_migrations( migrations_dir, @@ -1572,9 +1589,7 @@ class ProxyExtrasDBManager: mark_all_applied=False, ) else: - logger.info( - f"Found failed migration: {failed_migration}, marking as rolled back" - ) + logger.info(f"Found failed migration: {failed_migration}, marking as rolled back") # Mark the failed migration as rolled back prisma_toolchain.run_prisma( [ @@ -1587,23 +1602,14 @@ class ProxyExtrasDBManager: timeout=prisma_command_timeout(), env=_get_prisma_env(), ) - logger.info( - f"✅ Migration {failed_migration} marked as rolled back... retrying" - ) - elif ( - "P3005" in stderr - and "database schema is not empty" in stderr - ): + logger.info(f"✅ Migration {failed_migration} marked as rolled back... retrying") + elif "P3005" in stderr and "database schema is not empty" in stderr: logger.info( "Database schema is not empty, creating baseline migration. In read-only file system, please set an environment variable `LITELLM_MIGRATION_DIR` to a writable directory to enable migrations. Learn more - https://docs.litellm.ai/docs/proxy/prod#read-only-file-system" ) ProxyExtrasDBManager._create_baseline_migration(schema_path) - logger.info( - "Baseline migration created, resolving all migrations" - ) - ProxyExtrasDBManager._resolve_all_migrations( - migrations_dir, schema_path - ) + logger.info("Baseline migration created, resolving all migrations") + ProxyExtrasDBManager._resolve_all_migrations(migrations_dir, schema_path) logger.info("✅ All migrations resolved.") return True elif "P3018" in stderr: @@ -1611,14 +1617,8 @@ class ProxyExtrasDBManager: if ProxyExtrasDBManager._is_permission_error(stderr): # Permission errors should NOT be marked as applied # Extract migration name for logging - migration_match = re.search( - r"Migration name: (\d+_.*)", stderr - ) - migration_name = ( - migration_match.group(1) - if migration_match - else "unknown" - ) + migration_match = re.search(r"Migration name: (\d+_.*)", stderr) + migration_name = migration_match.group(1) if migration_match else "unknown" logger.error( f"❌ Migration {migration_name} failed due to insufficient permissions. " @@ -1628,16 +1628,10 @@ class ProxyExtrasDBManager: # Mark as rolled back and exit with error if migration_match: try: - ProxyExtrasDBManager._roll_back_migration( - migration_name - ) - logger.info( - f"Migration {migration_name} marked as rolled back" - ) + ProxyExtrasDBManager._roll_back_migration(migration_name) + logger.info(f"Migration {migration_name} marked as rolled back") except Exception as rollback_error: - logger.warning( - f"Failed to mark migration as rolled back: {rollback_error}" - ) + logger.warning(f"Failed to mark migration as rolled back: {rollback_error}") # Re-raise the error to prevent silent failures raise RuntimeError( @@ -1652,18 +1646,12 @@ class ProxyExtrasDBManager: "resolving as applied" ) # Extract the migration name from the error message - migration_match = re.search( - r"Migration name: (\d+_.*)", stderr - ) + migration_match = re.search(r"Migration name: (\d+_.*)", stderr) if migration_match: migration_name = migration_match.group(1) try: - logger.info( - f"Rolling back migration {migration_name}" - ) - ProxyExtrasDBManager._roll_back_migration( - migration_name - ) + logger.info(f"Rolling back migration {migration_name}") + ProxyExtrasDBManager._roll_back_migration(migration_name) except ( subprocess.CalledProcessError, subprocess.TimeoutExpired, @@ -1677,9 +1665,7 @@ class ProxyExtrasDBManager: f"Resolving migration {migration_name} that failed " f"due to existing schema objects" ) - ProxyExtrasDBManager._resolve_specific_migration( - migration_name - ) + ProxyExtrasDBManager._resolve_specific_migration(migration_name) logger.info( f"✅ Migration {migration_name} resolved, " f"retrying to apply remaining migrations" @@ -1688,9 +1674,7 @@ class ProxyExtrasDBManager: subprocess.CalledProcessError, subprocess.TimeoutExpired, ) as resolve_err: - logger.warning( - f"Failed to resolve migration {migration_name}: {resolve_err}" - ) + logger.warning(f"Failed to resolve migration {migration_name}: {resolve_err}") # Apply any schema drift not covered by the marked-as-applied migration ProxyExtrasDBManager._resolve_all_migrations( migrations_dir, @@ -1732,14 +1716,8 @@ class ProxyExtrasDBManager: time.sleep(random.randrange(5, 15)) except subprocess.CalledProcessError as e: attempts_left = 3 - attempt - retry_msg = ( - f" Retrying... ({attempts_left} attempts left)" - if attempts_left > 0 - else "" - ) - stderr_detail: Final = ( - f" stderr: {_redact_credentials(str(e.stderr))}" if e.stderr else "" - ) + retry_msg = f" Retrying... ({attempts_left} attempts left)" if attempts_left > 0 else "" + stderr_detail: Final = f" stderr: {_redact_credentials(str(e.stderr))}" if e.stderr else "" logger.error( f"The process failed to execute. Details: {_redact_command_error(e)}.{stderr_detail}{retry_msg}" ) diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index 53971ed412c..3d783d8aba1 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -1126,6 +1126,18 @@ class PrismaManager: return False return ProxyExtrasDBManager.build_request_log_indexes() + @staticmethod + def migration_data_check_report(use_migrate: bool = True) -> str | None: + """The read-only pre-migration row check on its own, for `--check_migrations_against_existing_data`: the + refusal the migration step would print, or None when the pending schema changes + can be applied to the rows the database has.""" + from litellm_proxy_extras.utils import ( + ProxyExtrasDBManager, # pyright: ignore[reportUnknownVariableType] # litellm-proxy-extras is an optional install + ) + + refusal: Final[object] = ProxyExtrasDBManager.migration_data_check_report(use_migrate=use_migrate) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # litellm-proxy-extras is an optional install + return refusal if isinstance(refusal, str) else None + @staticmethod def start_request_log_index_build() -> None: """Build the request-log indexes on a daemon thread, for a serving proxy that ran the diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 7f68b72a522..fc33ab1bd0b 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -218,6 +218,23 @@ class ProxyInitializationHelpers: _run_health_check = run_health_check + @staticmethod + def run_migration_data_check(use_migrate: bool) -> int: + """`--check_migrations_against_existing_data`: the pre-migration row check against DATABASE_URL, nothing applied. + 0 when the pending schema changes fit the rows, 2 with the refusal when they do not.""" + if os.getenv("DATABASE_URL") is None: + raise click.UsageError("--check_migrations_against_existing_data requires DATABASE_URL") + from litellm.proxy.db.prisma_client import PrismaManager + + refusal: Final[str | None] = PrismaManager.migration_data_check_report(use_migrate=use_migrate) # pyright: ignore[reportUnknownVariableType] # litellm-proxy-extras is an optional install + if refusal is None: + print("LiteLLM Proxy: the pending migrations and indexes can be applied to the rows this database has.") + return 0 + print( + f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {refusal}\033[0m", file=sys.stderr, flush=True + ) + return 2 + @staticmethod def run_config_validation(config: str | None) -> None: if config is None: @@ -978,6 +995,13 @@ class ProxyInitializationHelpers: default=False, help="Load and validate the config file (including mcp_servers) without starting the server, then exit. Exit code 1 on any config error.", ) +@click.option( + "--check_migrations_against_existing_data", + is_flag=True, + default=False, + help="Check, read-only, that the rows in DATABASE_URL can take this version's pending migrations and indexes, " + "print what would block them, then exit without applying anything. Exit code 2 when something blocks.", +) @click.option( "--keepalive_timeout", default=None, @@ -1127,6 +1151,7 @@ def run_server( use_prisma_db_push: bool, skip_server_startup, validate_config: bool, + check_migrations_against_existing_data: bool, keepalive_timeout, timeout_worker_healthcheck, max_requests_before_restart: int | None, @@ -1172,6 +1197,8 @@ def run_server( if validate_config is True: ProxyInitializationHelpers.run_config_validation(config) return + if check_migrations_against_existing_data is True: + sys.exit(ProxyInitializationHelpers.run_migration_data_check(use_migrate=not use_prisma_db_push)) if enforce_prisma_migration_check: print( "\033[1;33mLiteLLM Proxy: --enforce_prisma_migration_check is " diff --git a/tests/integration/database/migrations/test_request_log_indexes.py b/tests/integration/database/migrations/test_request_log_indexes.py index 3b6d1eaf21a..bf37ca80665 100644 --- a/tests/integration/database/migrations/test_request_log_indexes.py +++ b/tests/integration/database/migrations/test_request_log_indexes.py @@ -1,5 +1,6 @@ import os import queue +import secrets import shutil import subprocess import sys @@ -14,7 +15,7 @@ from typing import Final import psycopg import pytest -from litellm_proxy_extras import request_log_indexes +from litellm_proxy_extras import migration_data_checks, request_log_indexes from litellm_proxy_extras.migration_lock import MIGRATION_LOCK_KEY, migration_lock from litellm_proxy_extras.migration_recovery import roll_back_failed_inert_migration from litellm_proxy_extras.request_log_indexes import ( @@ -408,6 +409,12 @@ def test_a_migration_without_a_failed_row_is_not_touched(partitioned_database: s assert _ledger(partitioned_database) == before +def _backend_pid(connection: "psycopg.Connection[tuple[object, ...]]") -> int: + row: Final = connection.execute("SELECT pg_backend_pid()").fetchone() + assert row is not None and isinstance(row[0], int), row + return row[0] + + def _pin_a_snapshot_on(database_url: str, table: str) -> "psycopg.Connection[tuple[object, ...]]": pin: Final = psycopg.connect(database_url) pin.isolation_level = psycopg.IsolationLevel.REPEATABLE_READ @@ -532,7 +539,9 @@ def test_an_invalid_child_index_left_by_an_interrupted_build_is_rebuilt_and_atta @requires_db -def test_partitioning_a_table_the_index_job_already_covered_moves_its_call_id_index_aside(scratch_database: str) -> None: +def test_partitioning_a_table_the_index_job_already_covered_moves_its_call_id_index_aside( + scratch_database: str, +) -> None: """The runbook renames the Prisma indexes off the legacy table so the partitioned table can take their names. The call-id index is built by the index job, not Prisma, and a deployment that partitions after that job ran kept the name on the legacy table, so every later job run reported it as on another table.""" @@ -541,7 +550,9 @@ def test_partitioning_a_table_the_index_job_already_covered_moves_its_call_id_in _partition_spend_logs(scratch_database) - assert _indexed_table(scratch_database, "LiteLLM_SpendLogs_legacy_litellm_call_id_idx") == "LiteLLM_SpendLogs_legacy" + assert ( + _indexed_table(scratch_database, "LiteLLM_SpendLogs_legacy_litellm_call_id_idx") == "LiteLLM_SpendLogs_legacy" + ) assert ensure_request_log_indexes(scratch_database, "public") is True _assert_index_covers_every_partition(scratch_database, CALL_ID_INDEX, "litellm_call_id_idx") @@ -737,7 +748,7 @@ def test_inserts_are_never_held_back_while_attach_partition_waits_for_a_reader_o reader.execute( sql.SQL('SELECT count(*) FROM {} WHERE "litellm_call_id" IS NULL').format(sql.Identifier(partition)) ).fetchone() - reader_pid: Final = reader.execute("SELECT pg_backend_pid()").fetchone()[0] + reader_pid: Final = _backend_pid(reader) with psycopg.connect(partitioned_database, autocommit=True) as inspector: child_lock: Final = inspector.execute( "SELECT 1 FROM pg_locks WHERE pid = %s AND relation = to_regclass(%s) " @@ -800,7 +811,7 @@ def test_an_attach_that_never_gets_its_lock_is_left_for_the_next_index_build( reader.execute( sql.SQL('SELECT count(*) FROM {} WHERE "litellm_call_id" IS NULL').format(sql.Identifier(partition)) ).fetchone() - reader_pid: Final = reader.execute("SELECT pg_backend_pid()").fetchone()[0] + reader_pid: Final = _backend_pid(reader) with psycopg.connect(partitioned_database, autocommit=True) as inspector: child_lock: Final = inspector.execute( "SELECT 1 FROM pg_locks WHERE pid = %s AND relation = to_regclass(%s) " @@ -929,3 +940,183 @@ def test_a_second_copy_of_a_managed_index_is_reported_with_its_drop_statement_an assert 'remove it with: DROP INDEX CONCURRENTLY "public"."call_id_copy"' in caplog.text assert _index_oids(scratch_database)["call_id_copy"] == copy + + +def _insert_spend_log_with_key(conn: "psycopg.Connection[tuple[object, ...]]", request_id: str, api_key: str) -> None: + conn.execute( + 'INSERT INTO "LiteLLM_SpendLogs" ("request_id", "call_type", "startTime", "endTime", "api_key") ' + "VALUES (%s, %s, %s, %s, %s)", + (request_id, "acompletion", "2025-06-03 14:02:11", "2025-06-03 14:02:13", api_key), + ) + + +def _legacy_bearer_token() -> str: + """A raw JWT-sized bearer token the way a pre-v1.99.0 proxy stored it: random, so + Postgres cannot compress it into the 2704-byte btree entry limit.""" + return "Bearer " + secrets.token_hex(1600) + + +def _hash_sql_from(refusal: str) -> str: + return refusal[refusal.index('UPDATE "LiteLLM_SpendLogs"') : refusal.index("{64}$';") + len("{64}$';")] + + +@requires_db +@RESOLVERS +@pytest.mark.parametrize("release", (API_KEY_INDEX_MIGRATION,), ids=("v1.102.1",)) +def test_the_migration_job_refuses_before_the_first_migration_when_spend_log_keys_are_too_long_for_the_index( + scratch_database: str, use_v2_resolver: bool +) -> None: + with psycopg.connect(scratch_database, autocommit=True) as conn: + _insert_spend_log_with_key(conn, "legacy-1", _legacy_bearer_token()) + _insert_spend_log_with_key(conn, "legacy-2", _legacy_bearer_token()) + _insert_spend_log_with_key(conn, "hashed-1", "a" * 64) + ledger_before: Final = _ledger(scratch_database) + + with pytest.raises(RuntimeError) as refused: + ProxyExtrasDBManager.run_migration_job(use_migrate=True, use_v2_resolver=use_v2_resolver) + + refusal: Final = str(refused.value) + assert ( + '"LiteLLM_SpendLogs".api_key, startTime: 2 row(s) with an index entry over 2704 bytes, largest 3232 bytes ' + "(api_key up to 3207 bytes)" + ) in refusal, refusal + assert f'blocks: CREATE INDEX "{API_KEY_INDEX}"' in refusal, refusal + assert _ledger(scratch_database) == ledger_before, "a refused job must not apply a single migration" + assert API_KEY_INDEX not in _index_validity(scratch_database, "api_key_startTime_idx") + + with psycopg.connect(scratch_database, autocommit=True) as conn: + conn.execute(_hash_sql_from(refusal)) # pyright: ignore[reportArgumentType] # the refusal's own SQL + assert ProxyExtrasDBManager.run_migration_job(use_migrate=True, use_v2_resolver=use_v2_resolver) is True + assert _index_validity(scratch_database, "api_key_startTime_idx") == {API_KEY_INDEX: True} + assert len(_ledger(scratch_database)) > len(ledger_before) + + +@requires_db +@pytest.mark.parametrize("release", (API_KEY_INDEX_MIGRATION,), ids=("v1.102.1",)) +def test_a_serving_proxy_index_build_that_meets_too_long_keys_reports_the_rows_instead_of_the_postgres_error( + scratch_database: str, caplog: pytest.LogCaptureFixture +) -> None: + assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) is True + with psycopg.connect(scratch_database, autocommit=True) as conn: + _insert_spend_log_with_key(conn, "legacy-1", _legacy_bearer_token()) + + with caplog.at_level("WARNING", logger="litellm_proxy_extras"): + built: Final = ensure_request_log_indexes(scratch_database, "public") + + assert built is False + assert f"Not building index {API_KEY_INDEX}" in caplog.text, caplog.text + assert '"LiteLLM_SpendLogs".api_key, startTime: 1 row(s) with an index entry over 2704 bytes' in caplog.text, ( + caplog.text + ) + assert "exceeds btree version 4 maximum" not in caplog.text, caplog.text + assert API_KEY_INDEX not in _index_validity(scratch_database, "api_key_startTime_idx") + + +_PROBE_STATEMENTS: Final = ( + 'CREATE UNIQUE INDEX "probe_b_key" ON "preflight_probe" ("b")', + 'ALTER TABLE "preflight_probe" ALTER COLUMN "b" SET NOT NULL', + 'ALTER TABLE "preflight_probe" ADD COLUMN "c" INTEGER NOT NULL', + 'ALTER TABLE "preflight_probe" ADD CONSTRAINT "probe_r_fkey" FOREIGN KEY ("r") REFERENCES "preflight_parent"("id")', + 'ALTER TABLE "preflight_probe" ADD CONSTRAINT "probe_a_check" CHECK ("a" <> \'bad\')', + 'CREATE INDEX "probe_a_idx" ON "preflight_probe" ("a")', +) + + +@requires_db +def test_each_check_kind_counts_exactly_the_rows_postgres_would_reject(scratch_database: str) -> None: + with psycopg.connect(scratch_database, autocommit=True) as conn: + conn.execute('CREATE TABLE "preflight_parent" ("id" INTEGER PRIMARY KEY)') + conn.execute('CREATE TABLE "preflight_probe" ("a" TEXT, "b" INTEGER, "r" INTEGER)') + conn.execute('INSERT INTO "preflight_parent" VALUES (1)') + conn.execute( + 'INSERT INTO "preflight_probe" VALUES (%s, 1, 1), (%s, 1, 1), (%s, NULL, 2), (%s, 2, NULL), (%s, 3, 1)', + ("ok", "ok", "bad", "bad", secrets.token_hex(1400)), + ) + checks: Final = tuple( + check + for statement in _PROBE_STATEMENTS + for check in migration_data_checks.checks_for_statement(statement, "probe") + ) + findings: Final = migration_data_checks.find_blockers(conn, "public", checks) + for statement in _PROBE_STATEMENTS: + with conn.transaction(force_rollback=True), pytest.raises(psycopg.Error): + conn.execute(statement) # pyright: ignore[reportArgumentType] # test SQL + + counted: Final = { + (f.check.kind, f.check.columns): f.rows for f in findings if isinstance(f, migration_data_checks.BlockedChange) + } + assert counted == { + ("duplicates", ("b",)): 1, + ("nulls", ("b",)): 1, + ("any_row", ("c",)): 5, + ("orphans", ("r",)): 1, + ("check_violations", ()): 2, + ("too_long", ("a",)): 1, + } + + +@requires_db +@pytest.mark.parametrize( + ("index_columns", "s_value", "t_value", "longest_fitting_key", "described_as"), + ( + (("k", "t"), None, "2025-06-03 14:02:11", 2684, "k up to 2685 bytes"), + (("s", "k"), "q" * 20, None, 2668, "s up to 20 bytes, k up to 2669 bytes"), + (("s", "k"), None, None, 2684, "k up to 2685 bytes"), + ), + ids=("key then timestamp", "20-byte text then key", "NULL then key"), +) +def test_the_entry_size_check_draws_the_line_exactly_where_postgres_does( + scratch_database: str, + index_columns: tuple[str, ...], + s_value: str | None, + t_value: str | None, + longest_fitting_key: int, + described_as: str, +) -> None: + """One row's key is the longest that still fits the index entry, the other's is one byte + longer. Where that line falls depends on the other indexed column, its alignment and the + null bitmap, so the same two rows go through the check and through Postgres.""" + create_index: Final = sql.SQL('CREATE INDEX "entry_probe_idx" ON "entry_probe" ({})').format( + sql.SQL(", ").join(sql.Identifier(column) for column in index_columns) + ) + with psycopg.connect(scratch_database, autocommit=True) as conn: + conn.execute('CREATE TABLE "entry_probe" ("s" TEXT, "k" TEXT, "t" TIMESTAMP(3))') + conn.execute( + 'INSERT INTO "entry_probe" ("s", "k", "t") VALUES (%s, %s, %s), (%s, %s, %s)', + ( + s_value, + secrets.token_hex(longest_fitting_key)[:longest_fitting_key], + t_value, + s_value, + secrets.token_hex(longest_fitting_key)[: longest_fitting_key + 1], + t_value, + ), + ) + findings: Final = migration_data_checks.find_blockers( + conn, "public", migration_data_checks.checks_for_statement(create_index.as_string(conn), "probe") + ) + with pytest.raises(psycopg.errors.ProgramLimitExceeded, match="exceeds btree version 4 maximum 2704"): + conn.execute(create_index) + conn.execute('DELETE FROM "entry_probe" WHERE octet_length("k") > %s', (longest_fitting_key,)) + conn.execute(create_index) + + (finding,) = findings + assert isinstance(finding, migration_data_checks.BlockedChange) + assert finding.rows == 1, "the check must count the row one byte over the line and not the row on it" + assert finding.detail == f"with an index entry over 2704 bytes, largest 2712 bytes ({described_as})" + + +@requires_db +def test_a_scan_that_cannot_finish_in_time_is_reported_as_unverified_rather_than_passed(scratch_database: str) -> None: + with psycopg.connect(scratch_database, autocommit=True) as conn: + conn.execute('CREATE TABLE "preflight_slow" ("a" INTEGER)') + conn.execute('INSERT INTO "preflight_slow" SELECT generate_series(1, 50)') + (check,) = migration_data_checks.checks_for_statement( + 'ALTER TABLE "preflight_slow" ADD CONSTRAINT "slow" CHECK (pg_sleep(0.1) IS NULL)', "probe" + ) + findings: Final = migration_data_checks.find_blockers(conn, "public", (check,), timeout_seconds=0.5) + timeout_afterwards: Final = conn.execute("SHOW statement_timeout").fetchone() + + assert findings == (migration_data_checks.CouldNotVerify(check, 0.5),) + assert timeout_afterwards == ("0",), "the preflight must hand the session back with the timeout it found" + assert "could not be checked within 0.5 s" in migration_data_checks.report(findings) diff --git a/tests/unit/litellm_proxy_extras/test_migration_data_checks.py b/tests/unit/litellm_proxy_extras/test_migration_data_checks.py new file mode 100644 index 00000000000..7897a388474 --- /dev/null +++ b/tests/unit/litellm_proxy_extras/test_migration_data_checks.py @@ -0,0 +1,156 @@ +from pathlib import Path +from typing import Final + +import pytest +from litellm_proxy_extras.migration_data_checks import ( + DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS, + MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR, + BlockedChange, + Check, + CouldNotVerify, + btree_max_entry_bytes, + checks_for_index, + checks_for_migration, + checks_for_statement, + migration_data_check_timeout_seconds, + report, + statements, +) + +MIGRATIONS: Final = Path(__file__).resolve().parents[3] / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations" +API_KEY_INDEX_AS_SHIPPED_IN_V1_103_0: Final = 'CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx" ON "LiteLLM_SpendLogs"("api_key", "startTime")' + + +def _shape(checks: tuple[Check, ...]) -> tuple[tuple[str, str, tuple[str, ...]], ...]: + return tuple((check.kind, check.table, check.columns) for check in checks) + + +@pytest.mark.parametrize( + ("statement", "expected"), + ( + ( + API_KEY_INDEX_AS_SHIPPED_IN_V1_103_0, + (("too_long", "LiteLLM_SpendLogs", ("api_key", "startTime")),), + ), + ( + 'CREATE UNIQUE INDEX CONCURRENTLY "T_a_b_key" ON "public"."T" USING btree ("a", "b" DESC)', + (("too_long", "T", ("a", "b")), ("duplicates", "T", ("a", "b"))), + ), + ('CREATE INDEX "T_a_idx" ON "T" USING gin ("a")', ()), + ('CREATE INDEX "T_a_idx" ON "T" (lower("a"))', ()), + ('ALTER TABLE "T" ALTER COLUMN "a" SET NOT NULL', (("nulls", "T", ("a",)),)), + ('ALTER TABLE "T" ADD COLUMN "a" TEXT NOT NULL', (("any_row", "T", ("a",)),)), + ('ALTER TABLE "T" ADD COLUMN "a" TEXT NOT NULL DEFAULT \'x\'', ()), + ('ALTER TABLE "T" ADD COLUMN "a" TEXT', ()), + ( + 'ALTER TABLE "T" ADD CONSTRAINT "T_r_fkey" FOREIGN KEY ("r") REFERENCES "R"("id") ON DELETE CASCADE', + (("orphans", "T", ("r",)),), + ), + ('ALTER TABLE "T" ADD CONSTRAINT "T_r_fkey" FOREIGN KEY ("r") REFERENCES "R"("id") NOT VALID', ()), + ('ALTER TABLE "T" ADD CONSTRAINT "T_a_key" UNIQUE ("a", "b")', (("duplicates", "T", ("a", "b")),)), + ( + 'ALTER TABLE "T" ADD CONSTRAINT "T_a_check" CHECK ("a" > 0 AND "b" IS NOT NULL)', + (("check_violations", "T", ()),), + ), + ('ALTER TABLE "T" ADD CONSTRAINT "T_a_check" CHECK ("a" > 0) NOT VALID', ()), + ( + 'ALTER TABLE "T" ADD COLUMN "c" TEXT, ALTER COLUMN "b" SET NOT NULL, ADD CONSTRAINT "u" UNIQUE ("c")', + (("nulls", "T", ("b",)), ("duplicates", "T", ("c",))), + ), + ('CREATE TABLE "T" ("id" TEXT NOT NULL, CONSTRAINT "T_pkey" PRIMARY KEY ("id"))', ()), + ('DROP INDEX IF EXISTS "T_a_idx"', ()), + ("SELECT 1", ()), + ), +) +def test_a_statement_gets_one_check_per_way_existing_rows_can_make_it_fail( + statement: str, expected: tuple[tuple[str, str, tuple[str, ...]], ...] +) -> None: + assert _shape(checks_for_statement(statement, "migration x")) == expected + + +def test_a_foreign_key_check_keeps_the_referenced_table_and_columns() -> None: + (check,) = checks_for_statement( + 'ALTER TABLE "T" ADD CONSTRAINT "fk" FOREIGN KEY ("r", "s") REFERENCES "R"("id", "sub")', "migration x" + ) + assert (check.referenced_table, check.referenced_columns) == ("R", ("id", "sub")) + + +def test_a_check_constraint_keeps_its_expression_for_the_count_query() -> None: + (check,) = checks_for_statement('ALTER TABLE "T" ADD CONSTRAINT "c" CHECK ("a" > 0)', "migration x") + assert check.expression == '"a" > 0' + + +def test_comments_and_blank_statements_are_dropped_before_matching() -> None: + migration_sql: Final = "-- CreateIndex\n" + API_KEY_INDEX_AS_SHIPPED_IN_V1_103_0 + ";\n\n-- done; really\n" + assert statements(migration_sql) == (API_KEY_INDEX_AS_SHIPPED_IN_V1_103_0,) + assert _shape(checks_for_migration("x", migration_sql)) == _shape( + checks_for_statement(API_KEY_INDEX_AS_SHIPPED_IN_V1_103_0, "x") + ) + + +def test_every_shipped_migration_is_readable_and_the_later_indexes_on_text_columns_get_a_check() -> None: + derived: Final = { + folder.name: checks_for_migration(folder.name, (folder / "migration.sql").read_text()) + for folder in MIGRATIONS.iterdir() + if folder.is_dir() + } + assert len(derived) > 200 + everywhere: Final = tuple(check for checks in derived.values() for check in checks) # comprehension-ok: flattening + assert any( + check.kind == "too_long" and check.table == "LiteLLM_SpendLogs" and "session_id" in check.columns + for check in everywhere + ) + + +def test_a_request_log_index_gets_one_entry_size_check_over_all_its_columns() -> None: + assert _shape( + checks_for_index("LiteLLM_SpendLogs", "LiteLLM_SpendLogs_api_key_startTime_idx", ("api_key", "startTime")) + ) == (("too_long", "LiteLLM_SpendLogs", ("api_key", "startTime")),) + + +def test_the_btree_entry_limit_on_the_default_page_is_the_number_in_postgres_error() -> None: + # "index row size 2976 exceeds btree version 4 maximum 2704", Postgres 14, 8 KB page (BTMaxItemSize, nbtree.h) + assert btree_max_entry_bytes(8192) == 2704 + + +def test_the_report_names_the_rows_the_statement_and_the_fix_and_says_nothing_was_applied() -> None: + (check,) = checks_for_index( + "LiteLLM_SpendLogs", "LiteLLM_SpendLogs_api_key_startTime_idx", ("api_key", "startTime") + ) + text: Final = report( + (BlockedChange(check, 2, "with an index entry over 2704 bytes, largest 3232 bytes (api_key up to 3207 bytes)"),) + ) + assert ( + '"LiteLLM_SpendLogs".api_key, startTime: 2 row(s) with an index entry over 2704 bytes, largest 3232 bytes ' + "(api_key up to 3207 bytes)" + ) in text + assert 'blocks: CREATE INDEX "LiteLLM_SpendLogs_api_key_startTime_idx" (request-log index)' in text + assert 'UPDATE "LiteLLM_SpendLogs" SET api_key = encode(sha256(' in text + assert text.startswith("Existing rows would make a pending schema change fail:") + assert "No schema change was made" in text + + +def test_a_scan_that_timed_out_is_reported_as_unverified_with_the_knob_to_turn() -> None: + (check,) = checks_for_statement('ALTER TABLE "T" ALTER COLUMN "a" SET NOT NULL', "migration 20270101000000_x") + text: Final = report((CouldNotVerify(check, 600.0),)) + assert '"T".a: could not be checked within 600 s, refusing rather than guessing' in text + assert 'blocks: ALTER TABLE "T" ALTER COLUMN "a" SET NOT NULL (migration 20270101000000_x)' in text + assert MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR in text + + +@pytest.mark.parametrize( + ("raw", "expected"), + ( + (None, DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS), + ("30", 30.0), + ("nope", DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS), + ), +) +def test_the_scan_timeout_comes_from_the_environment_and_falls_back_on_garbage( + monkeypatch: pytest.MonkeyPatch, raw: str | None, expected: float +) -> None: + if raw is None: + monkeypatch.delenv(MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR, raising=False) + else: + monkeypatch.setenv(MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR, raw) + assert migration_data_check_timeout_seconds() == expected From 02605a14ada6c0e8cbc57258c28535e42c81a6c9 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 27 Oct 2026 12:16:15 -0700 Subject: [PATCH 009/102] fix(anthropic): estimate initial input tokens for bridged message streams (#48618) * fix(anthropic): estimate initial input tokens for bridged message streams Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(anthropic): count iterable prompt blocks in initial usage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(anthropic): avoid fetching images during initial usage estimation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../anthropic/messages/adapters/handler.py | 2 + .../messages/adapters/streaming_iterator.py | 48 ++++-- .../messages/adapters/transformation.py | 2 + .../test_streaming_iterator_message_id.py | 150 ++++++++++++++++++ 4 files changed, 189 insertions(+), 13 deletions(-) diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index e5dd1d3c94f..71ccfbaf915 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -806,6 +806,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: tool_name_mapping=tool_name_mapping, polyfill_result=polyfill_result, is_async=True, + completion_request=completion_kwargs, litellm_logging_obj=logging_obj, safeguards_evaluator=safeguards_evaluator, ) @@ -952,6 +953,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: tool_name_mapping=tool_name_mapping, polyfill_result=polyfill_result, is_async=False, + completion_request=completion_kwargs, litellm_logging_obj=logging_obj, ) if transformed_stream is not None: diff --git a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py index 13acb19e454..d1853bb6bd5 100644 --- a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py @@ -22,6 +22,7 @@ from typing import ( from pydantic import TypeAdapter from typing_extensions import assert_never +import litellm from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError @@ -41,19 +42,25 @@ from litellm.types.llms.anthropic import ( UsageDelta, UsageIteration, ) -from litellm.types.llms.openai import ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionRedactedThinkingBlock, + ChatCompletionThinkingBlock, + ChatCompletionToolParam, +) from litellm.types.utils import AdapterCompletionStreamWrapper, ChatCompletionDeltaToolCall, Delta, Function if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator - from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponseStream _STREAMING_DELTA_TYPES: Final = frozenset(get_args(StreamingContentBlockDeltaType)) _SSE_EVENT: Final = TypeAdapter(Mapping[str, object]) _THINKING_BLOCK: Final = TypeAdapter(ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock) +_PROMPT_MESSAGES: Final = TypeAdapter(list[AllMessageValues]) +_PROMPT_TOOLS: Final = TypeAdapter(list[ChatCompletionToolParam]) class _UsageDeltaWithIterations(UsageDelta, total=False): @@ -548,6 +555,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): iterations_usage: list[UsageIteration] | None = None, litellm_logging_obj: "LiteLLMLoggingObject | None" = None, safeguards_evaluator: "SafeguardsEvaluator | None" = None, + completion_request: Mapping[str, object] | None = None, ): # Wrap the upstream stream so chunks that carry both content and a # finish_reason (fake-streamed providers) are split into two — see @@ -556,6 +564,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): super().__init__(combined_chunk_splitter) self._combined_chunk_splitter: Final = combined_chunk_splitter self.model = model + self.completion_request: Final = completion_request self._message_id: str = f"msg_{uuid.uuid4()}" if litellm_logging_obj is not None: litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id) @@ -752,25 +761,38 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): def _create_initial_usage_delta(self) -> UsageDelta: """ - Create the initial UsageDelta for the message_start event. + Estimate prompt tokens for message_start without consuming upstream chunks. - Initializes cache token fields (cache_creation_input_tokens, cache_read_input_tokens) - to 0 to indicate to clients (like Claude Code) that prompt caching is supported. - - The actual cache token values will be provided in the message_delta event at the - end of the stream, since Bedrock Converse API only returns usage data in the final - response chunk. - - Returns: - UsageDelta with all token counts initialized to 0. + Output and cache counts start at zero. Provider usage in message_delta + remains authoritative. Failed estimates fall back to zero. """ return UsageDelta( - input_tokens=0, + input_tokens=self._estimate_initial_input_tokens(), output_tokens=0, cache_creation_input_tokens=0, cache_read_input_tokens=0, ) + def _estimate_initial_input_tokens(self) -> int: + try: + request: Final = self.completion_request or {} + messages: Final = _PROMPT_MESSAGES.validate_python(request.get("messages", self.messages)) + if not messages: + return 0 + materialized_messages: Final = cast( # cast-ok: JSON-mode serialization preserves validated message types and materializes iterable content + list[AllMessageValues], _PROMPT_MESSAGES.dump_python(messages, mode="json") + ) + model: Final = request.get("model", self.model) + tools: Final = request.get("tools") + return litellm.token_counter( # pyright: ignore[reportUnknownMemberType] # token_counter's public signature is partially untyped + model=model if isinstance(model, str) else self.model, + messages=materialized_messages, + tools=_PROMPT_TOOLS.validate_python(tools) if tools is not None else None, + use_default_image_token_count=True, + ) + except Exception: # noqa: BLE001 # prompt estimation must not interrupt the stream + return 0 + def __next__(self): from .transformation import LiteLLMAnthropicMessagesAdapter diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 5ab112d0bfe..1ed552d46e3 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -322,6 +322,7 @@ class AnthropicAdapter: is_async: bool = True, litellm_logging_obj: "LiteLLMLoggingObject | None" = None, safeguards_evaluator: "SafeguardsEvaluator | None" = None, + completion_request: Mapping[str, object] | None = None, ) -> AsyncIterator[bytes] | Iterator[bytes] | None: """ Translate OpenAI streaming response to Anthropic format. @@ -350,6 +351,7 @@ class AnthropicAdapter: iterations_usage=iterations_usage, litellm_logging_obj=litellm_logging_obj, safeguards_evaluator=safeguards_evaluator, + completion_request=completion_request, ) # Return the SSE-wrapped version for proper event formatting. if is_async: diff --git a/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_message_id.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_message_id.py index b98877f25aa..ec9146bc023 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_message_id.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_message_id.py @@ -7,9 +7,12 @@ to be keyed on that same value rather than on the provider's own completion id. import datetime import json +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence +from typing import Final import pytest import respx +from pydantic import TypeAdapter import litellm from litellm.llms.anthropic.messages.adapters.handler import ( @@ -18,6 +21,153 @@ from litellm.llms.anthropic.messages.adapters.handler import ( from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam +from litellm.types.utils import ModelResponseStream + + +_EVENT: Final = TypeAdapter(Mapping[str, object]) +_MESSAGES: Final = TypeAdapter(list[AllMessageValues]) +_TOOLS: Final = TypeAdapter(list[ChatCompletionToolParam]) +_BLOCK_PROMPTS: Final[tuple[list[AllMessageValues], ...]] = ( + [{"role": "user", "content": [{"type": "text", "text": "Count every word. " * 100}]}], + [{"role": "tool", "content": [{"type": "text", "text": "Count every word. " * 100}], "tool_call_id": "call_1"}], + [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": "https://image.example/test.png", "detail": "high"}} + ], + } + ], +) + + +def _assert_estimated_start_and_provider_final_usage(raw_events: Sequence[bytes], request_body: bytes) -> None: + request: Final = _EVENT.validate_json(request_body) + events: Final = tuple(_EVENT.validate_json(chunk.decode().split("data: ", 1)[1]) for chunk in raw_events) + start: Final = _EVENT.validate_python( + next(event for event in events if event["type"] == "message_start")["message"] + ) + final: Final = next(event for event in events if event["type"] == "message_delta") + expected: Final = litellm.token_counter( + model="kimi-k2", + messages=_MESSAGES.validate_python(request["messages"]), + tools=_TOOLS.validate_python(request["tools"]), + ) + assert expected > 0 + assert start["usage"] == { + "input_tokens": expected, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + } + assert final["usage"] == {"input_tokens": 3, "output_tokens": 4} + + +def test_sync_message_start_estimates_prompt_without_replacing_provider_usage(respx_mock: respx.MockRouter) -> None: + stream: Final = LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( + max_tokens=16, + messages=MESSAGES, + system="Reply with a short greeting.", + tools=[{"name": "greet", "description": "Greet a person", "input_schema": {"type": "object"}}], + model="groq/kimi-k2", + stream=True, + custom_llm_provider="groq", + ) + _assert_estimated_start_and_provider_final_usage(list(stream), respx_mock.calls.last.request.content) + + +@pytest.mark.asyncio +async def test_async_message_start_estimates_prompt_without_replacing_provider_usage( + respx_mock: respx.MockRouter, +) -> None: + stream: Final = await LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler( + max_tokens=16, + messages=MESSAGES, + system="Reply with a short greeting.", + tools=[{"name": "greet", "description": "Greet a person", "input_schema": {"type": "object"}}], + model="groq/kimi-k2", + stream=True, + custom_llm_provider="groq", + ) + raw: Final = [chunk async for chunk in stream] + _assert_estimated_start_and_provider_final_usage(raw, respx_mock.calls.last.request.content) + + +class _FailingUpstream: + def __iter__(self) -> Iterator[ModelResponseStream]: + raise RuntimeError("upstream failed before its first chunk") + + def __aiter__(self) -> AsyncIterator[ModelResponseStream]: + raise RuntimeError("upstream failed before its first chunk") + + +@pytest.mark.parametrize("messages", _BLOCK_PROMPTS) +def test_sync_initial_estimate_counts_content_blocks( + messages: list[AllMessageValues], respx_mock: respx.MockRouter +) -> None: + image: Final = respx_mock.get("https://image.example/test.png").respond(503) + wrapper: Final = AnthropicStreamWrapper(iter([]), model="gpt-4o", completion_request={"messages": messages}) + expected: Final = litellm.token_counter(model="gpt-4o", messages=messages, use_default_image_token_count=True) + first: Final = _EVENT.validate_python(next(wrapper)) + message: Final = _EVENT.validate_python(first["message"]) + assert message["usage"] == { + "input_tokens": expected, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + } + assert not image.called + + +@pytest.mark.asyncio +@pytest.mark.parametrize("messages", _BLOCK_PROMPTS) +async def test_async_initial_estimate_counts_content_blocks( + messages: list[AllMessageValues], respx_mock: respx.MockRouter +) -> None: + image: Final = respx_mock.get("https://image.example/test.png").respond(503) + wrapper: Final = AnthropicStreamWrapper(iter([]), model="gpt-4o", completion_request={"messages": messages}) + expected: Final = litellm.token_counter(model="gpt-4o", messages=messages, use_default_image_token_count=True) + first: Final = _EVENT.validate_python(await anext(wrapper)) + message: Final = _EVENT.validate_python(first["message"]) + assert message["usage"] == { + "input_tokens": expected, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + } + assert not image.called + + +@pytest.mark.parametrize("prompt", [{}, {"messages": []}, {"messages": "invalid"}]) +def test_sync_initial_estimate_fallback_does_not_consume_upstream(prompt: Mapping[str, object]) -> None: + wrapper: Final = AnthropicStreamWrapper(_FailingUpstream(), model="kimi-k2", completion_request=prompt) + first: Final = _EVENT.validate_python(next(wrapper)) + message: Final = _EVENT.validate_python(first["message"]) + assert message["usage"] == { + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + } + assert list(wrapper) == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prompt", [{}, {"messages": []}, {"messages": "invalid"}]) +async def test_async_initial_estimate_fallback_does_not_consume_upstream(prompt: Mapping[str, object]) -> None: + wrapper: Final = AnthropicStreamWrapper(_FailingUpstream(), model="kimi-k2", completion_request=prompt) + first: Final = _EVENT.validate_python(await anext(wrapper)) + message: Final = _EVENT.validate_python(first["message"]) + assert message["usage"] == { + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + } + with pytest.raises(RuntimeError, match="upstream failed before its first chunk"): + [chunk async for chunk in wrapper] + MESSAGES = [{"role": "user", "content": "hello"}] From 32297c917784a11309d08958b0bae8b86019235a Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 27 Oct 2026 12:16:34 -0700 Subject: [PATCH 010/102] fix(logging): match extra_spend_tag_headers case-insensitively (#48620) * test(logging): cover extra spend tag header casing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): match spend tag headers case-insensitively Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 12 ++-- .../test_litellm_logging.py | 67 +++++++++++++++++++ 2 files changed, 73 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 55f0f5d49f6..ee55c9ae030 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -7141,12 +7141,12 @@ class StandardLoggingPayloadSetup: request_headers: Final = cast( # cast-ok: request headers are untyped framework data dict[str, str], headers ) - - header_tags: Final = [] - for header_name in extra_headers: - header_value = request_headers.get(header_name) - if header_value: - header_tags.append(f"{header_name}: {header_value}") + lowercase_request_headers: Final = {str(name).lower(): value for name, value in request_headers.items()} + header_tags: Final = [ + f"{header_name}: {header_value}" + for header_name in extra_headers + if (header_value := lowercase_request_headers.get(header_name.lower())) + ] return header_tags if header_tags else None diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 317d3c35664..a1f674b467f 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -3672,6 +3672,73 @@ def test_get_extra_header_tags(): delattr(litellm, "extra_spend_tag_headers") +def test_get_extra_header_tags_matches_title_case_config_with_lowercase_request_header( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "extra_spend_tag_headers", ["X-Org-Id"], raising=False) + + assert StandardLoggingPayloadSetup._get_extra_header_tags( + proxy_server_request={"headers": {"x-org-id": "org_1"}} + ) == ["X-Org-Id: org_1"] + + +def test_get_extra_header_tags_matches_mixed_case_request_header_with_lowercase_config( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "extra_spend_tag_headers", ["x-customer-id"], raising=False) + + assert StandardLoggingPayloadSetup._get_extra_header_tags( + proxy_server_request={"headers": {"X-Customer-Id": "v"}} + ) == ["x-customer-id: v"] + + +def test_get_extra_header_tags_keeps_lowercase_label( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "extra_spend_tag_headers", ["x-customer-id"], raising=False) + + assert StandardLoggingPayloadSetup._get_extra_header_tags( + proxy_server_request={"headers": {"x-customer-id": "v"}} + ) == ["x-customer-id: v"] + + +def test_get_extra_header_tags_returns_none_for_absent_or_empty_header( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "extra_spend_tag_headers", ["x-org-id"], raising=False) + + assert StandardLoggingPayloadSetup._get_extra_header_tags(proxy_server_request={"headers": {}}) is None + assert StandardLoggingPayloadSetup._get_extra_header_tags( + proxy_server_request={"headers": {"x-org-id": ""}} + ) is None + + +def test_get_extra_header_tags_preserves_configured_header_order( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "extra_spend_tag_headers", ["x-second", "x-first"], raising=False) + + assert StandardLoggingPayloadSetup._get_extra_header_tags( + proxy_server_request={"headers": {"x-first": "1", "x-second": "2"}} + ) == ["x-second: 2", "x-first: 1"] + + +def test_get_request_tags_includes_extra_header_tag_with_user_agent_tags( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "extra_spend_tag_headers", ["X-Org-Id"], raising=False) + + assert StandardLoggingPayloadSetup.get_request_tags( + litellm_params={}, + proxy_server_request={ + "headers": { + "user-agent": "litellm/1.0", + "x-org-id": "org_1", + } + }, + ) == ["User-Agent: litellm", "User-Agent: litellm/1.0", "X-Org-Id: org_1"] + + def test_response_cost_calculator_with_response_cost_in_hidden_params(logging_obj): from litellm import Router From 3d92124d9631bd6fef7edcb0ffdb08a445f242f9 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:24:26 -0700 Subject: [PATCH 011/102] fix(vertex-ai): drop the floor retirement date from gemini-3.1-flash-lite-image (#48587) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 2 -- model_prices_and_context_window.json | 2 -- 2 files changed, 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b820b058eb3..390e9298489 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -28402,7 +28402,6 @@ "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, - "deprecation_date": "2027-06-28", "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, @@ -50878,7 +50877,6 @@ "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, - "deprecation_date": "2027-06-28", "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b820b058eb3..390e9298489 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -28402,7 +28402,6 @@ "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, - "deprecation_date": "2027-06-28", "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, @@ -50878,7 +50877,6 @@ "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, - "deprecation_date": "2027-06-28", "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, From 872bfd51024f9038af3cf196924da219a1e642a9 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 27 Oct 2026 12:32:48 -0700 Subject: [PATCH 012/102] fix(proxy): return 403 naming the Enterprise licence for get_spend_routes without a licence (#48623) * test(proxy): cover enterprise spend permission errors on key and user creation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): return 403 naming the Enterprise licence for get_spend_routes without a licence Co-authored-by: er-s-an <92594860+er-s-an@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): narrow parsed permissions before checking spend access Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ci): allow full-history database checkout to finish --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: er-s-an <92594860+er-s-an@users.noreply.github.com> --- .github/workflows/test-unit.yml | 2 +- .../key_management_endpoints.py | 10 ++- .../test_key_management_endpoints.py | 79 +++++++++++++++++-- 3 files changed, 82 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 9658e834c42..4d47a882891 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -952,7 +952,7 @@ jobs: dist: loadscope steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - timeout-minutes: 3 + timeout-minutes: ${{ matrix.shard == 'db-and-spend' && 10 || 3 }} with: persist-credentials: false fetch-depth: ${{ matrix.shard == 'db-and-spend' && '0' || '1' }} diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index b97d409a6c3..f7b72b263d4 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -167,6 +167,7 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports handle_exception_on_proxy, hash_token_if_needed, is_valid_api_key, + require_enterprise_license, ) from litellm.repositories.base_repository import BaseRepository, record_to_dict from litellm.repositories.budget_repository import BudgetRepository @@ -5259,10 +5260,13 @@ async def generate_key_helper_fn( if isinstance(saved_token["metadata"], str): saved_token["metadata"] = json.loads(saved_token["metadata"]) if isinstance(saved_token["permissions"], str): - if "get_spend_routes" in saved_token["permissions"] and premium_user is not True: - raise ValueError("get_spend_routes permission is only available for LiteLLM Enterprise users") - saved_token["permissions"] = json.loads(saved_token["permissions"]) + if ( + isinstance(saved_token["permissions"], dict) + and "get_spend_routes" in saved_token["permissions"] + and premium_user is not True + ): + require_enterprise_license("get_spend_routes") if isinstance(saved_token["model_max_budget"], str): saved_token["model_max_budget"] = json.loads(saved_token["model_max_budget"]) router_settings = cast(dict | None, saved_token.get("router_settings")) diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index 4266c2ed6d1..9634f1efce8 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -21,6 +21,7 @@ from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.models.object_permission import LiteLLM_ObjectPermissionTable from litellm.models.project import LiteLLM_ProjectTable from litellm.proxy._types import ( + CommonProxyErrors, GenerateKeyRequest, GenerateKeyResponse, KeyManagementRoutes, @@ -21354,8 +21355,9 @@ async def test_key_generation_premium_permission_error_precedes_project_ownershi team_table=None, ) - assert error.value.status_code == 500 - assert error.value.detail == {"error": "Internal Server Error."} + assert error.value.status_code == 403 + assert "get_spend_routes" in str(error.value.detail) + assert CommonProxyErrors.not_premium_user.value in str(error.value.detail).replace("\\n", "\n") mock_prisma_client.insert_data.assert_not_awaited() @@ -23210,6 +23212,76 @@ def test_key_info_unexpected_database_exception_sanitized( proxy_server.app.dependency_overrides.pop(user_api_key_auth, None) +@pytest.fixture +def spend_permission_client(monkeypatch: pytest.MonkeyPatch) -> TestClient: + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + prisma_client: Final = _configure_key_endpoints(monkeypatch, UserApiKeyCache()) + prisma_client.db.litellm_usertable = MagicMock() + prisma_client.db.litellm_usertable.count = AsyncMock(return_value=0) + monkeypatch.setattr(litellm, "default_key_generate_params", None) + monkeypatch.setattr(proxy_server, "premium_user", False) + monkeypatch.setitem( + proxy_server.app.dependency_overrides, + user_api_key_auth, + lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin"), + ) + return TestClient(proxy_server.app) + + +@pytest.mark.parametrize("route", ("/key/generate", "/user/new")) +@pytest.mark.parametrize("permission", ("true", "false", False), ids=("string_true", "string_false", "bool_false")) +def test_spend_permission_enterprise_error_regression( + spend_permission_client: TestClient, + route: str, + permission: str | bool, +) -> None: + response: Final = spend_permission_client.post(route, json={"permissions": {"get_spend_routes": permission}}) + + assert response.status_code == 403, response.text + assert "get_spend_routes" in response.text + assert CommonProxyErrors.not_premium_user.value in response.json()["error"]["message"].replace("\\n", "\n") + + +@pytest.mark.parametrize( + "permissions", + ({}, {"other": True}, {"other": "get_spend_routes"}, {"get_spend_routes_extra": True}), + ids=("empty", "unrelated", "permission_in_value", "permission_in_other_key"), +) +def test_non_enterprise_permissions_guard( + spend_permission_client: TestClient, + permissions: dict[str, str | bool], +) -> None: + response: Final = spend_permission_client.post("/key/generate", json={"permissions": permissions}) + + assert response.status_code == 200, response.text + assert response.json()["permissions"] == permissions + assert response.json()["key"].startswith("sk-") + + +def test_enterprise_spend_permission_guard( + spend_permission_client: TestClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + response: Final = spend_permission_client.post( + "/key/generate", json={"permissions": {"get_spend_routes": True}} + ) + + assert response.status_code == 200, response.text + assert response.json()["permissions"] == {"get_spend_routes": True} + assert response.json()["key"].startswith("sk-") + + +def test_guardrails_enterprise_error_guard(spend_permission_client: TestClient) -> None: + response: Final = spend_permission_client.post("/key/generate", json={"guardrails": ["g"]}) + + assert response.status_code == 403, response.text + assert "guardrails" in response.text + assert CommonProxyErrors.not_premium_user.value in response.json()["error"]["message"].replace("\\n", "\n") + + def test_key_info_missing_key_preserves_404( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -23300,6 +23372,3 @@ def test_key_info_database_not_connected_sanitized_500( assert "Database not connected" in caplog.text finally: proxy_server.app.dependency_overrides.pop(user_api_key_auth, None) - - - From 2dc4847579bb9ab4b8bff45b49022b2752658f3b Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 27 Oct 2026 12:33:48 -0700 Subject: [PATCH 013/102] fix(token_counter): count Anthropic-format tool_choice without KeyError (#48624) * test(token_counter): cover Anthropic tool_choice counting Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(token_counter): count Anthropic-format tool_choice without KeyError Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 23 ++++++++----- .../litellm_core_utils/test_token_counter.py | 34 ++++++++++++++++++- .../proxy/spend_tracking/test_input_tokens.py | 26 ++++++++++++++ 3 files changed, 73 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index ee5f70fb394..edb5ecf0f8d 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -59,7 +59,6 @@ from litellm.types.llms.anthropic import ( from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionDocumentObject, - ChatCompletionNamedToolChoiceParam, ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock, ChatCompletionToolParam, @@ -456,7 +455,7 @@ def token_counter( messages: Sequence[AllMessageValues | Message] | None = None, count_response_tokens: bool | None = False, tools: list[ChatCompletionToolParam] | None = None, - tool_choice: ChatCompletionNamedToolChoiceParam | None = None, + tool_choice: str | Mapping[str, object] | None = None, use_default_image_token_count: bool | None = False, default_token_count: int | None = None, ) -> int: @@ -636,7 +635,7 @@ def _count_messages( def _count_extra( count_function: TokenCounterFunction, tools: list[ChatCompletionToolParam] | None, - tool_choice: ChatCompletionNamedToolChoiceParam | None, + tool_choice: str | Mapping[str, object] | None, includes_system_message: bool, tool_section: ChatTemplateToolSection | None, ) -> int: @@ -673,13 +672,19 @@ def _count_tool_definitions( def _count_tool_choice( count_function: TokenCounterFunction, - tool_choice: ChatCompletionNamedToolChoiceParam | None, + tool_choice: str | Mapping[str, object] | None, ) -> int: - if tool_choice == "none": - return 1 - if isinstance(tool_choice, dict): - return 7 + count_function(str(tool_choice["function"]["name"])) - return 0 + match tool_choice: + case "none": + return 1 + case {"function": {"name": str() as name}}: + return 7 + count_function(name) + case {"type": "none"}: + return 1 + case {"name": str() as name}: + return 7 + count_function(name) + case _: + return 0 def _count_chat_template_tools( diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index f222a257b8f..9921bbe343a 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -14,7 +14,7 @@ from collections.abc import Callable, Mapping from concurrent.futures import Future, ThreadPoolExecutor, wait from itertools import accumulate, chain from pathlib import Path -from typing import Final +from typing import Final, cast from unittest.mock import MagicMock, sentinel import anyio.to_thread @@ -42,6 +42,7 @@ from litellm.litellm_core_utils.token_counter import ( ) from litellm.litellm_core_utils.token_counter import token_counter as token_counter_new from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, @@ -658,6 +659,37 @@ def test_token_counter_tool_choice_adds_nothing_for_deepseek_v4(tool_choice: obj ) == token_counter_new(model="DeepSeek-V4-Flash-0731", messages=list(_USER_MESSAGES), tools=tools) +@pytest.mark.parametrize( + ("tool_choice", "expected_tool_choice"), + [ + pytest.param({"type": "auto"}, None, id="anthropic-auto"), + pytest.param({"type": "any"}, None, id="anthropic-any"), + pytest.param({"type": "none"}, "none", id="anthropic-none"), + pytest.param( + {"type": "tool", "name": "get_weather"}, + {"type": "function", "function": {"name": "get_weather"}}, + id="anthropic-tool", + ), + pytest.param( + {"type": "function", "name": "get_weather"}, + {"type": "function", "function": {"name": "get_weather"}}, + id="responses-function", + ), + ], +) +def test_token_counter_counts_anthropic_tool_choices_like_openai_choices( + tool_choice: Mapping[str, object], expected_tool_choice: str | Mapping[str, object] | None +) -> None: + messages: Final = cast(list[AllMessageValues], list(_USER_MESSAGES)) + tools: Final = cast(list[ChatCompletionToolParam], [_openai_tool(_WEATHER_FUNCTION)]) + + assert token_counter_new( + model="gpt-4", messages=messages, tools=tools, tool_choice=tool_choice + ) == token_counter_new( + model="gpt-4", messages=messages, tools=tools, tool_choice=expected_tool_choice + ) + + @pytest.mark.parametrize("model", ["DeepSeek-V4-Flash-0731", "databricks-qwen35-122b-a10b"]) @pytest.mark.parametrize( "anthropic_tool", diff --git a/tests/unit/proxy/spend_tracking/test_input_tokens.py b/tests/unit/proxy/spend_tracking/test_input_tokens.py index 71f4dec2345..c7f13651471 100644 --- a/tests/unit/proxy/spend_tracking/test_input_tokens.py +++ b/tests/unit/proxy/spend_tracking/test_input_tokens.py @@ -30,6 +30,32 @@ async def test_large_input_is_still_counted() -> None: assert isinstance(counts, MappingProxyType) +def test_count_input_tokens_for_model_counts_anthropic_auto_tool_choice() -> None: + model: Final = "claude-sonnet-5-5" + body_without_tool_choice: Final = { + "model": model, + "messages": [{"role": "user", "content": "weather in pune?"}], + "tools": [ + { + "name": "get_weather", + "description": "Get the weather for a location", + "input_schema": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + } + ], + } + body_with_tool_choice: Final = body_without_tool_choice | {"tool_choice": {"type": "auto"}} + + expected: Final = count_input_tokens_for_model(request_body=body_without_tool_choice, model=model) + actual: Final = count_input_tokens_for_model(request_body=body_with_tool_choice, model=model) + + assert isinstance(expected, int) + assert actual == expected + + _COMPACTION_ITEM: Final = { "type": "compaction", "id": "cmp_1", From 189ab854389f49017c7568571de19b964b8e8e01 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:38:14 +0000 Subject: [PATCH 014/102] fix(bedrock): correct gpt max_input_tokens and add Mantle token counting for claude-opus-5-5 (#48479) * fix(bedrock): set gpt-6.1-terra max_input_tokens to the 1,050,000 Bedrock card context window * fix(bedrock): set gpt-5.x and gpt-6 max_input_tokens to the card context window minus max output * fix(bedrock): add Mantle token_count_model to bedrock_mantle claude-opus-5-5 rows Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: kerry --- ...odel_prices_and_context_window_backup.json | 107 ++++++++++++------ model_prices_and_context_window.json | 107 ++++++++++++------ .../common_utils/test_token_count_settings.py | 2 +- 3 files changed, 149 insertions(+), 67 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 390e9298489..7cd4eadca6e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -57957,7 +57957,7 @@ "search_context_size_medium": 0.012 }, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -58000,7 +58000,7 @@ "search_context_size_medium": 0.012 }, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -58108,7 +58108,7 @@ "search_context_size_medium": 0.012 }, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -58148,7 +58148,7 @@ "output_cost_per_token_above_272k_tokens": 3.3e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html", @@ -58184,7 +58184,7 @@ "output_cost_per_token_above_272k_tokens": 3e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html", @@ -58220,7 +58220,7 @@ "output_cost_per_token_above_272k_tokens": 1.98e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html", @@ -58256,7 +58256,7 @@ "output_cost_per_token_above_272k_tokens": 1.8e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html", @@ -58292,7 +58292,7 @@ "output_cost_per_token_above_272k_tokens": 1.98e-06, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html", @@ -58325,7 +58325,7 @@ "output_cost_per_token_above_272k_tokens": 2.475e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html", @@ -58356,7 +58356,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html", @@ -58387,7 +58387,7 @@ "output_cost_per_token_above_272k_tokens": 4.95e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html", @@ -58418,7 +58418,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html", @@ -58452,7 +58452,7 @@ "output_cost_per_token_above_272k_tokens": 1.8e-06, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html", @@ -58492,7 +58492,7 @@ "output_cost_per_token": 5.5e-05, "output_cost_per_token_above_272k_tokens": 8.25e-05, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token_above_272k_tokens_ultrafast": 0.000495, @@ -58532,7 +58532,7 @@ "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -58570,7 +58570,7 @@ "output_cost_per_token": 5.5e-07, "output_cost_per_token_above_272k_tokens": 8.25e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -58616,7 +58616,7 @@ "output_cost_per_token_above_272k_tokens": 8.25e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_ultrafast": 0.000495, @@ -58656,7 +58656,7 @@ "output_cost_per_token_above_272k_tokens": 1.65e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "supported_modalities": [ @@ -58694,7 +58694,7 @@ "output_cost_per_token_above_272k_tokens": 8.25e-07, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "supported_modalities": [ @@ -58738,7 +58738,7 @@ "output_cost_per_token_above_272k_tokens": 7.5e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_ultrafast": 0.00045, @@ -58777,7 +58777,7 @@ "output_cost_per_token_above_272k_tokens": 1.5e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "supported_modalities": [ @@ -58811,7 +58811,7 @@ "output_cost_per_token_above_272k_tokens": 1.5e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "supported_modalities": [ @@ -58848,7 +58848,7 @@ "output_cost_per_token_above_272k_tokens": 7.5e-07, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "supported_modalities": [ @@ -58882,7 +58882,7 @@ "output_cost_per_token_above_272k_tokens": 7.5e-07, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "supported_modalities": [ @@ -58921,7 +58921,7 @@ "search_context_size_medium": 0.012 }, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -58960,7 +58960,7 @@ "search_context_size_medium": 0.012 }, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -59181,7 +59181,8 @@ "prompt_cache_min_tokens": 512, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html", "thinking_always_on": true, - "supports_forced_tool_use": false + "supports_forced_tool_use": false, + "token_count_model": {"model":"anthropic.claude-opus-5-5","litellm_provider":"bedrock_mantle","region":"us-east-1"} }, "bedrock_mantle/anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -66036,7 +66037,7 @@ }, "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-terra": { "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -66071,7 +66072,7 @@ }, "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-luna": { "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -66106,7 +66107,7 @@ }, "bedrock_mantle/us-gov-west-1/openai.gpt-5.4": { "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -66297,7 +66298,8 @@ "supports_xhigh_reasoning_effort": true, "thinking_always_on": true, "supports_forced_tool_use": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html", + "token_count_model": {"model":"anthropic.claude-opus-5-5","litellm_provider":"bedrock_mantle","region":"us-gov-west-1"} }, "bedrock_mantle/us-gov-west-1/anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -66336,7 +66338,7 @@ }, "bedrock_mantle/us-gov-east-1/openai.gpt-5.4": { "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -66360,7 +66362,8 @@ "supports_vision": true, "input_cost_per_token": 3.3e-06, "cache_read_input_token_cost": 3.3e-07, - "output_cost_per_token": 1.98e-05 + "output_cost_per_token": 1.98e-05, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.md" }, "bedrock_mantle/us-gov-east-1/xai.grok-4.6": { "use_openai_responses_path": true, @@ -76003,5 +76006,43 @@ "high", "max" ] + }, + "bedrock_mantle/openai.gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-06, + "cache_read_input_token_cost": 5.5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-07, + "input_cost_per_token": 1.1e-06, + "input_cost_per_token_above_272k_tokens": 2.2e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 5.5e-06, + "output_cost_per_token_above_272k_tokens": 8.25e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-terra.html", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "use_openai_responses_path": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 390e9298489..7cd4eadca6e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -57957,7 +57957,7 @@ "search_context_size_medium": 0.012 }, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -58000,7 +58000,7 @@ "search_context_size_medium": 0.012 }, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -58108,7 +58108,7 @@ "search_context_size_medium": 0.012 }, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -58148,7 +58148,7 @@ "output_cost_per_token_above_272k_tokens": 3.3e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html", @@ -58184,7 +58184,7 @@ "output_cost_per_token_above_272k_tokens": 3e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html", @@ -58220,7 +58220,7 @@ "output_cost_per_token_above_272k_tokens": 1.98e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html", @@ -58256,7 +58256,7 @@ "output_cost_per_token_above_272k_tokens": 1.8e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html", @@ -58292,7 +58292,7 @@ "output_cost_per_token_above_272k_tokens": 1.98e-06, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html", @@ -58325,7 +58325,7 @@ "output_cost_per_token_above_272k_tokens": 2.475e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html", @@ -58356,7 +58356,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html", @@ -58387,7 +58387,7 @@ "output_cost_per_token_above_272k_tokens": 4.95e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html", @@ -58418,7 +58418,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html", @@ -58452,7 +58452,7 @@ "output_cost_per_token_above_272k_tokens": 1.8e-06, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html", @@ -58492,7 +58492,7 @@ "output_cost_per_token": 5.5e-05, "output_cost_per_token_above_272k_tokens": 8.25e-05, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "output_cost_per_token_above_272k_tokens_ultrafast": 0.000495, @@ -58532,7 +58532,7 @@ "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -58570,7 +58570,7 @@ "output_cost_per_token": 5.5e-07, "output_cost_per_token_above_272k_tokens": 8.25e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -58616,7 +58616,7 @@ "output_cost_per_token_above_272k_tokens": 8.25e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_ultrafast": 0.000495, @@ -58656,7 +58656,7 @@ "output_cost_per_token_above_272k_tokens": 1.65e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "supported_modalities": [ @@ -58694,7 +58694,7 @@ "output_cost_per_token_above_272k_tokens": 8.25e-07, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "supported_modalities": [ @@ -58738,7 +58738,7 @@ "output_cost_per_token_above_272k_tokens": 7.5e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_ultrafast": 0.00045, @@ -58777,7 +58777,7 @@ "output_cost_per_token_above_272k_tokens": 1.5e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "supported_modalities": [ @@ -58811,7 +58811,7 @@ "output_cost_per_token_above_272k_tokens": 1.5e-05, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "supported_modalities": [ @@ -58848,7 +58848,7 @@ "output_cost_per_token_above_272k_tokens": 7.5e-07, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "supported_modalities": [ @@ -58882,7 +58882,7 @@ "output_cost_per_token_above_272k_tokens": 7.5e-07, "litellm_provider": "bedrock_converse", "supports_bedrock_converse_tool_result_images": false, - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "supported_modalities": [ @@ -58921,7 +58921,7 @@ "search_context_size_medium": 0.012 }, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -58960,7 +58960,7 @@ "search_context_size_medium": 0.012 }, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -59181,7 +59181,8 @@ "prompt_cache_min_tokens": 512, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html", "thinking_always_on": true, - "supports_forced_tool_use": false + "supports_forced_tool_use": false, + "token_count_model": {"model":"anthropic.claude-opus-5-5","litellm_provider":"bedrock_mantle","region":"us-east-1"} }, "bedrock_mantle/anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -66036,7 +66037,7 @@ }, "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-terra": { "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -66071,7 +66072,7 @@ }, "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-luna": { "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -66106,7 +66107,7 @@ }, "bedrock_mantle/us-gov-west-1/openai.gpt-5.4": { "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -66297,7 +66298,8 @@ "supports_xhigh_reasoning_effort": true, "thinking_always_on": true, "supports_forced_tool_use": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html", + "token_count_model": {"model":"anthropic.claude-opus-5-5","litellm_provider":"bedrock_mantle","region":"us-gov-west-1"} }, "bedrock_mantle/us-gov-west-1/anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -66336,7 +66338,7 @@ }, "bedrock_mantle/us-gov-east-1/openai.gpt-5.4": { "litellm_provider": "bedrock_mantle", - "max_input_tokens": 1050000, + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, @@ -66360,7 +66362,8 @@ "supports_vision": true, "input_cost_per_token": 3.3e-06, "cache_read_input_token_cost": 3.3e-07, - "output_cost_per_token": 1.98e-05 + "output_cost_per_token": 1.98e-05, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.md" }, "bedrock_mantle/us-gov-east-1/xai.grok-4.6": { "use_openai_responses_path": true, @@ -76003,5 +76006,43 @@ "high", "max" ] + }, + "bedrock_mantle/openai.gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-06, + "cache_read_input_token_cost": 5.5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-07, + "input_cost_per_token": 1.1e-06, + "input_cost_per_token_above_272k_tokens": 2.2e-06, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 5.5e-06, + "output_cost_per_token_above_272k_tokens": 8.25e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-terra.html", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "use_openai_responses_path": true } } diff --git a/tests/unit/proxy/common_utils/test_token_count_settings.py b/tests/unit/proxy/common_utils/test_token_count_settings.py index d7314f1a6d3..366d8b010d0 100644 --- a/tests/unit/proxy/common_utils/test_token_count_settings.py +++ b/tests/unit/proxy/common_utils/test_token_count_settings.py @@ -183,7 +183,7 @@ def test_every_shipped_token_count_model_resolves_to_a_mapped_aws_model() -> Non for name, count_model in count_models.items(): model: Final = count_model["model"] assert get_bedrock_base_model(model) in aws_models - assert cost_map[name]["litellm_provider"] in ("bedrock", "bedrock_converse") + assert cost_map[name]["litellm_provider"] in ("bedrock", "bedrock_converse", "bedrock_mantle") assert "region" not in count_model or count_model["region"].strip() From 788868a52f1901d8c1570e58fb79014d572647c2 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Tue, 27 Oct 2026 12:40:59 -0700 Subject: [PATCH 015/102] fix(security)!: build proxy-only failure logs from an allowlist and drop caller guardrail info (#48599) * fix(proxy): build proxy-only failure logs from an allowlist and ignore request-time GCS credentials * fix(proxy): drop caller metadata from proxy-only failure logs metadata and litellm_metadata leave the proxy-only failure-log allowlist, so a rejected request can no longer point its failure log at its own Langsmith or Langfuse host through nested metadata. * fix(proxy): keep allowlisted metadata in proxy-only failure logs * test(proxy): isolate proxy-only failure log metadata tests * fix(logging): ignore non-list request tags when building the logging payload A guardrail-rejected request with metadata tags set to a non-list value crashed the standard logging payload, so its spend row lost model attribution. * fix(proxy): keep router attribution and drop unauthenticated tags in proxy-only failure logs A request rejected before routing now logs the configured deployment and model group even when its metadata or litellm_metadata carries its own values. Requests rejected at auth no longer write caller tags into failure logs and tag spend. * fix(proxy): always strip client guardrail info and mirror failure attribution into litellm_metadata allow_client_pricing_override no longer keeps a caller-supplied standard_logging_guardrail_information on the chat or pass-through paths, since its guardrail_cost is billed. Proxy-only failure logs now also carry key and team attribution in litellm_metadata. * refactor(proxy): keep the proxy-only failure-log fixes inside the type gates Types the picked helpers so basedpyright and the type-discipline gate stay at main's counts. No runtime change. * fix(proxy): keep the WS frame guardrail-info strip and string interaction tags as sent The guardrail-info key left the pricing strip list, so the Responses WS frame guard now drops it by name. get_request_tags no longer raises on string tags, so background interaction polling checks for them before keeping them as sent. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../gcs_bucket/gcs_bucket_base.py | 49 +----- .../interactions/background_cost_polling.py | 32 +++- litellm/litellm_core_utils/litellm_logging.py | 20 +-- litellm/proxy/litellm_pre_call_utils.py | 19 ++- .../pass_through_endpoints.py | 8 +- .../proxy/response_api_endpoints/endpoints.py | 2 +- litellm/proxy/utils.py | 114 +++++++++++-- .../gcs_bucket/test_gcs_bucket_base.py | 18 ++ .../test_litellm_logging.py | 15 ++ .../files/test_vertex_ai_files_handler.py | 84 ---------- .../test_pass_through_endpoints.py | 34 ++++ tests/unit/proxy/test_pricing_field_strip.py | 31 +++- ..._utils_model_creation_and_error_logging.py | 157 +++++++++++++----- .../test_post_call_failure_hook.py | 122 +++++++++++++- 14 files changed, 494 insertions(+), 211 deletions(-) diff --git a/litellm/integrations/gcs_bucket/gcs_bucket_base.py b/litellm/integrations/gcs_bucket/gcs_bucket_base.py index 63c4fe92eca..092372b5d15 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket_base.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket_base.py @@ -20,7 +20,7 @@ from litellm.llms.custom_httpx.http_handler import ( httpxSpecialProvider, ) from litellm.types.integrations.gcs_bucket import * -from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPayload +from litellm.types.utils import StandardLoggingPayload if TYPE_CHECKING: from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -140,49 +140,16 @@ class GCSBucketBase(CustomBatchLogger): async def get_gcs_logging_config(self, kwargs: dict[str, Any] | None = {}) -> GCSLoggingConfig: """ - This function is used to get the GCS logging config for the GCS Bucket Logger. - It checks if the dynamic parameters are provided in the kwargs and uses them to get the GCS logging config. - If no dynamic parameters are provided, it uses the default values. + Returns the bucket and service account this logger was configured with. """ - if kwargs is None: - kwargs = {} - - standard_callback_dynamic_params: Final[StandardCallbackDynamicParams | None] = kwargs.get( - "standard_callback_dynamic_params", None - ) - - bucket_name: str - path_service_account: str | None - if standard_callback_dynamic_params is not None: - verbose_logger.debug("Using dynamic GCS logging") - verbose_logger.debug("standard_callback_dynamic_params: %s", standard_callback_dynamic_params) - - _bucket_name: str | None = standard_callback_dynamic_params.get("gcs_bucket_name", None) or self.BUCKET_NAME - _path_service_account: Final[str | None] = ( - standard_callback_dynamic_params.get("gcs_path_service_account", None) or self.path_service_account_json + if self.BUCKET_NAME is None: + raise ValueError( + "GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment." ) - - if _bucket_name is None: - raise ValueError( - "GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment." - ) - bucket_name = _bucket_name - path_service_account = _path_service_account - vertex_instance = await self.get_or_create_vertex_instance(credentials=path_service_account) - else: - # If no dynamic parameters, use the default instance - if self.BUCKET_NAME is None: - raise ValueError( - "GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment." - ) - bucket_name = self.BUCKET_NAME - path_service_account = self.path_service_account_json - vertex_instance = await self.get_or_create_vertex_instance(credentials=path_service_account) - return GCSLoggingConfig( - bucket_name=bucket_name, - vertex_instance=vertex_instance, - path_service_account=path_service_account, + bucket_name=self.BUCKET_NAME, + vertex_instance=await self.get_or_create_vertex_instance(credentials=self.path_service_account_json), + path_service_account=self.path_service_account_json, ) async def get_or_create_vertex_instance(self, credentials: str | None) -> VertexBase: diff --git a/litellm/interactions/background_cost_polling.py b/litellm/interactions/background_cost_polling.py index 275eb17787e..0b8c128286a 100644 --- a/litellm/interactions/background_cost_polling.py +++ b/litellm/interactions/background_cost_polling.py @@ -313,19 +313,35 @@ def _spend_metadata(logging_obj: "LiteLLMLoggingObj") -> Mapping[str, object]: } +def _sent_tags_are_a_list(litellm_params: Mapping[str, object]) -> bool: + """Whether the first non-empty ``tags`` in ``metadata`` then ``litellm_metadata`` is absent or a list.""" + buckets: Final = (litellm_params.get("metadata"), litellm_params.get("litellm_metadata")) + sent: Final = next( + ( + tags + for bucket in buckets + if isinstance(bucket, Mapping) and (tags := _OBJECT_MAPPING.validate_python(bucket).get("tags")) + ), + None, + ) + return sent is None or isinstance(sent, list) + + def _request_tags(litellm_params: Mapping[str, object]) -> Mapping[str, object]: from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup try: - return { - "tags": StandardLoggingPayloadSetup.get_request_tags( - litellm_params=_OBJECT_DICT.validate_python(litellm_params), - proxy_server_request=_OBJECT_DICT.validate_python(litellm_params.get("proxy_server_request") or {}), - ) - } + if _sent_tags_are_a_list(litellm_params): + return { + "tags": StandardLoggingPayloadSetup.get_request_tags( + litellm_params=_OBJECT_DICT.validate_python(litellm_params), + proxy_server_request=_OBJECT_DICT.validate_python(litellm_params.get("proxy_server_request") or {}), + ) + } except AttributeError: - verbose_logger.debug("Keeping a background interaction's request tags as sent: they are not a list") - return {} + pass + verbose_logger.debug("Keeping a background interaction's request tags as sent: they are not a list") + return {} def _create_context(logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str) -> BackgroundInteractionCreateContext: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index ee55c9ae030..48e8a44b9a4 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -7161,19 +7161,17 @@ class StandardLoggingPayloadSetup: litellm_metadata: Final = cast( # cast-ok: request metadata is caller-provided Mapping[str, object], litellm_params.get("litellm_metadata") or {} ) - if metadata.get("tags", []): - request_tags = cast(list[str], metadata.get("tags", [])).copy() # cast-ok: tags are caller-provided - elif litellm_metadata.get("tags", []): - request_tags = cast(list[str], litellm_metadata.get("tags", [])).copy() # cast-ok: tags are caller-provided - else: - request_tags = [] + request_tags: Final = next( + ( + cast(list[str], tags) # cast-ok: tags are caller-provided + for tags in (metadata.get("tags"), litellm_metadata.get("tags")) + if isinstance(tags, list) and tags + ), + (), + ) user_agent_tags: Final = StandardLoggingPayloadSetup._get_user_agent_tags(proxy_server_request) additional_header_tags: Final = StandardLoggingPayloadSetup._get_extra_header_tags(proxy_server_request) - if user_agent_tags is not None: - request_tags.extend(user_agent_tags) - if additional_header_tags is not None: - request_tags.extend(additional_header_tags) - return request_tags + return [*request_tags, *(user_agent_tags or ()), *(additional_header_tags or ())] _get_request_tags = get_request_tags diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index baa150efc86..1e7cdc44c70 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -466,10 +466,7 @@ _ALLOW_CLIENT_MESSAGE_REDACTION_OPT_OUT_METADATA_KEY: Final = "allow_client_mess _CLIENT_PRICING_CONTROL_FIELDS: Final = frozenset(CustomPricingLiteLLMParams.model_fields.keys()) | {"base_model"} # ``model_info`` carries the same pricing fields when read by # ``use_custom_pricing_for_model``; strip from metadata for the same reason. -# ``standard_logging_guardrail_information`` is proxy-written telemetry summed -# into response_cost and spend; a client seeding it forges (even negative) -# guardrail cost. -_CLIENT_PRICING_METADATA_FIELDS: Final = frozenset({"model_info", "standard_logging_guardrail_information"}) +_CLIENT_PRICING_METADATA_FIELDS: Final = frozenset({"model_info"}) # ``attempted_fallbacks`` and ``original_model_group`` are written by the router # and read by spend logs as fact; a client value has no legitimate meaning and no # key or team setting keeps it, so the strip is never gated. @@ -799,6 +796,19 @@ def _strip_client_pricing_overrides(data: dict[str, object]) -> None: ) +def strip_client_guardrail_information(data: dict[str, object]) -> None: # mutable-ok: strips the request body in place + """Guardrail hooks write this field and its cost is billed, so no key or team setting keeps a caller's value.""" + for metadata_key in ("metadata", "litellm_metadata"): + if ( + isinstance(metadata := data.get(metadata_key), dict) + and "standard_logging_guardrail_information" in metadata + ): + del metadata["standard_logging_guardrail_information"] + verbose_proxy_logger.debug( + "Stripped client-supplied guardrail information from request body: %s", metadata_key + ) + + def _strip_router_reserved_metadata( data: dict[str, object], # mutable-ok: strips in place on the request body the pre-call pipeline threads through ) -> None: @@ -2695,6 +2705,7 @@ async def add_litellm_data_to_request( # would silently skip the field. if not _key_or_team_allows_client_pricing_override(user_api_key_dict): _strip_client_pricing_overrides(data) + strip_client_guardrail_information(data) _strip_router_reserved_metadata(data) # Same reason as the strips above: runs after the metadata string-to-dict parse diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index fc0e5be3f4c..97f2c10afb1 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -119,6 +119,7 @@ from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy modul _strip_client_pricing_overrides, # pyright: ignore[reportPrivateUsage] # sanitize before trusted hooks add guardrail costs get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage] # shared proxy helper, same import style as _read_request_body above redact_credential_headers, + strip_client_guardrail_information, ) from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.route_priority import ( @@ -1223,10 +1224,11 @@ async def pass_through_request( _parsed_body, ) + client_body: Final = TypeAdapter(dict[str, object]).validate_python(_parsed_body) + strip_client_guardrail_information(client_body) if not _key_or_team_allows_client_pricing_override(user_api_key_dict): - pricing_body: Final = TypeAdapter(dict[str, object]).validate_python(_parsed_body) - _strip_client_pricing_overrides(pricing_body) - _parsed_body = pricing_body + _strip_client_pricing_overrides(client_body) + _parsed_body = client_body if custom_llm_provider in ("laya", "bespoke"): decision_request: Final = TypeAdapter(Mapping[str, object]).validate_python(_parsed_body) checkpoint: Final = validate_oss_request(custom_llm_provider, decision_request) diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 312c5124ca5..97bbf893e3e 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -2116,7 +2116,7 @@ def _without_untrusted_metadata(key: str, value: object) -> object: if not name.startswith("user_api_key_") and name not in _UNTRUSTED_METADATA_CONTROL_FIELDS and name not in _CLIENT_PRICING_METADATA_FIELDS - and name not in ("headers", "pii_tokens", *_CLIENT_GUARDRAIL_SELECTORS) + and name not in ("headers", "pii_tokens", _GUARDRAIL_INFORMATION_KEY, *_CLIENT_GUARDRAIL_SELECTORS) } diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ce7e6afde24..d3202368d76 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1387,7 +1387,8 @@ def _stamp_deployment_attribution( callbacks key off it and must not count a proxy-side reject against the deployment. A failure after the provider handoff keeps the metadata the router stamped; a request that never reached a provider is flagged ``PROXY_REJECTED_BEFORE_ROUTING_KEY`` (deployment metrics key off it) whatever - its metadata says, since ``metadata.model_info`` can be caller supplied.""" + its metadata says, since ``metadata.model_info`` can be caller supplied. Its ``deployment`` and + ``model_group`` always come from the router config, never from the caller's metadata.""" attribution: Final = _deployment_attribution_for_model_group(model_group, team_id) if "custom_llm_provider" in attribution: litellm_params["custom_llm_provider"] = attribution["custom_llm_provider"] @@ -1402,9 +1403,13 @@ def _stamp_deployment_attribution( if not isinstance(metadata, dict): return attribution metadata.setdefault("model_info", attribution["model_info"]) - metadata.setdefault("deployment", attribution["deployment"]) - if isinstance(model_group, str): - metadata.setdefault("model_group", model_group) + routing: Final = MappingProxyType( + {"deployment": attribution["deployment"], **({"model_group": model_group} if model_group else {})} + ) + metadata.update(routing) + litellm_metadata: Final = litellm_params.get("litellm_metadata") + if isinstance(litellm_metadata, dict): + litellm_metadata.update(routing) return attribution @@ -1484,6 +1489,9 @@ object rather than ``True`` because ``request_data`` is the client body: JSON ca client can neither hide a guardrail block nor force a row by sending the key or a ``litellm_call_id`` itself.""" _PROXY_ONLY_LLM_API_ERRORS: Final = (HTTPException, ProxyException, GuardrailRaisedException) +_PROXY_ONLY_ERROR_LOGGED_FIELDS: Final = frozenset( + {"model", "messages", "input", "prompt", "user", "stream", "litellm_call_id"} +) _LOG_DB_METRICS_CALL_TYPES: Final = frozenset(("get_data", "insert_data", "update_data", "delete_data")) @@ -1518,6 +1526,57 @@ def _provider_billed_the_failure(exception: Exception) -> bool: return _timed_out_reading_from_provider(exception) or _client_disconnected(exception) +def _as_str_mapping(value: object) -> Mapping[str, object]: + if not isinstance(value, Mapping): + return MappingProxyType({}) + return cast( # cast-ok: the runtime Mapping check validates the container shape + Mapping[str, object], value + ) + + +def _proxy_only_error_logged_metadata_keys(*, authenticated: bool, past_pre_call: bool) -> frozenset[str]: + return frozenset( + key + for key, logged in (("tags", authenticated), ("standard_logging_guardrail_information", past_pre_call)) + if logged + ) + + +def _is_logged_metadata_value(key: str, value: object) -> bool: + if key != "tags": + return True + tags: Final = cast(list[object], value) if isinstance(value, list) else None # cast-ok: isinstance checked the list + return tags is not None and all(isinstance(item, str) for item in tags) + + +def _proxy_only_error_log_metadata( + value: object, logged_keys: frozenset[str] +) -> dict[str, object] | None: # mutable-ok: router attribution stamps the logged metadata in place + metadata: Final = _as_str_mapping(value) + if not isinstance(value, Mapping): + return None + return { + key: value for key, value in metadata.items() if key in logged_keys and _is_logged_metadata_value(key, value) + } + + +def _proxy_only_error_log_fields( + request_data: Mapping[str, object], logged_metadata_keys: frozenset[str] +) -> Mapping[str, object]: + metadata_fields: Final = { + key: filtered_metadata + for key in ("metadata", "litellm_metadata") + if (filtered_metadata := _proxy_only_error_log_metadata(request_data.get(key), logged_metadata_keys)) + is not None + } + return MappingProxyType( + { + **{key: value for key, value in request_data.items() if key in _PROXY_ONLY_ERROR_LOGGED_FIELDS}, + **metadata_fields, + } + ) + + def _failure_fields_to_lift( request_data: Mapping[str, object], provider_billed: bool = False, dispatched_call_type: str | None = None ) -> Mapping[str, object]: @@ -3778,6 +3837,7 @@ class ProxyLogging: user_api_key_dict=user_api_key_dict, route=route, original_exception=original_exception, + error_type=error_type, ) request_data.update( @@ -3872,6 +3932,7 @@ class ProxyLogging: user_api_key_dict: UserAPIKeyAuth, route: str | None = None, original_exception: Exception | None = None, + error_type: ProxyErrorTypes | None = None, ): """ Handle logging for proxy only errors by calling `litellm_logging_obj.async_failure_handler` @@ -3880,19 +3941,25 @@ class ProxyLogging: """ litellm_logging_obj: Logging | None = request_data.get("litellm_logging_obj", None) set_up_by_request: Final = litellm_logging_obj is not None + request_fields: Final = cast(Mapping[str, object], request_data) # cast-ok: request bodies are str-keyed JSON + logged_metadata_keys: Final = _proxy_only_error_logged_metadata_keys( + authenticated=error_type != ProxyErrorTypes.auth_error, past_pre_call=set_up_by_request + ) + user_api_key_logged_metadata: Final = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( + user_api_key_dict=user_api_key_dict + ) if litellm_logging_obj is None: from litellm._uuid import uuid request_data.setdefault("litellm_call_id", str(uuid.uuid4())) - user_api_key_logged_metadata: Final = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( - user_api_key_dict=user_api_key_dict - ) litellm_logging_obj, data = litellm.utils.function_setup( original_function=route or "IGNORE_THIS", rules_obj=litellm.utils.Rules(), start_time=datetime.now(), - **request_data, + **_proxy_only_error_log_fields( # pyright: ignore[reportArgumentType] # the allowlist never carries is_async_call + request_fields, logged_metadata_keys + ), ) request_data["litellm_logging_obj"] = litellm_logging_obj # rebind-ok: lifted then popped by the caller request_data.update({REJECTED_BEFORE_REQUEST_SETUP_KEY: REJECTED_BEFORE_REQUEST_SETUP}) @@ -3902,18 +3969,39 @@ class ProxyLogging: if litellm_logging_obj is not None: ## UPDATE LOGGING INPUT - _optional_params: Final = {} - _litellm_params: Final = {} + _optional_params: Final[dict[str, object]] = {} # mutable-ok: filled below, then handed to the logger + _litellm_params: Final[dict[str, object]] = {} # mutable-ok: filled below, then handed to the logger litellm_param_keys: Final = LoggedLiteLLMParams.__annotations__.keys() - for k, v in request_data.items(): + for k, v in _proxy_only_error_log_fields(request_fields, logged_metadata_keys).items(): if k in litellm_param_keys: _litellm_params[k] = v elif k not in ("model", "user", "litellm_logging_obj"): _optional_params[k] = v + existing: Final = _as_str_mapping( + litellm_logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped + ) + litellm_params: Final = { + **_litellm_params, + "metadata": { + **_as_str_mapping(existing.get("metadata")), + **_as_str_mapping(_litellm_params.get("metadata")), + **user_api_key_logged_metadata, + }, + **{ + key: { + **_as_str_mapping(existing.get(key)), + **_as_str_mapping(value), + **user_api_key_logged_metadata, + } + for key, value in _litellm_params.items() + if key == "litellm_metadata" + }, + } + attribution: Final = _stamp_deployment_attribution( - _litellm_params, + litellm_params, request_data.get("model"), user_api_key_dict.team_id, dispatched=_reached_deployment(litellm_logging_obj), @@ -3923,7 +4011,7 @@ class ProxyLogging: model=request_data.get("model", ""), user=request_data.get("user", ""), optional_params=_optional_params, - litellm_params=_litellm_params, + litellm_params=litellm_params, **( {"custom_llm_provider": attribution["custom_llm_provider"]} if "custom_llm_provider" in attribution diff --git a/tests/unit/integrations/gcs_bucket/test_gcs_bucket_base.py b/tests/unit/integrations/gcs_bucket/test_gcs_bucket_base.py index 07874bdbeb5..7ab683e9618 100644 --- a/tests/unit/integrations/gcs_bucket/test_gcs_bucket_base.py +++ b/tests/unit/integrations/gcs_bucket/test_gcs_bucket_base.py @@ -120,6 +120,24 @@ class TestGCSBucketBase: custom_llm_provider="vertex_ai", ) + @pytest.mark.asyncio + async def test_gcs_logging_config_ignores_request_time_bucket_and_credentials(self, monkeypatch): + monkeypatch.setenv("GCS_PATH_SERVICE_ACCOUNT", "/admin/sa.json") + handler = GCSBucketBase(bucket_name="admin-bucket") + + with patch.object(handler, "get_or_create_vertex_instance", new_callable=AsyncMock) as mock_vertex: + config = await handler.get_gcs_logging_config( + { + "standard_callback_dynamic_params": { + "gcs_bucket_name": "caller-bucket", + "gcs_path_service_account": "/caller/sa.json", + } + } + ) + + assert (config["bucket_name"], config["path_service_account"]) == ("admin-bucket", "/admin/sa.json") + mock_vertex.assert_awaited_once_with(credentials="/admin/sa.json") + @pytest.mark.asyncio async def test_log_json_data_on_gcs_url_encodes_object_name(self): handler = GCSBucketBase(bucket_name="test-bucket") diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index a1f674b467f..e786753e1a8 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -3520,6 +3520,21 @@ def test_get_request_tags_from_metadata_and_litellm_metadata(): assert "User-Agent: litellm/1.0.0" in tags +@pytest.mark.parametrize("unusable_metadata_tags", [123, "tag", {"tag": True}, []]) +def test_get_request_tags_falls_back_to_litellm_metadata_when_metadata_tags_are_unusable(unusable_metadata_tags): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + tags = StandardLoggingPayloadSetup._get_request_tags( + litellm_params={ + "metadata": {"tags": unusable_metadata_tags}, + "litellm_metadata": {"tags": ["litellm-metadata-tag"]}, + }, + proxy_server_request={}, + ) + + assert tags == ["litellm-metadata-tag"] + + def test_get_request_tags_does_not_mutate_original_tags(): """ Test that _get_request_tags does not mutate the original tags list in metadata. diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_handler.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_handler.py index 9b7cd127b83..39310836bf2 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_handler.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_handler.py @@ -260,87 +260,3 @@ class TestVertexAIFilesHandler: ) assert service_account == '{"type": "service_account", "project_id": "p"}' - - @pytest.mark.asyncio - async def test_afile_content_honors_per_model_bucket_over_env(self, monkeypatch): - """ - Regression for #32640: a batch output written to a per-model gcs_bucket_name must be - readable even when the global GCS_BUCKET_NAME points at a different bucket. Before the - fix the read path resolved the bucket from env only and raised - "file_id bucket does not match the configured storage bucket". - """ - monkeypatch.setenv("GCS_BUCKET_NAME", "env-default-bucket") - monkeypatch.delenv("GCS_PATH_SERVICE_ACCOUNT", raising=False) - - file_id = "gs%3A%2F%2Fmy-model-bucket%2Flitellm-vertex-files%2Fuploads%2Fabc-batch-output.jsonl" - file_content_request = FileContentRequest(file_id=file_id, extra_headers=None, extra_body=None) - - with ( - patch.object(self.handler, "download_gcs_object", new_callable=AsyncMock) as mock_download, - patch.object( - self.handler, - "get_or_create_vertex_instance", - new_callable=AsyncMock, - return_value=object(), - ), - ): - mock_download.return_value = b"batch output" - - result = await self.handler.afile_content( - file_content_request=file_content_request, - vertex_credentials="/model/sa.json", - vertex_project="test-project", - vertex_location="us-central1", - timeout=60.0, - max_retries=0, - litellm_params={ - "gcs_bucket_name": "my-model-bucket", - "vertex_credentials": "/model/sa.json", - }, - ) - - assert isinstance(result, HttpxBinaryResponseContent) - assert result.response.content == b"batch output" - - dynamic_params = mock_download.call_args.kwargs["standard_callback_dynamic_params"] - assert dynamic_params["gcs_bucket_name"] == "my-model-bucket" - assert dynamic_params["gcs_path_service_account"] == "/model/sa.json" - assert mock_download.call_args.kwargs["object_name"] == "litellm-vertex-files/uploads/abc-batch-output.jsonl" - - @pytest.mark.asyncio - async def test_afile_content_reads_without_global_env_bucket(self, monkeypatch): - """ - Regression for #32640: with no global GCS_BUCKET_NAME set, a model-group-level - deployment (per-model gcs_bucket_name) must still be readable. Before the fix the read - path raised "GCS_BUCKET_NAME is not set in the environment". - """ - monkeypatch.delenv("GCS_BUCKET_NAME", raising=False) - monkeypatch.delenv("GCS_PATH_SERVICE_ACCOUNT", raising=False) - - file_id = "gs%3A%2F%2Fmy-model-bucket%2Flitellm-vertex-files%2Fuploads%2Fabc-batch-output.jsonl" - file_content_request = FileContentRequest(file_id=file_id, extra_headers=None, extra_body=None) - - with ( - patch.object(self.handler, "download_gcs_object", new_callable=AsyncMock) as mock_download, - patch.object( - self.handler, - "get_or_create_vertex_instance", - new_callable=AsyncMock, - return_value=object(), - ), - ): - mock_download.return_value = b"batch output" - - result = await self.handler.afile_content( - file_content_request=file_content_request, - vertex_credentials="/model/sa.json", - vertex_project="test-project", - vertex_location="us-central1", - timeout=60.0, - max_retries=0, - litellm_params={"gcs_bucket_name": "my-model-bucket"}, - ) - - assert isinstance(result, HttpxBinaryResponseContent) - dynamic_params = mock_download.call_args.kwargs["standard_callback_dynamic_params"] - assert dynamic_params["gcs_bucket_name"] == "my-model-bucket" diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index db0a74d360a..112e4f6cb6c 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1922,6 +1922,40 @@ async def test_pass_through_request_uses_resolved_timeout(): assert mock_get_client.call_args[1]["params"]["timeout"] == 1500 +@pytest.mark.asyncio +@pytest.mark.parametrize("allow_client_pricing_override", [True, False]) +async def test_pass_through_request_strips_client_guardrail_information(allow_client_pricing_override: bool): + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.headers = Headers({}) + mock_request.query_params = QueryParams({}) + + with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: + mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=RuntimeError("stop before upstream")) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + with pytest.raises(ProxyException): + await pass_through_request( + request=mock_request, + target="http://test.com", + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth( + metadata={"allow_client_pricing_override": allow_client_pricing_override} + ), + custom_body={ + "model": "gpt-4o", + "input_cost_per_token": 0.5, + "metadata": { + "trace_id": "keep-me", + "standard_logging_guardrail_information": [{"guardrail_name": "forged", "guardrail_cost": 5.0}], + }, + }, + ) + + hooked_body: Final = mock_proxy_logging.pre_call_hook.call_args.kwargs["data"] + assert hooked_body["metadata"] == {"trace_id": "keep-me"} + assert ("input_cost_per_token" in hooked_body) is allow_client_pricing_override + + @pytest.mark.asyncio async def test_create_pass_through_route_forwards_timeout(): unique_path = "/test/path/unique/timeout" diff --git a/tests/unit/proxy/test_pricing_field_strip.py b/tests/unit/proxy/test_pricing_field_strip.py index 3e9bee9217d..c4a14b39f54 100644 --- a/tests/unit/proxy/test_pricing_field_strip.py +++ b/tests/unit/proxy/test_pricing_field_strip.py @@ -22,6 +22,7 @@ from litellm.proxy.litellm_pre_call_utils import ( _CLIENT_PRICING_METADATA_FIELDS, _strip_client_pricing_overrides, add_litellm_data_to_request, + strip_client_guardrail_information, ) from litellm.types.utils import CustomPricingLiteLLMParams @@ -133,7 +134,7 @@ class TestStripClientPricingOverrides: "standard_logging_guardrail_information": [{"guardrail_cost": 5.0}], }, } - _strip_client_pricing_overrides(data) + strip_client_guardrail_information(data) assert data["metadata"] == {"user_session": "keep-me"} assert data["litellm_metadata"] == {} @@ -164,7 +165,6 @@ class TestStripClientPricingOverrides: def test_metadata_field_set_contains_model_info(self): assert "model_info" in _CLIENT_PRICING_METADATA_FIELDS - assert "standard_logging_guardrail_information" in _CLIENT_PRICING_METADATA_FIELDS def test_strip_emits_debug_log_listing_dropped_fields(self, caplog): # Operators need a paper trail so they can diagnose why a previously @@ -315,6 +315,33 @@ async def test_add_litellm_data_to_request_skips_strip_with_key_opt_in(): assert updated["base_model"] == "gpt-4o-mini" +@pytest.mark.asyncio +@pytest.mark.parametrize("opt_in_field", ["metadata", "team_metadata"]) +async def test_add_litellm_data_to_request_strips_guardrail_information_despite_pricing_opt_in(opt_in_field): + data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "hi"}], + "input_cost_per_token": 0.0001, + "metadata": { + "user_session": "keep-me", + "standard_logging_guardrail_information": [{"guardrail_name": "forged", "guardrail_cost": 5.0}], + }, + } + + updated = await add_litellm_data_to_request( + data=data, + request=_make_request_mock(), + user_api_key_dict=_user_api_key_auth(**{opt_in_field: {"allow_client_pricing_override": True}}), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated["input_cost_per_token"] == 0.0001 + assert updated["metadata"]["user_session"] == "keep-me" + assert "standard_logging_guardrail_information" not in updated["metadata"] + + @pytest.mark.asyncio async def test_add_litellm_data_to_request_strips_json_string_litellm_metadata(): """``litellm_metadata`` may arrive as a JSON-encoded string (multipart/ diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index b5940e4c212..bc13595df6c 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -127,51 +127,132 @@ async def test_proxy_only_error_log_marks_no_upstream_llm_call(): @pytest.mark.asyncio -async def test_proxy_only_error_log_keeps_litellm_metadata_in_litellm_params(): - """Responses API requests carry guardrail info under ``litellm_metadata`` - (not ``metadata``). It must land in litellm_params so - ``merge_litellm_metadata`` can surface ``guardrail_information`` in the - spend-log failure row, matching the chat completions path.""" - from litellm.proxy._types import UserAPIKeyAuth +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +async def test_proxy_only_error_log_drops_caller_guardrail_information_without_logging_object( + metadata_field: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "input_callback", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + guardrail_info: Final = [ + { + "guardrail_name": "caller-guardrail", + "guardrail_status": "guardrail_intervened", + "guardrail_cost": 123.45, + } + ] + request_data: Final[dict[str, object]] = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + metadata_field: {"tags": ["t"], "standard_logging_guardrail_information": guardrail_info}, + } - proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) - captured = {} - guardrail_info = [{"guardrail_name": "test-guard", "guardrail_status": "blocked"}] + await ProxyLogging(user_api_key_cache=DualCache())._handle_logging_proxy_only_error( + request_data=request_data, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-rejected", team_id="team-a", request_route="/v1/chat/completions"), + route="/v1/chat/completions", + original_exception=HTTPException(status_code=400, detail="blocked"), + ) - def fake_update_environment_variables(self, *args, **kwargs): - captured["litellm_params"] = kwargs.get("litellm_params") - captured["optional_params"] = kwargs.get("optional_params") + details: Final = request_data["litellm_logging_obj"].model_call_details + assert details["standard_logging_object"] is not None + assert details["litellm_params"][metadata_field]["tags"] == ["t"] + assert "standard_logging_guardrail_information" not in details["litellm_params"][metadata_field] + assert "guardrail_cost" not in str(details["standard_logging_object"]) + assert "litellm_metadata" not in details["optional_params"] - from litellm.litellm_core_utils.litellm_logging import Logging - orig_update_env = Logging.update_environment_variables - orig_pre_call = Logging.pre_call - orig_async_failure = Logging.async_failure_handler +@pytest.mark.asyncio +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +async def test_proxy_only_error_log_keeps_tags_and_drops_caller_logging_fields( + metadata_field: str, +) -> None: + request_data: Final[dict[str, object]] = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + metadata_field: { + "langsmith_base_url": "https://caller.invalid", + "langsmith_api_key": "caller-key", + "tags": ["t"], + "user_api_key_team_id": "caller-team", + "user_api_key_hash": "caller-hash", + "user_api_key_auth_metadata": {"caller": "metadata"}, + }, + "langsmith_base_url": "https://caller.invalid", + "standard_callback_dynamic_params": {"gcs_path_service_account": "/caller/sa.json"}, + } - async def _noop_async_failure(self, *args, **kwargs): - return None + await ProxyLogging(user_api_key_cache=DualCache())._handle_logging_proxy_only_error( + request_data=request_data, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-rejected", team_id="team-a", request_route="/v1/chat/completions"), + route="/v1/chat/completions", + original_exception=Exception("bad key"), + ) - Logging.update_environment_variables = fake_update_environment_variables - Logging.pre_call = lambda self, *args, **kwargs: None - Logging.async_failure_handler = _noop_async_failure - try: - await proxy_logging_obj._handle_logging_proxy_only_error( - request_data={ - "model": "gpt-4o", - "input": "blocked prompt", - "litellm_metadata": {"standard_logging_guardrail_information": guardrail_info}, - }, - user_api_key_dict=UserAPIKeyAuth(api_key="sk-9876", request_route="/v1/responses"), - route="/v1/responses", - original_exception=HTTPException(status_code=400, detail="blocked"), - ) - finally: - Logging.update_environment_variables = orig_update_env - Logging.pre_call = orig_pre_call - Logging.async_failure_handler = orig_async_failure + details: Final = request_data["litellm_logging_obj"].model_call_details + assert "caller" not in str(details["standard_callback_dynamic_params"]) + assert "langsmith_base_url" not in details + assert "caller" not in str(details["litellm_params"]) + assert details["litellm_params"][metadata_field]["tags"] == ["t"] + assert details["litellm_params"]["metadata"]["user_api_key_team_id"] == "team-a" + assert details["litellm_params"][metadata_field]["user_api_key_team_id"] == "team-a" - assert captured["litellm_params"]["litellm_metadata"]["standard_logging_guardrail_information"] == guardrail_info - assert "litellm_metadata" not in captured["optional_params"] + +@pytest.mark.asyncio +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +async def test_proxy_only_error_log_drops_malformed_metadata_tags(metadata_field: str) -> None: + guardrail_info: Final = [{"guardrail_name": "g", "guardrail_status": "guardrail_intervened"}] + request_data: Final[dict[str, object]] = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + metadata_field: {"tags": 123, "standard_logging_guardrail_information": guardrail_info}, + } + + await ProxyLogging(user_api_key_cache=DualCache())._handle_logging_proxy_only_error( + request_data=request_data, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-rejected", team_id="team-a", request_route="/v1/chat/completions"), + route="/v1/chat/completions", + original_exception=Exception("bad key"), + ) + + details: Final = request_data["litellm_logging_obj"].model_call_details + assert details["standard_logging_object"] is not None + assert "tags" not in details["litellm_params"][metadata_field] + assert "standard_logging_guardrail_information" not in details["litellm_params"][metadata_field] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +async def test_proxy_only_error_log_builds_payload_when_logging_object_has_malformed_tags( + metadata_field: str, +) -> None: + request_data: Final[dict[str, object]] = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + "litellm_call_id": "malformed-tags-call", + metadata_field: {"tags": 123}, + } + logging_obj, _ = litellm.utils.function_setup( + original_function="acompletion", + rules_obj=litellm.utils.Rules(), + start_time=real_datetime.datetime.now(), + **request_data, + ) + + await ProxyLogging(user_api_key_cache=DualCache())._handle_logging_proxy_only_error( + request_data={**request_data, "litellm_logging_obj": logging_obj}, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-blocked", team_id="team-a", request_route="/v1/chat/completions"), + route="/v1/chat/completions", + original_exception=HTTPException(status_code=400, detail="Content blocked"), + ) + + standard_logging_object: Final = logging_obj.model_call_details["standard_logging_object"] + assert standard_logging_object is not None + assert standard_logging_object["model"] == request_data["model"] + assert standard_logging_object["metadata"]["user_api_key_team_id"] == "team-a" + assert 123 not in standard_logging_object["request_tags"] @pytest.mark.asyncio diff --git a/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py b/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py index 6557f0d1446..f880d3548f7 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py +++ b/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py @@ -402,6 +402,103 @@ async def test_post_call_failure_hook_flags_pre_routing_reject_despite_caller_mo assert kwargs["litellm_params"]["custom_llm_provider"] == "openai" +@pytest.mark.asyncio +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +async def test_pre_routing_reject_logs_the_configured_deployment_over_caller_metadata( + proxy_logging, make_user_api_key_auth, monkeypatch, metadata_field: str +): + from litellm.proxy import proxy_server + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "internal-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + } + ] + ), + ) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + request_data: Final[dict[str, object]] = { + "model": "internal-model", + "messages": [{"role": "user", "content": "hi"}], + "litellm_call_id": "caller-routing-metadata", + metadata_field: {"deployment": "openai/caller-chosen", "model_group": "caller-group"}, + } + logging_obj, _ = litellm.utils.function_setup( + original_function="acompletion", rules_obj=litellm.utils.Rules(), start_time=datetime.now(), **request_data + ) + + await proxy_logging.post_call_failure_hook( + request_data={**request_data, "litellm_logging_obj": logging_obj}, + original_exception=HTTPException(status_code=400, detail="Content blocked"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + now: Final = datetime.now() + payload: Final = get_logging_payload( + kwargs={**recorded[0], "completion_start_time": now}, response_obj=None, start_time=now, end_time=now + ) + standard_logging_object: Final = recorded[0]["standard_logging_object"] + assert { + "logged_model": standard_logging_object["model"], + "logged_model_group": standard_logging_object["model_group"], + "spend_model": payload["model"], + "spend_model_group": payload["model_group"], + } == { + "logged_model": "openai/gpt-4.1", + "logged_model_group": "internal-model", + "spend_model": "openai/gpt-4.1", + "spend_model_group": "internal-model", + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("error_type", "logged_tags"), [(ProxyErrorTypes.auth_error, []), (None, ["checkout"])]) +async def test_proxy_only_failure_log_keeps_caller_tags_only_after_auth( + proxy_logging, monkeypatch, error_type: ProxyErrorTypes | None, logged_tags: list[str] +): + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={ + "model": "internal-model", + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"tags": ["checkout"]}, + }, + original_exception=HTTPException(status_code=401, detail="Invalid proxy server token passed"), + user_api_key_dict=UserAPIKeyAuth(request_route="/chat/completions"), + error_type=error_type, + route="/chat/completions", + ) + + assert len(recorded) == 1 + assert recorded[0]["standard_logging_object"]["request_tags"] == logged_tags + + @pytest.mark.asyncio async def test_post_call_failure_hook_attribution_does_not_count_against_the_deployment( proxy_logging, make_user_api_key_auth, monkeypatch @@ -574,9 +671,8 @@ async def test_post_call_failure_hook_omits_provider_when_a_deployment_does_not_ async def test_handle_logging_proxy_only_path_attributes_with_read_only_metadata( proxy_logging, make_user_api_key_auth, monkeypatch ): - """With a logging object already on the request, its metadata is taken as given; - a read-only mapping there must not crash the stamp, and the failure handler - still receives the provider attribution.""" + """With a logging object already on the request, allowlisted metadata is kept, + a read-only mapping must not crash the stamp, and the failure handler receives provider attribution.""" from litellm.proxy import proxy_server monkeypatch.setattr( @@ -595,16 +691,26 @@ async def test_handle_logging_proxy_only_path_attributes_with_read_only_metadata logging_obj = MagicMock() logging_obj.call_type = "acompletion" logging_obj.model_call_details = {} + logging_obj.litellm_params = {"metadata": {"preexisting": "logged"}} logging_obj.async_failure_handler = AsyncMock() + guardrail_info: Final = [ + { + "guardrail_name": "local-block", + "guardrail_status": "guardrail_intervened", + "guardrail_cost": 0.25, + } + ] await proxy_logging._handle_logging_proxy_only_error( request_data={ "litellm_logging_obj": logging_obj, "model": "internal-model", "messages": [{"role": "user", "content": "hi"}], - "metadata": MappingProxyType({"user_api_key_alias": "frozen"}), + "metadata": MappingProxyType( + {"tags": ["checkout"], "standard_logging_guardrail_information": guardrail_info} + ), }, - user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions", key_alias="frozen"), route="/chat/completions", original_exception=HTTPException(status_code=403, detail="blocked"), ) @@ -613,7 +719,11 @@ async def test_handle_logging_proxy_only_path_attributes_with_read_only_metadata update_kwargs = logging_obj.update_environment_variables.call_args.kwargs assert update_kwargs["custom_llm_provider"] == "openai" assert update_kwargs["litellm_params"]["custom_llm_provider"] == "openai" - assert update_kwargs["litellm_params"]["metadata"] == {"user_api_key_alias": "frozen"} + assert { + key: update_kwargs["litellm_params"]["metadata"][key] for key in ("preexisting", "user_api_key_alias") + } == {"preexisting": "logged", "user_api_key_alias": "frozen"} + assert update_kwargs["litellm_params"]["metadata"]["tags"] == ["checkout"] + assert update_kwargs["litellm_params"]["metadata"]["standard_logging_guardrail_information"] == guardrail_info @pytest.mark.asyncio From b766aaf010442204c3770b72bccdb87b1ea97479 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 27 Oct 2026 12:43:04 -0700 Subject: [PATCH 016/102] fix(ui): expose the authorization MCP auth type in create and edit forms (#48629) * test(ui): cover the authorization MCP auth type in create and edit forms Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): expose the authorization MCP auth type in create and edit forms Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../CreateMCPServer.integration.test.tsx | 40 +++++++++++++++++++ .../_components/createServerPayload.ts | 1 + .../editServerPayload.differential.cases.ts | 1 + .../mcp_server_edit.integration.test.tsx | 5 +++ .../_components/mcp_server_edit.tsx | 2 +- .../src/components/mcp_tools/types.tsx | 2 + 6 files changed, 50 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx index 65656a053ce..3b3269b87cc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.integration.test.tsx @@ -396,6 +396,46 @@ describe("CreateMCPServer", () => { expect(payload.credentials).toEqual({ auth_value: "my-secret-key" }); }); + it("creates an Authorization (raw header) server with the header value sent verbatim", async () => { + await selectHttpTransport(); + + fireEvent.change(getServerNameInput(), { target: { value: "Raw_Auth_Server" } }); + fireEvent.change(screen.getByPlaceholderText("https://your-mcp-server.com"), { + target: { value: "https://example.com/mcp" }, + }); + + await selectOption("Authentication", "Authorization (raw header)"); + + fireEvent.change(await screen.findByPlaceholderText("Enter token or secret"), { + target: { value: "ApiKey raw-key-123" }, + }); + + vi.mocked(networking.createMCPServer).mockResolvedValue({ + server_id: "new-server-2", + server_name: "Raw_Auth_Server", + alias: "Raw_Auth_Server", + url: "https://example.com/mcp", + transport: "http", + auth_type: "authorization", + created_at: "2024-01-01T00:00:00Z", + created_by: "user-1", + updated_at: "2024-01-01T00:00:00Z", + updated_by: "user-1", + }); + + await act(async () => { + fireEvent.click(screen.getByRole("button", { name: "Add MCP Server" })); + }); + + await waitFor(() => { + expect(networking.createMCPServer).toHaveBeenCalledTimes(1); + }); + + const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0]; + expect(payload.auth_type).toBe("authorization"); + expect(payload.credentials).toEqual({ auth_value: "ApiKey raw-key-123" }); + }); + it("does not write the browser-authorized token into form.credentials for true_passthrough", async () => { await selectHttpTransport(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.ts index 30c4c502c42..5536b4b1bc4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createServerPayload.ts @@ -15,6 +15,7 @@ export const AUTH_TYPES_REQUIRING_AUTH_VALUE = [ AUTH_TYPE.BEARER_TOKEN, AUTH_TYPE.TOKEN, AUTH_TYPE.BASIC, + AUTH_TYPE.AUTHORIZATION, ]; export const AUTH_TYPES_REQUIRING_CREDENTIALS = [ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editServerPayload.differential.cases.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editServerPayload.differential.cases.ts index 4219113918b..e9f32b5c4db 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editServerPayload.differential.cases.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/editServerPayload.differential.cases.ts @@ -82,6 +82,7 @@ const AUTH_TYPES = [ "bearer_token", "token", "basic", + "authorization", "oauth2", "oauth2_token_exchange", "oauth2_id_jag", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.integration.test.tsx index 9f91c06e9e5..15b09961a84 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.integration.test.tsx @@ -135,6 +135,11 @@ const CASES: readonly Case[] = [ server: { ...BASE, auth_type: "basic", credentials: CREDS }, expected: { ...EXPECTED_BASE, auth_type: "basic", credentials: { auth_value: "secret-value" } }, }, + { + label: "authorization sends only auth_value under credentials", + server: { ...BASE, auth_type: "authorization", credentials: CREDS }, + expected: { ...EXPECTED_BASE, auth_type: "authorization", credentials: { auth_value: "secret-value" } }, + }, { label: "oauth2 m2m mounts token_url and omits the interactive endpoints entirely", server: { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx index f35e3ab3d3a..57985fd8cbb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx @@ -51,6 +51,7 @@ import OAuthFormFields from "./OAuthFormFields"; import MCPLogoSelector from "./MCPLogoSelector"; import EnvVarsSection from "./EnvVarsSection"; import { validateMCPServerUrl, validateMCPServerName, normalizeToolOverrideMap } from "./utils"; +import { AUTH_TYPES_REQUIRING_AUTH_VALUE } from "./createServerPayload"; import { EditServerFormValues, buildEditServerPayload, editPayloadErrorMessage } from "./editServerPayload"; import { DUPLICATE_IDENTIFIER_MESSAGE, findDuplicateMcpServer, mcpSubmitErrorReason } from "./duplicateServerCheck"; import { toast } from "@/lib/toast"; @@ -94,7 +95,6 @@ interface MCPServerEditProps { stdioEnabled?: boolean; } -const AUTH_TYPES_REQUIRING_AUTH_VALUE = [AUTH_TYPE.API_KEY, AUTH_TYPE.BEARER_TOKEN, AUTH_TYPE.TOKEN, AUTH_TYPE.BASIC]; export const EDIT_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-edit-state"; const MCPServerEdit: React.FC = ({ diff --git a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx index 22fd64f10d8..61c81c030c5 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx @@ -38,6 +38,7 @@ export const AUTH_TYPE = { BEARER_TOKEN: "bearer_token", TOKEN: "token", BASIC: "basic", + AUTHORIZATION: "authorization", OAUTH2: "oauth2", OAUTH2_TOKEN_EXCHANGE: "oauth2_token_exchange", OAUTH2_ID_JAG: "oauth2_id_jag", @@ -52,6 +53,7 @@ export const AUTH_TYPE_ITEMS = [ { value: AUTH_TYPE.BEARER_TOKEN, label: "Bearer Token" }, { value: AUTH_TYPE.TOKEN, label: "Token" }, { value: AUTH_TYPE.BASIC, label: "Basic Auth" }, + { value: AUTH_TYPE.AUTHORIZATION, label: "Authorization (raw header)" }, { value: AUTH_TYPE.OAUTH2, label: "OAuth" }, { value: AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE, label: "OAuth Token Exchange (OBO)" }, { value: AUTH_TYPE.OAUTH2_ID_JAG, label: "ID-JAG (Okta Cross App Access)" }, From 29946e4b75421d5a91e4e3e7f8df913a4f956032 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 27 Oct 2026 12:43:36 -0700 Subject: [PATCH 017/102] feat(hosted_vllm): accept images on decision deployments (#48621) * test(decisions): cover hosted_vllm decision images Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(hosted_vllm): accept images on decision deployments Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../hosted_vllm/decisions/transformation.py | 2 + tests/unit/decisions/test_main.py | 124 ++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/litellm/llms/hosted_vllm/decisions/transformation.py b/litellm/llms/hosted_vllm/decisions/transformation.py index ef99f2de173..fab07d2e7db 100644 --- a/litellm/llms/hosted_vllm/decisions/transformation.py +++ b/litellm/llms/hosted_vllm/decisions/transformation.py @@ -9,6 +9,8 @@ class HostedVLLMDecisionsConfig(BaseDecisionsConfig): api_key_env = ("HOSTED_VLLM_API_KEY",) api_base_env = ("HOSTED_VLLM_API_BASE",) api_key_required = False + supports_images = True + inlines_remote_images = True health_check_questions = MappingProxyType( { "reachable": MappingProxyType( diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 0bc18080ef4..b9813289749 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -4026,3 +4026,127 @@ async def test_lm_studio_decisions_sends_inline_images_to_systemone( "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, } assert response.answers[0].probability == 0.9 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("remote_image", (False, True), ids=("inline", "remote")) +async def test_hosted_vllm_decisions_sends_input_images_to_systemone( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + remote_image: bool, + async_only_image_fetch: AsyncOnlyImageFetch, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + image: Final[str] = "https://images.example/input.png" if remote_image else "data:image/png;base64,AA==" + expected_image: Final[str] = async_only_image_fetch.data_url if remote_image else image + route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_RESPONSE) + + response: Final = await litellm.adecisions( + model="hosted_vllm/Qwen/Qwen3-0.6B", + input=[ + { + "role": "user", + "content": [{"type": "input_text", "text": "review"}, {"type": "input_image", "image_url": image}], + } + ], + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + api_base="http://vllm.local:8000/v1", + ) + + assert json.loads(route.calls.last.request.content) == { + "model": "Qwen/Qwen3-0.6B", + "state": "review", + "images": [expected_image], + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert response.answers[0].probability == 0.9 + assert async_only_image_fetch.fetched == (["https://images.example/input.png"] if remote_image else []) + + +def test_hosted_vllm_systemone_forwards_images( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + image: Final[str] = "data:image/png;base64,AA==" + route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_RESPONSE) + + litellm.systemone( + model="hosted_vllm/Qwen/Qwen3-0.6B", + state="review", + images=[image], + questions={ + "is_defect": { + "type": "choice", + "instructions": "Is this a defect?", + "criteria": {"yes": "defective", "no": "intact"}, + } + }, + api_base="http://vllm.local:8000", + ) + + assert json.loads(route.calls.last.request.content) == { + "model": "Qwen/Qwen3-0.6B", + "state": "review", + "images": [image], + "questions": { + "is_defect": { + "type": "choice", + "instructions": "Is this a defect?", + "criteria": {"yes": "defective", "no": "intact"}, + } + }, + } + + +@pytest.mark.asyncio +async def test_hosted_vllm_decisions_sends_image_only_input_to_systemone( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + image: Final[str] = "data:image/png;base64,AA==" + route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_RESPONSE) + + await litellm.adecisions( + model="hosted_vllm/Qwen/Qwen3-0.6B", + input=[{"role": "user", "content": [{"type": "input_image", "image_url": image}]}], + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + api_base="http://vllm.local:8000", + ) + + assert json.loads(route.calls.last.request.content) == { + "model": "Qwen/Qwen3-0.6B", + "state": "", + "images": [image], + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + + +def test_hosted_vllm_systemone_maps_image_rejection_to_bad_request( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + message: Final[str] = "this model does not take image inputs" + route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond( + status_code=400, + json={"error": {"message": message, "type": "BadRequestError", "code": 400}}, + ) + + with pytest.raises(litellm.BadRequestError, match=re.escape(message)) as caught: + litellm.systemone( + model="hosted_vllm/Qwen/Qwen3-0.6B", + state="review", + images=["data:image/png;base64,AA=="], + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_base="http://vllm.local:8000", + num_retries=0, + ) + + assert caught.value.status_code == 400 + assert route.call_count == 1 From 618d5aec71880362a78984a839e080e403ec99d3 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:46:19 -0700 Subject: [PATCH 018/102] fix(mcp): load deprecated package aliases on demand (#48615) * fix(mcp): load deprecated package aliases on demand * fix(mcp): preserve lazy child access and module reloads --- .../litellm_core_utils/deprecated_package.py | 84 ++++++++++++------- .../unit/experimental_mcp_client/test_init.py | 2 + .../_experimental/mcp_server/test_init.py | 49 +++++++++++ 3 files changed, 103 insertions(+), 32 deletions(-) diff --git a/litellm/litellm_core_utils/deprecated_package.py b/litellm/litellm_core_utils/deprecated_package.py index 542686ecf19..f9915af4880 100644 --- a/litellm/litellm_core_utils/deprecated_package.py +++ b/litellm/litellm_core_utils/deprecated_package.py @@ -1,53 +1,73 @@ import importlib +import importlib.abc +import importlib.machinery +import importlib.util import sys import warnings -from itertools import chain -from pathlib import Path +from collections.abc import Sequence +from dataclasses import dataclass +from functools import partial from types import ModuleType from typing import Final -def _dotted_name(root: Path, path: Path, package_name: str) -> str: - parts: Final = path.relative_to(root).with_suffix("").parts - return ".".join((package_name, *parts)).removesuffix(".__init__") +@dataclass(frozen=True, slots=True) +class _AliasLoader(importlib.abc.Loader): + new_dotted: str + canonical_spec: importlib.machinery.ModuleSpec + old_dotted: str + + def create_module(self, spec: importlib.machinery.ModuleSpec) -> ModuleType: + return importlib.import_module(self.new_dotted) + + def exec_module(self, module: ModuleType) -> None: + module.__spec__ = self.canonical_spec # rebind-ok: [LIT011] restore import metadata overwritten by importlib + if self.canonical_spec.submodule_search_locations is not None: + _alias_attributes(self.old_dotted, self.new_dotted) -def _parents_and_self(dotted: str, package_name: str) -> tuple[str, ...]: - parts: Final = dotted.removeprefix(f"{package_name}.").split(".") - return tuple(f"{package_name}.{'.'.join(parts[:depth])}" for depth in range(1, len(parts) + 1)) +@dataclass(frozen=True, slots=True) +class _AliasFinder(importlib.abc.MetaPathFinder): + old_name: str + new_name: str - -def _module_names(package: ModuleType) -> tuple[str, ...]: - roots: Final = tuple(Path(root) for root in package.__path__) - files: Final = tuple( - (root, path) for root in roots for path in root.rglob("*.py") - ) # comprehension-ok: nested walk over each package root - leaves: Final = tuple( - name - for name in (_dotted_name(root, path, package.__name__) for root, path in files) - if name != package.__name__ - ) - return tuple( - sorted( - set(chain.from_iterable(_parents_and_self(leaf, package.__name__) for leaf in leaves)) - {package.__name__} + def find_spec( + self, + fullname: str, + path: Sequence[str] | None = None, + target: ModuleType | None = None, + ) -> importlib.machinery.ModuleSpec | None: + if not fullname.startswith(f"{self.old_name}."): + return None + new_dotted: Final = f"{self.new_name}{fullname.removeprefix(self.old_name)}" + new_spec: Final = importlib.util.find_spec(new_dotted) + if new_spec is None: + return None + return importlib.util.spec_from_loader( + fullname, + _AliasLoader(new_dotted, new_spec, fullname), + is_package=new_spec.submodule_search_locations is not None, ) - ) -def _alias(old_name: str, new_package_name: str, new_dotted: str) -> None: - module: Final = importlib.import_module(new_dotted) - old_dotted: Final = f"{old_name}{new_dotted.removeprefix(new_package_name)}" - old_parent, _, child = old_dotted.rpartition(".") - sys.modules[old_dotted] = module - setattr(sys.modules[old_parent], child, module) +def _deprecated_attribute(old_name: str, new_name: str, name: str) -> ModuleType: + if importlib.util.find_spec(f"{new_name}.{name}") is None: + raise AttributeError(f"module {old_name!r} has no attribute {name!r}") + return importlib.import_module(f"{old_name}.{name}") + + +def _alias_attributes(old_name: str, new_name: str) -> None: + if "__getattr__" not in vars(sys.modules[old_name]): + sys.modules[old_name].__getattr__ = partial(_deprecated_attribute, old_name, new_name) def alias_deprecated_package(old_name: str, new_package: ModuleType) -> None: - """Make every module under ``new_package`` importable and attribute-reachable under ``old_name`` and warn once.""" warnings.warn( f"{old_name} is deprecated, import {new_package.__name__} instead", DeprecationWarning, stacklevel=3, ) - for new_dotted in _module_names(new_package): - _alias(old_name, new_package.__name__, new_dotted) + finder: Final = _AliasFinder(old_name, new_package.__name__) + if finder not in sys.meta_path: + sys.meta_path.insert(0, finder) + _alias_attributes(old_name, new_package.__name__) diff --git a/tests/unit/experimental_mcp_client/test_init.py b/tests/unit/experimental_mcp_client/test_init.py index 9d2f1137470..1ce2a236d9d 100644 --- a/tests/unit/experimental_mcp_client/test_init.py +++ b/tests/unit/experimental_mcp_client/test_init.py @@ -19,6 +19,8 @@ def test_old_package_warns_and_deep_imports_resolve_to_mcp_client(monkeypatch: p with pytest.warns(DeprecationWarning, match="litellm.mcp_client"): import litellm.experimental_mcp_client as old + assert old.client is litellm.mcp_client.client + assert old.tools is litellm.mcp_client.tools assert old.__all__ == litellm.mcp_client.__all__ assert old.load_mcp_tools is litellm.mcp_client.load_mcp_tools assert old.call_openai_tool is litellm.mcp_client.call_openai_tool diff --git a/tests/unit/proxy/_experimental/mcp_server/test_init.py b/tests/unit/proxy/_experimental/mcp_server/test_init.py index 2f933e11351..5ecb79b9813 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_init.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_init.py @@ -1,6 +1,7 @@ import importlib import sys from collections.abc import Mapping +from typing import Final import pytest @@ -16,9 +17,17 @@ def test_old_package_warns_and_deep_imports_resolve_to_proxy_mcp_server(monkeypa if name.startswith(OLD_PACKAGE): monkeypatch.delitem(sys.modules, name) + monkeypatch.delattr(litellm.proxy.mcp_server.auth, "user_api_key_auth_mcp") + monkeypatch.delitem(sys.modules, "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp") + with pytest.warns(DeprecationWarning, match="litellm.proxy.mcp_server"): old = importlib.import_module(OLD_PACKAGE) + assert old.server is litellm.proxy.mcp_server.server + assert old.auth is litellm.proxy.mcp_server.auth + assert old.auth.user_api_key_auth_mcp is importlib.import_module( + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp" + ) assert importlib.import_module(f"{OLD_PACKAGE}.server") is litellm.proxy.mcp_server.server assert importlib.import_module(f"{OLD_PACKAGE}.utils") is litellm.proxy.mcp_server.utils assert ( @@ -28,3 +37,43 @@ def test_old_package_warns_and_deep_imports_resolve_to_proxy_mcp_server(monkeypa attrs: Mapping[str, object] = vars(old) assert attrs["server"] is litellm.proxy.mcp_server.server assert attrs["auth"] is litellm.proxy.mcp_server.auth + + +def test_old_stdio_helper_import_works_without_optional_mcp_dependency(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delattr(litellm.proxy, "mcp_server") + old_parent: Final = importlib.import_module("litellm.proxy._experimental") + monkeypatch.delattr(old_parent, "mcp_server", raising=False) + monkeypatch.delattr(litellm, "mcp_client", raising=False) + prefixes: Final = (OLD_PACKAGE, "litellm.proxy.mcp_server", "litellm.mcp_client", "mcp") + for name in tuple(sys.modules): + if any(name == prefix or name.startswith(f"{prefix}.") for prefix in prefixes): + monkeypatch.delitem(sys.modules, name) + monkeypatch.setitem(sys.modules, "mcp", None) + + with pytest.warns(DeprecationWarning, match="litellm.proxy.mcp_server"): + old: Final = importlib.import_module(f"{OLD_PACKAGE}.stdio_gate") + new: Final = importlib.import_module("litellm.proxy.mcp_server.stdio_gate") + + monkeypatch.setenv("LITELLM_ENABLE_MCP_STDIO", "true") + assert old.is_mcp_stdio_enabled() is True + monkeypatch.setenv("LITELLM_ENABLE_MCP_STDIO", "false") + assert old.is_mcp_stdio_enabled() is False + assert old is new + + +def test_legacy_child_import_keeps_canonical_module_reload_working(monkeypatch: pytest.MonkeyPatch) -> None: + for name in tuple(sys.modules): + if name.startswith(OLD_PACKAGE): + monkeypatch.delitem(sys.modules, name) + + helper: Final = importlib.import_module("litellm.proxy.mcp_server.stdio_gate") + with pytest.warns(DeprecationWarning, match="litellm.proxy.mcp_server"): + old: Final = importlib.import_module(f"{OLD_PACKAGE}.stdio_gate") + + monkeypatch.setattr(helper, "MCP_STDIO_ENABLED_ENV_VAR", "RELOAD_PROBE") + monkeypatch.setenv("LITELLM_ENABLE_MCP_STDIO", "true") + monkeypatch.setenv("RELOAD_PROBE", "false") + importlib.reload(helper) + + assert old is helper + assert old.is_mcp_stdio_enabled() is True From 34134d7ae5b7909c6b75aedb0efc6eff9aab281f Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 27 Oct 2026 12:46:22 -0700 Subject: [PATCH 019/102] fix(auth): log routine 401 rejections as one warning without a traceback (#48613) A missing or malformed Authorization header, a blocked key, a non-admin key on an admin-only or master-key-only route and a SCIM-deactivated user were raised as bare exceptions, so the auth failure handler could not see the 401 and logged each one at ERROR with a full traceback, which flooded the log under scanner traffic They now raise a typed 401 carrying the quiet-log marker that the non-sk bearer rejection already used, and the unknown sk- key rejection joins the same path, so each lands as one WARNING line on stdout with no traceback. The client response is unchanged --- litellm/proxy/auth/auth_exception_handler.py | 7 +-- litellm/proxy/auth/auth_utils.py | 20 +++++---- litellm/proxy/auth/route_checks.py | 14 ++++-- litellm/proxy/auth/user_api_key_auth.py | 45 ++++++++++++++++--- .../proxy/auth/test_auth_exception_handler.py | 29 ++++++++---- tests/unit/proxy/auth/test_route_checks.py | 2 +- .../unit/proxy/auth/test_user_api_key_auth.py | 4 +- 7 files changed, 88 insertions(+), 33 deletions(-) diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index 2eb25cc6105..ed743764fce 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -24,6 +24,7 @@ from litellm.proxy.auth.auth_utils import ( # noqa: F401 # legacy module expor _get_request_ip_address, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export get_request_ip_address, is_invalid_virtual_key_error, + is_unauthorized_error, mark_invalid_virtual_key_error, normalize_request_route, ) @@ -200,11 +201,11 @@ class UserAPIKeyAuthExceptionHandler: user_agent: Final = _get_user_agent(request) # Log authentication failures before identity seeding and callbacks, so the log - # survives a raising callback pipeline. Classify and route malformed virtual-key - # rejections to WARNING on stdout (suppressible via LITELLM_LOG=ERROR). + # survives a raising callback pipeline. Route 401 rejections to WARNING on stdout + # (suppressible via LITELLM_LOG=ERROR); anything else is unexpected and logs at ERROR. log_extra: Final = {"requester_ip": requester_ip} is_invalid_virtual_key: Final = is_invalid_virtual_key_error(e) - is_quiet_log: Final = is_invalid_virtual_key and not litellm.log_client_error_tracebacks + is_quiet_log: Final = is_unauthorized_error(e) and not litellm.log_client_error_tracebacks logger: Final = verbose_proxy_stdout_logger if is_quiet_log else verbose_proxy_logger logger.log( logging.WARNING if is_quiet_log else logging.ERROR, diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 30ede935cfa..aa6ddae438d 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -48,6 +48,16 @@ from litellm.types.utils import CustomPricingLiteLLMParams, oauth_token_exchange from litellm.types.videos.main import DecodedVideoId +def is_unauthorized_error(exception: BaseException | None) -> bool: + """True when the proxy itself rejected the caller's credentials with a 401.""" + if not isinstance(exception, (HTTPException, ProxyException)): + return False + + code: Final[object] = getattr(exception, "code", None) + status_code: Final[object] = code if code is not None else getattr(exception, "status_code", None) + return str(status_code) == str(status.HTTP_401_UNAUTHORIZED) + + def is_invalid_virtual_key_error(exception: BaseException | None) -> bool: """True when an authentication error rejects a malformed virtual key. @@ -57,15 +67,7 @@ def is_invalid_virtual_key_error(exception: BaseException | None) -> bool: match would let a request body demote an authorization failure to the quiet log path. """ - if not isinstance(exception, (HTTPException, ProxyException)): - return False - - code: Final[object] = getattr(exception, "code", None) - status_code: Final[object] = code if code is not None else getattr(exception, "status_code", None) - if str(status_code) != str(status.HTTP_401_UNAUTHORIZED): - return False - - return getattr(exception, INVALID_VIRTUAL_KEY_ERROR_MARKER, False) is True + return is_unauthorized_error(exception) and getattr(exception, INVALID_VIRTUAL_KEY_ERROR_MARKER, False) is True def log_model_access_denial(exc: BaseException) -> None: diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index c402f111563..74e4c4c6d11 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -14,6 +14,8 @@ from litellm.proxy._types import ( LiteLLM_UserTable, LiteLLMRoutes, LitellmUserRoles, + ProxyErrorTypes, + ProxyException, UserAPIKeyAuth, ) @@ -284,7 +286,7 @@ class RouteChecks: route (str): The route being accessed Raises: - Exception: With user role and masked user_id information + ProxyException: 401 with user role and masked user_id information """ user_role = "unknown" user_id = "unknown" @@ -293,8 +295,14 @@ class RouteChecks: user_id = user_obj.user_id or "unknown" masked_user_id: Final = RouteChecks._mask_user_id(user_id) - raise Exception( - f"Only proxy admin can be used to generate, delete, update info for new keys/users/teams. Route={route}. Your role={user_role}. Your user_id={masked_user_id}" + raise ProxyException( + message=( + "Authentication Error, Only proxy admin can be used to generate, delete, update info for new " + f"keys/users/teams. Route={route}. Your role={user_role}. Your user_id={masked_user_id}" + ), + type=ProxyErrorTypes.auth_error, + param="None", + code=status.HTTP_401_UNAUTHORIZED, ) @staticmethod diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index b1e8bdbe787..cb490b55352 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -2513,10 +2513,20 @@ async def user_api_key_auth_builder( parent_otel_span=parent_otel_span, ) elif api_key is None: # only require api key if master key is set - raise Exception("No api key passed in.") + raise ProxyException( + message="Authentication Error, No api key passed in.", + type=ProxyErrorTypes.auth_error, + param="None", + code=status.HTTP_401_UNAUTHORIZED, + ) elif api_key == "": # missing 'Bearer ' prefix - raise Exception("Malformed API Key passed in. Ensure Key has `Bearer ` prefix.") + raise ProxyException( + message="Authentication Error, Malformed API Key passed in. Ensure Key has `Bearer ` prefix.", + type=ProxyErrorTypes.auth_error, + param="None", + code=status.HTTP_401_UNAUTHORIZED, + ) if route == "/user/auth": if general_settings.get("allow_user_auth", False) is True: @@ -2752,7 +2762,12 @@ async def user_api_key_auth_builder( ## IF it's not a master key ## Route should not be in master_key_only_routes if route in LiteLLMRoutes.master_key_only_routes.value: - raise Exception(f"Tried to access route={route}, which is only for MASTER KEY") + raise ProxyException( + message=f"Authentication Error, Tried to access route={route}, which is only for MASTER KEY", + type=ProxyErrorTypes.auth_error, + param="None", + code=status.HTTP_401_UNAUTHORIZED, + ) ## Check DB @@ -2880,7 +2895,12 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e ## base case ## key is disabled if valid_token.blocked is True: - raise Exception("Key is blocked. Update via `/key/unblock` if you're an admin.") + raise ProxyException( + message="Authentication Error, Key is blocked. Update via `/key/unblock` if you're an admin.", + type=ProxyErrorTypes.auth_error, + param="None", + code=status.HTTP_401_UNAUTHORIZED, + ) await enforce_key_and_fallback_model_access( valid_token=valid_token, request_data=request_data, @@ -2923,8 +2943,14 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e and isinstance(user_obj.metadata, dict) and user_obj.metadata.get("scim_active") is False ): - raise Exception( - f"User={valid_token.user_id} has been deactivated via SCIM. Keys owned by this user cannot be used." + raise ProxyException( + message=( + f"Authentication Error, User={valid_token.user_id} has been deactivated via SCIM. " + "Keys owned by this user cannot be used." + ), + type=ProxyErrorTypes.auth_error, + param="None", + code=status.HTTP_401_UNAUTHORIZED, ) # Check 2a. Check if model has zero cost - if so, skip all budget checks @@ -3241,7 +3267,12 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e if valid_token is None: # No token was found when looking up in the DB - raise Exception("Invalid proxy server token passed") + raise ProxyException( + message="Authentication Error, Invalid proxy server token passed", + type=ProxyErrorTypes.auth_error, + param="None", + code=status.HTTP_401_UNAUTHORIZED, + ) if valid_token_dict is not None: virtual_key_auth_obj: Final = await _return_user_api_key_auth_obj( user_obj=user_obj, diff --git a/tests/unit/proxy/auth/test_auth_exception_handler.py b/tests/unit/proxy/auth/test_auth_exception_handler.py index e21491fd996..92b4c93118f 100644 --- a/tests/unit/proxy/auth/test_auth_exception_handler.py +++ b/tests/unit/proxy/auth/test_auth_exception_handler.py @@ -668,7 +668,7 @@ async def test_expired_key_error_log_names_the_key_owner( ): verbose_proxy_logger.propagate = True try: - with caplog.at_level("ERROR", logger="LiteLLM Proxy"), pytest.raises(ProxyException): + with caplog.at_level("WARNING", logger="LiteLLM Proxy"), pytest.raises(ProxyException): await handler.handle_authentication_error( expired_key_error, MagicMock(), @@ -714,8 +714,8 @@ async def test_expired_key_error_log_names_the_key_owner( async def test_unknown_key_error_log_keeps_the_operator_wording_the_401_hides( looked_up, logged_fragment, absent_fragment, caplog ): - """A wrong key gets a generic 401, while the one auth failure log line keeps the - operator wording and the hash that was looked up, next to the requester IP.""" + """A wrong key gets a generic 401, while the one auth failure log line, a WARNING with no + traceback, keeps the operator wording and the hash that was looked up, next to the requester IP.""" from litellm.constants import INVALID_API_KEY_ERROR_MESSAGE from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError @@ -733,7 +733,7 @@ async def test_unknown_key_error_log_keeps_the_operator_wording_the_401_hides( ): verbose_proxy_logger.propagate = True try: - with caplog.at_level("ERROR", logger="LiteLLM Proxy"), pytest.raises(ProxyException) as exc_info: + with caplog.at_level("WARNING", logger="LiteLLM Proxy"), pytest.raises(ProxyException) as exc_info: await UserAPIKeyAuthExceptionHandler().handle_authentication_error( KeyNotFoundError(looked_up), MagicMock(), @@ -748,6 +748,7 @@ async def test_unknown_key_error_log_keeps_the_operator_wording_the_401_hides( assert exc_info.value.message == INVALID_API_KEY_ERROR_MESSAGE records = [r for r in caplog.records if "user_api_key_auth(): Exception occured" in r.getMessage()] assert len(records) == 1, [r.getMessage() for r in caplog.records] + assert records[0].levelname == "WARNING" and records[0].exc_info is None logged = records[0].getMessage() assert "Requester IP Address:" in logged and logged_fragment in logged, logged if absent_fragment is not None: @@ -1148,10 +1149,16 @@ def _marked_malformed_key_error() -> HTTPException: message="Authentication Error", type=ProxyErrorTypes.auth_error, param=None, code=401 ), False, - "ERROR", - id="expected_401_no_traceback", + "WARNING", + id="proxy_exception_401_warning_no_traceback", ), pytest.param(ValueError("unexpected internal error"), True, "ERROR", id="unexpected_error_keeps_traceback"), + pytest.param( + Exception("No api key passed in."), + True, + "ERROR", + id="bare_exception_coerced_to_401_still_keeps_traceback", + ), pytest.param( _marked_malformed_key_error(), False, @@ -1161,8 +1168,14 @@ def _marked_malformed_key_error() -> HTTPException: pytest.param( HTTPException(status_code=401, detail="LiteLLM Virtual Key expected. Received=test"), False, + "WARNING", + id="http_exception_401_warning_no_traceback", + ), + pytest.param( + ProxyException(message="Forbidden", type=ProxyErrorTypes.auth_error, param=None, code=403), + False, "ERROR", - id="phrase_without_marker_stays_loud", + id="403_stays_error_without_traceback", ), ], ) @@ -1189,7 +1202,7 @@ async def test_handle_authentication_error_traceback_only_for_unexpected_errors( try: try: raise auth_error - except (ProxyException, ValueError, HTTPException) as caught: + except Exception as caught: with caplog.at_level(expect_level, logger="LiteLLM Proxy"), pytest.raises((ProxyException, HTTPException)): await handler.handle_authentication_error( caught, diff --git a/tests/unit/proxy/auth/test_route_checks.py b/tests/unit/proxy/auth/test_route_checks.py index 6de54aa4af6..b2c67b4e009 100644 --- a/tests/unit/proxy/auth/test_route_checks.py +++ b/tests/unit/proxy/auth/test_route_checks.py @@ -3742,7 +3742,7 @@ def test_team_spend_by_user_reachable_by_non_admin(user_role): return "allowed" assert outcome("/team/spend/by_user") == "allowed" - assert outcome("/team/spend/by_key").startswith("denied: Only proxy admin") + assert outcome("/team/spend/by_key").startswith("denied: Authentication Error, Only proxy admin") def test_user_daily_activity_aggregated_not_covered_by_prefix_match(): diff --git a/tests/unit/proxy/auth/test_user_api_key_auth.py b/tests/unit/proxy/auth/test_user_api_key_auth.py index 60a8dd73a34..092805a9cac 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth.py @@ -2031,9 +2031,9 @@ async def test_user_api_key_auth_websocket_rejection_adds_no_traceback_for_other assert rejection.value.status_code == 403 assert "expired key" in str(rejection.value.detail).lower() assert sent == [{"type": "websocket.close", "code": status.WS_1008_POLICY_VIOLATION, "reason": ""}] - proxy_records: Final = [record for record in caplog.records if record.name == verbose_proxy_logger.name] + proxy_records: Final = [record for record in caplog.records if record.name.startswith(verbose_proxy_logger.name)] assert [record.getMessage() for record in proxy_records if record.exc_info is not None] == [] - assert [record.levelno for record in proxy_records if record.levelno >= logging.WARNING] == [logging.ERROR] + assert [record.levelno for record in proxy_records if record.levelno >= logging.WARNING] == [logging.WARNING] @pytest.mark.asyncio From d65587593f04f6bb004f33e39c9f61d1fc542ed7 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 27 Oct 2026 12:49:23 -0700 Subject: [PATCH 020/102] ci: spend-logs-guard requires kerry-berri approval on LiteLLM_SpendLogs schema and migration changes (#48518) * ci: spend-logs-guard requires kerry-berri approval on LiteLLM_SpendLogs schema and migration changes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci(spend-logs-guard): flatten paginated reviews with jq, check out the base ref Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci(spend-logs-guard): read the diff from the pulls/files API instead of fetching PR commits Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * Revert "ci(spend-logs-guard): read the diff from the pulls/files API instead of fetching PR commits" This reverts commit 36da4fe034426194431aa8af92f502b819dff981. --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/spend-logs-guard.yml | 49 ++++++++++ ci_cd/spend_logs_guard.py | 119 +++++++++++++++++++++++++ tests/unit/test_spend_logs_guard.py | 79 ++++++++++++++++ 3 files changed, 247 insertions(+) create mode 100644 .github/workflows/spend-logs-guard.yml create mode 100644 ci_cd/spend_logs_guard.py create mode 100644 tests/unit/test_spend_logs_guard.py diff --git a/.github/workflows/spend-logs-guard.yml b/.github/workflows/spend-logs-guard.yml new file mode 100644 index 00000000000..80a503679ec --- /dev/null +++ b/.github/workflows/spend-logs-guard.yml @@ -0,0 +1,49 @@ +name: Spend logs guard + +on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the PR's diff is read as data and never executed + pull_request_target: + branches: + - main + pull_request_review: + types: + - submitted + - dismissed + +permissions: + contents: read + pull-requests: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + spend-logs-guard: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + ref: ${{ github.event.pull_request.base.ref }} + persist-credentials: false + - name: Fetch the pull request head, its merge base and its reviews + id: revisions + env: + GH_TOKEN: ${{ github.token }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')" + git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" + gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate --slurp | jq 'add // []' > "$RUNNER_TEMP/reviews.json" + echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + - name: Run the guard + env: + MERGE_BASE: ${{ steps.revisions.outputs.merge_base }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + uv run --frozen python ci_cd/spend_logs_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --approver kerry-berri --reviews "$RUNNER_TEMP/reviews.json" diff --git a/ci_cd/spend_logs_guard.py b/ci_cd/spend_logs_guard.py new file mode 100644 index 00000000000..a04999c47c0 --- /dev/null +++ b/ci_cd/spend_logs_guard.py @@ -0,0 +1,119 @@ +"""Required check: a pull request that touches LiteLLM_SpendLogs in schema.prisma or a migration +needs an approving review from the spend-log owner before it can merge. + +CODEOWNERS can only match file paths, and every schema change edits the same schema.prisma copies, +so a path rule would put every migration on the owner. This gate reads the diff instead: when no +added line in the watched files mentions the table it passes at once, otherwise it passes only once +the owner's latest review on the pull request is an approval. The workflow re-runs on every review. +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from collections.abc import Sequence +from pathlib import Path +from typing import Final + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +WATCHED: Final = ( + "schema.prisma", + "litellm/proxy/schema.prisma", + "litellm-proxy-extras/litellm_proxy_extras/schema.prisma", +) +MIGRATIONS_PREFIX: Final = "litellm-proxy-extras/litellm_proxy_extras/migrations/" +TABLE: Final = re.compile(r"(? bool: + return path in WATCHED or (path.startswith(MIGRATIONS_PREFIX) and path.endswith("/migration.sql")) + + +def _added_lines(section: str) -> tuple[str, ...]: + body: Final = section.split("\n@@", 1) + return tuple(line[1:] for line in body[1].splitlines() if line.startswith("+")) if len(body) == 2 else () + + +def spend_log_hits(diff: str) -> tuple[str, ...]: + """`path: added line` for every added line in a watched file that names the table.""" + sections: Final = re.split(r"^diff --git [^\n]*\n", diff, flags=re.MULTILINE) + return tuple(f"{path}: {line.strip()}" for path, line in _watched_added_lines(sections) if TABLE.search(line)) + + +def _watched_added_lines(sections: Sequence[str]) -> tuple[tuple[str, str], ...]: + pairs: Final = tuple((_path_of(section), section) for section in sections) + return tuple( + (path, line) for path, section in pairs if watched(path) for line in _added_lines(section) + ) # comprehension-ok: path then its lines + + +def _path_of(section: str) -> str: + match: Final = re.search(r"^\+\+\+ b/(.+)$", section, flags=re.MULTILINE) + return match.group(1) if match else "" + + +def approved_by(reviews: Sequence[Review], login: str) -> bool: + """True when `login`'s most recent review on the pull request is an approval.""" + states: Final = tuple(review.state for review in reviews if review.user.login == login) + return bool(states) and states[-1] == "APPROVED" + + +def added_diff(base: str, head: str) -> str: + result: Final = subprocess.run( + ("git", "diff", "--unified=0", "--no-color", f"{base}..{head}", "--", *WATCHED, MIGRATIONS_PREFIX), + check=True, + capture_output=True, + text=True, + ) + return result.stdout + + +class Arguments(BaseModel): + model_config = ConfigDict(frozen=True) + base: str + head: str + approver: str + reviews: Path + + +def main(argv: Sequence[str]) -> int: + parser: Final = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", required=True, help="merge base of the pull request") + parser.add_argument("--head", required=True, help="head sha of the pull request") + parser.add_argument("--approver", required=True, help="GitHub login whose approval unlocks the merge") + parser.add_argument("--reviews", required=True, help="JSON file with the pull request's reviews") + args: Final = Arguments.model_validate(vars(parser.parse_args(argv))) + hits: Final = spend_log_hits(added_diff(args.base, args.head)) + if not hits: + sys.stdout.write("No added line in schema.prisma or a migration names LiteLLM_SpendLogs; nothing to review.\n") + return 0 + if approved_by(_REVIEWS.validate_json(args.reviews.read_bytes()), args.approver): + sys.stdout.write(f"LiteLLM_SpendLogs change approved by {args.approver}: {', '.join(hits)}\n") + return 0 + listed: Final = "".join(f" {hit}\n" for hit in hits) + sys.stdout.write( + f"This pull request changes LiteLLM_SpendLogs and needs an approving review from @{args.approver}:\n" + f"{listed}The check re-runs when a review is submitted.\n" + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/unit/test_spend_logs_guard.py b/tests/unit/test_spend_logs_guard.py new file mode 100644 index 00000000000..9a033045dd0 --- /dev/null +++ b/tests/unit/test_spend_logs_guard.py @@ -0,0 +1,79 @@ +import importlib.util +import sys +from pathlib import Path +from typing import Final + +_SCRIPT: Final = Path(__file__).resolve().parents[2] / "ci_cd" / "spend_logs_guard.py" +_SPEC: Final = importlib.util.spec_from_file_location("spend_logs_guard", _SCRIPT) +assert _SPEC is not None and _SPEC.loader is not None +guard: Final = importlib.util.module_from_spec(_SPEC) +sys.modules["spend_logs_guard"] = guard +_SPEC.loader.exec_module(guard) + + +def _diff(path: str, added: str, start: int = 10) -> str: + return f"diff --git a/{path} b/{path}\n--- a/{path}\n+++ b/{path}\n@@ -{start},0 +{start},1 @@\n+{added}\n" + + +class TestSpendLogHits: + def test_an_added_schema_line_naming_the_table_is_a_hit(self): + diff: Final = _diff( + "schema.prisma", ' @@index([api_key, startTime], map: "LiteLLM_SpendLogs_api_key_startTime_idx")', 740 + ) + assert guard.spend_log_hits(diff) == ( + 'schema.prisma: @@index([api_key, startTime], map: "LiteLLM_SpendLogs_api_key_startTime_idx")', + ) + + def test_an_added_migration_line_is_a_hit_in_any_case(self): + path: Final = "litellm-proxy-extras/litellm_proxy_extras/migrations/20260101000000_x/migration.sql" + hits: Final = guard.spend_log_hits(_diff(path, 'CREATE INDEX "i" ON "litellm_spendlogs"("api_key");', 3)) + assert hits == (f'{path}: CREATE INDEX "i" ON "litellm_spendlogs"("api_key");',) + + def test_a_removed_line_or_another_table_or_an_unwatched_file_is_not_a_hit(self): + removed: Final = "--- a/schema.prisma\n+++ b/schema.prisma\n@@ -1,1 +1,0 @@\n-model LiteLLM_SpendLogs {\n" + other: Final = _diff("schema.prisma", " team_id String // LiteLLM_TeamTable, not MyLiteLLM_SpendLogs", 5) + elsewhere: Final = _diff("litellm/proxy/utils.py", 'table = "LiteLLM_SpendLogs"', 5) + assert guard.spend_log_hits(removed + other + elsewhere) == () + + def test_two_files_in_one_diff_are_attributed_to_their_own_path(self): + diff: Final = _diff("litellm/proxy/schema.prisma", " a LiteLLM_SpendLogs", 1) + _diff( + "schema.prisma", " b String", 1 + ) + assert guard.spend_log_hits(diff) == ("litellm/proxy/schema.prisma: a LiteLLM_SpendLogs",) + + +class TestApprovedBy: + def test_the_latest_review_of_the_approver_decides(self): + reviews: Final = tuple( + guard.Review(state=state, user=guard.Reviewer(login=login)) + for login, state in ( + ("kerry-berri", "CHANGES_REQUESTED"), + ("someone-else", "APPROVED"), + ("kerry-berri", "APPROVED"), + ) + ) + assert guard.approved_by(reviews, "kerry-berri") is True + assert guard.approved_by(reviews[::-1], "kerry-berri") is False + + def test_no_review_from_the_approver_is_not_an_approval(self): + other: Final = guard.Review(state="APPROVED", user=guard.Reviewer(login="someone-else")) + assert guard.approved_by((other,), "kerry-berri") is False + assert guard.approved_by((), "kerry-berri") is False + + +class TestExitCode: + def test_a_pull_request_without_spend_log_lines_passes_without_reviews(self, tmp_path, monkeypatch, capsys): + monkeypatch.setattr(guard, "added_diff", lambda base, head: _diff("schema.prisma", " foo String", 1)) + reviews: Final = tmp_path / "r.json" + reviews.write_text("[]") + assert guard.main(["--base", "a", "--head", "b", "--approver", "kerry-berri", "--reviews", str(reviews)]) == 0 + assert "nothing to review" in capsys.readouterr().out + + def test_a_spend_log_change_fails_until_the_approver_approves(self, tmp_path, monkeypatch, capsys): + monkeypatch.setattr(guard, "added_diff", lambda base, head: _diff("schema.prisma", " x LiteLLM_SpendLogs", 7)) + reviews: Final = tmp_path / "r.json" + reviews.write_text('[{"user": {"login": "someone-else"}, "state": "APPROVED"}]') + assert guard.main(["--base", "a", "--head", "b", "--approver", "kerry-berri", "--reviews", str(reviews)]) == 1 + assert "schema.prisma: x LiteLLM_SpendLogs" in capsys.readouterr().out + reviews.write_text('[{"user": {"login": "kerry-berri"}, "state": "APPROVED"}]') + assert guard.main(["--base", "a", "--head", "b", "--approver", "kerry-berri", "--reviews", str(reviews)]) == 0 From 1bff4e3dae792f5884e9c366376ee147f94fd97e Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 27 Oct 2026 12:57:21 -0700 Subject: [PATCH 021/102] fix(anthropic): make opening usage estimates opt-in (#48633) * fix(anthropic): make opening usage estimates opt-in * docs(anthropic): follow README paragraph style --- litellm/__init__.py | 1 + litellm/anthropic_interface/readme.md | 36 ++++++++++- .../messages/adapters/streaming_iterator.py | 8 ++- .../test_streaming_iterator_message_id.py | 64 +++++++++++++++---- 4 files changed, 94 insertions(+), 15 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index c74292c5888..41e3fa34323 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -234,6 +234,7 @@ bedrock_neutralize_orphaned_tool_blocks: bool = True use_chat_completions_url_for_anthropic_messages: bool = bool( os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False) ) # When True, routes OpenAI /v1/messages requests to chat/completions instead of the Responses API +anthropic_messages_estimate_initial_input_tokens: bool = False # When True, strip the OpenAI-flavored `usage.total_tokens` field that # LiteLLM injects into non-streaming /v1/messages responses, bringing the # wire response into line with the Anthropic spec (matches the streaming diff --git a/litellm/anthropic_interface/readme.md b/litellm/anthropic_interface/readme.md index a864e2572e6..99e18e17de0 100644 --- a/litellm/anthropic_interface/readme.md +++ b/litellm/anthropic_interface/readme.md @@ -57,6 +57,40 @@ async for chunk in response: print(chunk) ``` +### Optional opening token estimate + +When `/v1/messages` streams through the Chat Completions adapter, the opening +`message_start.usage.input_tokens` defaults to `0`. To show an estimated prompt +size while a response streams, enable `anthropic_messages_estimate_initial_input_tokens`. +This setting defaults to `false` and applies to all requests using this adapter + +For an existing proxy deployment, add the setting under `litellm_settings` in +`config.yaml` and restart the proxy. Use a YAML boolean, without quotes + +```yaml +litellm_settings: + anthropic_messages_estimate_initial_input_tokens: true +``` + +For the Python SDK, set the option before making streaming Messages requests + +```python +import litellm + +litellm.anthropic_messages_estimate_initial_input_tokens = True +``` + +Send a streaming Messages request to a Chat Completions-backed model, such as a +`hosted_vllm/` model. The opening `message_start` event includes the estimated +input count. If estimation fails, the count remains `0`. Set the option to +`false` in YAML or `False` in Python to disable estimation and skip token counting + +The opening count is approximate and is not suitable for billing or enforcing +exact context limits. Opening output and cache counts remain `0`. Final +`message_delta` usage still comes from the provider and takes precedence over the +opening estimate. Native Anthropic streams, the Responses adapter, and +non-streaming responses are unaffected. This setting does not select the adapter + ### LiteLLM Proxy Server @@ -113,4 +147,4 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/messages' \ ], "max_tokens": 100 }' -``` \ No newline at end of file +``` diff --git a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py index d1853bb6bd5..62a1e5f0a97 100644 --- a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py @@ -761,13 +761,17 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): def _create_initial_usage_delta(self) -> UsageDelta: """ - Estimate prompt tokens for message_start without consuming upstream chunks. + Optionally estimate prompt tokens without consuming upstream chunks. Output and cache counts start at zero. Provider usage in message_delta remains authoritative. Failed estimates fall back to zero. """ return UsageDelta( - input_tokens=self._estimate_initial_input_tokens(), + input_tokens=( + self._estimate_initial_input_tokens() + if litellm.anthropic_messages_estimate_initial_input_tokens is True + else 0 + ), output_tokens=0, cache_creation_input_tokens=0, cache_read_input_tokens=0, diff --git a/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_message_id.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_message_id.py index ec9146bc023..985cd38ef3c 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_message_id.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_message_id.py @@ -9,6 +9,7 @@ import datetime import json from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from typing import Final +from unittest.mock import Mock import pytest import respx @@ -24,7 +25,6 @@ from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam from litellm.types.utils import ModelResponseStream - _EVENT: Final = TypeAdapter(Mapping[str, object]) _MESSAGES: Final = TypeAdapter(list[AllMessageValues]) _TOOLS: Final = TypeAdapter(list[ChatCompletionToolParam]) @@ -42,7 +42,9 @@ _BLOCK_PROMPTS: Final[tuple[list[AllMessageValues], ...]] = ( ) -def _assert_estimated_start_and_provider_final_usage(raw_events: Sequence[bytes], request_body: bytes) -> None: +def _assert_start_and_provider_final_usage( + raw_events: Sequence[bytes], request_body: bytes, estimate_enabled: bool +) -> None: request: Final = _EVENT.validate_json(request_body) events: Final = tuple(_EVENT.validate_json(chunk.decode().split("data: ", 1)[1]) for chunk in raw_events) start: Final = _EVENT.validate_python( @@ -56,7 +58,7 @@ def _assert_estimated_start_and_provider_final_usage(raw_events: Sequence[bytes] ) assert expected > 0 assert start["usage"] == { - "input_tokens": expected, + "input_tokens": expected if estimate_enabled else 0, "output_tokens": 0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, @@ -64,7 +66,12 @@ def _assert_estimated_start_and_provider_final_usage(raw_events: Sequence[bytes] assert final["usage"] == {"input_tokens": 3, "output_tokens": 4} -def test_sync_message_start_estimates_prompt_without_replacing_provider_usage(respx_mock: respx.MockRouter) -> None: +@pytest.mark.parametrize("estimate_enabled", [None, False, True, "false"]) +def test_sync_message_start_opt_in_preserves_provider_final_usage( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, estimate_enabled: bool | str | None +) -> None: + if estimate_enabled is not None: + monkeypatch.setattr(litellm, "anthropic_messages_estimate_initial_input_tokens", estimate_enabled) stream: Final = LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( max_tokens=16, messages=MESSAGES, @@ -74,13 +81,18 @@ def test_sync_message_start_estimates_prompt_without_replacing_provider_usage(re stream=True, custom_llm_provider="groq", ) - _assert_estimated_start_and_provider_final_usage(list(stream), respx_mock.calls.last.request.content) + _assert_start_and_provider_final_usage( + list(stream), respx_mock.calls.last.request.content, estimate_enabled is True + ) @pytest.mark.asyncio -async def test_async_message_start_estimates_prompt_without_replacing_provider_usage( - respx_mock: respx.MockRouter, +@pytest.mark.parametrize("estimate_enabled", [None, False, True, "false"]) +async def test_async_message_start_opt_in_preserves_provider_final_usage( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, estimate_enabled: bool | str | None ) -> None: + if estimate_enabled is not None: + monkeypatch.setattr(litellm, "anthropic_messages_estimate_initial_input_tokens", estimate_enabled) stream: Final = await LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler( max_tokens=16, messages=MESSAGES, @@ -91,7 +103,7 @@ async def test_async_message_start_estimates_prompt_without_replacing_provider_u custom_llm_provider="groq", ) raw: Final = [chunk async for chunk in stream] - _assert_estimated_start_and_provider_final_usage(raw, respx_mock.calls.last.request.content) + _assert_start_and_provider_final_usage(raw, respx_mock.calls.last.request.content, estimate_enabled is True) class _FailingUpstream: @@ -102,10 +114,31 @@ class _FailingUpstream: raise RuntimeError("upstream failed before its first chunk") +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True]) +async def test_disabled_initial_estimate_skips_token_counting(monkeypatch: pytest.MonkeyPatch, is_async: bool) -> None: + counter: Final = Mock(side_effect=AssertionError("disabled estimation must not count tokens")) + monkeypatch.setattr(litellm, "anthropic_messages_estimate_initial_input_tokens", False) + monkeypatch.setattr(litellm, "token_counter", counter) + wrapper: Final = AnthropicStreamWrapper( + _FailingUpstream(), model="kimi-k2", completion_request={"messages": MESSAGES} + ) + first: Final = _EVENT.validate_python(await anext(wrapper) if is_async else next(wrapper)) + message: Final = _EVENT.validate_python(first["message"]) + assert message["usage"] == { + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + } + counter.assert_not_called() + + @pytest.mark.parametrize("messages", _BLOCK_PROMPTS) def test_sync_initial_estimate_counts_content_blocks( - messages: list[AllMessageValues], respx_mock: respx.MockRouter + messages: list[AllMessageValues], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: + monkeypatch.setattr(litellm, "anthropic_messages_estimate_initial_input_tokens", True) image: Final = respx_mock.get("https://image.example/test.png").respond(503) wrapper: Final = AnthropicStreamWrapper(iter([]), model="gpt-4o", completion_request={"messages": messages}) expected: Final = litellm.token_counter(model="gpt-4o", messages=messages, use_default_image_token_count=True) @@ -123,8 +156,9 @@ def test_sync_initial_estimate_counts_content_blocks( @pytest.mark.asyncio @pytest.mark.parametrize("messages", _BLOCK_PROMPTS) async def test_async_initial_estimate_counts_content_blocks( - messages: list[AllMessageValues], respx_mock: respx.MockRouter + messages: list[AllMessageValues], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: + monkeypatch.setattr(litellm, "anthropic_messages_estimate_initial_input_tokens", True) image: Final = respx_mock.get("https://image.example/test.png").respond(503) wrapper: Final = AnthropicStreamWrapper(iter([]), model="gpt-4o", completion_request={"messages": messages}) expected: Final = litellm.token_counter(model="gpt-4o", messages=messages, use_default_image_token_count=True) @@ -140,7 +174,10 @@ async def test_async_initial_estimate_counts_content_blocks( @pytest.mark.parametrize("prompt", [{}, {"messages": []}, {"messages": "invalid"}]) -def test_sync_initial_estimate_fallback_does_not_consume_upstream(prompt: Mapping[str, object]) -> None: +def test_sync_initial_estimate_fallback_does_not_consume_upstream( + prompt: Mapping[str, object], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "anthropic_messages_estimate_initial_input_tokens", True) wrapper: Final = AnthropicStreamWrapper(_FailingUpstream(), model="kimi-k2", completion_request=prompt) first: Final = _EVENT.validate_python(next(wrapper)) message: Final = _EVENT.validate_python(first["message"]) @@ -155,7 +192,10 @@ def test_sync_initial_estimate_fallback_does_not_consume_upstream(prompt: Mappin @pytest.mark.asyncio @pytest.mark.parametrize("prompt", [{}, {"messages": []}, {"messages": "invalid"}]) -async def test_async_initial_estimate_fallback_does_not_consume_upstream(prompt: Mapping[str, object]) -> None: +async def test_async_initial_estimate_fallback_does_not_consume_upstream( + prompt: Mapping[str, object], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "anthropic_messages_estimate_initial_input_tokens", True) wrapper: Final = AnthropicStreamWrapper(_FailingUpstream(), model="kimi-k2", completion_request=prompt) first: Final = _EVENT.validate_python(await anext(wrapper)) message: Final = _EVENT.validate_python(first["message"]) From 4846d9c2ed38795b931cd397e70705e45090b280 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:24:56 +0000 Subject: [PATCH 022/102] fix(cost-map): add 2026-11-06 deprecation_date to two together_ai rows (#48651) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 2 ++ model_prices_and_context_window.json | 2 ++ 2 files changed, 4 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7cd4eadca6e..39500330096 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -47102,6 +47102,7 @@ }, "together_ai/MiniMaxAI/MiniMax-M3": { "cache_read_input_token_cost": 6e-08, + "deprecation_date": "2026-11-06", "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 524288, @@ -47295,6 +47296,7 @@ }, "together_ai/thinkingmachines/Inkling": { "cache_read_input_token_cost": 1.7e-07, + "deprecation_date": "2026-11-06", "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", "max_input_tokens": 524288, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7cd4eadca6e..39500330096 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -47102,6 +47102,7 @@ }, "together_ai/MiniMaxAI/MiniMax-M3": { "cache_read_input_token_cost": 6e-08, + "deprecation_date": "2026-11-06", "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 524288, @@ -47295,6 +47296,7 @@ }, "together_ai/thinkingmachines/Inkling": { "cache_read_input_token_cost": 1.7e-07, + "deprecation_date": "2026-11-06", "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", "max_input_tokens": 524288, From d6edd37810944752561ed49d9a8e85085f8dce81 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 27 Oct 2026 13:33:03 -0700 Subject: [PATCH 023/102] fix(aiohttp_transport): close an owned aiohttp session when its transport is collected (#48650) An OpenAI or Azure SDK client litellm built and evicted from the client cache is usually garbage collected inside the closer's 900 second grace window, so the closer's weak reference dies and the client's aiohttp session is never closed. LiteLLMAiohttpTransport.__del__ now hands an owned, still-open ClientSession to _close_recycled_session, which already covers the current-loop, foreign-loop and closed-loop cases. Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: BangerSoul --- .../llms/custom_httpx/aiohttp_transport.py | 7 ++ .../sdk/test_aiohttp_session_rebuild_wire.py | 86 ++++++++++++++++++- .../unit/caching/test_llm_caching_handler.py | 33 +++++++ .../custom_httpx/test_aiohttp_transport.py | 73 ++++++++++++++++ 4 files changed, 197 insertions(+), 2 deletions(-) diff --git a/litellm/llms/custom_httpx/aiohttp_transport.py b/litellm/llms/custom_httpx/aiohttp_transport.py index b8cb183c02f..0e4d97fd770 100644 --- a/litellm/llms/custom_httpx/aiohttp_transport.py +++ b/litellm/llms/custom_httpx/aiohttp_transport.py @@ -289,6 +289,13 @@ class LiteLLMAiohttpTransport(AiohttpTransport): cls._background_close_tasks.add(task) task.add_done_callback(cls._on_close_task_done) + def __del__(self) -> None: + session: Final = getattr(self, "client", None) + if not getattr(self, "_owns_session", False) or not isinstance(session, ClientSession): + return + with contextlib.suppress(Exception): + self._close_recycled_session(session) + def get_valid_client_session(self) -> ClientSession: """ Helper to get a valid ClientSession for the current event loop. diff --git a/tests/integration/sdk/test_aiohttp_session_rebuild_wire.py b/tests/integration/sdk/test_aiohttp_session_rebuild_wire.py index 1ec1f486bed..f1a18ba389a 100644 --- a/tests/integration/sdk/test_aiohttp_session_rebuild_wire.py +++ b/tests/integration/sdk/test_aiohttp_session_rebuild_wire.py @@ -11,11 +11,30 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Final import pytest -from pydantic import JsonValue, TypeAdapter +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter CONFIGURED_KEEPALIVE_SECONDS: Final = 1 IDLE_SECONDS: Final = 2 +EVICTED_CLIENTS: Final = 30 RESPONSES: Final = TypeAdapter(list[dict[str, JsonValue]]) +CHAT_COMPLETION: Final = { + "id": "chatcmpl-scripted", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-5.4-nano", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +class EvictionReport(BaseModel): + model_config = ConfigDict(frozen=True) + + cache_size: int + cached_clients: int + established_connections: int + unclosed_messages: tuple[str, ...] + REBUILT_SESSION_EXCHANGE: Final = textwrap.dedent( """ @@ -38,6 +57,47 @@ REBUILT_SESSION_EXCHANGE: Final = textwrap.dedent( ) +EVICTED_CLIENTS_EXCHANGE: Final = textwrap.dedent( + """ + import asyncio, gc, json, sys + import psutil + import litellm + from litellm.llms.custom_httpx.aiohttp_transport import LiteLLMAiohttpTransport + + unclosed = [] + + def established(port: int) -> int: + connections = psutil.Process().net_connections("tcp") + return sum(1 for c in connections if c.status == "ESTABLISHED" and c.raddr and c.raddr.port == port) + + async def one_call(base_url: str, index: int) -> None: + await litellm.acompletion( + model="openai/gpt-5.4-nano", + api_key=f"sk-distinct-{index}", + api_base=base_url, + messages=[{"role": "user", "content": "hi"}], + ) + + async def main(base_url: str, port: int, evicted: int) -> None: + asyncio.get_running_loop().set_exception_handler(lambda loop, ctx: unclosed.append(ctx["message"])) + cache = litellm.in_memory_llm_clients_cache + for index in range(cache.max_size_in_memory + evicted): + await one_call(base_url, index) + gc.collect() + await asyncio.gather(*LiteLLMAiohttpTransport._background_close_tasks) + await asyncio.sleep(0) + print(json.dumps({ + "cache_size": cache.max_size_in_memory, + "cached_clients": len(cache.cache_dict), + "established_connections": established(port), + "unclosed_messages": unclosed, + })) + + asyncio.run(main(sys.argv[1], int(sys.argv[2]), int(sys.argv[3]))) + """ +) + + class _ConnectionCountingPeer(ThreadingHTTPServer): daemon_threads = True @@ -62,7 +122,8 @@ class _ConnectionHandler(BaseHTTPRequestHandler): def do_POST(self) -> None: self.rfile.read(int(self.headers["Content-Length"])) - body: Final = json.dumps({"connection": self.connection_number}).encode() + payload: Final = CHAT_COMPLETION if self.path.endswith("/chat/completions") else {"connection": self.connection_number} + body: Final = json.dumps(payload).encode() self.send_response(200) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) @@ -107,3 +168,24 @@ def test_rebuilt_shared_session_drops_idle_connection_after_configured_keepalive ) -> None: observed: Final = _rebuilt_session_exchange(connection_counting_peer) assert observed == [{"connection": 1}, {"connection": 2}], observed + + +def _evicted_clients_exchange(base_url: str) -> EvictionReport: + completed: Final = subprocess.run( + [sys.executable, "-P", "-c", EVICTED_CLIENTS_EXCHANGE, base_url, base_url.rsplit(":", 1)[1], str(EVICTED_CLIENTS)], + capture_output=True, + text=True, + timeout=300, + check=False, + ) + assert completed.returncode == 0, completed.stderr + return EvictionReport.model_validate_json(completed.stdout) + + +def test_sdk_clients_evicted_from_a_full_client_cache_leave_no_open_session_or_connection( + connection_counting_peer: str, +) -> None: + report: Final = _evicted_clients_exchange(connection_counting_peer) + assert report.cached_clients == report.cache_size, report + assert report.unclosed_messages == (), report + assert report.established_connections <= report.cache_size, report diff --git a/tests/unit/caching/test_llm_caching_handler.py b/tests/unit/caching/test_llm_caching_handler.py index 0c0fe8ca3c5..a3efd5d97aa 100644 --- a/tests/unit/caching/test_llm_caching_handler.py +++ b/tests/unit/caching/test_llm_caching_handler.py @@ -11,11 +11,14 @@ See: https://github.com/BerriAI/litellm/pull/22247 import asyncio, gc, httpx, importlib, litellm, os, threading, time, weakref import warnings +import aiohttp import pytest +from openai import AsyncOpenAI from litellm.caching.evicted_client_closer import EvictedClientCloser from litellm.caching.llm_caching_handler import LLMClientCache +from litellm.llms.custom_httpx.aiohttp_transport import LiteLLMAiohttpTransport from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.custom_httpx.http_handler import( @@ -204,6 +207,36 @@ async def test_evicted_litellm_owned_client_is_closed_once_the_grace_window_elap assert client.closed is True +@pytest.mark.asyncio +async def test_an_evicted_owned_client_collected_inside_its_grace_window_still_closes_its_aiohttp_session(): + """ + The closer holds an evicted client weakly for the whole grace window. An SDK + process that exits, or a proxy that evicts and never calls the cache again, + collects the client first, so only its own finalizer can still close the + aiohttp session (LIT-9767). + """ + clock = _FakeClock() + cache = LLMClientCache( + max_size_in_memory=2, + evicted_client_closer=EvictedClientCloser(grace_seconds=900.0, clock=clock), + ) + session = aiohttp.ClientSession() + client = AsyncOpenAI( + api_key="sk-evicted", + http_client=httpx.AsyncClient(transport=LiteLLMAiohttpTransport(client=session)), + ) + cache.set_cache("client-key", client, litellm_owned_client=True, ttl=600) + + cache.ttl_dict = {key: 0 for key in cache.ttl_dict} + cache.expiration_heap = [(0, key) for _, key in cache.expiration_heap] + cache.evict_cache() + del client + gc.collect() + await asyncio.gather(*LiteLLMAiohttpTransport._background_close_tasks) + + assert session.closed, "the evicted client was collected inside the grace window and left its session open" + + @pytest.mark.asyncio async def test_evicted_caller_supplied_client_is_never_closed(): """litellm does not own a client the caller passed in, so it must stay open.""" diff --git a/tests/unit/llms/custom_httpx/test_aiohttp_transport.py b/tests/unit/llms/custom_httpx/test_aiohttp_transport.py index 0765371dc4a..c7d5518ef5b 100644 --- a/tests/unit/llms/custom_httpx/test_aiohttp_transport.py +++ b/tests/unit/llms/custom_httpx/test_aiohttp_transport.py @@ -1,8 +1,10 @@ import aiohttp as aiohttp_aiohttp_handler, asyncio, importlib, litellm import concurrent.futures +import gc import socket import sys import struct +import threading from typing import Final import aiohttp @@ -1310,3 +1312,74 @@ async def test_reset_after_request_is_not_replayed(monkeypatch: pytest.MonkeyPat assert len(received_requests) == 1 finally: await handler.close() + + +async def _new_session() -> aiohttp.ClientSession: + return aiohttp.ClientSession() + + +async def _until_closed(session: aiohttp.ClientSession) -> None: + while not session.closed: + await asyncio.sleep(0) + + +@pytest.mark.asyncio +async def test_a_collected_transport_closes_the_session_it_owns(): + """An SDK client evicted from the cache is collected long before the closer's grace window ends.""" + session = aiohttp.ClientSession() + transport = LiteLLMAiohttpTransport(client=session) + + del transport + gc.collect() + await asyncio.gather(*LiteLLMAiohttpTransport._background_close_tasks) + + assert session.closed, "the owned session outlived the transport that was collected before closing it" + + +@pytest.mark.asyncio +async def test_a_collected_transport_leaves_a_shared_session_open(): + session = aiohttp.ClientSession() + try: + transport = LiteLLMAiohttpTransport(client=session, owns_session=False) + + del transport + gc.collect() + await asyncio.gather(*LiteLLMAiohttpTransport._background_close_tasks) + + assert not session.closed, "a transport closed a session it does not own" + finally: + await session.close() + + +def test_a_transport_collected_after_its_loop_closed_still_closes_its_session(caplog: pytest.LogCaptureFixture): + """asyncio.run() closes the loop the session lives on before the evicted client is collected.""" + loop = asyncio.new_event_loop() + session = loop.run_until_complete(_new_session()) + loop.close() + transport = LiteLLMAiohttpTransport(client=session) + + del transport + gc.collect() + + assert session.closed, "the owned session stayed open after its transport was collected on a closed loop" + del session + gc.collect() + assert "Unclosed client session" not in caplog.text, caplog.text + + +def test_a_transport_collected_while_its_loop_runs_elsewhere_closes_its_session_on_that_loop(): + loop = asyncio.new_event_loop() + thread = threading.Thread(target=loop.run_forever, daemon=True) + thread.start() + session = asyncio.run_coroutine_threadsafe(_new_session(), loop).result(timeout=10) + transport = LiteLLMAiohttpTransport(client=session) + try: + del transport + gc.collect() + asyncio.run_coroutine_threadsafe(_until_closed(session), loop).result(timeout=10) + + assert session.closed, "the owned session stayed open after its transport was collected from another thread" + finally: + loop.call_soon_threadsafe(loop.stop) + thread.join(timeout=10) + loop.close() From 9e01b3c37b0736806aa70dcde72f5d85079be3e8 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 27 Oct 2026 13:33:32 -0700 Subject: [PATCH 024/102] fix(complexity_router): keep the held user_turn route for Codex compaction requests (#48627) * test(complexity_router): Codex compaction requests hold the user_turn decision Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(complexity_router): treat Codex compaction traffic as user_turn continuation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(complexity_router): validate Codex turn metadata with frozen models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(complexity_router): flatten body system text through _message_text Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(complexity_router): drop unrelated reformatting from the mapped test file Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(complexity_router): skip pin escalation on Codex compaction turns Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../complexity_router/complexity_router.py | 97 ++++++++++++++-- ...test_complexity_router_codex_compaction.py | 109 ++++++++++++++++++ .../router_strategy/test_complexity_router.py | 96 +++++++++++++++ 3 files changed, 290 insertions(+), 12 deletions(-) create mode 100644 tests/integration/routing/test_complexity_router_codex_compaction.py diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index cc592bbcbc5..12968bf7d4a 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -28,7 +28,7 @@ from threading import Lock from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast -from pydantic import BaseModel, TypeAdapter, ValidationError, create_model +from pydantic import BaseModel, Field, TypeAdapter, ValidationError, create_model from pydantic_core import ErrorDetails from typing_extensions import ReadOnly, TypedDict @@ -773,6 +773,56 @@ def _newest_turn_is_human_ask( return _last_human_ask_index(messages, marker_pairs) == newest_activity +_CODEX_TURN_METADATA_KEY: Final = "x-codex-turn-metadata" +_CODEX_COMPACTION_SUMMARY_PREFIX: Final = ( + "Another language model started to solve this problem and produced a summary of its thinking process." +) + + +class _CodexTurnMetadata(BaseModel, frozen=True): + request_kind: str | None = None + + +class _CodexTurnMetadataCarrier(BaseModel, frozen=True): + turn_metadata: str | None = Field(default=None, alias=_CODEX_TURN_METADATA_KEY) + + +class _CodexProxyRequest(BaseModel, frozen=True): + headers: _CodexTurnMetadataCarrier | None = None + + +class _CodexRequestMetadata(BaseModel, frozen=True): + """Codex sends its turn metadata in the Responses body's `client_metadata` and projects it onto a + header of the same name for transports without that field""" + + client_metadata: _CodexTurnMetadataCarrier | None = None + proxy_server_request: _CodexProxyRequest | None = None + + def turn_metadata_values(self) -> tuple[str | None, ...]: + headers: Final = self.proxy_server_request.headers if self.proxy_server_request else None + return ( + self.client_metadata.turn_metadata if self.client_metadata else None, + headers.turn_metadata if headers else None, + ) + + +def _codex_request_kind(value: str | None) -> str | None: + if value is None: + return None + try: + return _CodexTurnMetadata.model_validate_json(value).request_kind + except ValidationError: + return None + + +def _is_codex_compaction_request(request_kwargs: Mapping[str, object]) -> bool: + try: + request_metadata: Final = _CodexRequestMetadata.model_validate(request_kwargs) + except ValidationError: + return False + return any(_codex_request_kind(value) == "compaction" for value in request_metadata.turn_metadata_values()) + + def _iter_system_scope_texts( body_system: object, messages: Sequence[Mapping[str, object]], @@ -787,10 +837,9 @@ def _iter_system_scope_texts( must go through the staleness-aware tail scan instead -- scanning it here would floor every turn of a session that once planned, for any pattern whose client injects mid-conversation. """ - if isinstance(body_system, str): - yield body_system - elif isinstance(body_system, list): - yield _message_text(body_system) + body_system_text: Final = _message_text(body_system) + if body_system_text: + yield body_system_text for msg in messages: if msg.get("role") != "system": return @@ -4263,14 +4312,31 @@ class ComplexityRouter(CustomLogger): def _reminder_markers_for_request(self, request_kwargs: Mapping[str, object]) -> tuple[tuple[str, str], ...]: if self.config.reminder_markers is not None: return self._reminder_markers - if any( + return _CODEX_REMINDER_MARKERS if self._is_codex_request(request_kwargs) else _DEFAULT_REMINDER_MARKERS + + @staticmethod + def _is_codex_request(request_kwargs: Mapping[str, object]) -> bool: + return any( is_codex_user_agent(user_agent) for metadata_key in ("litellm_metadata", "metadata") if isinstance(metadata := request_kwargs.get(metadata_key), Mapping) if isinstance(user_agent := metadata.get("user_agent"), str) - ): - return _CODEX_REMINDER_MARKERS - return _DEFAULT_REMINDER_MARKERS + ) + + def _is_codex_compaction_lifecycle( + self, + request_kwargs: Mapping[str, object], + resolved_messages: Sequence[Mapping[str, object]] | None, + marker_pairs: tuple[tuple[str, str], ...], + ) -> bool: + """Codex's compaction call and the first request after it carry a synthetic user turn (the + summarization prompt, then the summary), so `user_turn` must read both as continuations.""" + if _is_codex_compaction_request(request_kwargs): + return True + if not resolved_messages or not self._is_codex_request(request_kwargs): + return False + newest_ask: Final = _newest_turn_ask(resolved_messages, marker_pairs) + return newest_ask is not None and newest_ask.startswith(_CODEX_COMPACTION_SUMMARY_PREFIX) @staticmethod def _iter_metadata_dicts(request_kwargs: dict) -> list[dict]: @@ -4380,15 +4446,22 @@ class ComplexityRouter(CustomLogger): # In 'user_turn' mode a held pin is replayed only on continuation turns; a new human # ask falls through and re-classifies. session_affinity restores pin-first for asks too. - pin_replay_allowed: Final = bool(self.config.session_affinity) or not _newest_turn_is_human_ask( - resolved_messages, marker_pairs + codex_compaction: Final = self._is_codex_compaction_lifecycle(request_kwargs, resolved_messages, marker_pairs) + pin_replay_allowed: Final = ( + bool(self.config.session_affinity) + or not _newest_turn_is_human_ask(resolved_messages, marker_pairs) + or codex_compaction ) if cache_key is not None and pin_replay_allowed: pinned_value: Final = await self.litellm_router_instance.cache.async_get_cache(key=cache_key) pinned_pin: Final = _parse_session_affinity_pin(pinned_value, self.config.tier_names()) if pinned_pin is not None: - user_message: Final = _newest_turn_ask(resolved_messages, marker_pairs) if resolved_messages else None + user_message: Final = ( + _newest_turn_ask(resolved_messages, marker_pairs) + if resolved_messages and not codex_compaction + else None + ) pin_escalation_keyword: Final = ( self._matched_escalation_keyword(user_message) if user_message is not None else None ) diff --git a/tests/integration/routing/test_complexity_router_codex_compaction.py b/tests/integration/routing/test_complexity_router_codex_compaction.py new file mode 100644 index 00000000000..2f436901cd0 --- /dev/null +++ b/tests/integration/routing/test_complexity_router_codex_compaction.py @@ -0,0 +1,109 @@ +import json +import uuid +from collections.abc import Callable +from pathlib import Path +from typing import Final + +import yaml +from integration._support.client import Gateway +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + +_ASK: Final = {"role": "user", "content": "Design a Byzantine fault tolerant consensus protocol and prove it safe"} +_COMPACTION_PROMPT: Final = {"role": "user", "content": "Hello! Summarize the conversation so far."} + + +def _completion(content: str) -> Reply: + return Reply( + body=json.dumps( + { + "id": "chatcmpl-" + uuid.uuid4().hex[:8], + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 3, "total_tokens": 12}, + } + ).encode() + ) + + +def _answering(content: str) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=json.dumps({"object": "list", "data": []}).encode()) + return _completion(content) + + return respond + + +def _classifier(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=json.dumps({"object": "list", "data": []}).encode()) + body: Final = request.body.decode() + tier: Final = "SIMPLE" if "Summarize" in body else "REASONING" + return _completion(json.dumps({"tier": tier})) + + +def test_codex_compaction_request_stays_on_the_held_user_turn_model(gateway: Gateway, tmp_path: Path) -> None: + with ( + wire_server(_classifier) as judge, + wire_server(_answering("simple answer")) as simple, + wire_server(_answering("reasoning answer")) as reasoning, + ): + router: Final = "router-" + uuid.uuid4().hex[:8] + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["model_list"] = [ + { + "model_name": name, + "litellm_params": {"model": "openai/gpt-4o-mini", "api_base": url, "api_key": "synthetic"}, + } + for name, url in (("judge", judge.url), ("simple", simple.url), ("reasoning", reasoning.url)) + ] + [ + { + "model_name": router, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "classifier_type": "llm", + "classifier_llm_config": {"model": "judge", "timeout_ms": 20000}, + "classification_mode": "user_turn", + "tiers": { + "SIMPLE": "simple", + "MEDIUM": "simple", + "COMPLEX": "reasoning", + "REASONING": "reasoning", + }, + }, + }, + } + ] + path: Final = tmp_path / "complexity_router.yaml" + path.write_text(yaml.safe_dump(config)) + session: Final = uuid.uuid4().hex + + def codex_headers(request_kind: str) -> dict[str, str]: + return { + "User-Agent": "codex-tui/0.164.0", + "session_id": session, + "x-codex-turn-metadata": json.dumps({"session_id": session, "request_kind": request_kind}), + } + + with owned_proxy(gateway, tmp_path, {}, config=path) as candidate: + ask: Final = candidate.request( + "POST", "/v1/chat/completions", {"model": router, "messages": [_ASK]}, headers=codex_headers("turn") + ) + assert ask.status_code == 200, ask.text + assert ask.json()["choices"][0]["message"]["content"] == "reasoning answer", ask.text + compaction: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": router, "messages": [_ASK, {"role": "assistant", "content": "done"}, _COMPACTION_PROMPT]}, + headers=codex_headers("compaction"), + ) + assert compaction.status_code == 200, compaction.text + assert compaction.json()["choices"][0]["message"]["content"] == "reasoning answer", compaction.text + assert len([call for call in judge.drain() if call.method == "POST"]) == 1 + assert [call for call in simple.drain() if call.method == "POST"] == [] diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index a46f415e10a..2d27a025ffa 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -7820,6 +7820,102 @@ class TestClassificationMode: assert pure.routing_decision["cause"] == "user_turn_continuation" assert hybrid.model == "gpt-4o-mini" + CODEX_USER_AGENT = "codex-tui/0.164.0 (Ubuntu 22.4.0; x86_64) screen (codex-tui; 0.164.0)" + COMPACTION_PROMPT = {"role": "user", "content": "Hello! Summarize the conversation so far."} + COMPACTION_SUMMARY = { + "role": "user", + "content": "Another language model started to solve this problem and produced a summary of its thinking " + "process. You also have access to the state of the tools that were used by that language model.\nHi.", + } + + def _codex_kwargs( + self, session_id: str, request_kind: str, transport: str = "client_metadata" + ) -> dict[str, object]: + turn_metadata: Final = json.dumps({"session_id": session_id, "request_kind": request_kind}) + carrier: Final = ( + {"client_metadata": {"x-codex-turn-metadata": turn_metadata}} + if transport == "client_metadata" + else {"proxy_server_request": {"headers": {"x-codex-turn-metadata": turn_metadata}}} + ) + return {"metadata": {"session_id": session_id, "user_agent": self.CODEX_USER_AGENT}, **carrier} + + @pytest.mark.asyncio + @pytest.mark.parametrize("transport", ["client_metadata", "header"]) + async def test_codex_compaction_request_replays_held_decision( + self, mock_router_instance, user_turn_config, transport + ): + """Codex's compaction call carries a synthetic user prompt; it must hold the session's + decision instead of classifying the prompt as a new human ask.""" + router = self._router(mock_router_instance, user_turn_config) + session = f"codex-compact-{transport}" + await router.async_pre_routing_hook( + model="test-model", request_kwargs=self._codex_kwargs(session, "turn"), messages=[self.REASONING_ASK] + ) + compaction = await router.async_pre_routing_hook( + model="test-model", + request_kwargs=self._codex_kwargs(session, "compaction", transport), + messages=[self.REASONING_ASK, self.ASSISTANT_ANSWER, self.COMPACTION_PROMPT], + ) + assert compaction.model == "o1-preview" + assert compaction.routing_decision["cause"] == "user_turn_continuation" + + @pytest.mark.asyncio + async def test_codex_post_compaction_summary_is_a_continuation(self, mock_router_instance, user_turn_config): + """The first request after a mid-turn compaction ends on Codex's summary fragment, which + is not a human ask, so the held decision carries on.""" + router = self._router(mock_router_instance, user_turn_config) + await router.async_pre_routing_hook( + model="test-model", + request_kwargs=self._codex_kwargs("codex-summary", "turn"), + messages=[self.REASONING_ASK], + ) + resumed = await router.async_pre_routing_hook( + model="test-model", + request_kwargs=self._codex_kwargs("codex-summary", "turn"), + messages=[self.REASONING_ASK, self.COMPACTION_SUMMARY], + ) + assert resumed.model == "o1-preview" + assert resumed.routing_decision["cause"] == "user_turn_continuation" + + @pytest.mark.asyncio + async def test_codex_summary_quoting_escalation_keyword_keeps_held_tier( + self, mock_router_instance, user_turn_config + ): + router = self._router(mock_router_instance, user_turn_config) + await router.async_pre_routing_hook( + model="test-model", request_kwargs=self._codex_kwargs("codex-quote", "turn"), messages=[self.SIMPLE_ASK] + ) + quoted_summary: Final = { + "role": "user", + "content": f"{self.COMPACTION_SUMMARY['content']} The user said LITELLM ESCALATE", + } + resumed = await router.async_pre_routing_hook( + model="test-model", + request_kwargs=self._codex_kwargs("codex-quote", "turn"), + messages=[self.SIMPLE_ASK, quoted_summary], + ) + assert resumed.model == "gpt-4o-mini" + assert resumed.routing_decision["cause"] == "user_turn_continuation" + + @pytest.mark.asyncio + async def test_codex_human_ask_after_compaction_reclassifies(self, mock_router_instance, user_turn_config): + router = self._router(mock_router_instance, user_turn_config) + await router.async_pre_routing_hook( + model="test-model", request_kwargs=self._codex_kwargs("codex-next", "turn"), messages=[self.REASONING_ASK] + ) + await router.async_pre_routing_hook( + model="test-model", + request_kwargs=self._codex_kwargs("codex-next", "compaction"), + messages=[self.REASONING_ASK, self.ASSISTANT_ANSWER, self.COMPACTION_PROMPT], + ) + next_ask = await router.async_pre_routing_hook( + model="test-model", + request_kwargs=self._codex_kwargs("codex-next", "turn"), + messages=[self.REASONING_ASK, self.COMPACTION_SUMMARY, self.SIMPLE_ASK], + ) + assert next_ask.model == "gpt-4o-mini" + assert next_ask.routing_decision["cause"] != "user_turn_continuation" + @pytest.mark.asyncio async def test_session_affinity_wins_when_both_knobs_are_on(self, mock_router_instance, user_turn_config): """With session_affinity also on, the pin short-circuits new asks too and keeps its own From 34c10829b1f2aba5ee44887a413bac12d77e2bc0 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 27 Oct 2026 13:33:47 -0700 Subject: [PATCH 025/102] fix(ui): percent-encode request id in spend log detail URL (#48630) * test: cover request ids with URL special characters in log detail fetches Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): percent-encode request id in spend log detail URL Co-authored-by: Charlie Freeman <243496084+cf3901646@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Charlie Freeman <243496084+cf3901646@users.noreply.github.com> --- litellm/proxy/client/cli/commands/debug.py | 3 ++- .../proxy/client/cli/test_debug_commands.py | 18 ++++++++++++++++ .../src/components/networking.test.ts | 21 +++++++++++++++++++ .../src/components/networking.tsx | 4 ++-- 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/client/cli/commands/debug.py b/litellm/proxy/client/cli/commands/debug.py index 8728a9323b7..89b93e2bbb8 100644 --- a/litellm/proxy/client/cli/commands/debug.py +++ b/litellm/proxy/client/cli/commands/debug.py @@ -15,6 +15,7 @@ from datetime import datetime, timezone from pathlib import Path from types import MappingProxyType from typing import Final +from urllib.parse import quote import click import requests @@ -177,7 +178,7 @@ class SpendLogsFetcher: return DebugFailure(f"Unexpected {uri} response: {e}") def payload(self, request_id: str) -> RequestResponsePayload | None | DebugFailure: - uri: Final = f"/spend/logs/ui/{request_id}" + uri: Final = f"/spend/logs/ui/{quote(request_id, safe='')}" raw: Final = self._get(uri) if isinstance(raw, DebugFailure): return raw diff --git a/tests/unit/proxy/client/cli/test_debug_commands.py b/tests/unit/proxy/client/cli/test_debug_commands.py index 419fd821cfb..311edde0af9 100644 --- a/tests/unit/proxy/client/cli/test_debug_commands.py +++ b/tests/unit/proxy/client/cli/test_debug_commands.py @@ -1,6 +1,8 @@ import json import os +import re import time +from urllib.parse import unquote import pytest import requests @@ -114,6 +116,22 @@ def test_recent_bodies_fetches_latest_turns_even_when_successful(): assert _called_paths() == ["/spend/logs/session/ui", "/spend/logs/ui/req-ok"] +@pytest.mark.parametrize("request_id", ["req#a1", "req?a=1", "req%41b", "req a&b"]) +@responses.activate +def test_bodies_are_fetched_for_request_ids_with_url_special_characters(request_id): + def payload_for_stored_id(request): + stored_id = unquote(request.path_url.split("?")[0].removeprefix("/spend/logs/ui/")) + found = stored_id == request_id + return 200, {}, json.dumps({"response": {"id": "msg_special"}} if found else None) + + _mock_proxy([{**OK_ROW, "request_id": request_id}], {}) + responses.add_callback(responses.GET, re.compile(f"{PROXY}/spend/logs/ui/.*"), callback=payload_for_stored_id) + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION, "--no-save"]) + + assert result.exit_code == 0, result.output + assert "msg_special" in result.output + + @responses.activate def test_bodies_are_truncated_to_max_chars(): _mock_proxy([OK_ROW], {"req-ok": {"proxy_server_request": {"body": "a" * 5000}, "response": None}}) diff --git a/ui/litellm-dashboard/src/components/networking.test.ts b/ui/litellm-dashboard/src/components/networking.test.ts index 8b0f0389772..874cdb52fad 100644 --- a/ui/litellm-dashboard/src/components/networking.test.ts +++ b/ui/litellm-dashboard/src/components/networking.test.ts @@ -602,6 +602,27 @@ describe("uiSpendLogsCall exclude_internal_health_checks serialization", () => { }); }); +describe("uiSpendLogDetailsCall", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it.each(["chatcmpl-123#a1", "chatcmpl-123?a=1", "chatcmpl-123%41b", "chatcmpl-123 a&b"])( + "requests the details of %s with the full request id and start date", + async (logId) => { + const mockFetch = vi.fn().mockResolvedValue({ ok: true, json: vi.fn().mockResolvedValue({ id: logId }) }); + vi.stubGlobal("fetch", mockFetch); + + await Networking.uiSpendLogDetailsCall("token", logId, "2026-01-01 00:00:00"); + + const url = new URL(String(mockFetch.mock.calls.at(-1)?.[0]), "http://example.com"); + expect(decodeURIComponent(url.pathname.split("/spend/logs/ui/")[1])).toBe(logId); + expect(url.hash).toBe(""); + expect([...url.searchParams.entries()]).toEqual([["start_date", "2026-01-01 00:00:00"]]); + }, + ); +}); + describe("sessionSpendLogsCall", () => { const originalFetch = global.fetch; diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 38d150f1d63..486ec2be068 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -4709,8 +4709,8 @@ export const uiSpendLogDetailsCall = async (accessToken: string, logId: string, try { // Construct base URL let url = proxyBaseUrl - ? `${proxyBaseUrl}/spend/logs/ui/${logId}?start_date=${encodeURIComponent(start_date)}` - : `/spend/logs/ui/${logId}?start_date=${encodeURIComponent(start_date)}`; + ? `${proxyBaseUrl}/spend/logs/ui/${encodeURIComponent(logId)}?start_date=${encodeURIComponent(start_date)}` + : `/spend/logs/ui/${encodeURIComponent(logId)}?start_date=${encodeURIComponent(start_date)}`; const response = await fetch(url, { method: "GET", From 52562c47f83be9df28ecf6ca9f9fb620286d04b9 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 27 Oct 2026 13:53:23 -0700 Subject: [PATCH 026/102] refactor(proxy-extras): drop the spend-log api_key special case from the row check remedy (#48636) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_proxy_extras/migration_data_checks.py | 8 -------- .../database/migrations/test_request_log_indexes.py | 9 ++++++--- .../litellm_proxy_extras/test_migration_data_checks.py | 2 +- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py b/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py index a2d3728349a..7b01e64dd73 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py +++ b/litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py @@ -42,12 +42,6 @@ _OCTET_LENGTH_TYPES: Final = frozenset({"text", "varchar", "bpchar", "bytea", "n MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR: Final = "LITELLM_MIGRATION_DATA_CHECK_TIMEOUT" DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS: Final = 600.0 REFUSAL: Final = "Existing rows would make a pending schema change fail:" -LEGACY_SPEND_LOG_KEY_REMEDY: Final = ( - "Rows written before v1.99.0 hold the raw key. Hash them in place (one-way, idempotent):\n" - ' UPDATE "LiteLLM_SpendLogs" SET api_key = encode(sha256(convert_to(' - "regexp_replace(api_key, '^[Bb][Ee][Aa][Rr][Ee][Rr] ', ''), 'UTF8')), 'hex')\n" - " WHERE api_key <> '' AND api_key !~ '^[0-9a-fA-F]{64}$' AND api_key !~ '^hashed-jwt-[0-9A-Fa-f]{64}$';" -) Kind = Literal["too_long", "duplicates", "nulls", "any_row", "orphans", "check_violations"] @@ -571,8 +565,6 @@ def find_blockers( def _remedy(check: Check) -> str: - if check.kind == "too_long" and check.table == "LiteLLM_SpendLogs" and "api_key" in check.columns: - return LEGACY_SPEND_LOG_KEY_REMEDY joined: Final = ", ".join(check.columns) match check.kind: case "too_long": diff --git a/tests/integration/database/migrations/test_request_log_indexes.py b/tests/integration/database/migrations/test_request_log_indexes.py index bf37ca80665..1de2090b487 100644 --- a/tests/integration/database/migrations/test_request_log_indexes.py +++ b/tests/integration/database/migrations/test_request_log_indexes.py @@ -956,8 +956,11 @@ def _legacy_bearer_token() -> str: return "Bearer " + secrets.token_hex(1600) -def _hash_sql_from(refusal: str) -> str: - return refusal[refusal.index('UPDATE "LiteLLM_SpendLogs"') : refusal.index("{64}$';") + len("{64}$';")] +_HASH_LEGACY_KEYS: Final = ( + 'UPDATE "LiteLLM_SpendLogs" SET api_key = encode(sha256(convert_to(' + "regexp_replace(api_key, '^[Bb][Ee][Aa][Rr][Ee][Rr] ', ''), 'UTF8')), 'hex') " + "WHERE api_key <> '' AND api_key !~ '^[0-9a-fA-F]{64}$' AND api_key !~ '^hashed-jwt-[0-9A-Fa-f]{64}$'" +) @requires_db @@ -985,7 +988,7 @@ def test_the_migration_job_refuses_before_the_first_migration_when_spend_log_key assert API_KEY_INDEX not in _index_validity(scratch_database, "api_key_startTime_idx") with psycopg.connect(scratch_database, autocommit=True) as conn: - conn.execute(_hash_sql_from(refusal)) # pyright: ignore[reportArgumentType] # the refusal's own SQL + conn.execute(_HASH_LEGACY_KEYS) assert ProxyExtrasDBManager.run_migration_job(use_migrate=True, use_v2_resolver=use_v2_resolver) is True assert _index_validity(scratch_database, "api_key_startTime_idx") == {API_KEY_INDEX: True} assert len(_ledger(scratch_database)) > len(ledger_before) diff --git a/tests/unit/litellm_proxy_extras/test_migration_data_checks.py b/tests/unit/litellm_proxy_extras/test_migration_data_checks.py index 7897a388474..f3b787beda1 100644 --- a/tests/unit/litellm_proxy_extras/test_migration_data_checks.py +++ b/tests/unit/litellm_proxy_extras/test_migration_data_checks.py @@ -125,7 +125,7 @@ def test_the_report_names_the_rows_the_statement_and_the_fix_and_says_nothing_wa "(api_key up to 3207 bytes)" ) in text assert 'blocks: CREATE INDEX "LiteLLM_SpendLogs_api_key_startTime_idx" (request-log index)' in text - assert 'UPDATE "LiteLLM_SpendLogs" SET api_key = encode(sha256(' in text + assert 'fix: Shorten or hash the long "LiteLLM_SpendLogs" (api_key, startTime) values' in text assert text.startswith("Existing rows would make a pending schema change fail:") assert "No schema change was made" in text From e0d1e64591dc0f4243a17014ce216943ffa0d186 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Tue, 27 Oct 2026 17:04:30 -0400 Subject: [PATCH 027/102] feat(router): respect caller output budgets in complexity routing (#48632) Co-authored-by: stuti612 --- litellm/router.py | 96 ++++---- .../complexity_router/README.md | 14 ++ .../complexity_router/complexity_router.py | 63 ++++-- .../complexity_router/config.py | 5 +- .../common_utils/test_cache_aware_routing.py | 18 +- .../router_strategy/test_complexity_router.py | 206 +++++++++++++++++- .../add_model/ComplexityRouterConfig.tsx | 2 +- .../add_model/ResponseFormatControls.tsx | 56 ++++- .../build_complexity_router_config.test.ts | 20 ++ .../build_complexity_router_config.ts | 8 +- ...d_updated_complexity_router_config.test.ts | 23 ++ ...dit_auto_router_modal.integration.test.tsx | 22 +- .../hydrate_complexity_router_config.ts | 3 +- .../src/lib/autorouter_presets.test.ts | 7 + .../src/lib/autorouter_presets.ts | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +- 16 files changed, 448 insertions(+), 100 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 0d4045e6ca8..c8153b596d6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1122,6 +1122,27 @@ def as_output_cap(value: object) -> int | None: return cap if cap >= 0 else None +def stamped_client_ceiling(request_kwargs: Mapping[str, object]) -> Mapping[str, int] | None: + """None means unstamped; an empty mapping means the original caller supplied no limit.""" + stamped: Final = next( + ( + bucket.get(CLIENT_OUTPUT_CEILING_METADATA_KEY) + for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")) + if isinstance(bucket, dict) and CLIENT_OUTPUT_CEILING_METADATA_KEY in bucket + ), + None, + ) + if not isinstance(stamped, dict): + return None + return MappingProxyType( + { + carrier: cap + for carrier, value in stamped.items() + if carrier in OUTPUT_TOKEN_CEILING_PARAMS and (cap := as_output_cap(value)) is not None + } + ) + + def _exact_base_model_max_input_tokens(base_model: str) -> int | None: try: model_info: Final = litellm.get_model_info(model=base_model) @@ -15235,9 +15256,7 @@ class Router: request_kwargs: dict, responses_call: bool, ) -> bool: - """Apply a routing strategy's per-tier litellm_params on top of the request and report - whether they pinned an output ceiling, so the caller can hand the request its own ceiling - back on a routing pass that pins none.""" + """Apply per-tier litellm_params and report whether they pinned an output ceiling.""" if not tier_litellm_params: return False reject_api_key_secret_reference(tier_litellm_params) @@ -15250,34 +15269,26 @@ class Router: return not OUTPUT_TOKEN_CEILING_PARAMS.isdisjoint(surface_tier_params) @staticmethod - def _restore_client_ceiling_no_tier_pins(request_kwargs: MutableMapping[str, object]) -> None: - """A model-group fallback re-enters routing with the kwargs an earlier auto-router pass - already rewrote, so a ceiling sized for that pass's tier would ride onto a group no tier - chose. When this pass pins none, hand the request back exactly the carriers the caller - sent, which the first pinning pass stamped. The stamp lives in a metadata bucket a - caller can also write, so the proxy strips the key at ingestion and this read takes - nothing but the three ceiling carriers as integers: no other key ever reaches kwargs.""" - stamped: Final = next( - ( - bucket.get(CLIENT_OUTPUT_CEILING_METADATA_KEY) - for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")) - if isinstance(bucket, dict) and CLIENT_OUTPUT_CEILING_METADATA_KEY in bucket - ), - None, - ) - if not isinstance(stamped, dict): + def _restore_client_output_ceiling(request_kwargs: MutableMapping[str, object]) -> None: + """Restore caller intent before routing, so neither a previous tier pin nor alias defaults + become the next attempt's budget. The proxy strips caller-written stamps at ingestion.""" + callers_ceiling: Final = stamped_client_ceiling(request_kwargs) + if callers_ceiling is None: return - callers_ceiling: Final = MappingProxyType( - { - carrier: cap - for carrier, value in stamped.items() - if carrier in OUTPUT_TOKEN_CEILING_PARAMS and (cap := as_output_cap(value)) is not None - } - ) for carrier in OUTPUT_TOKEN_CEILING_PARAMS: request_kwargs.pop(carrier, None) request_kwargs.update(callers_ceiling) + @staticmethod + def _stamp_client_output_ceiling( + request_kwargs: dict[str, object], # mutable-ok: preserve request-owned metadata across routing attempts + ) -> None: + _, metadata_bucket = get_or_create_metadata_bucket(request_kwargs) + metadata_bucket.setdefault( + CLIENT_OUTPUT_CEILING_METADATA_KEY, + {carrier: request_kwargs[carrier] for carrier in OUTPUT_TOKEN_CEILING_PARAMS if carrier in request_kwargs}, + ) + @staticmethod def _drop_client_carriers_a_tier_pin_supersedes( request_kwargs: dict[str, object], @@ -15294,15 +15305,7 @@ class Router: one setting under three names, and a provider handed two of them either rejects the request or picks one by iteration order.""" if not OUTPUT_TOKEN_CEILING_PARAMS.isdisjoint(tier_litellm_params): - _, metadata_bucket = get_or_create_metadata_bucket(request_kwargs) - metadata_bucket.setdefault( - CLIENT_OUTPUT_CEILING_METADATA_KEY, - { - carrier: request_kwargs[carrier] - for carrier in OUTPUT_TOKEN_CEILING_PARAMS - if carrier in request_kwargs - }, - ) + Router._stamp_client_output_ceiling(request_kwargs) for carrier in OUTPUT_TOKEN_CEILING_PARAMS: request_kwargs.pop(carrier, None) if "reasoning_effort" not in tier_litellm_params: @@ -15374,14 +15377,12 @@ class Router: model = pre_routing_hook_response.model messages = pre_routing_hook_response.messages record_pre_routing_selection(request_kwargs, model) - tier_pins_ceiling: Final = self._pin_tier_params_onto_request( + self._pin_tier_params_onto_request( model=model, tier_litellm_params=pre_routing_hook_response.litellm_params if pre_routing_hook_response else None, request_kwargs=request_kwargs, responses_call=responses_call, ) - if not tier_pins_ceiling: - self._restore_client_ceiling_no_tier_pins(request_kwargs) ######################################################### # Resolve the strategy and logger AFTER the pre-routing hook, since @@ -15539,14 +15540,12 @@ class Router: model = pre_routing_hook_response.model messages = pre_routing_hook_response.messages record_pre_routing_selection(request_kwargs, model) - tier_pins_ceiling: Final = self._pin_tier_params_onto_request( + self._pin_tier_params_onto_request( model=model, tier_litellm_params=pre_routing_hook_response.litellm_params if pre_routing_hook_response else None, request_kwargs=request_kwargs, responses_call=responses_call, ) - if not tier_pins_ceiling: - self._restore_client_ceiling_no_tier_pins(request_kwargs) # 2. Get healthy deployments healthy_deployments: Final = await self.async_get_healthy_deployments( @@ -15896,6 +15895,7 @@ class Router: spend metadata is stamped before routing and the response carries the tier group the strategy picked. """ + self._restore_client_output_ceiling(request_kwargs) self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) requested_registered_model_name: Final = self.get_model_from_alias(model=model) or model registered_model_name: Final = await self._resolve_claude_code_session_router( @@ -16062,7 +16062,8 @@ class Router: # them) until `_update_kwargs_with_deployment` drops any the selected # deployment sets itself (its own `aws_region_name` beats the marker's). # Per-tier `litellm_params` on the hook response are deliberate overrides - # the caller applies on top, so those keys are never forwarded here. + # the caller applies on top, so those keys and equivalent output-limit + # names are never tracked as alias defaults here. marker_params: Final = ( self._forwardable_alias_marker_params( model=registered_model_name, strategy_tags=selected_strategy.tags, request_kwargs=request_kwargs @@ -16073,9 +16074,16 @@ class Router: tier_param_keys: Final = ( tuple(pre_routing_hook_response.litellm_params or ()) if pre_routing_hook_response is not None else () ) - newly_forwarded: Final = tuple( - (key, value) for key, value in marker_params if key not in request_kwargs and key not in tier_param_keys + overridden_marker_keys: Final = frozenset(tier_param_keys) | ( + OUTPUT_TOKEN_CEILING_PARAMS if not OUTPUT_TOKEN_CEILING_PARAMS.isdisjoint(tier_param_keys) else frozenset() ) + newly_forwarded: Final = tuple( + (key, value) + for key, value in marker_params + if key not in request_kwargs and key not in overridden_marker_keys + ) + if pre_routing_hook_response is not None: + self._stamp_client_output_ceiling(request_kwargs) request_kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, None) request_kwargs.update(newly_forwarded) if newly_forwarded: diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index 7567a0ef37e..08d39841b61 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -107,6 +107,20 @@ Session affinity, user-turn classification, adaptive routing, routing plugins, c When cache costs change the model, the routing decision reports `cause: prompt_cache_cost`. Its signals include the original model, classification cause and both estimated costs +### Output token limits + +Set `max_tokens_from_tier_model` under `complexity_router_config` to choose how a routed request's output limit is determined: + +| Setting | Effective output limit | +| --- | --- | +| `true` (default) | Replace the caller's limit with the selected model group's ceiling | +| `"clamp"` | Use the smaller of the caller's positive limit and the selected model group's ceiling | +| `false` | Keep the caller's limit | + +Clamp reads `max_tokens`, `max_completion_tokens` and `max_output_tokens`. If several are supplied, it uses the smallest positive value. With no positive caller limit, it uses the model ceiling. For example, a caller requesting 100 tokens keeps that limit on a model with a 32,768-token ceiling; a request for 100,000 is capped at 32,768. Rerouting uses the original caller budget, and cache-aware cost estimates use the same effective limit. + +All modes preserve explicit output limits in a tier's own `litellm_params`. An unknown ceiling leaves the caller's value unchanged. In the dashboard, select **Respect caller budget** under **Advanced settings > Compatibility > Output token limit** to enable clamp. + ### Capability forecasting Set `classifier_type: capability` to use diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 12968bf7d4a..f5fd4ae9c98 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -23,6 +23,7 @@ import random import re import time from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence +from functools import partial from itertools import accumulate, chain, islice, takewhile from threading import Lock from types import MappingProxyType @@ -161,6 +162,17 @@ def _tier_name(tier: ComplexityTier | str) -> str: return tier.value if isinstance(tier, ComplexityTier) else tier +def _callers_output_cap(request_kwargs: Mapping[str, object]) -> int | None: + """The smallest positive output ceiling the caller sent under any of its three names, read + from the stamp when an earlier routing pass already rewrote the kwargs.""" + from litellm.router import as_output_cap, stamped_client_ceiling + + stamped: Final = stamped_client_ceiling(request_kwargs) + carriers: Final = request_kwargs if stamped is None else stamped + caps: Final = tuple(as_output_cap(carriers.get(carrier)) for carrier in OUTPUT_TOKEN_CEILING_PARAMS) + return min((cap for cap in caps if cap is not None and cap > 0), default=None) + + def _built_in_tier_or_none(tier_name: str) -> ComplexityTier | None: """The built-in tier a `tiers` key names, or None when the key is an operator-defined name.""" return ComplexityTier.__members__.get(tier_name) @@ -3028,7 +3040,9 @@ class ComplexityRouter(CustomLogger): raise ValueError(f"No model configured for tier {tier_key} and no default_model set") - def _litellm_params_for_model(self, tier: ComplexityTier | str | None, model: str) -> Mapping[str, object]: + def _litellm_params_for_model( + self, tier: ComplexityTier | str | None, model: str, request_kwargs: Mapping[str, object] + ) -> Mapping[str, object]: entries: Final = self.config.tier_model_configs.get(_tier_name(tier), ()) if tier is not None else () entry: Final = next((candidate for candidate in entries if candidate.model_name == model), None) explicit: Final = entry.litellm_params if entry is not None else MappingProxyType({}) @@ -3037,7 +3051,12 @@ class ComplexityRouter(CustomLogger): ceiling: Final = self._group_output_ceiling(model) if ceiling is None: return explicit - return MappingProxyType({**explicit, "max_tokens": ceiling}) + callers_cap: Final = ( + _callers_output_cap(request_kwargs) if self.config.max_tokens_from_tier_model == "clamp" else None + ) + return MappingProxyType( + {**explicit, "max_tokens": ceiling if callers_cap is None else min(callers_cap, ceiling)} + ) @staticmethod def _pick_from_tier_value(model: str | Sequence[str], tier_key: str) -> str: @@ -3768,7 +3787,7 @@ class ComplexityRouter(CustomLogger): response: PreRoutingHookResponse, messages: list[dict[str, object]] | None, # mutable-ok: forwarded verbatim to the list-typed re-pick resolved_messages: Sequence[Mapping[str, object]] | None, - request_kwargs: dict, # mutable-ok: same shape the hook receives + request_kwargs: dict[str, object], # mutable-ok: same shape the hook receives context_fit: _RequestContextFit | None = None, retained_pin: _SessionAffinityPin | None = None, ) -> PreRoutingHookResponse: @@ -3865,7 +3884,7 @@ class ComplexityRouter(CustomLogger): conversation_continuing=bool(decision.get("conversation_continuing", True)) if decision is not None else True, - tier_litellm_params=self._litellm_params_for_model(new_tier, new_model), + tier_litellm_params=self._litellm_params_for_model(new_tier, new_model, request_kwargs), context_escalation_original_tier=( decision.get("context_escalation_original_tier") if decision is not None else None ), @@ -3876,7 +3895,7 @@ class ComplexityRouter(CustomLogger): return HookResponse( model=new_model, messages=response.messages, - litellm_params=self._litellm_params_for_model(new_tier, new_model), + litellm_params=self._litellm_params_for_model(new_tier, new_model, request_kwargs), routing_decision=new_decision, ) @@ -3904,7 +3923,7 @@ class ComplexityRouter(CustomLogger): model_name: str, messages: list[dict[str, Any]] | None, # mutable-ok: forwarded verbatim to the router's own probe input: str | list | None, # mutable-ok: mirrors the owner's own input parameter, which this forwards verbatim - request_kwargs: dict, # mutable-ok: same shape the hook receives + request_kwargs: dict[str, object], # mutable-ok: same shape the hook receives ) -> bool: """Whether the router would find a deployment for this group ON THIS REQUEST. @@ -3962,7 +3981,7 @@ class ComplexityRouter(CustomLogger): messages: list[dict[str, object]] | None, # mutable-ok: forwarded verbatim to the list-typed re-pick input: str | list | None, # mutable-ok: mirrors the owner's own input parameter, which this forwards verbatim resolved_messages: Sequence[Mapping[str, object]] | None, - request_kwargs: dict, # mutable-ok: same shape the hook receives + request_kwargs: dict[str, object], # mutable-ok: same shape the hook receives context_fit: _RequestContextFit | None = None, retained_pin: _SessionAffinityPin | None = None, ) -> PreRoutingHookResponse: @@ -4045,14 +4064,14 @@ class ComplexityRouter(CustomLogger): classifier_model=decision.get("classifier_model"), classifier_cost=decision.get("classifier_cost"), conversation_continuing=bool(decision.get("conversation_continuing", True)), - tier_litellm_params=self._litellm_params_for_model(candidate_tier, new_model), + tier_litellm_params=self._litellm_params_for_model(candidate_tier, new_model, request_kwargs), context_escalation_original_tier=decision.get("context_escalation_original_tier"), previous_decision=decision, ) return response.model_copy( update={ "model": new_model, - "litellm_params": self._litellm_params_for_model(candidate_tier, new_model), + "litellm_params": self._litellm_params_for_model(candidate_tier, new_model, request_kwargs), "routing_decision": new_decision, } ) @@ -4090,14 +4109,14 @@ class ComplexityRouter(CustomLogger): classifier_model=decision.get("classifier_model"), classifier_cost=decision.get("classifier_cost"), conversation_continuing=bool(decision.get("conversation_continuing", True)), - tier_litellm_params=self._litellm_params_for_model(None, default_model), + tier_litellm_params=self._litellm_params_for_model(None, default_model, request_kwargs), context_escalation_original_tier=decision.get("context_escalation_original_tier"), previous_decision=decision, ) return response.model_copy( update={ "model": default_model, - "litellm_params": self._litellm_params_for_model(None, default_model), + "litellm_params": self._litellm_params_for_model(None, default_model, request_kwargs), "routing_decision": default_decision, } ) @@ -4580,7 +4599,9 @@ class ComplexityRouter(CustomLogger): if pin_placement is not None and pin_context_original_tier is not None else pin_source_tier ) - session_tier_litellm_params: Final = self._litellm_params_for_model(routed_pin_tier, routed_model) + session_tier_litellm_params: Final = self._litellm_params_for_model( + routed_pin_tier, routed_model, request_kwargs + ) has_original_messages: Final = messages is not None and len(messages) > 0 return self._with_session_deployment_affinity( await self._gate_response_health( @@ -4678,14 +4699,14 @@ class ComplexityRouter(CustomLogger): choice: Final = await choose_cached_model( router=self.litellm_router_instance, config=self.config, - params_for_model=self._litellm_params_for_model, + params_for_model=partial(self._litellm_params_for_model, request_kwargs=request_kwargs), response=response, request_kwargs=request_kwargs, messages=messages, ) if choice is None or not context_fit.accepts(choice.model): return response - params: Final = self._litellm_params_for_model(choice.tier, choice.model) + params: Final = self._litellm_params_for_model(choice.tier, choice.model, request_kwargs) decision: Final[StandardLoggingRoutingDecision] = { **response.routing_decision, "routed_model": choice.model, @@ -4714,7 +4735,7 @@ class ComplexityRouter(CustomLogger): async def _classify_and_route( self, model: str, - request_kwargs: dict, + request_kwargs: dict[str, object], messages: list[dict[str, object]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, @@ -4772,7 +4793,7 @@ class ComplexityRouter(CustomLogger): ComplexityTier.MEDIUM, messages, resolved_messages, request_kwargs ) fallback_tier: Final = None if default_model_first else ComplexityTier.MEDIUM - default_tier_params: Final = self._litellm_params_for_model(fallback_tier, routed_model) + default_tier_params: Final = self._litellm_params_for_model(fallback_tier, routed_model, request_kwargs) return PreRoutingHookResponse( model=routed_model, messages=messages if has_original_messages else None, @@ -4810,7 +4831,7 @@ class ComplexityRouter(CustomLogger): _tier_name(plan_floor), routed_model, ) - plan_tier_params: Final = self._litellm_params_for_model(plan_floor, routed_model) + plan_tier_params: Final = self._litellm_params_for_model(plan_floor, routed_model, request_kwargs) return PreRoutingHookResponse( model=routed_model, messages=messages if has_original_messages else None, @@ -4839,7 +4860,9 @@ class ComplexityRouter(CustomLogger): ) keyword_plan_floored: Final = routed_tier != escalated_tier routed_model = await self._pick_model_for_tier(routed_tier, messages, resolved_messages, request_kwargs) - keyword_tier_litellm_params: Final = self._litellm_params_for_model(routed_tier, routed_model) + keyword_tier_litellm_params: Final = self._litellm_params_for_model( + routed_tier, routed_model, request_kwargs + ) keyword_cause: Final[RoutingDecisionCause] = ( "plan_mode" if keyword_plan_floored @@ -4919,7 +4942,7 @@ class ComplexityRouter(CustomLogger): outcome.signals, fallback_model, ) - fallback_tier_params: Final = self._litellm_params_for_model(None, fallback_model) + fallback_tier_params: Final = self._litellm_params_for_model(None, fallback_model, request_kwargs) return PreRoutingHookResponse( model=fallback_model, messages=messages if has_original_messages else None, @@ -5001,7 +5024,7 @@ class ComplexityRouter(CustomLogger): routed_model, ) - tier_litellm_params: Final = self._litellm_params_for_model(tier, routed_model) + tier_litellm_params: Final = self._litellm_params_for_model(tier, routed_model, request_kwargs) classifier_model: Final = ( f"{outcome.jev_verdict.provider}/{outcome.jev_verdict.model}" if outcome.cause == "jev_classifier" and outcome.jev_verdict is not None diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index ec960a19f79..020f2d33742 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -1462,7 +1462,7 @@ class ComplexityRouterConfig(LiteLLMBaseModel): "wording the built-ins don't cover, or after a client release changes its strings." ), ) - max_tokens_from_tier_model: bool = Field( + max_tokens_from_tier_model: bool | Literal["clamp"] = Field( default=True, description=( "Set max_tokens on every routed request to the output ceiling of the tier model it " @@ -1473,7 +1473,8 @@ class ComplexityRouterConfig(LiteLLMBaseModel): "from each deployment's model_info and then the model cost map; a tier model with a " "deployment whose ceiling is unknown keeps the caller's value. A max_tokens, " "max_completion_tokens or max_output_tokens in the tier's own litellm_params still " - "wins. Set false to forward the caller's value unchanged." + 'wins. Set "clamp" to keep a smaller positive caller value and cap a larger one at ' + "the ceiling. Set false to forward the caller's value unchanged." ), ) route_housekeeping_to_cheapest_tier: bool = Field( diff --git a/tests/unit/proxy/common_utils/test_cache_aware_routing.py b/tests/unit/proxy/common_utils/test_cache_aware_routing.py index 24fc1f22a6a..4eb22b2e34f 100644 --- a/tests/unit/proxy/common_utils/test_cache_aware_routing.py +++ b/tests/unit/proxy/common_utils/test_cache_aware_routing.py @@ -1,5 +1,6 @@ from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass +from functools import partial from typing import Final import pytest @@ -148,7 +149,7 @@ async def _select( return await select_cached_model( router=router, config=config, - params_for_model=complexity._litellm_params_for_model, + params_for_model=partial(complexity._litellm_params_for_model, request_kwargs=request_kwargs), response=response, body=body, request_kwargs=request_kwargs, @@ -234,7 +235,9 @@ async def test_disabled_setting_does_not_access_prediction_services() -> None: await choose_cached_model( router=_router(), config=config, - params_for_model=ComplexityRouter("smart", _router(), config.model_dump())._litellm_params_for_model, + params_for_model=partial( + ComplexityRouter("smart", _router(), config.model_dump())._litellm_params_for_model, request_kwargs={} + ), response=_response(), request_kwargs={}, messages=None, @@ -348,6 +351,7 @@ def test_repeated_model_in_multiple_tiers_is_only_considered_once() -> None: (True, "custom_endpoint", "cheap"), (True, "compaction", "cheap"), (True, "guardrail", "cheap"), + (True, "clamp", "strong"), ], ) async def test_router_applies_opt_in_and_preserves_failure_semantics( @@ -368,7 +372,9 @@ async def test_router_applies_opt_in_and_preserves_failure_semantics( from litellm.router_strategy.complexity_router.context_compaction import initialize_compaction_state config: Final = _config( - cache_aware_routing=enabled, cache_aware_routing_timeout_ms=1 if behavior == "deadline" else 2000 + cache_aware_routing=enabled, + cache_aware_routing_timeout_ms=1 if behavior == "deadline" else 2000, + max_tokens_from_tier_model="clamp" if behavior == "clamp" else True, ) models: Final = _router( strong_output_rate=0.000048 if behavior == "tier_cost" else 0.000015, @@ -423,6 +429,7 @@ async def test_router_applies_opt_in_and_preserves_failure_semantics( body: Final = { **_body(), **({"max_tokens": 1000} if behavior in ("tier_cost", "tier_context") else {}), + **({"max_tokens": 100} if behavior == "clamp" else {}), **({"thinking": {"type": "enabled", "budget_tokens": 10000}} if behavior == "unsupported_shape" else {}), } kwargs: Final = { @@ -437,6 +444,7 @@ async def test_router_applies_opt_in_and_preserves_failure_semantics( else {} ), **({"guardrails": ["test-guardrail"]} if behavior == "guardrail" else {}), + **({"max_tokens": 100} if behavior == "clamp" else {}), } if expected is None: with pytest.raises(asyncio.CancelledError): @@ -458,6 +466,10 @@ async def test_router_applies_opt_in_and_preserves_failure_semantics( if enabled and behavior == "success": assert requests.qsize() == 4 assert response.routing_decision is not None + if behavior == "clamp": + assert "cache-aware:estimated-cost=0.01950000;original-cost=0.06390000" in ( + response.routing_decision["signals"] or () + ), "both arms must be estimated at the caller's 100 output tokens, not the 1024 default" assert response.routing_decision["cause"] == ( "prompt_cache_cost" if expected == "strong" else "heuristic_scorer" ) diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index 2d27a025ffa..78dc2eaf314 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -36,11 +36,12 @@ from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import ( + CLIENT_OUTPUT_CEILING_METADATA_KEY, OUTPUT_TOKEN_CEILING_PARAMS, RETURN_RAW_MODEL_NAME_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY, ) -from litellm.router import as_output_cap +from litellm.router import as_output_cap, stamped_client_ceiling from litellm.router_strategy.complexity_router.complexity_router import ( _CLASSIFICATION_CURRENT_MESSAGE_ONLY, _CLASSIFICATION_WITH_CONVERSATION, @@ -13733,7 +13734,7 @@ def test_tier_model_params_are_used_by_pools_and_savings_baseline(mock_router_in assert router._tier_pools() == {"SIMPLE": ["mini"], "REASONING": ["opus", "abc"]} assert router._hardest_tier_models() == ("opus", "abc") - assert router._litellm_params_for_model(ComplexityTier.REASONING, "opus") == {"reasoning_effort": "xhigh"} + assert router._litellm_params_for_model(ComplexityTier.REASONING, "opus", {}) == {"reasoning_effort": "xhigh"} @pytest.mark.asyncio @@ -16917,9 +16918,10 @@ class TestMaxTokensFromTierModel: @staticmethod def _router( tier_litellm_params: dict | None = None, - max_tokens_from_tier_model: bool | None = None, + max_tokens_from_tier_model: bool | Literal["clamp"] | None = None, simple_deployments: list[dict] | None = None, extra_config: dict | None = None, + marker_params: Mapping[str, object] | None = None, ) -> Router: simple_tier: dict = {"model_name": "small"} if tier_litellm_params: @@ -16934,7 +16936,11 @@ class TestMaxTokensFromTierModel: model_list=[ { "model_name": "smart-router", - "litellm_params": {"model": "auto_router/complexity_router", "complexity_router_config": config}, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": config, + **(marker_params or {}), + }, }, *(simple_deployments or [TestMaxTokensFromTierModel.SMALL]), { @@ -17016,6 +17022,167 @@ class TestMaxTokensFromTierModel: assert sent["max_tokens"] == 8192 + @pytest.mark.asyncio + @pytest.mark.parametrize( + "prompt, client_kwargs, expected", + [ + (COMPLEX_PROMPT, {"max_tokens": 2000}, ("anthropic/claude-sonnet-5", 2000)), + ("hi", {"max_tokens": 100000}, ("anthropic/claude-haiku-4-5", 8192)), + (COMPLEX_PROMPT, {}, ("anthropic/claude-sonnet-5", 64000)), + (COMPLEX_PROMPT, {"max_tokens": 0}, ("anthropic/claude-sonnet-5", 64000)), + (COMPLEX_PROMPT, {"max_completion_tokens": 2000}, ("anthropic/claude-sonnet-5", 2000)), + (COMPLEX_PROMPT, {"max_tokens": 3000, "max_completion_tokens": 2000}, ("anthropic/claude-sonnet-5", 2000)), + ], + ids=["below-ceiling", "above-ceiling", "omitted", "zero", "other-carrier", "smallest-carrier"], + ) + async def test_clamp_keeps_the_smaller_of_the_client_value_and_the_ceiling( + self, prompt: str, client_kwargs: dict[str, int], expected: tuple[str, int] + ) -> None: + sent: Final = await self._routed(self._router(max_tokens_from_tier_model="clamp"), prompt, **client_kwargs) + + assert (sent["model"], sent["max_tokens"]) == expected + assert not (OUTPUT_TOKEN_CEILING_PARAMS - {"max_tokens"}) & sent.keys() + + @pytest.mark.asyncio + async def test_clamp_on_the_responses_surface_keeps_its_own_name(self) -> None: + router: Final = self._router(max_tokens_from_tier_model="clamp") + + sent: Final = await self._routed_responses(router, self.COMPLEX_PROMPT, max_output_tokens=2000) + + assert (sent["model"], sent["max_output_tokens"]) == ("anthropic/claude-sonnet-5", 2000) + assert "max_tokens" not in sent + + @pytest.mark.asyncio + @pytest.mark.parametrize("tier_carrier", sorted(OUTPUT_TOKEN_CEILING_PARAMS)) + @pytest.mark.parametrize("marker_carrier", sorted(OUTPUT_TOKEN_CEILING_PARAMS)) + @pytest.mark.parametrize("surface", ["chat", "responses"]) + @pytest.mark.parametrize("caller_cap", [None, 100]) + async def test_clamp_leaves_the_operators_own_tier_ceiling_in_charge( + self, + tier_carrier: str, + marker_carrier: str, + surface: str, + caller_cap: int | None, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + surface_key: Final = "max_output_tokens" if surface == "responses" else "max_tokens" + caller_key: Final = "max_tokens" if surface == "responses" else "max_output_tokens" + request_caps: Final = {caller_key: caller_cap} if caller_cap is not None else {} + router: Final = self._router( + tier_litellm_params={tier_carrier: 4321}, + max_tokens_from_tier_model="clamp", + marker_params={marker_carrier: 2000}, + simple_deployments=[{ + "model_name": "small", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_key": "test-only", + "api_base": "https://tier.test/v1", + surface_key: 9000, + }, + "model_info": {"max_output_tokens": 16384}, + }], + ) + + with respx.mock(assert_all_mocked=True) as upstream: + upstream.post(host="tier.test").mock(side_effect=TestHealthFallbackDispatch._http_response) + if surface == "responses": + await router.aresponses(model="smart-router", input="hi", **request_caps) + else: + await router.acompletion( + model="smart-router", messages=[{"role": "user", "content": "hi"}], **request_caps + ) + + assert len(upstream.calls) == 1 + sent: Final = json.loads(upstream.calls[0].request.content) + wire_key: Final = "max_output_tokens" if surface == "responses" else "max_completion_tokens" + assert {key: value for key, value in sent.items() if key in OUTPUT_TOKEN_CEILING_PARAMS} == {wire_key: 4321} + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "callers_original, expected", + [({"max_tokens": 2000}, 2000), ({}, 64000)], + ids=["caller-sent-a-value", "caller-sent-nothing"], + ) + async def test_clamp_on_a_rerouted_request_uses_the_callers_original_value( + self, callers_original: dict[str, int], expected: int + ) -> None: + """A retry re-enters routing with kwargs an earlier pass already rewrote; the stamp holds + what the caller actually sent.""" + router: Final = self._router(max_tokens_from_tier_model="clamp") + rewritten: Final[dict[str, object]] = { + "max_tokens": 8192, + "metadata": {CLIENT_OUTPUT_CEILING_METADATA_KEY: callers_original}, + } + + sent: Final = await self._routed(router, self.COMPLEX_PROMPT, **rewritten) + + assert sent["max_tokens"] == expected + + @pytest.mark.asyncio + @pytest.mark.parametrize("marker_carrier", sorted(OUTPUT_TOKEN_CEILING_PARAMS)) + @pytest.mark.parametrize("caller_cap", [None, 3000]) + @pytest.mark.parametrize("tier_cap", [None, 4321]) + @pytest.mark.parametrize("known_ceiling", [True, False]) + async def test_rerouting_preserves_caller_budget_without_adopting_marker_defaults( + self, marker_carrier: str, caller_cap: int | None, tier_cap: int | None, known_ceiling: bool + ) -> None: + router: Final = self._router( + max_tokens_from_tier_model="clamp", + marker_params={marker_carrier: 2000}, + tier_litellm_params={"max_tokens": tier_cap} if tier_cap else None, + simple_deployments=None if known_ceiling else [{ + "model_name": "small", + "litellm_params": {"model": "openai/not-in-any-map", "api_key": "k"}, + }], + ) + request: Final[dict[str, object]] = {} if caller_cap is None else {"max_output_tokens": caller_cap} + await router.async_get_available_deployment( + model="smart-router", request_kwargs=request, messages=[{"role": "user", "content": "hi"}] + ) + first: Final = {key: request[key] for key in OUTPUT_TOKEN_CEILING_PARAMS if key in request} + await router.async_get_available_deployment( + model="smart-router", request_kwargs=request, + messages=[{"role": "user", "content": self.COMPLEX_PROMPT}], + ) + + expected_first: Final = ( + {"max_tokens": tier_cap or caller_cap or 8192} + if known_ceiling or tier_cap else + {marker_carrier: 2000, **({"max_output_tokens": caller_cap} if caller_cap else {})} + ) + assert first == expected_first + assert request["max_tokens"] == (caller_cap or 64000) + assert not (OUTPUT_TOKEN_CEILING_PARAMS - {"max_tokens"}) & request.keys() + + await router.async_get_available_deployment( + model="smart-router", request_kwargs=request, messages=[{"role": "user", "content": "hi"}] + ) + assert {key: request[key] for key in OUTPUT_TOKEN_CEILING_PARAMS if key in request} == first + + @pytest.mark.asyncio + @pytest.mark.parametrize("mode", [False, "clamp"]) + @pytest.mark.parametrize("marker_carrier", sorted(OUTPUT_TOKEN_CEILING_PARAMS)) + async def test_unpinned_output_limits_still_use_marker_defaults( + self, mode: bool | Literal["clamp"], marker_carrier: str + ) -> None: + router: Final = self._router( + max_tokens_from_tier_model=mode, + marker_params={marker_carrier: 2000}, + simple_deployments=[{ + "model_name": "small", + "litellm_params": {"model": "openai/not-in-any-map", "api_key": "k"}, + }], + ) + sent: Final = await self._routed(router) + + assert {key: sent[key] for key in OUTPUT_TOKEN_CEILING_PARAMS if key in sent} == {marker_carrier: 2000} + + def test_an_unknown_mode_is_rejected(self) -> None: + with pytest.raises(ValidationError): + ComplexityRouterConfig(tiers={"SIMPLE": "small"}, max_tokens_from_tier_model="sometimes") + @pytest.mark.asyncio async def test_a_tier_model_with_an_unknown_ceiling_keeps_the_client_value(self): unmapped: dict = {"model_name": "small", "litellm_params": {"model": "openai/not-in-any-map", "api_key": "k"}} @@ -17112,7 +17279,7 @@ class TestMaxTokensFromTierModel: complexity_router_config={"tiers": {"SIMPLE": "small"}, "default_model": "big"}, ) - assert dict(strategy._litellm_params_for_model(None, "big")) == {"max_tokens": 64000} + assert dict(strategy._litellm_params_for_model(None, "big", {})) == {"max_tokens": 64000} @pytest.mark.asyncio async def test_a_fallback_into_a_plain_group_gets_the_callers_ceiling_back(self): @@ -17228,29 +17395,52 @@ class TestMaxTokensFromTierModel: def test_a_client_cap_is_read_as_an_integer_or_ignored(self, value, expected): assert as_output_cap(value) == expected + @pytest.mark.parametrize( + "request_kwargs, expected", + [ + ({"max_tokens": 8192}, None), + ({"metadata": {CLIENT_OUTPUT_CEILING_METADATA_KEY: {}}}, {}), + ({"litellm_metadata": {CLIENT_OUTPUT_CEILING_METADATA_KEY: {"max_tokens": "2000"}}}, {"max_tokens": 2000}), + ( + {"metadata": {CLIENT_OUTPUT_CEILING_METADATA_KEY: {"max_output_tokens": 5, "api_key": "x", "max_tokens": True}}}, + {"max_output_tokens": 5}, + ), + ({"metadata": {CLIENT_OUTPUT_CEILING_METADATA_KEY: "not-a-dict"}}, None), + ], + ids=["no-stamp", "empty-stamp", "litellm-metadata-bucket", "only-integer-carriers", "malformed-stamp"], + ) + def test_the_stamp_reads_back_only_the_callers_integer_carriers( + self, request_kwargs: dict[str, object], expected: dict[str, int] | None + ) -> None: + stamped: Final = stamped_client_ceiling(request_kwargs) + + assert (None if stamped is None else dict(stamped)) == expected + def test_restoring_the_callers_ceiling_reads_the_stamp_and_replaces_every_carrier(self): stamped: dict = {"max_output_tokens": 500, "metadata": {"_client_output_ceiling": {"max_tokens": 8192}}} - Router._restore_client_ceiling_no_tier_pins(stamped) + Router._restore_client_output_ceiling(stamped) assert {k: v for k, v in stamped.items() if k != "metadata"} == {"max_tokens": 8192} coerced: dict = { "max_tokens": 64000, "metadata": {"_client_output_ceiling": {"max_tokens": "8192", "max_completion_tokens": 100.0}}, } - Router._restore_client_ceiling_no_tier_pins(coerced) + Router._restore_client_output_ceiling(coerced) assert {k: v for k, v in coerced.items() if k != "metadata"} == { "max_tokens": 8192, "max_completion_tokens": 100, } unstamped: dict = {"max_tokens": 64000, "metadata": {}} - Router._restore_client_ceiling_no_tier_pins(unstamped) + Router._restore_client_output_ceiling(unstamped) assert unstamped["max_tokens"] == 64000 @pytest.mark.asyncio async def test_pinning_stamps_the_callers_carriers_once(self): router = self._router() request_kwargs: dict = {"max_completion_tokens": 8192} + router._stamp_client_output_ceiling(request_kwargs) + assert stamped_client_ceiling(request_kwargs) == {"max_completion_tokens": 8192} first = router._pin_tier_params_onto_request( model="big", tier_litellm_params={"max_tokens": 64000}, request_kwargs=request_kwargs, responses_call=False diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 72a08c53267..b59aa051e38 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -452,7 +452,7 @@ export interface ComplexityRouterConfigValue { route_housekeeping_to_cheapest_tier?: boolean; housekeeping_patterns?: string[]; reminder_markers?: ReminderMarkerPair[]; - max_tokens_from_tier_model?: boolean; + max_tokens_from_tier_model?: boolean | "clamp"; classifier_plugin_timeout_ms?: number; } diff --git a/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx b/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx index 6e7b1489a4c..f596ecfdc62 100644 --- a/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx @@ -1,7 +1,28 @@ import { Switch } from "@/components/ui/switch"; +import { Label } from "@/components/ui/label"; +import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"; import React from "react"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +const OUTPUT_TOKEN_POLICIES = [ + { + value: true, + label: "Use model ceiling", + description: "Replace the caller's output limit with the selected model's ceiling. This is the default.", + }, + { + value: "clamp", + label: "Respect caller budget", + description: + "Use the smaller of the caller's output limit and the model ceiling. Use the ceiling when no limit is sent.", + }, + { + value: false, + label: "Pass through caller limit", + description: "Forward the caller's output limit unchanged.", + }, +] as const; + const ResponseFormatControls: React.FC<{ value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; @@ -18,18 +39,31 @@ const ResponseFormatControls: React.FC<{ Return the resolved underlying model name in responses instead of the autorouter alias. -
- onChange({ ...value, max_tokens_from_tier_model: enabled })} - aria-label="Cap max_tokens at the tier model's output ceiling" - /> - Cap max_tokens at the tier model's output ceiling +
+ Output token limit + { + if (typeof mode === "boolean" || mode === "clamp") { + onChange({ ...value, max_tokens_from_tier_model: mode }); + } + }} + aria-label="Output token limit" + > + {OUTPUT_TOKEN_POLICIES.map((policy) => ( + + ))} + + + Explicit tier output limits take precedence. Unknown model ceilings keep the caller's limit. +
- - Replace the caller's max_tokens with the routed tier model's output ceiling so one client value fits - every tier. Off forwards the caller's value unchanged. - ); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 89d419d0487..b25baa0d1e4 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -1315,6 +1315,17 @@ describe("buildComplexityRouterConfig with an edited tier set", () => { expect(payload.fallback_tier).toBe("CASUAL"); }); + it("keeps clamp and explicit output limits on custom tiers", () => { + const payload = build({ + maxTokensFromTierModel: "clamp", + tierModelParams: { sec: { "o1-preview": { max_completion_tokens: 700 } } }, + }); + expect(payload.max_tokens_from_tier_model).toBe("clamp"); + expect(payload.tier_model_configs?.SECURITY_REVIEW).toEqual([ + { model_name: "o1-preview", litellm_params: { max_completion_tokens: 700 } }, + ]); + }); + it("forces the LLM classifier even when the form still holds heuristic, which the backend rejects", () => { expect(build({ classifierType: "heuristic" }).classifier_type).toBe("llm"); }); @@ -1595,6 +1606,15 @@ describe("classifier vision wire payload", () => { }); describe("advanced complexity router fields", () => { + it.each([undefined, true, false, "clamp"] as const)("serializes output token policy %s", (mode) => { + const payload = buildComplexityRouterConfig({ ...baseParams, maxTokensFromTierModel: mode }); + if (mode === false || mode === "clamp") { + expect(payload.max_tokens_from_tier_model).toBe(mode); + } else { + expect(payload).not.toHaveProperty("max_tokens_from_tier_model"); + } + }); + it("normalizes lists, reminder markers, and explicit false values", () => { const payload = buildComplexityRouterConfig({ ...baseParams, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 09bf0ea41a9..511a1653aae 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -226,7 +226,7 @@ export interface BuildComplexityRouterConfigParams { routeHousekeepingToCheapestTier?: boolean; housekeepingPatterns?: string[]; reminderMarkers?: ReminderMarkerPair[]; - maxTokensFromTierModel?: boolean; + maxTokensFromTierModel?: ComplexityRouterConfigValue["max_tokens_from_tier_model"]; classifierPluginTimeoutMs?: number; } @@ -310,7 +310,7 @@ export interface ComplexityRouterConfigPayload { route_housekeeping_to_cheapest_tier?: boolean; housekeeping_patterns?: string[]; reminder_markers?: ReminderMarkerPair[]; - max_tokens_from_tier_model?: boolean; + max_tokens_from_tier_model?: ComplexityRouterConfigValue["max_tokens_from_tier_model"]; classifier_plugin_timeout_ms?: number; } @@ -827,7 +827,9 @@ export const buildComplexityRouterConfig = ({ ...cleanedLists, ...(routeHousekeepingToCheapestTier === false && { route_housekeeping_to_cheapest_tier: false }), ...(cleanedReminderMarkers && cleanedReminderMarkers.length > 0 && { reminder_markers: cleanedReminderMarkers }), - ...(maxTokensFromTierModel === false && { max_tokens_from_tier_model: false }), + ...((maxTokensFromTierModel === false || maxTokensFromTierModel === "clamp") && { + max_tokens_from_tier_model: maxTokensFromTierModel, + }), ...(hasValidCustomClassifierTimeout && { classifier_plugin_timeout_ms: classifierPluginTimeoutMs }), ...scorerKnobs, }; diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index 9445e18913e..6238e30ce07 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -47,6 +47,29 @@ const hydratedState: KeywordMatchingState = { matchThreshold: 0.72, }; +describe("output token policy edits", () => { + it.each([undefined, true, false, "clamp"] as const)("preserves and edits stored mode %s", (mode) => { + const stored = { + ...STORED, + ...(mode !== undefined && { max_tokens_from_tier_model: mode }), + tier_model_configs: { SIMPLE: [{ model_name: "gpt-4o-mini", litellm_params: { max_tokens: 700 } }] }, + }; + const hydrated = hydrateComplexityRouterConfig(stored, undefined); + expect(hydrated.max_tokens_from_tier_model).toBe(mode); + const unchanged = buildUpdatedComplexityRouterConfig(stored, { ...hydrated, return_raw_model_name: true }); + expect(unchanged.max_tokens_from_tier_model).toBe(mode); + expect("max_tokens_from_tier_model" in unchanged).toBe(mode !== undefined); + for (const nextMode of [true, false, "clamp"] as const) { + const saved = buildUpdatedComplexityRouterConfig(stored, { + ...hydrated, + max_tokens_from_tier_model: nextMode, + }); + expect(saved.max_tokens_from_tier_model).toBe(nextMode === true && mode !== true ? undefined : nextMode); + expect(saved.tier_model_configs).toEqual(stored.tier_model_configs); + } + }); +}); + describe("buildUpdatedComplexityRouterConfig keyword matching", () => { it.each([undefined, false, true])("preserves cache settings through edit and save: %s", (enabled) => { const stored = { diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx index 72d557d1f8f..e8c9288cfc5 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx @@ -472,9 +472,17 @@ describe("EditAutoRouterModal advanced field round trips", () => { modelPatchUpdateCall.mockClear(); }); - it("hydrates housekeeping and reminder fields, then omits the default max-token value after editing", async () => { + it("reopens clamp and saves the selected output policy without changing other advanced settings", async () => { const user = userEvent.setup(); - renderAdvancedModal(); + renderAdvancedModal({ + modelData: { + ...MODEL_DATA, + litellm_params: { + ...MODEL_DATA.litellm_params, + complexity_router_config: { ...storedAdvancedConfig, max_tokens_from_tier_model: "clamp" }, + }, + }, + }); openAutoRouterAdvanced("Housekeeping Routing"); expect(screen.getByRole("switch", { name: "Route housekeeping calls to the cheapest tier" })).not.toBeChecked(); @@ -485,13 +493,17 @@ describe("EditAutoRouterModal advanced field round trips", () => { expect(screen.getByLabelText("Closing tag")).toHaveValue(""); openAutoRouterAdvanced("Response Format"); - const maxTokensSwitch = screen.getByRole("switch", { name: "Cap max_tokens at the tier model's output ceiling" }); - await user.click(maxTokensSwitch); + expect(screen.getByRole("radio", { name: /^Respect caller budget/ })).toBeChecked(); + await user.click(screen.getByRole("radio", { name: /^Pass through caller limit/ })); + expect(screen.getByRole("radio", { name: /^Pass through caller limit/ })).toBeChecked(); + await user.click(screen.getByRole("radio", { name: /^Use model ceiling/ })); + expect(screen.getByRole("radio", { name: /^Use model ceiling/ })).toBeChecked(); + await user.click(screen.getByRole("radio", { name: /^Respect caller budget/ })); await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /save changes/i })); await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce()); - expect(savedConfig()).not.toHaveProperty("max_tokens_from_tier_model"); + expect(savedConfig().max_tokens_from_tier_model).toBe("clamp"); expect(savedConfig()).toMatchObject({ route_housekeeping_to_cheapest_tier: false, housekeeping_patterns: ["conversation title"], diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts b/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts index 233b5aed922..75fd6e15bc1 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts @@ -184,7 +184,8 @@ export const hydrateComplexityRouterConfig = ( ? parsedConfig.reminder_markers.filter(isReminderMarkerPair) : undefined, max_tokens_from_tier_model: - typeof parsedConfig.max_tokens_from_tier_model === "boolean" + typeof parsedConfig.max_tokens_from_tier_model === "boolean" || + parsedConfig.max_tokens_from_tier_model === "clamp" ? parsedConfig.max_tokens_from_tier_model : undefined, classifier_plugin_timeout_ms: diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index b398780d14e..f4e45e473df 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -877,6 +877,13 @@ describe("autorouter_presets", () => { }); describe("buildPresetPrefill", () => { + it.each([undefined, true, false, "clamp"] as const)("preserves a preset's output token policy %s", (mode) => { + const preset = getPresetByKey("anthropic_family")!; + const config = { ...preset.complexity_router_config, max_tokens_from_tier_model: mode }; + const prefill = buildPresetPrefill(config, groupsOnly(getRequiredModelsInPreset(preset))); + expect(prefill.complexityRouterConfig.max_tokens_from_tier_model).toBe(mode); + }); + it("preserves Laya settings and drops inactive classifier settings when prefilling", () => { const config = { tiers: { SIMPLE: ["fast"], MEDIUM: [], COMPLEX: [], REASONING: [] }, diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index 0a83a06050b..fd400a8163a 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -364,6 +364,7 @@ export const buildPresetPrefill = ( tier_distance_penalty: config.tier_distance_penalty, adaptive_eligible: config.adaptive_eligible, return_raw_model_name: config.return_raw_model_name, + max_tokens_from_tier_model: config.max_tokens_from_tier_model, dimension_weights: hydrateDimensionWeights(config.dimension_weights), custom_dimensions: hydrateCustomDimensions(config.custom_dimensions), tier_boundaries: hydrateTierBoundaries(config.tier_boundaries), diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d03f4b4cffd..e3f80b37498 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -42620,10 +42620,10 @@ export interface components { match_threshold: number; /** * Max Tokens From Tier Model - * @description Set max_tokens on every routed request to the output ceiling of the tier model it lands on, replacing whatever the caller sent. A caller behind an auto-router cannot pick one value that fits every tier: the smallest tier's ceiling starves a bigger tier's thinking budget, and a bigger tier's ceiling is rejected by the smallest. The ceiling is the smallest max_output_tokens across the tier model's deployments, read from each deployment's model_info and then the model cost map; a tier model with a deployment whose ceiling is unknown keeps the caller's value. A max_tokens, max_completion_tokens or max_output_tokens in the tier's own litellm_params still wins. Set false to forward the caller's value unchanged. + * @description Set max_tokens on every routed request to the output ceiling of the tier model it lands on, replacing whatever the caller sent. A caller behind an auto-router cannot pick one value that fits every tier: the smallest tier's ceiling starves a bigger tier's thinking budget, and a bigger tier's ceiling is rejected by the smallest. The ceiling is the smallest max_output_tokens across the tier model's deployments, read from each deployment's model_info and then the model cost map; a tier model with a deployment whose ceiling is unknown keeps the caller's value. A max_tokens, max_completion_tokens or max_output_tokens in the tier's own litellm_params still wins. Set "clamp" to keep a smaller positive caller value and cap a larger one at the ceiling. Set false to forward the caller's value unchanged. * @default true */ - max_tokens_from_tier_model: boolean; + max_tokens_from_tier_model: boolean | "clamp"; /** * Modality Pin Override * @description Let modality_routing replace a kept session-affinity pin on the turns that carry an image. Without this, a session pinned to a text-only model fails every image turn with a provider 400, since the pin is exempt from the modality gate. When enabled, such a turn routes to a capable model for that request only and the stored pin is left untouched, so the next text turn replays the session's own model; the override is reported as cause modality_pin_override and is never itself pinned. Inert unless modality_routing is also enabled. From c23cff7e459aa83373209e273179481227155005 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Tue, 27 Oct 2026 17:30:48 -0400 Subject: [PATCH 028/102] fix(mcp): enforce deny-first tool policy consistently (#48625) --- .../proxy/mcp_server/mcp_server_manager.py | 25 ++++- litellm/proxy/mcp_server/operations.py | 27 ++--- litellm/proxy/mcp_server/utils.py | 9 ++ .../mcp_server/test_mcp_server_manager.py | 102 ++++++++++++------ .../test_mcp_server_tool_calls_and_headers.py | 35 +++--- 5 files changed, 129 insertions(+), 69 deletions(-) diff --git a/litellm/proxy/mcp_server/mcp_server_manager.py b/litellm/proxy/mcp_server/mcp_server_manager.py index 8e613a8a6e8..f7146ac9428 100644 --- a/litellm/proxy/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/mcp_server/mcp_server_manager.py @@ -1705,6 +1705,24 @@ def warn_on_shared_identifier_prefixes(servers: Iterable[MCPServer]) -> None: ) +def _warn_overlapping_tool_policies(server: MCPServer) -> None: + overlapping_tools: Final = tuple( + entry + for entry in server.allowed_tools or () + if any( + match_known_tool_name(name, server, server.disallowed_tools or ()) is not None + for name in (entry, strip_known_server_prefix(entry, server)) + ) + ) + if overlapping_tools: + verbose_logger.warning( + "MCP server %s has entries in both allowed_tools and disallowed_tools: %s. " + "disallowed_tools takes precedence; matching tools will be blocked.", + server.name, + ", ".join(overlapping_tools), + ) + + def _warn_legacy_delegate_auth_if_applicable(server: MCPServer, *, source: str) -> None: """Direct legacy delegated OAuth configurations to the admitted replacement.""" if server.auth_type != MCPAuth.oauth2: @@ -2854,6 +2872,7 @@ class MCPServerManager: oauth_identity_binding=server_config.get("oauth_identity_binding", None), ) self.assign_unique_short_prefix(new_server) + _warn_overlapping_tool_policies(new_server) _warn_legacy_delegate_auth_if_applicable(new_server, source="config") _warn_config_id_jag_server_outruns_sso(new_server) self.invalidate_server_definition_caches(server_id) @@ -5943,12 +5962,10 @@ class MCPServerManager: cut a second boundary out of a native name that opens with the server prefix. """ from litellm.proxy.mcp_server.utils import ( - server_applies_tool_allowlist, + is_tool_allowed_for_server, ) - if server_applies_tool_allowlist(server): - return match_known_tool_name(tool_name, server, server.allowed_tools or ()) is not None - return match_known_tool_name(tool_name, server, server.disallowed_tools or ()) is None + return is_tool_allowed_for_server(tool_name, server) def validate_allowed_params(self, tool_name: str, arguments: _ToolArguments, server: MCPServer) -> None: """ diff --git a/litellm/proxy/mcp_server/operations.py b/litellm/proxy/mcp_server/operations.py index 763e3578a96..4c2cbe2dfbf 100644 --- a/litellm/proxy/mcp_server/operations.py +++ b/litellm/proxy/mcp_server/operations.py @@ -800,28 +800,17 @@ def filter_tools_by_allowed_tools( Filtered list of tools """ from litellm.proxy.mcp_server.utils import ( + is_tool_allowed_for_server, server_applies_tool_allowlist, ) - tools_to_return = tools - - # Filter by allowed_tools (whitelist) - if server_applies_tool_allowlist(mcp_server): - if not mcp_server.allowed_tools: - return [] - tools_to_return = [ - tool for tool in tools if _tool_name_matches(tool.name, mcp_server.allowed_tools, mcp_server) - ] - - # Filter by disallowed_tools (blacklist) - if mcp_server.disallowed_tools: - tools_to_return = [ - tool - for tool in tools_to_return - if not _tool_name_matches(tool.name, mcp_server.disallowed_tools, mcp_server) - ] - - return tools_to_return + if not server_applies_tool_allowlist(mcp_server) and not mcp_server.disallowed_tools: + return tools + return [ + tool + for tool in tools + if is_tool_allowed_for_server(strip_known_server_prefix(tool.name, mcp_server), mcp_server) + ] def apply_display_name_overrides( diff --git a/litellm/proxy/mcp_server/utils.py b/litellm/proxy/mcp_server/utils.py index ee8f3ea3b30..3f72804ff35 100644 --- a/litellm/proxy/mcp_server/utils.py +++ b/litellm/proxy/mcp_server/utils.py @@ -266,6 +266,15 @@ def server_applies_tool_allowlist(mcp_server: object) -> bool: return is_server_tool_allowlist_enforced(mcp_server) or bool(allowed_tools) +def is_tool_allowed_for_server(tool_name: str, server: MCPServer) -> bool: + """Evaluate server tool policy for a bare name, with explicit denials taking precedence.""" + if match_known_tool_name(tool_name, server, server.disallowed_tools or ()) is not None: + return False + return not server_applies_tool_allowlist(server) or ( + match_known_tool_name(tool_name, server, server.allowed_tools or ()) is not None + ) + + def validate_and_normalize_mcp_server_payload(payload: McpServerPayloadLike) -> None: """ Validate and normalize MCP server payload fields (server_name, alias, and diff --git a/tests/unit/proxy/mcp_server/test_mcp_server_manager.py b/tests/unit/proxy/mcp_server/test_mcp_server_manager.py index c76326c51f6..c42579dc7f7 100644 --- a/tests/unit/proxy/mcp_server/test_mcp_server_manager.py +++ b/tests/unit/proxy/mcp_server/test_mcp_server_manager.py @@ -71,6 +71,7 @@ from litellm.proxy.mcp_server.legacy_callbacks import ElicitationCallback from litellm.proxy.mcp_server.mcp_context import active_mcp_request_ctx_var from litellm.proxy.mcp_server.mcp_server_manager import ( ListedToolsCaller, + MCPServerConfig, MCPServerManager, _deserialize_json_dict, _deserialize_json_list, @@ -1076,6 +1077,50 @@ class TestMCPServerManager: assert server.alias == "friendly_alias" assert server.server_name == "validserver" + @pytest.mark.asyncio + @pytest.mark.parametrize( + "allowed_entry,denied_entry,expect_warning", + [ + ("read_status", "read_status", True), + ("policy-read_status", "read_status", True), + ("read_status", "policy-read_status", True), + ("policy-read_status", "policy-read_status", True), + ("read_status", "READ_STATUS", False), + ("read_status", "delete_records", False), + ], + ) + async def test_config_tool_policy_overlap_warns_without_changing_lists( + self, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + allowed_entry: str, + denied_entry: str, + expect_warning: bool, + ) -> None: + monkeypatch.delenv("LITELLM_ENABLE_MCP_STDIO", raising=False) + manager: Final = MCPServerManager() + config: Final[dict[str, MCPServerConfig]] = { + "policy_server": { + "alias": "policy", + "transport": MCPTransport.stdio, + "command": "python", + "allowed_tools": [allowed_entry], + "disallowed_tools": [denied_entry], + } + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + await manager.load_servers_from_config(config) + + server: Final = next(iter(manager.config_mcp_servers.values())) + warnings: Final = tuple( + message for message in caplog.messages if "both allowed_tools and disallowed_tools" in message + ) + assert len(warnings) == int(expect_warning) + assert all("will be blocked" in message for message in warnings) + assert server.allowed_tools == [allowed_entry] + assert server.disallowed_tools == [denied_entry] + def _oauth2_config(self, **overrides): base = { "url": "https://example.com/mcp", @@ -6431,53 +6476,42 @@ class TestMCPServerManager: ) @pytest.mark.asyncio - async def test_pre_call_tool_check_allowed_tools_takes_precedence(self): - """Test that allowed_tools list takes precedence over disallowed_tools list""" - manager = MCPServerManager() - - # Create server with both allowed_tools and disallowed_tools - # Note: The logic in check_allowed_or_banned_tools prioritizes allowed_tools - server = MCPServer( + @pytest.mark.parametrize("tool_name,denied", [("tool1", False), ("tool2", True), ("tool3", True)]) + async def test_pre_call_tool_check_disallowed_tools_takes_precedence( + self, tool_name: str, denied: bool + ) -> None: + manager: Final = MCPServerManager() + server: Final = MCPServer( server_id="test-server", name="test-server", transport=MCPTransport.stdio, allowed_tools=["tool1", "tool2"], - disallowed_tools=["tool2", "tool3"], # tool2 is in both lists + disallowed_tools=["tool2", "tool3"], ) - - # Mock dependencies - set object_permission and object_permission_id to None - # so permission checks return None (no restrictions) user_api_key_auth: Final = UserAPIKeyAuth() - proxy_logging_obj = _mock_proxy_logging() - - # Mock the async methods that pre_call_tool_check calls - proxy_logging_obj.create_mcp_request_object_from_kwargs = MagicMock(return_value={}) - proxy_logging_obj.convert_mcp_to_llm_format = MagicMock(return_value={}) - proxy_logging_obj.pre_call_hook = AsyncMock(return_value={}) - - # tool2 should be allowed since it's in allowed_tools (takes precedence) - await manager.pre_call_tool_check( - name="tool2", - arguments={"param": "value"}, - server_name="test-server", - user_api_key_auth=user_api_key_auth, - proxy_logging_obj=proxy_logging_obj, - server=server, - ) - - # tool3 should be blocked since it's not in allowed_tools - with pytest.raises(HTTPException) as exc_info: + if not denied: await manager.pre_call_tool_check( - name="tool3", - arguments={"param": "value"}, + name=tool_name, + arguments={}, server_name="test-server", user_api_key_auth=user_api_key_auth, - proxy_logging_obj=proxy_logging_obj, + proxy_logging_obj=None, + server=server, + ) + return + + with pytest.raises(HTTPException) as exc_info: + await manager.pre_call_tool_check( + name=tool_name, + arguments={}, + server_name="test-server", + user_api_key_auth=user_api_key_auth, + proxy_logging_obj=None, server=server, ) assert exc_info.value.status_code == 403 - assert "Tool tool3 is not allowed for server test-server" in exc_info.value.detail["error"] + assert f"Tool {tool_name} is not allowed for server test-server" in exc_info.value.detail["error"] async def test_get_tools_from_server_add_prefix(self): """Verify _get_tools_from_server respects add_prefix True/False.""" diff --git a/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py b/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py index aa61deaf554..18907228b00 100644 --- a/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py +++ b/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py @@ -10941,7 +10941,7 @@ class TestListFiltersHonorThePrefixBoundary: assert _tool_name_matches("eiG-read_wiki_contents", ["deepwiki_cfg-read_wiki_contents"], server) - def test_discovery_hides_exactly_what_dispatch_refuses(self, monkeypatch): + def test_discovery_hides_exactly_what_dispatch_refuses(self, monkeypatch: pytest.MonkeyPatch) -> None: """Both halves of one decision, driven through both production paths. A spelling the blocklist enforces but the filter misses leaves a blocked @@ -10963,7 +10963,10 @@ class TestListFiltersHonorThePrefixBoundary: monkeypatch.setenv("LITELLM_USE_SHORT_MCP_TOOL_PREFIX", "true") - def _server(**overrides): + def _server( + allowed_tools: list[str] | None = None, + disallowed_tools: list[str] | None = None, + ) -> MCPServer: return MCPServer( server_id=self.SERVER_ID, name="deepwiki_prod", @@ -10972,27 +10975,35 @@ class TestListFiltersHonorThePrefixBoundary: short_prefix="eiG", url="http://127.0.0.1:5115/mcp", transport=MCPTransport.http, - **overrides, + allowed_tools=allowed_tools, + disallowed_tools=disallowed_tools, ) - manager = MCPServerManager() + manager: Final = MCPServerManager() manager._create_prefixed_tools( [MCPTool(name="read_wiki_contents", description="", inputSchema={"type": "object"})], _server(), ) - registered = sorted(manager.tool_name_to_mcp_server_name_mapping) + registered: Final = sorted(manager.tool_name_to_mcp_server_name_mapping) assert len(registered) > 1 - published = MCPTool(name="eiG-read_wiki_contents", description="", inputSchema={"type": "object"}) + published: Final = MCPTool(name="eiG-read_wiki_contents", description="", inputSchema={"type": "object"}) for spelling in registered: - for entry, expected in ((spelling, True), (spelling.upper(), False)): - server = _server(disallowed_tools=[entry]) + policies: Final[tuple[tuple[list[str] | None, list[str], bool], ...]] = ( + (None, [spelling], True), + (None, [spelling.upper()], False), + (["read_wiki_contents"], [spelling], True), + ([spelling], ["read_wiki_contents"], True), + ([spelling], [spelling.upper()], False), + ) + for allowed_tools, disallowed_tools, expected in policies: + server: Final = _server(allowed_tools, disallowed_tools) - refused = not manager.check_allowed_or_banned_tools("read_wiki_contents", server) - hidden = filter_tools_by_allowed_tools([published], server) == [] + refused: Final = not manager.check_allowed_or_banned_tools("read_wiki_contents", server) + hidden: Final = filter_tools_by_allowed_tools([published], server) == [] - assert refused == hidden, entry - assert refused is expected, entry + assert refused == hidden, (allowed_tools, disallowed_tools) + assert refused is expected, (allowed_tools, disallowed_tools) @pytest.mark.asyncio @pytest.mark.parametrize( From ad4b35d791bd79b142f25a022e9f8a77230f0798 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Tue, 27 Oct 2026 17:46:15 -0400 Subject: [PATCH 029/102] fix(mcp): accept YAML OpenAPI documents (#48622) --- .../mcp_server/openapi_to_mcp_generator.py | 101 +++++++++++- .../mcp_server/test_mcp_server_manager.py | 37 ++++- .../mcp_server/test_openapi_spec_path_url.py | 152 +++++++++++++++++- 3 files changed, 284 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/mcp_server/openapi_to_mcp_generator.py index 96d178dbeb1..0fa904cb582 100644 --- a/litellm/proxy/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/mcp_server/openapi_to_mcp_generator.py @@ -5,15 +5,29 @@ This module is used to generate MCP tools from OpenAPI specs. import asyncio import contextvars import json +import math import os import re from collections.abc import Mapping, Sequence from pathlib import PurePosixPath -from typing import Final, Literal, Protocol, TypeAlias, TypedDict, overload +from typing import ( + ClassVar, + Final, + Literal, + Protocol, + TypeAlias, + TypedDict, + cast, # noqa: TID251 # existing OpenAPI Protocol fields have no runtime schema + overload, +) from urllib.parse import quote import httpx +import yaml +from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly, Required +from yaml.events import AliasEvent, ScalarEvent +from yaml.nodes import MappingNode, Node, ScalarNode from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError from litellm.proxy.mcp_server.exceptions import ( @@ -30,6 +44,18 @@ _OPENAPI_TOOL_NAME_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_-]") OPENAPI_TOOL_NAME_MAX_LEN: Final = 128 _OPENAPI_TOOL_NAME_MAX_LEN: Final = OPENAPI_TOOL_NAME_MAX_LEN +_OPENAPI_DOCUMENT: Final = TypeAdapter(dict[str, object]) +_MAX_YAML_INT_LENGTH: Final = 1024 +# YAML 1.2 core scalars; isolate these from PyYAML's global YAML 1.1 resolver. +_YAML_SCALARS: Final = { + "tag:yaml.org,2002:null": re.compile(r"^(?:null|Null|NULL|~|)$"), + "tag:yaml.org,2002:bool": re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$"), + "tag:yaml.org,2002:int": re.compile(r"^(?:[-+]?[0-9]+|0o[0-7]+|0x[0-9a-fA-F]+)$"), + "tag:yaml.org,2002:float": re.compile( + r"^(?:[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)(?:[eE][-+]?[0-9]+)?" + r"|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN))$" + ), +} def sanitize_openapi_tool_name(raw_name: str) -> str: @@ -199,6 +225,70 @@ def load_openapi_spec(filepath: str) -> _OpenAPISpec: return asyncio.run(load_openapi_spec_async(filepath)) +class _OpenAPISafeLoader(yaml.SafeLoader): + yaml_implicit_resolvers: ClassVar[Mapping[str | None, Sequence[tuple[str, re.Pattern[str]]]]] = { # pyright: ignore[reportIncompatibleVariableOverride] # PyYAML resolver tables are class state + None: list(_YAML_SCALARS.items()) + } + + def compose_node(self, parent: Node | None, index: int) -> Node | None: + event: Final[object] = self.peek_event() # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # incomplete PyYAML event stubs + if isinstance(event, AliasEvent): + raise ValueError("YAML aliases are not supported in OpenAPI specs") # noqa: TRY004 # callers handle malformed documents as ValueError + node: Final = super().compose_node(parent, index) + # PyYAML otherwise implicitly resolves even an explicit nonspecific ! tag. + if isinstance(event, ScalarEvent) and event.tag == "!" and isinstance(node, ScalarNode): + node.tag = "tag:yaml.org,2002:str" + return node + + def construct_object(self, node: Node, deep: bool = False) -> object: + pattern: Final = _YAML_SCALARS.get(node.tag) + if pattern is not None: + if not isinstance(node, ScalarNode): + raise ValueError("Invalid YAML OpenAPI spec") + scalar: Final[str] = node.value + if node.tag == "tag:yaml.org,2002:int" and len(scalar) > _MAX_YAML_INT_LENGTH: + raise ValueError("YAML integer is too long in OpenAPI spec") + if pattern.fullmatch(scalar) is None: + raise ValueError("Invalid YAML OpenAPI spec") + if node.tag == "tag:yaml.org,2002:int": + # YAML 1.2 decimal values with leading zeroes are not octal. + return int(scalar, 0 if scalar.startswith(("0o", "0x")) else 10) + if node.tag == "tag:yaml.org,2002:float": + value: Final = self.construct_yaml_float(node) + if not math.isfinite(value): + raise ValueError("Invalid YAML OpenAPI spec") + return value + elif node.tag not in {"tag:yaml.org,2002:str", "tag:yaml.org,2002:seq", "tag:yaml.org,2002:map"} or ( + node.tag == "tag:yaml.org,2002:str" and not isinstance(node, ScalarNode) + ): + raise ValueError("Invalid YAML OpenAPI spec") + if isinstance(node, MappingNode): + pairs: Final[Sequence[tuple[Node, Node]]] = node.value + if any(not isinstance(key, ScalarNode) or key.tag != "tag:yaml.org,2002:str" for key, _ in pairs): + raise ValueError("Invalid YAML OpenAPI spec") + return super().construct_object( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # incomplete PyYAML stubs + node, deep + ) + + +def _validate_openapi_spec(value: object) -> _OpenAPISpec: + try: + document: Final = _OPENAPI_DOCUMENT.validate_python(value) + except ValidationError as exc: + raise ValueError("OpenAPI spec must be a JSON or YAML object") from exc + return cast(_OpenAPISpec, document) # cast-ok: preserve existing lazy OpenAPI Protocol validation + + +def _parse_openapi_spec(content: str | bytes) -> _OpenAPISpec: + try: + return _validate_openapi_spec(json.loads(content)) + except json.JSONDecodeError: + try: + return _validate_openapi_spec(yaml.load(content, Loader=_OpenAPISafeLoader)) + except (yaml.YAMLError, RecursionError) as exc: + raise ValueError("Invalid YAML OpenAPI spec") from exc + + async def load_openapi_spec_async(filepath: str, *, max_bytes: int | None = None) -> _OpenAPISpec: if filepath.startswith("http://") or filepath.startswith("https://"): client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP) @@ -208,14 +298,17 @@ async def load_openapi_spec_async(filepath: str, *, max_bytes: int | None = None else await async_safe_get(client, filepath, max_response_bytes=max_bytes) ) r.raise_for_status() - return r.json() + return await asyncio.to_thread(_parse_openapi_spec, r.content) # fallback: local file - # Local filesystem path + return await asyncio.to_thread(_load_openapi_file, filepath) + + +def _load_openapi_file(filepath: str) -> _OpenAPISpec: if not os.path.exists(filepath): raise FileNotFoundError(f"OpenAPI spec not found at {filepath}") with open(filepath, "r", encoding="utf-8") as f: - return json.load(f) + return _parse_openapi_spec(f.read()) def get_base_url(spec: _OpenAPISpec, spec_path: str | None = None) -> str: diff --git a/tests/unit/proxy/mcp_server/test_mcp_server_manager.py b/tests/unit/proxy/mcp_server/test_mcp_server_manager.py index c42579dc7f7..ac79c10869b 100644 --- a/tests/unit/proxy/mcp_server/test_mcp_server_manager.py +++ b/tests/unit/proxy/mcp_server/test_mcp_server_manager.py @@ -5405,7 +5405,42 @@ class TestMCPServerManager: ( httpx.Response(200, text="secret invalid JSON body"), "unhealthy", - "OpenAPI specification could not be loaded (JSONDecodeError)", + "OpenAPI specification could not be loaded (ValueError)", + ), + ( + httpx.Response(200, text="paths: [secret invalid YAML body"), + "unhealthy", + "OpenAPI specification could not be loaded (ValueError)", + ), + ( + httpx.Response(200, text="paths: {}\nx: \x01"), + "unhealthy", + "OpenAPI specification could not be loaded (ValueError)", + ), + ( + httpx.Response(200, text="x: &a []\ny: *a"), + "unhealthy", + "OpenAPI specification could not be loaded (ValueError)", + ), + ( + httpx.Response(200, text="x: !!bool invalid"), + "unhealthy", + "OpenAPI specification could not be loaded (ValueError)", + ), + ( + httpx.Response(200, text="x: !!int ''"), + "unhealthy", + "OpenAPI specification could not be loaded (ValueError)", + ), + ( + httpx.Response(200, text="x: !!float ''"), + "unhealthy", + "OpenAPI specification could not be loaded (ValueError)", + ), + ( + httpx.Response(200, text="x: !!timestamp invalid"), + "unhealthy", + "OpenAPI specification could not be loaded (ValueError)", ), ], ) diff --git a/tests/unit/proxy/mcp_server/test_openapi_spec_path_url.py b/tests/unit/proxy/mcp_server/test_openapi_spec_path_url.py index aac85160eba..0f29a3bf20d 100644 --- a/tests/unit/proxy/mcp_server/test_openapi_spec_path_url.py +++ b/tests/unit/proxy/mcp_server/test_openapi_spec_path_url.py @@ -1,9 +1,15 @@ from __future__ import annotations -from typing import Any, Dict +import asyncio +import json +import threading +from pathlib import Path +from typing import Any, Dict, Final import httpx import pytest +import yaml +from respx import MockRouter from litellm.proxy.mcp_server import openapi_to_mcp_generator as gen @@ -98,3 +104,147 @@ def test_load_openapi_spec_supports_local_file_path( spec = gen.load_openapi_spec(str(p)) assert spec == expected + + +@pytest.mark.parametrize("source", ["url", "file"]) +@pytest.mark.parametrize("suffix", [".yaml", ".json", ""]) +@pytest.mark.parametrize("document_format", ["json", "yaml"]) +@pytest.mark.parametrize("use_async", [False, True]) +def test_openapi_formats_preserve_document_across_sources( + tmp_path: Path, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch, + source: str, suffix: str, document_format: str, use_async: bool, +) -> None: + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + expected: Final = { + "openapi": "3.0.0", + "paths": {"/ping": {"get": { + "operationId": "on", + "parameters": [ + {"name": "on", "in": "query", "schema": {"type": "string"}}, + {"name": "off", "in": "query", "schema": {"type": "string"}}, + ], + }}}, + "x-values": [100.0, "on", "off", "yes", "no", "2026-01-01", "1:20", True, False, None, 12, 10, 10], + } + content: Final = ( + json.dumps(expected).replace("100.0", "1e2") + if document_format == "json" + else "openapi: 3.0.0\npaths:\n /ping:\n get:\n operationId: on\n" + " parameters:\n" + " - {name: on, in: query, schema: {type: string}}\n" + " - {name: off, in: query, schema: {type: string}}\n" + "x-values: [1e2, on, off, yes, no, 2026-01-01, 1:20, TRUE, False, null, 012, 0o12, 0xA]\n" + ) + path: Final = tmp_path / ("spec" + suffix) + path.write_text(content, encoding="utf-8") + target: Final = str(path) if source == "file" else "https://93.184.216.34/spec" + suffix + if source == "url": + respx_mock.get(target).respond(200, text=content, headers={"content-type": "text/plain"}) + + spec: Final = asyncio.run(gen.load_openapi_spec_async(target)) if use_async else gen.load_openapi_spec(target) + + assert spec == expected + operation: Final = spec["paths"]["/ping"]["get"] + assert gen.sanitize_openapi_tool_name(operation["operationId"]) == "on" + assert gen.build_input_schema(operation) == { + "type": "object", + "properties": {"on": {"type": "string", "description": ""}, "off": {"type": "string", "description": ""}}, + "required": [], + } + + +_LONG_YAML_INTEGER: Final = "1" * 1025 + + +@pytest.mark.parametrize("content,message", [ + ("", "OpenAPI spec must be a JSON or YAML object"), + ("null", "OpenAPI spec must be a JSON or YAML object"), + ("[]", "OpenAPI spec must be a JSON or YAML object"), + ("- item", "OpenAPI spec must be a JSON or YAML object"), + ("plain text", "OpenAPI spec must be a JSON or YAML object"), + ("paths: [", "Invalid YAML OpenAPI spec"), + ("paths: {}\nx: \x01", "Invalid YAML OpenAPI spec"), + ("paths: " + "[" * 600 + "0" + "]" * 600, "Invalid YAML OpenAPI spec"), + ("x: !!python/object/apply:builtins.str [unsafe]", "Invalid YAML OpenAPI spec"), + ("x: &a []\ny: *a", "YAML aliases are not supported in OpenAPI specs"), + ("x: " + _LONG_YAML_INTEGER, "YAML integer is too long in OpenAPI spec"), + ("x: !!int " + _LONG_YAML_INTEGER, "YAML integer is too long in OpenAPI spec"), + ("x: !!int {=: '" + _LONG_YAML_INTEGER + "'}", "Invalid YAML OpenAPI spec"), + ("x: !!int [1]", "Invalid YAML OpenAPI spec"), + ("x: !!bool invalid", "Invalid YAML OpenAPI spec"), + ("x: !!bool []", "Invalid YAML OpenAPI spec"), + ("x: !!str []", "Invalid YAML OpenAPI spec"), + ("x: !!int ''", "Invalid YAML OpenAPI spec"), + ("x: !!float ''", "Invalid YAML OpenAPI spec"), + ("x: !!float {=: 1.2}", "Invalid YAML OpenAPI spec"), + ("x: !!timestamp invalid", "Invalid YAML OpenAPI spec"), + ("x: !!timestamp 2026-01-01", "Invalid YAML OpenAPI spec"), + ("x: !!set {a: null}", "Invalid YAML OpenAPI spec"), + ("x: !!binary YQ==", "Invalid YAML OpenAPI spec"), + ("x: {1: value}", "Invalid YAML OpenAPI spec"), + ("x: {[key]: value}", "Invalid YAML OpenAPI spec"), + ("x: .inf", "Invalid YAML OpenAPI spec"), + ("x: .nan", "Invalid YAML OpenAPI spec"), + ("x: 1e999", "Invalid YAML OpenAPI spec"), +], ids=[ + "empty", "null", "json-array", "yaml-sequence", "scalar", "syntax", "reader", "nesting", + "python-tag", "alias", "implicit-int", "explicit-int", "mapping-int", "sequence-int", + "invalid-bool", "sequence-bool", "sequence-str", "empty-int", "empty-float", "mapping-float", "invalid-timestamp", + "timestamp", "set", "binary", "non-string-key", "complex-key", "infinity", "nan", "float-overflow", +]) +def test_openapi_loader_rejects_invalid_or_unsafe_documents( + tmp_path: Path, content: str, message: str, +) -> None: + path: Final = tmp_path / "spec.yaml" + path.write_text(content, encoding="utf-8") + + with pytest.raises(ValueError, match="^" + message + "$"): + gen.load_openapi_spec(str(path)) + + +def test_openapi_yaml_integer_boundary_preserves_quoted_strings(tmp_path: Path) -> None: + integer: Final = "1" * 1024 + path: Final = tmp_path / "spec.yaml" + path.write_text("x-integer: " + integer + "\nx-string: '" + integer + "1'", encoding="utf-8") + + assert gen.load_openapi_spec(str(path)) == {"x-integer": int(integer), "x-string": integer + "1"} + + +def test_openapi_json_integer_preserved_even_with_yaml_extension(tmp_path: Path) -> None: + path: Final = tmp_path / "spec.yaml" + path.write_text('{"x-integer":' + _LONG_YAML_INTEGER + "}", encoding="utf-8") + + assert gen.load_openapi_spec(str(path)) == {"x-integer": int(_LONG_YAML_INTEGER)} + + +@pytest.mark.parametrize("value", ["! " + _LONG_YAML_INTEGER, "! '" + _LONG_YAML_INTEGER + "'"], ids=["plain", "quoted"]) +def test_openapi_nonspecific_tag_disables_scalar_resolution(tmp_path: Path, value: str) -> None: + path: Final = tmp_path / "spec.yaml" + path.write_text("x: " + value, encoding="utf-8") + + assert gen.load_openapi_spec(str(path)) == {"x": _LONG_YAML_INTEGER} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("source", ["url", "file"]) +async def test_openapi_yaml_parses_off_the_event_loop( + tmp_path: Path, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch, source: str, +) -> None: + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + event_loop_thread: Final = threading.get_ident() + parser_threads: Final[list[int]] = [] + original_load: Final = yaml.load + + def observed_load(stream: str | bytes, Loader: type[yaml.SafeLoader]) -> object: + parser_threads.append(threading.get_ident()) + return original_load(stream, Loader=Loader) + + monkeypatch.setattr(yaml, "load", observed_load) + path: Final = tmp_path / "spec.yaml" + path.write_text("paths: {}", encoding="utf-8") + target: Final = str(path) if source == "file" else "https://93.184.216.34/spec.yaml" + if source == "url": + respx_mock.get(target).respond(200, text="paths: {}") + + assert await gen.load_openapi_spec_async(target) == {"paths": {}} + assert len(parser_threads) == 1 and parser_threads[0] != event_loop_thread From 2a9c63018242f014b5a7416a1e7b065144d10b1e Mon Sep 17 00:00:00 2001 From: tin-berri Date: Tue, 27 Oct 2026 17:47:04 -0400 Subject: [PATCH 030/102] fix(proxy): release concurrency slots when synthetic streams complete (#48628) --- litellm/proxy/common_request_processing.py | 7 +- litellm/proxy/utils.py | 21 +--- .../agent_endpoints/test_a2a_endpoints.py | 3 +- .../test_streaming_model_restamp.py | 53 ++++---- .../hooks/test_parallel_request_limiter_v3.py | 18 ++- tests/unit/proxy/test_budget_reservation.py | 3 +- .../proxy/test_common_request_processing.py | 117 ++++++++++++++++++ 7 files changed, 168 insertions(+), 54 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 3b9c17baa02..740d8742e5a 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -4355,6 +4355,8 @@ class ProxyBaseLLMRequestProcessing: proxy_logging_obj: ProxyLogging | None = None, ) -> None: with anyio.CancelScope(shield=True): + if stream_completed and proxy_logging_obj is not None and user_api_key_dict is not None: + await proxy_logging_obj.arelease_max_parallel_requests(user_api_key_dict) should_record_client_disconnect: Final = client_disconnected or (not stream_completed) recorded_client_disconnect = False if should_record_client_disconnect: @@ -4369,10 +4371,7 @@ class ProxyBaseLLMRequestProcessing: # A disconnect-time success event (the deferred-guardrail flush # above, or the partial-spend billing below) releases the # request's max_parallel_requests slot through the limiter's - # own success callback. Release the slot explicitly only when - # no such event fires, so exactly one release happens; two - # concurrent releases would race and double-decrement under the - # limiter's in-memory fallback. + # own success callback. Release explicitly when no event fires. success_event_owns_slot_release = deferred_stream_logging_armed if not deferred_stream_logging_armed: success_event_owns_slot_release = await _bill_partial_streamed_spend_on_disconnect( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index d3202368d76..452434e8c81 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -4942,31 +4942,22 @@ class ProxyLogging: logging_obj._deferred_stream_complete_args = None return True - async def arelease_max_parallel_requests_on_disconnect( + async def arelease_max_parallel_requests( self, user_api_key_dict: UserAPIKeyAuth, ) -> None: """ - Release the api-key max_parallel_requests slot when a streaming - response is cancelled mid-flight (client disconnect) and no logging - callback fired for it. Neither the success nor failure callback runs on - the resulting CancelledError / GeneratorExit, so the pre-call +1 would - otherwise leak. - - Awaited from the shielded streaming cleanup rather than scheduled - fire-and-forget, so the caller can make it the single owner of the - release: when a disconnect-time success event does fire (partial-spend - billing or a deferred-guardrail flush), that event's own limiter - callback releases the slot and this is not called at all. Two - concurrent releases of the same acquisition would otherwise race and - double-decrement under the limiter's in-memory fallback. + Release an ended request's v3 concurrency acquisition independently of logging. + The request stash serializes release with callbacks and clears the acquisition, + so repeated cleanup cannot release another request's slot. """ limiter: Final = self.get_proxy_hook("parallel_request_limiter") if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3): return await limiter.async_release_max_parallel_requests_on_disconnect(user_api_key_dict) - _arelease_max_parallel_requests_on_disconnect = arelease_max_parallel_requests_on_disconnect + arelease_max_parallel_requests_on_disconnect = arelease_max_parallel_requests + _arelease_max_parallel_requests_on_disconnect = arelease_max_parallel_requests async def arelease_responses_websocket_slot(self, user_api_key_dict: UserAPIKeyAuth) -> None: """Release the slot a Responses WebSocket took at admission. Only call this once that admission succeeded: diff --git a/tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py b/tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py index a2448694dd7..a7d7ac5627c 100644 --- a/tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py +++ b/tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py @@ -1058,6 +1058,7 @@ async def test_subscribe_to_task_calls_pre_call_hook(): """tasks/resubscribe must run pre_call_hook so guardrails configured on the agent are enforced before streaming begins.""" from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.utils import ProxyLogging agent = _make_agent_mock() mock_request = _make_request_mock("tasks/resubscribe", {"id": "task-1"}) @@ -1083,7 +1084,7 @@ async def test_subscribe_to_task_calls_pre_call_hook(): async for chunk in response: yield chunk - mock_proxy_logging = MagicMock() + mock_proxy_logging: Final = MagicMock(spec=ProxyLogging) mock_proxy_logging.pre_call_hook = AsyncMock( side_effect=lambda user_api_key_dict, data, call_type, skip_guardrails=False: data ) diff --git a/tests/unit/proxy/anthropic_endpoints/test_streaming_model_restamp.py b/tests/unit/proxy/anthropic_endpoints/test_streaming_model_restamp.py index b7bc670c7f8..85bb8304b95 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_streaming_model_restamp.py +++ b/tests/unit/proxy/anthropic_endpoints/test_streaming_model_restamp.py @@ -3,15 +3,18 @@ Tests for restamping the public model on Anthropic Messages streaming chunks. """ import json -from unittest.mock import AsyncMock, MagicMock +from collections.abc import AsyncIterator, Sequence import pytest +from litellm.caching.caching import DualCache +from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.streaming_model_restamp import ( AnthropicStreamModelRestamper, restamp_anthropic_stream_chunk_model, ) from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.utils import ProxyLogging def _message_start_frame(model: str, line_end: str = "\n") -> bytes: @@ -22,15 +25,9 @@ def _message_start_frame(model: str, line_end: str = "\n") -> bytes: return f"event: message_start{line_end}data: {json.dumps(payload)}{line_end}{line_end}".encode() -def _proxy_logging_obj_streaming(frames: list[bytes]) -> MagicMock: - async def _iterator_hook(**_kwargs): - for frame in frames: - yield frame - - proxy_logging_obj = MagicMock() - proxy_logging_obj.async_post_call_streaming_iterator_hook = _iterator_hook - proxy_logging_obj.async_post_call_streaming_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["response"]) - return proxy_logging_obj +async def _stream_frames(frames: Sequence[bytes]) -> AsyncIterator[bytes]: + for frame in frames: + yield frame def _model_from_frame(frame: bytes | str) -> str: @@ -76,15 +73,14 @@ def test_leaves_chunks_without_a_model_untouched(chunk): async def test_sse_generator_publishes_requested_model_on_message_start(): """The message_start event reports the requested model, not the provider's.""" delta_frame = b'event: content_block_delta\ndata: {"type":"content_block_delta","delta":{"text":"hi"}}\n\n' - proxy_logging_obj = _proxy_logging_obj_streaming([_message_start_frame("claude-haiku-4-5-20251001"), delta_frame]) chunks = [ chunk async for chunk in ProxyBaseLLMRequestProcessing.async_sse_data_generator( - response=MagicMock(), - user_api_key_dict=MagicMock(), + response=_stream_frames((_message_start_frame("claude-haiku-4-5-20251001"), delta_frame)), + user_api_key_dict=UserAPIKeyAuth(), request_data={"model": "claude-auto-1"}, - proxy_logging_obj=proxy_logging_obj, + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), restamp_model="claude-auto-1", ) ] @@ -95,15 +91,13 @@ async def test_sse_generator_publishes_requested_model_on_message_start(): @pytest.mark.asyncio async def test_sse_generator_keeps_provider_model_when_restamping_is_off(): - proxy_logging_obj = _proxy_logging_obj_streaming([_message_start_frame("claude-haiku-4-5-20251001")]) - chunks = [ chunk async for chunk in ProxyBaseLLMRequestProcessing.async_sse_data_generator( - response=MagicMock(), - user_api_key_dict=MagicMock(), + response=_stream_frames((_message_start_frame("claude-haiku-4-5-20251001"),)), + user_api_key_dict=UserAPIKeyAuth(), request_data={"model": "claude-auto-1"}, - proxy_logging_obj=proxy_logging_obj, + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), ) ] @@ -175,15 +169,14 @@ def test_dict_message_start_disarms_after_restamp(): @pytest.mark.asyncio async def test_sse_generator_restamps_message_start_split_across_chunks(): frame = _message_start_frame("claude-haiku-4-5-20251001") - proxy_logging_obj = _proxy_logging_obj_streaming([frame[:30], frame[30:]]) chunks = [ chunk async for chunk in ProxyBaseLLMRequestProcessing.async_sse_data_generator( - response=MagicMock(), - user_api_key_dict=MagicMock(), + response=_stream_frames((frame[:30], frame[30:])), + user_api_key_dict=UserAPIKeyAuth(), request_data={"model": "claude-auto-1"}, - proxy_logging_obj=proxy_logging_obj, + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), restamp_model="claude-auto-1", ) ] @@ -250,15 +243,14 @@ def test_flush_disarms_the_restamper(): @pytest.mark.asyncio async def test_sse_generator_flushes_held_tail_at_end_of_stream(): unterminated = _message_start_frame("claude-haiku-4-5-20251001")[:-2] - proxy_logging_obj = _proxy_logging_obj_streaming([unterminated]) chunks = [ chunk async for chunk in ProxyBaseLLMRequestProcessing.async_sse_data_generator( - response=MagicMock(), - user_api_key_dict=MagicMock(), + response=_stream_frames((unterminated,)), + user_api_key_dict=UserAPIKeyAuth(), request_data={"model": "claude-auto-1"}, - proxy_logging_obj=proxy_logging_obj, + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), restamp_model="claude-auto-1", ) ] @@ -271,15 +263,14 @@ async def test_sse_generator_flushes_held_tail_at_end_of_stream(): async def test_sse_generator_restamps_crlf_stream(): frame = _message_start_frame("claude-haiku-4-5-20251001", line_end="\r\n") delta = b'event: content_block_delta\r\ndata: {"type":"content_block_delta","delta":{"text":"hi"}}\r\n\r\n' - proxy_logging_obj = _proxy_logging_obj_streaming([frame, delta]) chunks = [ chunk async for chunk in ProxyBaseLLMRequestProcessing.async_sse_data_generator( - response=MagicMock(), - user_api_key_dict=MagicMock(), + response=_stream_frames((frame, delta)), + user_api_key_dict=UserAPIKeyAuth(), request_data={"model": "claude-auto-1"}, - proxy_logging_obj=proxy_logging_obj, + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), restamp_model="claude-auto-1", ) ] diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index b8dba5e1e68..cd0d786f841 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -4378,9 +4378,12 @@ async def test_mcp_tool_call_reusing_request_slot_keeps_request_max_parallel_hea @pytest.mark.asyncio -async def test_concurrent_success_callbacks_release_parallel_slot_once_when_redis_fails() -> None: +@pytest.mark.parametrize("release_from", ["post_call", "stream_cleanup"]) +async def test_concurrent_success_callbacks_release_parallel_slot_once_when_redis_fails(release_from: str) -> None: from unittest.mock import AsyncMock + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + api_key = hash_token("sk-concurrent-release") local_cache = DualCache() handler = _PROXY_MaxParallelRequestsHandler( @@ -4409,9 +4412,20 @@ async def test_concurrent_success_callbacks_release_parallel_slot_once_when_redi counter_keys=[parallel_key], ) data = {"litellm_call_id": call_id} + logger: Final = ProxyLogging(user_api_key_cache=local_cache) + logger.proxy_hook_mapping["parallel_request_limiter"] = handler post_call_task = asyncio.create_task( - handler.async_post_call_success_hook( + ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup( + request=None, + request_data=data, + response=None, + stream_completed=True, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=logger, + ) + if release_from == "stream_cleanup" + else handler.async_post_call_success_hook( data=data, user_api_key_dict=user_api_key_dict, response=ResponsesAPIResponse( diff --git a/tests/unit/proxy/test_budget_reservation.py b/tests/unit/proxy/test_budget_reservation.py index 42b41a2e3f2..535c3da8c3a 100644 --- a/tests/unit/proxy/test_budget_reservation.py +++ b/tests/unit/proxy/test_budget_reservation.py @@ -4,6 +4,7 @@ import time from collections.abc import Mapping, Sequence from datetime import datetime, timedelta, timezone from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -3233,7 +3234,7 @@ async def test_streaming_slow_path_processes_and_yields_chunk(spend_counter_stat async def one_chunk(user_api_key_dict, response, request_data): yield {"content": "hi"} - streaming_logging_obj = MagicMock() + streaming_logging_obj: Final = MagicMock(spec=ProxyLogging) streaming_logging_obj.async_post_call_streaming_iterator_hook = one_chunk streaming_logging_obj.async_post_call_streaming_hook = AsyncMock( side_effect=lambda **kwargs: kwargs["response"] diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 5c99e5a01ea..52816dc50a8 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -1,4 +1,5 @@ import asyncio +import contextvars import copy import datetime import json @@ -12055,6 +12056,122 @@ class TestResponseCostFromLoggingObjForCacheHits: assert cost == "" +@pytest.mark.asyncio +@pytest.mark.parametrize("surface", ["messages", "openai"]) +@pytest.mark.parametrize("keepalive", ["disabled", "fast", "pinged"]) +async def test_completed_synthetic_stream_reuses_capacity_without_freeing_another_request( + surface: Literal["messages", "openai"], + keepalive: Literal["disabled", "fast", "pinged"], + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( + FakeAnthropicMessagesStreamIterator, + ) + from litellm.proxy import proxy_server + from litellm.proxy.common_utils.sse_keepalive import SSE_COMMENT_PING_BYTES + from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 + from litellm.proxy.utils import InternalUsageCache, hash_token + from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse + + def clock() -> datetime.datetime: + return datetime.datetime(2026, 1, 1) + + cache: Final = DualCache() + limiter: Final = PROXY_MaxParallelRequestsHandler_v3( + internal_usage_cache=InternalUsageCache(cache), time_provider=clock + ) + logger: Final = ProxyLogging(user_api_key_cache=cache) + logger.proxy_hook_mapping["parallel_request_limiter"] = limiter + auth: Final = ProxyUserAPIKeyAuth(api_key=hash_token("sk-synthetic-stream"), max_parallel_requests=2) + counter: Final = f"{{api_key:{auth.api_key}}}:max_parallel_requests" + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", logger) + + async def admit(call_id: str) -> None: + await limiter.async_pre_call_hook( + user_api_key_dict=auth, + cache=cache, + data={"model": "test-model", "litellm_call_id": call_id}, + call_type="anthropic_messages", + ) + + async def request(call_id: str) -> None: + upstream_answers: Final = asyncio.Event() + if keepalive != "pinged": + upstream_answers.set() + + async def produce() -> StreamingResponse: + await admit(call_id) + with pytest.raises(HTTPException) as rejected: + await contextvars.Context().run(asyncio.create_task, admit("overlapping")) + assert rejected.value.status_code == 429 + await upstream_answers.wait() + body: Final = AnthropicMessagesResponse( + id=call_id, + type="message", + role="assistant", + model="test-model", + content=[{"type": "text", "text": "Search completed"}], + stop_reason="end_turn", + stop_sequence=None, + usage={"input_tokens": 0, "output_tokens": 0}, + ) + source: Final = FakeAnthropicMessagesStreamIterator(body) + data: Final = {"model": "test-model", "litellm_call_id": call_id} + stream: Final = ( + ProxyBaseLLMRequestProcessing.async_sse_data_generator(source, auth, data, logger) + if surface == "messages" + else proxy_server.async_data_generator(source, auth, data) + ) + return StreamingResponse(stream) + + response: Final = await open_sse_before_first_byte( + produce(), None if keepalive == "disabled" else 0.001 if keepalive == "pinged" else 60.0 + ) + first: Final = await anext(response.body_iterator) + if keepalive == "pinged": + assert first == SSE_COMMENT_PING_BYTES + upstream_answers.set() + frames: Final = (first, *tuple([frame async for frame in response.body_iterator])) + wire: Final = b"".join(frame.encode() if isinstance(frame, str) else frame for frame in frames) + assert b'"type": "message_stop"' in wire + + await contextvars.Context().run(asyncio.create_task, admit("still-streaming")) + for index in range(4): + await contextvars.Context().run(asyncio.create_task, request(f"completed-{index}")) + occupied: Final = await cache.async_get_cache(counter) + assert isinstance(occupied, dict) and len(occupied) == 1, occupied + + +@pytest.mark.asyncio +async def test_completed_stream_cleanup_does_not_release_an_unowned_legacy_slot() -> None: + from litellm.proxy.hooks.parallel_request_limiter import PROXY_MaxParallelRequestsHandler + from litellm.proxy.utils import InternalUsageCache, hash_token + + def clock() -> datetime.datetime: + return datetime.datetime(2026, 1, 1) + + cache: Final = DualCache() + limiter: Final = PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache), clock=clock) + logger: Final = ProxyLogging(user_api_key_cache=cache) + logger.proxy_hook_mapping["parallel_request_limiter"] = limiter + auth: Final = ProxyUserAPIKeyAuth(api_key=hash_token("sk-legacy-stream"), max_parallel_requests=1) + await limiter.async_pre_call_hook(auth, cache, {}, "acompletion") + + await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup( + request=None, + request_data={}, + response=None, + stream_completed=True, + user_api_key_dict=auth, + proxy_logging_obj=logger, + ) + + with pytest.raises(HTTPException) as rejected: + await limiter.async_pre_call_hook(auth, cache, {}, "acompletion") + assert rejected.value.status_code == 429 + + def test_maybe_get_model_id_reads_the_chat_metadata_the_router_updated_in_place(): processing_obj: Final = ProxyBaseLLMRequestProcessing(data={"metadata": {"model_info": {"id": "declared-text"}}}) From a53ea5bd7784c51e4c0b726efe55de44c56c8b5d Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Tue, 27 Oct 2026 14:49:07 -0700 Subject: [PATCH 031/102] fix(proxy): keep every startup line JSON when json_logs is on (#47035) * fix(proxy): keep every startup line JSON when json_logs is on Boot-time messages the proxy printed straight to stdout (banner, feedback box, model list, cache and callback notices, policy table, migration notices, CLI launch notes) and the first records of uvicorn- or granian-spawned workers came out as plain text while json_logs was on, breaking log pipelines that parse every line as JSON. - print_startup_message() prints unchanged in plain mode and, with json_logs on, emits one INFO record on the proxy logger (ANSI stripped) that skips the logger and handler level checks like the print it replaces; show_banner() routes to it in JSON mode only - the CLI exports JSON_LOGS=true when the config sets json_logs so spawned workers log JSON from their first record - the --enforce_prisma_migration_check deprecation notice prints after the config is read so a config-only json_logs covers it Resolves LIT-1624 * fix(proxy): keep show_banner() SDK-free, pick the JSON banner in initialize() Since #46803 the `lite` shell calls show_banner() against an uninitialized litellm package, and reading litellm.json_logs inside it initialized the whole SDK at the banner (0.14 s to 0.83 s for a bare `lite` here, the seconds that PR removed on small VMs). show_banner() is back to its main version; the proxy's initialize() now emits the JSON banner record itself and calls show_banner() only in plain mode. The banner unit tests move to the initialize() call site. * fix(proxy): keep the JSON banner when an operator --log_config names the proxy logger uvicorn applies --log_config after proxy_cli has switched JSON logging on and before initialize() switches it on again through load_config(). A logging config that names "LiteLLM Proxy" removes that logger's handlers in the window between the two, which is where initialize() emits the banner and the feedback box, so at one worker both went nowhere; two-worker children import litellm after uvicorn's logging setup and were not affected. print_startup_message() now falls back to a one-off JSON handler, on the stream the JSON handler uses, when the proxy logger has no handler of its own. Live, json_logs on plus a log_config naming the proxy logger, one worker: main prints the banner and the feedback box as plaintext, e2303a3925 prints neither, this head prints both as one JSON record each. New unit test fails on e2303a3925; new integration cell test_json_logs_with_operator_log_config_keeps_every_line_json passes at one and two workers. * fix(proxy): keep a long startup listing whole in its JSON record print_startup_message ran its record through the proxy logger's filters, so in JSON mode the required-migration SQL list and the policy list were middle-truncated at MAX_STRING_LENGTH_STDOUT_LOG (4096 by default) with the litellm_truncated marker, where the prints they replace showed them whole. That cap bounds echoed request payloads; a startup message is operator text, so the record now gets the uncapped processing of a DEBUG line: secret redaction and base64 collapse still apply, the text cap does not. Red on 41f92b745a, green here: - tests/unit/test_logging.py: a listing longer than the cap comes out whole; a credential in a startup message is still redacted - tests/integration/configuration/test_json_logs_startup.py: a 60-policy config boots with json_logs on and the policy listing record (9762 chars) carries every policy with no truncation marker (4096 chars, marker, on the unfixed tree) * fix(proxy): keep the --enforce_prisma_migration_check notice on --health and --test This PR moved the deprecation notice after the config is read, so that a json_logs set only in the config applies to it; --health and --test return before that point and stopped printing it, a visible change main did not have. The notice is now also emitted before those two returns, through the same helper, so a deployment running `litellm --health --enforce_prisma_migration_check` still learns the flag is dead, as on main. A normal server start emits it once, after the config, as in the previous commit. Red on 2d7a2d6844, green here: tests/unit/proxy/test_proxy_cli.py asserts the notice on --health and on --test. * fix(proxy): keep the JSON startup records on stdout under an operator handler on the proxy logger An operator --log_config that gives "LiteLLM Proxy" a handler of its own, a NullHandler that silences it or a plain WARNING-level handler on stderr, is in force in the window where the banner, the feedback box and the cache notice are emitted: uvicorn applies it after the CLI process switched JSON on and before initialize() re-installs the JSON handler through load_config(). 41f92b745a sent the startup records through the proxy logger's handlers and fell back to a JSON stdout handler only when there were none, so under such a config the records landed in the operator's handler, as plaintext on stderr, or vanished, where the prints they replace reached stdout whatever the logging config said. print_startup_message() now writes every startup record through the proxy's own JSON handler on stdout, filters included, and never through the proxy logger's handlers; the operator's handler gets nothing from a startup message, as it got nothing from the print. The normal JSON handler is the same class, formatter and filters, so the records a pipeline reads are unchanged when no operator config is in play. Tests: tests/unit/test_logging.py gets the two handler shapes (red on f6f5957d16: the record went to stderr as plaintext or nowhere); tests/integration/configuration/ test_json_logs_startup.py gets a silencing operator --log_config cell at 1 and 2 workers (red on f6f5957d16: banner, feedback box and cache notice missing; green here). * fix(proxy): print the --enforce_prisma_migration_check notice when the config fails to load f6f5957d16 printed the deprecation notice after the config is read, so a json_logs set only in the config makes it a JSON record. A boot whose config file is missing or does not parse then exited before that point and never printed it, where main printed it before the error (10-20 gauntlet finding). The notice is printed where main printed it, unless a config file can still switch json_logs on: then it is held until the config is read and printed right after the json_logs switch, and a click close callback prints it on the way out if the boot fails first. functools.cache keeps it to one print either way. Tests: tests/unit/proxy/test_proxy_cli.py covers a failed config read (red on 75043aed83), a plain config printing it once and a config-only json_logs making it one JSON record; tests/integration/configuration/test_json_logs_startup.py adds the JSON cell and a missing and an unparseable config cell (red on 75043aed83, green here), through refused_boot_log, which now takes extra CLI arguments. * fix(proxy): keep startup records as printed instead of running secret redaction In JSON mode the startup messages ran secret redaction, which the prints they replace never had. Its sk- key pattern has no left boundary, so a model named risk-assessment-gpt4o was logged as riREDACTED in the model list (10-21 gauntlet finding). The records now skip redaction like the prints did; the processed record still counts as scrubbed, so the handler's secret filter and the formatter leave it alone. The stdout cap stays off and base64 collapse stays on, as before. Tests: tests/unit/test_logging.py checks the model name comes out whole (red on 7d17f126e5), replacing the test that asserted startup records were redacted; tests/integration/configuration/test_json_logs_startup.py adds a boot with a risk-assessment- model whose model list record carries the name (red on 7d17f126e5). * refactor(proxy): meet the AGENTS.md rules Greptile flagged on the startup-message change The policy listing builds its rows as tuples in two helpers instead of appending to a list; plain output is unchanged (same bytes as before across 3000 random policy and attachment configs). The explanatory comments added with this change are gone, and the docstring of print_startup_message now says the record is whole and unredacted. Tests: the migration-diff JSON test no longer runs the fake prisma CLI; it checks the skipped-diff notice in-process, and the fixture extension it needed is reverted. The json_logs export and deprecation-notice CLI tests run the real proxy module and a real config file instead of a mocked ProxyConfig and proxy module, and the --health/--test notice tests no longer patch our helpers. Each still fails when the export, the hold or call_on_close is removed. New test parameters are typed. --------- Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- litellm/_logging.py | 44 ++ litellm/proxy/db/check_migration.py | 19 +- litellm/proxy/policy_engine/init_policies.py | 111 ++--- litellm/proxy/proxy_cli.py | 113 +++-- litellm/proxy/proxy_server.py | 70 +-- tests/integration/_support/process.py | 2 + .../configuration/test_json_logs_startup.py | 397 ++++++++++++++++++ tests/unit/proxy/db/test_check_migration.py | 36 ++ .../proxy/policy_engine/test_init_policies.py | 116 +++++ tests/unit/proxy/test_proxy_cli.py | 138 ++++++ ...test_proxy_server_endpoints_and_startup.py | 62 ++- tests/unit/test_logging.py | 220 ++++++++++ 12 files changed, 1188 insertions(+), 140 deletions(-) create mode 100644 tests/integration/configuration/test_json_logs_startup.py create mode 100644 tests/unit/proxy/policy_engine/test_init_policies.py diff --git a/litellm/_logging.py b/litellm/_logging.py index d08254db3fd..8974f85f517 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -1,6 +1,7 @@ import ast import contextvars import functools +import inspect import itertools import logging import os @@ -897,6 +898,49 @@ verbose_proxy_logger.addFilter(_diagnostic_filter) verbose_proxy_stdout_logger.addFilter(_diagnostic_filter) verbose_logger.addFilter(_diagnostic_filter) +_ANSI_ESCAPE: Final = re.compile(r"\x1b\[[0-9;]*m") + + +def print_startup_message(message: str) -> None: + """Write a boot-time message meant for the operator's terminal. + + Plain mode prints it unchanged, colors and spacing included. With JSON logs on, every line of + stdout is read by a log pipeline, so the message goes out as one INFO record of the proxy logger + instead, on stdout like the print whatever handlers the logging config gave that logger, with the + ANSI color codes and surrounding blank lines removed, located at the caller and emitted at any log + level, whole and unredacted as the print showed it; a blank spacer produces no record. + """ + if not litellm.json_logs: + print(message) # noqa: T201 # plain mode: this is the operator-terminal print it replaces + return + text: Final = _ANSI_ESCAPE.sub("", message).strip("\n") + if not text: + return + frame: Final = inspect.currentframe() + caller: Final = frame.f_back if frame is not None else None + record: Final = verbose_proxy_logger.makeRecord( + verbose_proxy_logger.name, + logging.INFO, + caller.f_code.co_filename if caller is not None else __file__, + caller.f_lineno if caller is not None else 0, + text, + (), + None, + func=caller.f_code.co_name if caller is not None else None, + ) + if not _process_record(record, base64_limit=_get_max_base64_length_stdout_log(), text_limit=0, redact=False): + return + _startup_json_handler().handle(record) + + +def _startup_json_handler() -> logging.Handler: + """The proxy's JSON handler on stdout, filters included, for one startup message.""" + handler: Final = LevelRoutingStreamHandler() + handler.setFormatter(JsonFormatter()) + handler.addFilter(_secret_filter) + handler.addFilter(_correlation_filter) + return handler + def _suppress_loggers(): """Suppress noisy loggers at INFO level""" diff --git a/litellm/proxy/db/check_migration.py b/litellm/proxy/db/check_migration.py index 6e2a06c96e1..83cc4501466 100644 --- a/litellm/proxy/db/check_migration.py +++ b/litellm/proxy/db/check_migration.py @@ -4,7 +4,7 @@ import os import subprocess from typing import Final -from litellm._logging import verbose_logger +from litellm._logging import print_startup_message, verbose_logger def extract_sql_commands(diff_output: str) -> list[str]: @@ -63,8 +63,8 @@ def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]: run_prisma, ) except ImportError as e: - print( # noqa: T201 # boot-time operator output, same channel as this helper's other messages - f"Skipping the migration diff: litellm-proxy-extras has no Prisma runner. Error: {e}" + print_startup_message( + f"Skipping the migration diff: litellm-proxy-extras has no Prisma runner. Error: {e}", ) return False, [] @@ -89,22 +89,21 @@ def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]: sql_commands: Final = extract_sql_commands(result.stdout) if sql_commands: - print("Changes to DB Schema detected") # noqa: T201 - print("Required SQL commands:") # noqa: T201 - for command in sql_commands: - print(command) # noqa: T201 + print_startup_message( + "\n".join(("Changes to DB Schema detected", "Required SQL commands:", *sql_commands)), + ) return True, sql_commands else: return False, [] except subprocess.TimeoutExpired: - print( # noqa: T201 # boot-time operator output, same channel as this helper's other messages + print_startup_message( f"Timed out after {timeout}s generating the migration diff. " - f"Raise {PRISMA_COMMAND_TIMEOUT_ENV_VAR} if this database needs longer." + f"Raise {PRISMA_COMMAND_TIMEOUT_ENV_VAR} if this database needs longer.", ) return False, [] except subprocess.CalledProcessError as e: error_message: Final = f"Failed to generate migration diff. Error: {e.stderr}" - print(error_message) # noqa: T201 + print_startup_message(error_message) return False, [] diff --git a/litellm/proxy/policy_engine/init_policies.py b/litellm/proxy/policy_engine/init_policies.py index d75e651e6c7..59d15b0ba0a 100644 --- a/litellm/proxy/policy_engine/init_policies.py +++ b/litellm/proxy/policy_engine/init_policies.py @@ -6,10 +6,12 @@ Configuration structure: - policy_attachments: Define WHERE policies apply (teams, keys, models) """ +import sys from collections.abc import Mapping, Sequence +from itertools import chain from typing import TYPE_CHECKING, Any, Final, Optional -from litellm._logging import verbose_proxy_logger +from litellm._logging import print_startup_message, verbose_proxy_logger from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry from litellm.proxy.policy_engine.policy_registry import get_policy_registry from litellm.proxy.policy_engine.policy_validator import PolicyValidator @@ -25,6 +27,42 @@ _yellow_color_code: Final = "\033[93m" _reset_color_code: Final = "\033[0m" +def _policy_lines(policy_name: str, policy_data: Mapping[str, object]) -> tuple[str, ...]: + guardrails: Final = policy_data.get("guardrails", {}) + inherit: Final = policy_data.get("inherit") + condition: Final = policy_data.get("condition") + description: Final = policy_data.get("description") + + guardrails_add: Final = guardrails.get("add", []) if isinstance(guardrails, dict) else [] + guardrails_remove: Final = guardrails.get("remove", []) if isinstance(guardrails, dict) else [] + model_condition: Final = condition.get("model") if isinstance(condition, dict) else None + inherit_str: Final = f" (inherits: {inherit})" if inherit else "" + + return ( + f"{_blue_color_code} - {policy_name}{inherit_str}{_reset_color_code}", + *((f" description: {description}",) if description else ()), + *((f" guardrails.add: {guardrails_add}",) if guardrails_add else ()), + *((f" guardrails.remove: {guardrails_remove}",) if guardrails_remove else ()), + *((f" condition.model: {model_condition}",) if model_condition else ()), + ) + + +def _attachment_line(attachment: Mapping[str, object]) -> str: + policy: Final = attachment.get("policy", "unknown") + scope: Final = attachment.get("scope") + teams: Final = attachment.get("teams") + keys: Final = attachment.get("keys") + models: Final = attachment.get("models") + + scope_parts: Final = ( + *(("scope=* (global)",) if scope == "*" else ()), + *((f"teams={teams}",) if teams else ()), + *((f"keys={keys}",) if keys else ()), + *((f"models={models}",) if models else ()), + ) + return f" - {policy} -> {', '.join(scope_parts) if scope_parts else 'all'}" + + def _print_policies_on_startup( policies_config: Mapping[str, Mapping[str, object]], policy_attachments_config: Sequence[Mapping[str, object]] | None = None, @@ -32,67 +70,32 @@ def _print_policies_on_startup( """ Print loaded policies to console on startup (similar to model list). """ - import sys - - print( # noqa: T201 + header: Final = ( f"{_green_color_code}\nLiteLLM Policy Engine: Loaded {len(policies_config)} policies{_reset_color_code}\n" ) - sys.stdout.flush() - - for policy_name, policy_data in policies_config.items(): - guardrails = policy_data.get("guardrails", {}) - inherit = policy_data.get("inherit") - condition = policy_data.get("condition") - description = policy_data.get("description") - - guardrails_add = guardrails.get("add", []) if isinstance(guardrails, dict) else [] - guardrails_remove = guardrails.get("remove", []) if isinstance(guardrails, dict) else [] - inherit_str = f" (inherits: {inherit})" if inherit else "" - - print( # noqa: T201 - f"{_blue_color_code} - {policy_name}{inherit_str}{_reset_color_code}" + policy_lines: Final = tuple( + chain.from_iterable( + _policy_lines(policy_name, policy_data) for policy_name, policy_data in policies_config.items() ) - if description: - print(f" description: {description}") # noqa: T201 - if guardrails_add: - print(f" guardrails.add: {guardrails_add}") # noqa: T201 - if guardrails_remove: - print(f" guardrails.remove: {guardrails_remove}") # noqa: T201 - if condition: - model_condition = condition.get("model") if isinstance(condition, dict) else None - if model_condition: - print(f" condition.model: {model_condition}") # noqa: T201 + ) - # Print attachments if policy_attachments_config: - print( # noqa: T201 - f"\n{_yellow_color_code}Policy Attachments: {len(policy_attachments_config)} attachment(s){_reset_color_code}" + print_startup_message( + "\n".join( + ( + header, + *policy_lines, + f"\n{_yellow_color_code}Policy Attachments: {len(policy_attachments_config)} attachment(s){_reset_color_code}", + *(_attachment_line(attachment) for attachment in policy_attachments_config), + "", + ) + ) ) - for attachment in policy_attachments_config: - policy = attachment.get("policy", "unknown") - scope = attachment.get("scope") - teams = attachment.get("teams") - keys = attachment.get("keys") - models = attachment.get("models") - - scope_parts = [] - if scope == "*": - scope_parts.append("scope=* (global)") - if teams: - scope_parts.append(f"teams={teams}") - if keys: - scope_parts.append(f"keys={keys}") - if models: - scope_parts.append(f"models={models}") - scope_str = ", ".join(scope_parts) if scope_parts else "all" - - print(f" - {policy} -> {scope_str}") # noqa: T201 else: - print( # noqa: T201 - f"\n{_yellow_color_code}Warning: No policy_attachments configured. Policies will not be applied to any requests.{_reset_color_code}" + print_startup_message("\n".join((header, *policy_lines))) + print_startup_message( + f"\n{_yellow_color_code}Warning: No policy_attachments configured. Policies will not be applied to any requests.{_reset_color_code}\n", ) - - print() # noqa: T201 sys.stdout.flush() diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index fc33ab1bd0b..909f6de40cd 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -1,4 +1,5 @@ # ruff: noqa: T201 +import functools import importlib import json import os @@ -7,7 +8,7 @@ import re import subprocess import sys import urllib.parse as urlparse -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence from pathlib import Path from typing import TYPE_CHECKING, Any, Final @@ -18,6 +19,7 @@ from dotenv import load_dotenv from pydantic import ConfigDict import litellm +from litellm._logging import print_startup_message from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY from litellm.proxy.db.pgbouncer import ( PgBouncerError, @@ -202,6 +204,29 @@ def deprecated_v2_flag_passed_on_cli() -> bool: return ctx.get_parameter_source("use_v2_migration_resolver") is ParameterSource.COMMANDLINE +def _print_enforce_prisma_migration_check_notice() -> None: + print_startup_message( + "\033[1;33mLiteLLM Proxy: --enforce_prisma_migration_check is " + "deprecated and has no effect, because the proxy always exits " + "when database setup fails at startup. You can safely remove it.\033[0m", + ) + + +def _print_or_hold_enforce_prisma_migration_check_notice( + *, reads_config_file: bool, returns_before_config: bool +) -> Callable[[], None] | None: + """Print the notice now, or, while a config file can still switch json_logs on, return a printer + to call once it is read; a boot that fails before that prints the held notice on the way out.""" + if not reads_config_file or litellm.json_logs or returns_before_config: + _print_enforce_prisma_migration_check_notice() + return None + print_once: Final = functools.cache(_print_enforce_prisma_migration_check_notice) + ctx: Final = click.get_current_context(silent=True) + if ctx is not None: + ctx.call_on_close(print_once) + return print_once + + class ProxyInitializationHelpers: @staticmethod def echo_litellm_version(): @@ -335,7 +360,7 @@ class ProxyInitializationHelpers: "timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(), } if log_config is not None: - print(f"Using log_config: {log_config}") + print_startup_message(f"Using log_config: {log_config}") uvicorn_args["log_config"] = log_config elif litellm.json_logs: # Use JSON log config for uvicorn to ensure all logs (including exceptions) are JSON @@ -348,10 +373,10 @@ class ProxyInitializationHelpers: if "timeout_worker_healthcheck" in inspect.signature(uvicorn.Config.__init__).parameters: uvicorn_args["timeout_worker_healthcheck"] = timeout_worker_healthcheck else: - print( + print_startup_message( f"\033[1;33mLiteLLM Proxy: --timeout_worker_healthcheck " f"requires uvicorn>=0.37.0, but installed uvicorn=={uvicorn.__version__}. " - f"Ignoring the flag.\033[0m" + f"Ignoring the flag.\033[0m", ) return uvicorn_args @@ -371,18 +396,18 @@ class ProxyInitializationHelpers: import uvicorn if max_requests_before_restart is None: - print( + print_startup_message( "\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter " - "has no effect without --max_requests_before_restart\033[0m\n" + "has no effect without --max_requests_before_restart\033[0m\n", ) return if "limit_max_requests_jitter" in inspect.signature(uvicorn.Config.__init__).parameters: uvicorn_args["limit_max_requests_jitter"] = jitter else: - print( + print_startup_message( f"\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter " f"requires uvicorn>=0.41.0, but installed uvicorn=={uvicorn.__version__}. " - f"Ignoring the flag.\033[0m" + f"Ignoring the flag.\033[0m", ) _apply_uvicorn_max_requests_jitter = apply_uvicorn_max_requests_jitter @@ -506,12 +531,12 @@ class ProxyInitializationHelpers: from hypercorn.asyncio import serve from hypercorn.config import Config - print(f"\033[1;32mLiteLLM Proxy: Starting server on {host}:{port} using Hypercorn\033[0m\n") + print_startup_message(f"\033[1;32mLiteLLM Proxy: Starting server on {host}:{port} using Hypercorn\033[0m\n") config: Final = Config() config.bind = [f"{host}:{port}"] if ssl_certfile_path is not None and ssl_keyfile_path is not None: - print( + print_startup_message( f"\033[1;32mLiteLLM Proxy: Using SSL with certfile: {ssl_certfile_path} and keyfile: {ssl_keyfile_path}\033[0m\n" ) config.certfile = ssl_certfile_path @@ -544,14 +569,14 @@ class ProxyInitializationHelpers: from granian import Granian from granian.constants import Interfaces - print(f"\033[1;32mLiteLLM Proxy: Starting server on {host}:{port} using Granian\033[0m\n") + print_startup_message(f"\033[1;32mLiteLLM Proxy: Starting server on {host}:{port} using Granian\033[0m\n") if max_requests_before_restart is not None: - print( + print_startup_message( "\033[1;33mLiteLLM: --max_requests_before_restart is not supported by Granian " - "(Granian uses workers_lifetime in seconds, not a per-request limit).\033[0m\n" + "(Granian uses workers_lifetime in seconds, not a per-request limit).\033[0m\n", ) if ciphers is not None: - print("\033[1;33mLiteLLM: --ciphers is not applied when using --run_granian.\033[0m\n") + print_startup_message("\033[1;33mLiteLLM: --ciphers is not applied when using --run_granian.\033[0m\n") kwargs: Final[dict[str, object]] = { "target": "litellm.proxy.proxy_server:app", @@ -564,7 +589,7 @@ class ProxyInitializationHelpers: if granian_runtime_threads is not None: kwargs["runtime_threads"] = granian_runtime_threads if ssl_certfile_path is not None and ssl_keyfile_path is not None: - print( + print_startup_message( f"\033[1;32mLiteLLM Proxy: Using SSL with certfile: {ssl_certfile_path} and keyfile: {ssl_keyfile_path}\033[0m\n" ) kwargs["ssl_cert"] = Path(ssl_certfile_path) @@ -625,13 +650,15 @@ class ProxyInitializationHelpers: \n """ ) - print() - print( + print_startup_message("") + print_startup_message( '\033[1;34mLiteLLM: Test your local proxy with: "litellm --test" This runs an openai.ChatCompletion request to your proxy [In a new terminal tab]\033[0m\n' ) - print(f"\033[1;34mLiteLLM: Curl Command Test for your local proxy\n {curl_command} \033[0m\n") - print("\033[1;34mDocs: https://docs.litellm.ai/docs/simple_proxy\033[0m\n") - print(f"\033[1;34mSee all Router/Swagger docs on http://0.0.0.0:{port} \033[0m\n") + print_startup_message( + f"\033[1;34mLiteLLM: Curl Command Test for your local proxy\n {curl_command} \033[0m\n" + ) + print_startup_message("\033[1;34mDocs: https://docs.litellm.ai/docs/simple_proxy\033[0m\n") + print_startup_message(f"\033[1;34mSee all Router/Swagger docs on http://0.0.0.0:{port} \033[0m\n") def load_config(self): # note: This Loads the gunicorn config - has nothing to do with LiteLLM Proxy config @@ -651,7 +678,9 @@ class ProxyInitializationHelpers: # gunicorn app function return self.application - print(f"\033[1;32mLiteLLM Proxy: Starting server on {host}:{port} with {num_workers} workers\033[0m\n") + print_startup_message( + f"\033[1;32mLiteLLM Proxy: Starting server on {host}:{port} with {num_workers} workers\033[0m\n" + ) graceful_shutdown_timeout: Final = GracefulShutdownManager.get_timeout() gunicorn_options: Final = { "bind": f"{host}:{port}", @@ -669,9 +698,9 @@ class ProxyInitializationHelpers: gunicorn_options["max_requests"] = max_requests_before_restart if max_requests_before_restart_jitter is not None: if max_requests_before_restart is None: - print( + print_startup_message( "\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter " - "has no effect without --max_requests_before_restart\033[0m\n" + "has no effect without --max_requests_before_restart\033[0m\n", ) else: gunicorn_options["max_requests_jitter"] = max_requests_before_restart_jitter @@ -686,7 +715,7 @@ class ProxyInitializationHelpers: gunicorn_options["child_exit"] = child_exit if ssl_certfile_path is not None and ssl_keyfile_path is not None: - print( + print_startup_message( f"\033[1;32mLiteLLM Proxy: Using SSL with certfile: {ssl_certfile_path} and keyfile: {ssl_keyfile_path}\033[0m\n" ) gunicorn_options["certfile"] = ssl_certfile_path @@ -710,7 +739,7 @@ class ProxyInitializationHelpers: with open(os.devnull, "w") as devnull: subprocess.Popen(command, stdout=devnull, stderr=devnull) except Exception as e: - print(f""" + print_startup_message(f""" LiteLLM Warning: proxy started with `ollama` model\n`ollama serve` failed with Exception{e}. \nEnsure you run `ollama serve` """) @@ -779,7 +808,7 @@ class ProxyInitializationHelpers: os.makedirs(multiproc_dir, exist_ok=True) wipe_directory(multiproc_dir) action: Final = "Using existing" if configured_dir else "Auto-created" - print(f"LiteLLM: {action} PROMETHEUS_MULTIPROC_DIR={multiproc_dir}") + print_startup_message(f"LiteLLM: {action} PROMETHEUS_MULTIPROC_DIR={multiproc_dir}") return multiproc_dir _maybe_setup_prometheus_multiproc_dir = maybe_setup_prometheus_multiproc_dir @@ -1199,12 +1228,13 @@ def run_server( return if check_migrations_against_existing_data is True: sys.exit(ProxyInitializationHelpers.run_migration_data_check(use_migrate=not use_prisma_db_push)) - if enforce_prisma_migration_check: - print( - "\033[1;33mLiteLLM Proxy: --enforce_prisma_migration_check is " - "deprecated and has no effect, because the proxy always exits " - "when database setup fails at startup. You can safely remove it.\033[0m" + held_prisma_notice: Final = ( + _print_or_hold_enforce_prisma_migration_check_notice( + reads_config_file=config is not None, returns_before_config=health is True or test is True ) + if enforce_prisma_migration_check + else None + ) if model and "ollama" in model and api_base is None: ProxyInitializationHelpers.run_ollama_serve() if health is True: @@ -1320,8 +1350,11 @@ def run_server( import litellm litellm.json_logs = True + os.environ["JSON_LOGS"] = "true" litellm.turn_on_json() + if held_prisma_notice is not None: + held_prisma_notice() ### GENERAL SETTINGS ### general_settings = _config.get("general_settings", {}) if general_settings is None: @@ -1488,7 +1521,7 @@ def run_server( lifetime_params, ) ) - print( + print_startup_message( postgres_connection_budget_message( writer_limit=_url_query_value(os.getenv("DATABASE_URL"), "connection_limit") or f"{db_connection_pool_limit}", @@ -1520,19 +1553,19 @@ def run_server( env_value=os.getenv("USE_V2_MIGRATION_RESOLVER"), ) if deprecated_v2_flag_passed_on_cli() and use_v2_resolver: - print( + print_startup_message( "\033[1;33mLiteLLM Proxy: --use_v2_migration_resolver is " "deprecated and has no effect, because the v2 migration " "resolver is now the default. You can safely remove it. To " "opt back into the legacy v1 resolver, pass " - "--use_legacy_migration_resolver.\033[0m" + "--use_legacy_migration_resolver.\033[0m", ) if not use_v2_resolver: - print( + print_startup_message( "\033[1;33mLiteLLM Proxy: Using the legacy (v1) migration " "resolver. It performs the diff-and-force recovery that can " "cause schema thrashing during rolling deploys where two " - "LiteLLM versions contend for the same DB.\033[0m" + "LiteLLM versions contend for the same DB.\033[0m", ) try: migrated: Final = PrismaManager.setup_database( @@ -1600,7 +1633,7 @@ def run_server( # Skip server startup if requested (after all setup is done) if skip_server_startup: - print("LiteLLM: Setup complete. Skipping server startup as requested.") + print_startup_message("LiteLLM: Setup complete. Skipping server startup as requested.") return # Auto-create PROMETHEUS_MULTIPROC_DIR for multi-worker setups @@ -1619,7 +1652,7 @@ def run_server( ) except MetricsServerStartupError as error: raise click.ClickException(str(error)) from error - print( + print_startup_message( f"\033[1;32mLiteLLM: Serving Prometheus metrics on {host}:{prometheus_metrics_port}/metrics " f"(pid {metrics_process.pid})\033[0m" ) @@ -1642,7 +1675,7 @@ def run_server( run_gunicorn=run_gunicorn is True, ) if single_worker_warning is not None: - print(f"\033[1;33m{single_worker_warning}\033[0m\n") + print_startup_message(f"\033[1;33m{single_worker_warning}\033[0m\n") if run_gunicorn is False and run_hypercorn is False and run_granian is False: if limit_concurrency is not None: uvicorn_args["limit_concurrency"] = limit_concurrency @@ -1653,7 +1686,7 @@ def run_server( jitter=max_requests_before_restart_jitter, ) if ssl_certfile_path is not None and ssl_keyfile_path is not None: - print( + print_startup_message( f"\033[1;32mLiteLLM Proxy: Using SSL with certfile: {ssl_certfile_path} and keyfile: {ssl_keyfile_path}\033[0m\n" ) uvicorn_args["ssl_keyfile"] = ssl_keyfile_path diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bc79becf7bb..3b36ea5afce 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -273,25 +273,27 @@ def generate_feedback_box(): # Select a random message message: Final = random.choice(list_of_messages) - print() # noqa: T201 - print("\033[1;37m" + "#" + "-" * box_width + "#\033[0m") # noqa: T201 - print("\033[1;37m" + "#" + " " * box_width + "#\033[0m") # noqa: T201 - print("\033[1;37m" + f"# {message:^59} #\033[0m") # noqa: T201 - print( # noqa: T201 - "\033[1;37m" + "# {:^59} #\033[0m".format("https://github.com/BerriAI/litellm/issues/new") + print_startup_message( + "\n".join( + ( + "", + "\033[1;37m" + "#" + "-" * box_width + "#\033[0m", + "\033[1;37m" + "#" + " " * box_width + "#\033[0m", + "\033[1;37m" + f"# {message:^59} #\033[0m", + "\033[1;37m" + "# {:^59} #\033[0m".format("https://github.com/BerriAI/litellm/issues/new"), + "\033[1;37m" + "#" + " " * box_width + "#\033[0m", + "\033[1;37m" + "#" + "-" * box_width + "#\033[0m", + "", + " Thank you for using LiteLLM! - Krrish & Ishaan", + "", + "", + "", + "\033[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new\033[0m", + "", + "", + ) + ) ) - print("\033[1;37m" + "#" + " " * box_width + "#\033[0m") # noqa: T201 - print("\033[1;37m" + "#" + "-" * box_width + "#\033[0m") # noqa: T201 - print() # noqa: T201 - print(" Thank you for using LiteLLM! - Krrish & Ishaan") # noqa: T201 - print() # noqa: T201 - print() # noqa: T201 - print() # noqa: T201 - print( # noqa: T201 - "\033[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new\033[0m" - ) - print() # noqa: T201 - print() # noqa: T201 import contextlib @@ -304,6 +306,7 @@ import litellm._redis from litellm import Router from litellm._internal_context import service_target, with_service_target from litellm._logging import ( + print_startup_message, redact_string, verbose_proxy_logger, verbose_router_logger, @@ -6412,17 +6415,13 @@ class ProxyConfig: search_tools_parsed: Final[list[SearchToolTypedDict]] = [] - print( # noqa: T201 - "\033[32mLiteLLM: Proxy initialized with Search Tools:\033[0m" - ) + print_startup_message("\033[32mLiteLLM: Proxy initialized with Search Tools:\033[0m") for search_tool in search_tools_raw: # Display loaded search tool search_tool_name = search_tool.get("search_tool_name", "") search_provider = search_tool.get("litellm_params", {}).get("search_provider", "") - print( # noqa: T201 - f"\033[32m {search_tool_name} ({search_provider})\033[0m" - ) + print_startup_message(f"\033[32m {search_tool_name} ({search_provider})\033[0m") # Handle os.environ/ variables in litellm_params litellm_params = search_tool.get("litellm_params", {}) @@ -6598,7 +6597,7 @@ class ProxyConfig: reset_color_code: Final = "\033[0m" for key, value in litellm_settings.items(): if key == "cache" and value is True: - print(f"{blue_color_code}\nSetting Cache on Proxy") # noqa: T201 + print_startup_message(f"{blue_color_code}\nSetting Cache on Proxy") from litellm.caching.caching import Cache cache_params = {} @@ -6789,7 +6788,7 @@ class ProxyConfig: if PrometheusLogger is not None: verbose_proxy_logger.debug("mounting metrics endpoint") PrometheusLogger._mount_metrics_endpoint() - print( # noqa: T201 + print_startup_message( f"{blue_color_code} Initialized Success Callbacks - {litellm.success_callback} {reset_color_code}" ) elif key == "failure_callback": @@ -6808,7 +6807,7 @@ class ProxyConfig: # these are litellm callbacks - "langfuse", "sentry", "wandb" else: litellm.logging_callback_manager.add_litellm_failure_callback(callback) - print( # noqa: T201 + print_startup_message( f"{blue_color_code} Initialized Failure Callbacks - {litellm.failure_callback} {reset_color_code}" ) elif key == "audit_log_callbacks": @@ -6833,7 +6832,7 @@ class ProxyConfig: _store_audit_logs = litellm_settings.get("store_audit_logs", litellm.store_audit_logs) if is_audit_logging_enabled(store_audit_logs=_store_audit_logs): - print( # noqa: T201 + print_startup_message( f"{blue_color_code} Initialized Audit Log Callbacks - {litellm.audit_log_callbacks} {reset_color_code}" ) else: @@ -7232,9 +7231,7 @@ class ProxyConfig: if model_list: router_params["model_list"] = model_list validate_auto_router_capability_limits(model_list, limit=_license_check.auto_router_capability_limit()) - print( # noqa: T201 - "\033[32mLiteLLM: Proxy initialized with Config, Set models:\033[0m" - ) + print_startup_message("\033[32mLiteLLM: Proxy initialized with Config, Set models:\033[0m") for model in model_list: ### LOAD FROM os.environ/ ### for k, v in model["litellm_params"].items(): @@ -7251,7 +7248,7 @@ class ProxyConfig: complexity_router_config=complexity_router_config, config_file_path=config_file_path, ) - print(f"\033[32m {model.get('model_name', '')}\033[0m") # noqa: T201 + print_startup_message(f"\033[32m {model.get('model_name', '')}\033[0m") litellm_model_name = model["litellm_params"]["model"] litellm_model_api_base = model["litellm_params"].get("api_base", None) if "ollama" in litellm_model_name and litellm_model_api_base is None: @@ -9672,9 +9669,12 @@ async def initialize( master_key, \ user_custom_auth, \ prisma_client - from litellm.proxy.common_utils.banner import show_banner + from litellm.proxy.common_utils.banner import LITELLM_BANNER, show_banner - show_banner() + if litellm.json_logs: + print_startup_message(LITELLM_BANNER) + else: + show_banner() if os.getenv("LITELLM_DONT_SHOW_FEEDBACK_BOX", "").lower() != "true": generate_feedback_box() user_model = model @@ -11930,7 +11930,7 @@ class ProxyStartupEvent: and proxy_logging_obj.slack_alerting_instance.alerting is not None and prisma_client is not None ): - print("Alerting: Initializing Weekly/Monthly Spend Reports") # noqa: T201 + print_startup_message("Alerting: Initializing Weekly/Monthly Spend Reports") spend_report_frequency: Final[str] = general_settings.get("spend_report_frequency", "7d") or "7d" days: Final = int(spend_report_frequency[:-1]) diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index 052c52982b6..262114ee32b 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -987,6 +987,7 @@ def refused_boot_log( overrides: Mapping[str, str], *, config: Path | None = None, + extra_arguments: tuple[str, ...] = (), ) -> str: """Start the proxy and return its log once it exits non-zero instead of becoming ready.""" root: Final = _proxy_root() @@ -1004,6 +1005,7 @@ def refused_boot_log( "--num_workers", "1", *DB_PUSH, + *extra_arguments, ) launch: Final = _launch(command, root, environment, output) try: diff --git a/tests/integration/configuration/test_json_logs_startup.py b/tests/integration/configuration/test_json_logs_startup.py new file mode 100644 index 00000000000..de139b7986f --- /dev/null +++ b/tests/integration/configuration/test_json_logs_startup.py @@ -0,0 +1,397 @@ +"""JSON logging keeps every line the proxy writes during boot, serving and shutdown machine-parseable. + +A deployment that turns on JSON logs ships the proxy's stdout to a log pipeline, and every line that is +not a JSON object breaks that pipeline's parser. This contract boots an owned proxy with JSON logs on, +serves a chat request on every configured model plus one rejected request, stops the proxy and +rejects any line of its output that is not one JSON record, while checking that the +startup information those lines used to carry is still present as JSON records. JSON mode is switched +on through the environment and through the config, at one and two uvicorn workers, at the INFO, the +default and the WARNING log level, under a concurrent burst that lands on both workers, under an operator +--log_config that names the proxy logger or gives it a handler of its own, and with a policy list longer +than the stdout cap. The model list keeps a model name the secret patterns would match. The deprecated +--enforce_prisma_migration_check notice is one JSON record under a config-only json_logs, and a boot whose +config cannot be loaded still prints it before failing. +""" + +from __future__ import annotations + +import re +import uuid +from collections.abc import Mapping +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final + +import pytest +import yaml +from pydantic import JsonValue, TypeAdapter, ValidationError + +from tests.integration._support.client import Gateway +from tests.integration._support.process import owned_proxy_process, refused_boot_log + +JSON_RECORD: Final = TypeAdapter(dict[str, JsonValue]) +PLAIN_LOG_LINE: Final = re.compile(r"^\d\d:\d\d:\d\d - LiteLLM") +PRISMA_ENGINE_TEXT: Final = re.compile(r"query-engine [0-9a-f]{40}") +STARTED_WORKER: Final = "Started server process" +MODEL_LIST_HEADER: Final = "Proxy initialized with Config, Set models:" +CACHE_LINE: Final = "Setting Cache on Proxy" +CONNECTION_BUDGET_LINE: Final = "Postgres connection budget" +REDIS_AUTH_CACHE_LINE: Final = "enable_redis_auth_cache" +BANNER_FRAGMENT: Final = "██╗" +FEEDBACK_BOX_LINE: Final = "Give Feedback / Get Help" +WORKER_COUNTS: Final = (1, 2) +UNKNOWN_MODEL_STATUS: Final = 400 +BURST_SIZE: Final = 16 +CACHE_SETTINGS: Final[Mapping[str, JsonValue]] = {"cache": True, "cache_params": {"type": "redis"}} +STDOUT_CAP: Final = 4096 +POLICY_COUNT: Final = 60 +POLICY_LIST_HEADER: Final = "LiteLLM Policy Engine: Loaded" +TRUNCATION_MARKER: Final = "litellm_truncated" +DEPRECATED_FLAG: Final = "--enforce_prisma_migration_check" +DEPRECATED_FLAG_NOTICE: Final = "--enforce_prisma_migration_check is deprecated and has no effect" +UNPARSEABLE_CONFIG: Final = "model_list: [\n - bad: yaml: here\n" +OPERATOR_LOG_CONFIG: Final[Mapping[str, JsonValue]] = { + "version": 1, + "disable_existing_loggers": False, + "formatters": {"json": {"()": "litellm._logging.JsonFormatter"}}, + "handlers": {"stdout": {"class": "logging.StreamHandler", "stream": "ext://sys.stdout", "formatter": "json"}}, + "loggers": {"LiteLLM Proxy": {"level": "INFO"}}, + "root": {"level": "INFO", "handlers": ["stdout"]}, +} +OPERATOR_SILENCING_LOG_CONFIG: Final[Mapping[str, JsonValue]] = { + "version": 1, + "disable_existing_loggers": False, + "formatters": {"json": {"()": "litellm._logging.JsonFormatter"}}, + "handlers": { + "stdout": {"class": "logging.StreamHandler", "stream": "ext://sys.stdout", "formatter": "json"}, + "silence": {"class": "logging.NullHandler"}, + }, + "loggers": {"LiteLLM Proxy": {"level": "INFO", "handlers": ["silence"], "propagate": False}}, + "root": {"level": "INFO", "handlers": ["stdout"]}, +} + + +def _models() -> tuple[str, str]: + stamp: Final = uuid.uuid4().hex + return (f"json-logs-first-{stamp}", f"json-logs-second-{stamp}") + + +def _policies() -> dict[str, JsonValue]: + stamp: Final = uuid.uuid4().hex + return { + f"startup-listing-policy-{n:02d}-{stamp}": { + "description": f"policy {n:02d} of a listing that runs past the stdout cap meant for request payloads" + } + for n in range(POLICY_COUNT) + } + + +def _write_config( + directory: Path, + upstream_url: str, + models: tuple[str, ...], + litellm_settings: Mapping[str, JsonValue], + extra_sections: Mapping[str, JsonValue] | None = None, +) -> Path: + config: Final = directory / f"json_logs_{uuid.uuid4().hex}.yaml" + config.write_text( + yaml.safe_dump( + { + "model_list": [ + { + "model_name": model, + "litellm_params": { + "model": f"openai/{model}", + "api_base": f"{upstream_url}/v1", + "api_key": "integration-provider-key", + }, + } + for model in models + ], + "litellm_settings": dict(litellm_settings), + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "disable_prisma_schema_update": True, + }, + **(extra_sections or {}), + }, + sort_keys=False, + ) + ) + return config + + +def _record(line: str) -> dict[str, JsonValue] | None: + stripped: Final = line.strip() + if not stripped.startswith("{"): + return None + try: + return JSON_RECORD.validate_json(stripped) + except ValidationError: + return None + + +def _lines(log: str) -> tuple[str, ...]: + return tuple(line for line in PRISMA_ENGINE_TEXT.sub("", log).splitlines() if line.strip()) + + +def _messages(log: str) -> tuple[str, ...]: + records: Final = tuple(record for record in (_record(line) for line in _lines(log)) if record is not None) + return tuple(str(record["message"]) for record in records if "message" in record) + + +def _non_json_lines(log: str) -> tuple[str, ...]: + return tuple(line for line in _lines(log) if _record(line) is None) + + +def _boot_serve_and_stop( + gateway: Gateway, + directory: Path, + config: Path, + overrides: Mapping[str, str], + workers: int, + models: tuple[str, ...], + extra_arguments: tuple[str, ...] = (), +) -> str: + with owned_proxy_process( + gateway, + directory, + overrides, + config=config, + workers=workers, + database_setup=(), + extra_arguments=extra_arguments, + ) as owned: + _serve(owned.gateway, models) + log: Final = owned.log + return log.read_text() + + +def _serve(gateway: Gateway, models: tuple[str, ...]) -> None: + """One chat per model and one rejected request, so the log holds the request-time lines of served + and failed requests next to the boot lines.""" + for model in models: + reply: Final = gateway.chat(model, text=f"json logs {uuid.uuid4().hex}") + assert reply["model"] == model, reply + rejected: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": f"missing-{uuid.uuid4().hex}", "messages": [{"role": "user", "content": "json logs"}]}, + ) + assert rejected.status_code == UNKNOWN_MODEL_STATUS, rejected.text + + +def _assert_every_line_is_json( + log: str, workers: int, models: tuple[str, ...], *, redis_auth_cache_line: bool = True +) -> None: + plaintext: Final = _non_json_lines(log) + assert plaintext == (), "plaintext lines with JSON logs on:\n" + "\n".join(plaintext) + messages: Final = _messages(log) + assert sum(STARTED_WORKER in message for message in messages) == workers, messages + assert any(MODEL_LIST_HEADER in message for message in messages), messages + for model in models: + assert any(model in message for message in messages), (model, messages) + assert any(CACHE_LINE in message for message in messages), messages + if redis_auth_cache_line: + assert any(REDIS_AUTH_CACHE_LINE in message for message in messages), messages + assert any(CONNECTION_BUDGET_LINE in message for message in messages), messages + assert sum(BANNER_FRAGMENT in message for message in messages) == workers, messages + assert sum(FEEDBACK_BOX_LINE in message for message in messages) == workers, messages + + +@pytest.mark.parametrize("workers", WORKER_COUNTS) +def test_json_logs_env_keeps_every_line_json(gateway: Gateway, tmp_path: Path, workers: int) -> None: + models: Final = _models() + config: Final = _write_config(tmp_path, gateway.upstream_url, models, CACHE_SETTINGS) + log: Final = _boot_serve_and_stop( + gateway, tmp_path, config, {"JSON_LOGS": "true", "LITELLM_LOG": "INFO"}, workers, models + ) + _assert_every_line_is_json(log, workers, models) + + +@pytest.mark.parametrize("workers", WORKER_COUNTS) +def test_json_logs_config_keeps_every_line_json(gateway: Gateway, tmp_path: Path, workers: int) -> None: + models: Final = _models() + config: Final = _write_config(tmp_path, gateway.upstream_url, models, {**CACHE_SETTINGS, "json_logs": True}) + log: Final = _boot_serve_and_stop(gateway, tmp_path, config, {"LITELLM_LOG": "INFO"}, workers, models) + _assert_every_line_is_json(log, workers, models) + + +@pytest.mark.parametrize("workers", WORKER_COUNTS) +def test_json_logs_with_operator_log_config_keeps_every_line_json( + gateway: Gateway, tmp_path: Path, workers: int +) -> None: + """uvicorn applies an operator `--log_config` after the CLI process switched JSON on and before + `initialize()` switches it on again through the config; a config that names the proxy logger leaves it + without handlers in that window, where the banner and the feedback box are emitted. They still arrive, + as JSON, one per worker, and the operator's own formatter keeps uvicorn's lines JSON. The Redis + auth-cache line is a regular logger line written in that same window; this log_config removed the + proxy logger's handlers and left propagation off, so it is lost at one worker on main and here alike.""" + models: Final = _models() + config: Final = _write_config(tmp_path, gateway.upstream_url, models, {**CACHE_SETTINGS, "json_logs": True}) + log_config: Final = tmp_path / "operator_log_config.yaml" + log_config.write_text(yaml.safe_dump(dict(OPERATOR_LOG_CONFIG))) + log: Final = _boot_serve_and_stop( + gateway, tmp_path, config, {"LITELLM_LOG": "INFO"}, workers, models, ("--log_config", str(log_config)) + ) + _assert_every_line_is_json(log, workers, models, redis_auth_cache_line=False) + + +@pytest.mark.parametrize("workers", WORKER_COUNTS) +def test_json_logs_with_operator_handler_on_the_proxy_logger_keeps_the_startup_notices( + gateway: Gateway, tmp_path: Path, workers: int +) -> None: + """An operator `--log_config` that gives the proxy logger a handler of its own, here a NullHandler that + silences it, is in force in the window where the banner and the feedback box are emitted. The prints + they replace wrote to stdout whatever the logging config said, so both still arrive there as JSON, one + per worker. The Redis auth-cache line is a regular logger line written in that same window and goes + to the operator's handler, so it is silenced on main and here alike.""" + models: Final = _models() + config: Final = _write_config(tmp_path, gateway.upstream_url, models, {**CACHE_SETTINGS, "json_logs": True}) + log_config: Final = tmp_path / "operator_silencing_log_config.yaml" + log_config.write_text(yaml.safe_dump(dict(OPERATOR_SILENCING_LOG_CONFIG))) + log: Final = _boot_serve_and_stop( + gateway, tmp_path, config, {"LITELLM_LOG": "INFO"}, workers, models, ("--log_config", str(log_config)) + ) + _assert_every_line_is_json(log, workers, models, redis_auth_cache_line=False) + + +def test_json_logs_keeps_a_startup_listing_longer_than_the_stdout_cap_whole(gateway: Gateway, tmp_path: Path) -> None: + """A startup listing longer than MAX_STRING_LENGTH_STDOUT_LOG reaches the pipeline whole. The policy list + stands in for every such listing (the required migration SQL on a drifted database is the other one): + the print it replaces showed it whole, and the stdout cap exists for echoed request payloads, so the JSON + record must not be middle-truncated.""" + models: Final = _models() + policies: Final = _policies() + config: Final = _write_config( + tmp_path, gateway.upstream_url, models, {**CACHE_SETTINGS, "json_logs": True}, {"policies": policies} + ) + + log: Final = _boot_serve_and_stop(gateway, tmp_path, config, {"LITELLM_LOG": "INFO"}, 1, models) + + _assert_every_line_is_json(log, 1, models) + listings: Final = tuple(message for message in _messages(log) if POLICY_LIST_HEADER in message) + assert len(listings) == 1, listings + assert len(listings[0]) > STDOUT_CAP, len(listings[0]) + assert TRUNCATION_MARKER not in listings[0], listings[0] + for policy in policies: + assert policy in listings[0], (policy, listings[0]) + + +def test_json_logs_model_list_keeps_a_model_name_the_secret_patterns_would_match( + gateway: Gateway, tmp_path: Path +) -> None: + """The model list printed every model name as configured. The sk- key pattern matches inside a name such as + `risk-assessment-...`, so the JSON record must carry that name as the print did, not `riREDACTED`.""" + models: Final = (f"risk-assessment-{uuid.uuid4().hex}",) + config: Final = _write_config(tmp_path, gateway.upstream_url, models, {**CACHE_SETTINGS, "json_logs": True}) + + log: Final = _boot_serve_and_stop(gateway, tmp_path, config, {"LITELLM_LOG": "INFO"}, 1, models) + + _assert_every_line_is_json(log, 1, models) + assert f" {models[0]}" in _messages(log), _messages(log) + + +def test_json_logs_default_level_keeps_every_line_json(gateway: Gateway, tmp_path: Path) -> None: + """Without LITELLM_LOG the proxy loggers sit at WARNING; the startup notices a plain-mode operator + always sees still arrive, as JSON, and nothing arrives as plaintext.""" + models: Final = _models() + config: Final = _write_config(tmp_path, gateway.upstream_url, models, {**CACHE_SETTINGS, "json_logs": True}) + log: Final = _boot_serve_and_stop(gateway, tmp_path, config, {}, 1, models) + plaintext: Final = _non_json_lines(log) + assert plaintext == (), "plaintext lines with JSON logs on:\n" + "\n".join(plaintext) + messages: Final = _messages(log) + assert sum(STARTED_WORKER in message for message in messages) == 1, messages + for fragment in ( + BANNER_FRAGMENT, + FEEDBACK_BOX_LINE, + MODEL_LIST_HEADER, + CACHE_LINE, + CONNECTION_BUDGET_LINE, + *models, + ): + assert any(fragment in message for message in messages), (fragment, messages) + + +def test_json_logs_warning_level_keeps_startup_notices(gateway: Gateway, tmp_path: Path) -> None: + """`LITELLM_LOG=WARNING` drops the INFO logger lines; the startup notices a plain-mode operator sees at + every level still arrive, as JSON, and nothing arrives as plaintext.""" + models: Final = _models() + config: Final = _write_config(tmp_path, gateway.upstream_url, models, {**CACHE_SETTINGS, "json_logs": True}) + log: Final = _boot_serve_and_stop(gateway, tmp_path, config, {"LITELLM_LOG": "WARNING"}, 1, models) + plaintext: Final = _non_json_lines(log) + assert plaintext == (), "plaintext lines with JSON logs on:\n" + "\n".join(plaintext) + messages: Final = _messages(log) + assert not any(REDIS_AUTH_CACHE_LINE in message for message in messages), messages + for fragment in ( + BANNER_FRAGMENT, + FEEDBACK_BOX_LINE, + MODEL_LIST_HEADER, + CACHE_LINE, + CONNECTION_BUDGET_LINE, + *models, + ): + assert any(fragment in message for message in messages), (fragment, messages) + + +def test_json_logs_burst_on_two_workers_keeps_every_line_json(gateway: Gateway, tmp_path: Path) -> None: + """Concurrent requests spread over both workers interleave their log writes; every line must still be + one complete JSON record.""" + models: Final = _models() + config: Final = _write_config(tmp_path, gateway.upstream_url, models, {**CACHE_SETTINGS, "json_logs": True}) + with owned_proxy_process( + gateway, tmp_path, {"LITELLM_LOG": "INFO"}, config=config, workers=2, database_setup=() + ) as owned: + with ThreadPoolExecutor(max_workers=BURST_SIZE) as pool: + replies: Final = tuple( + pool.map(lambda index: owned.gateway.chat(models[index % 2], text=f"burst {index}"), range(BURST_SIZE)) + ) + assert tuple(reply["model"] for reply in replies) == tuple(models[index % 2] for index in range(BURST_SIZE)) + log: Final = owned.log + text: Final = log.read_text() + plaintext: Final = _non_json_lines(text) + assert plaintext == (), "plaintext lines with JSON logs on:\n" + "\n".join(plaintext) + assert sum(STARTED_WORKER in message for message in _messages(text)) == 2, _messages(text) + + +def test_json_logs_config_makes_the_deprecated_flag_notice_one_json_record(gateway: Gateway, tmp_path: Path) -> None: + """The notice waits until the config is read, so a json_logs set only there makes it a JSON record.""" + models: Final = _models() + config: Final = _write_config(tmp_path, gateway.upstream_url, models, {"json_logs": True}) + log: Final = _boot_serve_and_stop(gateway, tmp_path, config, {"LITELLM_LOG": "INFO"}, 1, models, (DEPRECATED_FLAG,)) + plaintext: Final = _non_json_lines(log) + assert plaintext == (), "plaintext lines with JSON logs on:\n" + "\n".join(plaintext) + assert sum(DEPRECATED_FLAG_NOTICE in message for message in _messages(log)) == 1, _messages(log) + + +@pytest.mark.parametrize("config_text", (None, UNPARSEABLE_CONFIG), ids=("missing", "unparseable")) +def test_a_config_that_fails_to_load_still_prints_the_deprecated_flag_notice( + gateway: Gateway, tmp_path: Path, config_text: str | None +) -> None: + """A boot with the deprecated flag and a config file that does not exist or does not parse exits + non-zero, and it still prints the notice once before the error, as it did before the notice waited + for the config.""" + config: Final = tmp_path / f"unloadable_{uuid.uuid4().hex}.yaml" + if config_text is not None: + config.write_text(config_text) + log: Final = refused_boot_log(gateway, tmp_path, {}, config=config, extra_arguments=(DEPRECATED_FLAG,)) + assert log.count(DEPRECATED_FLAG_NOTICE) == 1, log + + +def test_plain_logs_control_prints_startup_text(gateway: Gateway, tmp_path: Path) -> None: + models: Final = _models() + config: Final = _write_config(tmp_path, gateway.upstream_url, models, CACHE_SETTINGS) + log: Final = _boot_serve_and_stop(gateway, tmp_path, config, {"LITELLM_LOG": "INFO"}, 1, models) + plaintext: Final = _non_json_lines(log) + assert any(BANNER_FRAGMENT in line for line in plaintext), log + assert any(FEEDBACK_BOX_LINE in line for line in plaintext), log + assert any(CACHE_LINE in line for line in plaintext), log + assert any(MODEL_LIST_HEADER in line for line in plaintext), log + for model in models: + assert any(line.strip().endswith(model) or line.strip().endswith(f"{model}\x1b[0m") for line in plaintext), ( + model, + log, + ) + assert any(PLAIN_LOG_LINE.match(line) and REDIS_AUTH_CACHE_LINE in line for line in plaintext), log + assert any(line.startswith(f"LiteLLM Proxy: {CONNECTION_BUDGET_LINE}") for line in plaintext), log diff --git a/tests/unit/proxy/db/test_check_migration.py b/tests/unit/proxy/db/test_check_migration.py index 74a6841cb23..b5685676941 100644 --- a/tests/unit/proxy/db/test_check_migration.py +++ b/tests/unit/proxy/db/test_check_migration.py @@ -1,6 +1,12 @@ +import json +import logging +import sys +from typing import Final import pytest +import litellm +from litellm._logging import turn_on_json, verbose_logger, verbose_proxy_logger def test_check_migration_out_of_sync(mocker): @@ -69,3 +75,33 @@ def test_migrate_diff_without_the_prisma_runner_skips_instead_of_crashing_boot(m monkeypatch.setitem(sys.modules, "litellm_proxy_extras.prisma_toolchain", None) assert check_prisma_schema_diff_helper("postgresql://u:p@localhost:9/x") == (False, []) + + +def test_a_skipped_migration_diff_is_one_json_record_in_json_mode( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] +) -> None: + """ + With json_logs on, the notice that the migration diff was skipped is one INFO record on stdout, + also while the loggers sit at WARNING, as they do at boot without LITELLM_LOG. + """ + from litellm.proxy.db.check_migration import check_prisma_schema_diff_helper + + monkeypatch.setitem(sys.modules, "litellm_proxy_extras.prisma_toolchain", None) + monkeypatch.setattr(litellm, "json_logs", True) + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + turn_on_json() + verbose_logger.setLevel(logging.WARNING) + verbose_proxy_logger.setLevel(logging.WARNING) + + assert check_prisma_schema_diff_helper("postgresql://u:p@localhost:9/x") == (False, []) + + out, err = capfd.readouterr() + assert err == "" + lines: Final = [raw for raw in out.splitlines() if raw.strip()] + assert len(lines) == 1, lines + record: Final = json.loads(lines[0]) + assert (record["level"], record["message"]) == ( + "INFO", + "Skipping the migration diff: litellm-proxy-extras has no Prisma runner. " + "Error: import of litellm_proxy_extras.prisma_toolchain halted; None in sys.modules", + ) diff --git a/tests/unit/proxy/policy_engine/test_init_policies.py b/tests/unit/proxy/policy_engine/test_init_policies.py new file mode 100644 index 00000000000..b9b5f278252 --- /dev/null +++ b/tests/unit/proxy/policy_engine/test_init_policies.py @@ -0,0 +1,116 @@ +"""The policy table printed at boot keeps its terminal form in plain mode and is JSON when json_logs is on.""" + +import json +import sys +from collections.abc import Mapping, Sequence +from typing import Final + +import pytest + +import litellm +from litellm._logging import turn_on_json +from litellm.proxy.policy_engine.init_policies import _print_policies_on_startup + +POLICIES: Final[Mapping[str, Mapping[str, object]]] = { + "pii-policy": { + "inherit": "base", + "description": "Block PII", + "guardrails": {"add": ["pii-guard"], "remove": ["noop"]}, + "condition": {"model": "gpt-4o"}, + } +} +ATTACHMENTS: Final[Sequence[Mapping[str, object]]] = ({"policy": "pii-policy", "scope": "*", "teams": ["team-a"]},) +POLICY_ROWS: Final = ( + " - pii-policy (inherits: base)", + " description: Block PII", + " guardrails.add: ['pii-guard']", + " guardrails.remove: ['noop']", + " condition.model: gpt-4o", +) +PLAIN_POLICY_TABLE: Final = ( + "\x1b[92m\nLiteLLM Policy Engine: Loaded 1 policies\x1b[0m\n\n" + "\x1b[94m - pii-policy (inherits: base)\x1b[0m\n" + " description: Block PII\n" + " guardrails.add: ['pii-guard']\n" + " guardrails.remove: ['noop']\n" + " condition.model: gpt-4o\n" +) +NO_ATTACHMENTS_WARNING: Final = ( + "Warning: No policy_attachments configured. Policies will not be applied to any requests." +) + + +@pytest.mark.parametrize( + ("attachments", "expected"), + [ + ( + ATTACHMENTS, + PLAIN_POLICY_TABLE + + "\n\x1b[93mPolicy Attachments: 1 attachment(s)\x1b[0m\n" + + " - pii-policy -> scope=* (global), teams=['team-a']\n\n", + ), + (None, PLAIN_POLICY_TABLE + f"\n\x1b[93m{NO_ATTACHMENTS_WARNING}\x1b[0m\n\n"), + ], + ids=["with-attachments", "without-attachments"], +) +def test_policy_table_prints_unchanged_in_plain_mode( + monkeypatch: pytest.MonkeyPatch, + capfd: pytest.CaptureFixture[str], + attachments: Sequence[Mapping[str, object]] | None, + expected: str, +) -> None: + """Operators without JSON logs keep the colored policy table byte for byte.""" + monkeypatch.setattr(litellm, "json_logs", False) + + _print_policies_on_startup(POLICIES, attachments) + + assert capfd.readouterr() == (expected, "") + + +def test_policy_table_is_one_json_record_in_json_mode( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] +) -> None: + """With JSON logs on the whole table, attachments included, is one INFO record without color codes.""" + monkeypatch.setattr(litellm, "json_logs", True) + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + turn_on_json() + + _print_policies_on_startup(POLICIES, ATTACHMENTS) + + out, err = capfd.readouterr() + assert err == "" + lines: Final = [raw for raw in out.splitlines() if raw.strip()] + assert len(lines) == 1, lines + record: Final = json.loads(lines[0]) + assert record["level"] == "INFO" + assert record["message"] == "\n".join( + ( + "LiteLLM Policy Engine: Loaded 1 policies", + "", + *POLICY_ROWS, + "", + "Policy Attachments: 1 attachment(s)", + " - pii-policy -> scope=* (global), teams=['team-a']", + ) + ) + + +def test_missing_attachments_warning_is_a_second_json_record( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] +) -> None: + """Without attachments the table and the yellow warning are two INFO records on stdout, where the two + prints put them.""" + monkeypatch.setattr(litellm, "json_logs", True) + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + turn_on_json() + + _print_policies_on_startup(POLICIES, None) + + out, err = capfd.readouterr() + assert err == "" + records: Final = [json.loads(raw) for raw in out.splitlines() if raw.strip()] + assert [record["message"] for record in records] == [ + "\n".join(("LiteLLM Policy Engine: Loaded 1 policies", "", *POLICY_ROWS)), + NO_ATTACHMENTS_WARNING, + ] + assert {record["level"] for record in records} == {"INFO"} diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index 89a35923be0..d77731aa83b 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -1,4 +1,5 @@ import inspect +import json import logging import os from contextlib import nullcontext @@ -9,6 +10,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import click import fastapi +import httpx import pytest @@ -23,6 +25,8 @@ import yaml from uvicorn.config import LOOP_FACTORIES from uvicorn.importer import import_from_string +import litellm +from litellm._logging import turn_on_json from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS from litellm.proxy import proxy_cli from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server @@ -2875,6 +2879,26 @@ class TestRunServerDbSetup: mock_setup_database.assert_called_once_with(use_migrate=True, use_v2_resolver=True) assert ("--enforce_prisma_migration_check is deprecated and has no effect" in capsys.readouterr().out) is warned + def test_the_retired_enforce_prisma_migration_check_opt_in_warns_on_health( + self, capsys: pytest.CaptureFixture[str] + ) -> None: + """`--health` returns before the config is read, where a server start emits the deprecation line; + it still prints it, as it did when the notice came before that return.""" + with patch("httpx.get", return_value=httpx.Response(200, json={})) as mock_get: + run_server.main(["--health", "--enforce_prisma_migration_check"], standalone_mode=False) + + mock_get.assert_called_once_with(url="http://0.0.0.0:4000/health") + assert "--enforce_prisma_migration_check is deprecated and has no effect" in capsys.readouterr().out + + def test_the_retired_enforce_prisma_migration_check_opt_in_warns_on_test( + self, capsys: pytest.CaptureFixture[str] + ) -> None: + """`--test` returns before the config is read too, and prints the deprecation line before its request.""" + with pytest.raises(ValueError, match="Invalid test value"): + run_server.main(["--test", "--enforce_prisma_migration_check"], standalone_mode=False) + + assert "--enforce_prisma_migration_check is deprecated and has no effect" in capsys.readouterr().out + @patch("subprocess.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database", return_value=True) @@ -3935,3 +3959,117 @@ class TestValidateConfigFlag: assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" assert "config OK" in result.output mock_popen.assert_not_called() + + +def _boot_from_config_file(config_path: Path, *flags: str) -> str | None: + env: Final = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL", "JSON_LOGS")} + with patch.dict(os.environ, env, clear=True): + run_server.main(["--config", str(config_path), "--skip_server_startup", *flags], standalone_mode=False) + return os.environ.get("JSON_LOGS") + + +class TestJsonLogsConfigReachesSpawnedWorkers: + """`litellm_settings.json_logs: true` turns JSON on inside the CLI process, but the workers that + uvicorn or granian spawn are fresh interpreters that pick plain or JSON from the JSON_LOGS + environment variable at import. The CLI has to export it, or every worker logs plain text until + its own load_config reaches the key.""" + + def test_json_logs_true_exports_json_logs_to_spawned_workers( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "json_logs", False) + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + config_path: Final = tmp_path / "config.yaml" + config_path.write_text(yaml.dump({"model_list": [], "litellm_settings": {"json_logs": True}})) + + assert _boot_from_config_file(config_path) == "true" + + def test_plain_logs_config_leaves_json_logs_unset(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "json_logs", False) + config_path: Final = tmp_path / "config.yaml" + config_path.write_text(yaml.dump({"model_list": [], "litellm_settings": {}})) + + assert _boot_from_config_file(config_path) is None + + +class TestEnforcePrismaMigrationCheckNoticeAroundTheConfigRead: + """With a config file, the `--enforce_prisma_migration_check` deprecation notice waits until the config + is read, so a `json_logs: true` set only there makes it a JSON record. It is printed once either way, + and a boot whose config read fails still prints it before the error, as it did before the wait.""" + + NOTICE: Final = "--enforce_prisma_migration_check is deprecated and has no effect" + FLAG: Final = "--enforce_prisma_migration_check" + + def test_a_config_read_that_fails_still_prints_the_notice( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + monkeypatch.setattr(litellm, "json_logs", False) + + with pytest.raises(Exception, match="Config file not found"): + _boot_from_config_file(tmp_path / "missing.yaml", self.FLAG) + + assert capsys.readouterr().out.count(self.NOTICE) == 1 + + def test_a_plain_config_prints_the_notice_once( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + monkeypatch.setattr(litellm, "json_logs", False) + config_path: Final = tmp_path / "config.yaml" + config_path.write_text(yaml.dump({"model_list": [], "litellm_settings": {}})) + + _boot_from_config_file(config_path, self.FLAG) + + assert capsys.readouterr().out.count(self.NOTICE) == 1 + + def test_a_config_only_json_logs_makes_the_notice_one_json_record( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + monkeypatch.setattr(litellm, "json_logs", False) + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + config_path: Final = tmp_path / "config.yaml" + config_path.write_text(yaml.dump({"model_list": [], "litellm_settings": {"json_logs": True}})) + + _boot_from_config_file(config_path, self.FLAG) + + notice_lines: Final = [line for line in capsys.readouterr().out.splitlines() if self.NOTICE in line] + assert len(notice_lines) == 1, notice_lines + assert self.NOTICE in json.loads(notice_lines[0])["message"] + + +def test_granian_flag_notices_are_json_records_in_json_mode( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] +) -> None: + """The yellow notices for the flags Granian ignores (`--max_requests_before_restart`, `--ciphers`) are + printed while the launcher starts; with json_logs on they are INFO records on stdout like every other + line, not plaintext between the records.""" + pytest.importorskip("granian") + monkeypatch.setattr(litellm, "json_logs", True) + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + turn_on_json() + + with ( + patch("granian.Granian") as mock_granian_cls, + patch("granian.constants.Interfaces", SimpleNamespace(ASGI="asgi")), + ): + ProxyInitializationHelpers._init_granian_server( + host="0.0.0.0", + port=4000, + num_workers=2, + ssl_certfile_path=None, + ssl_keyfile_path=None, + max_requests_before_restart=100, + ciphers="ECDHE", + granian_runtime_threads=None, + ) + + mock_granian_cls.return_value.serve.assert_called_once() + out, err = capfd.readouterr() + assert err == "" + records: Final = [json.loads(raw) for raw in out.splitlines() if raw.strip()] + assert [record["message"] for record in records] == [ + "LiteLLM Proxy: Starting server on 0.0.0.0:4000 using Granian", + "LiteLLM: --max_requests_before_restart is not supported by Granian " + "(Granian uses workers_lifetime in seconds, not a per-request limit).", + "LiteLLM: --ciphers is not applied when using --run_granian.", + ] + assert {record["level"] for record in records} == {"INFO"} diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 8c62b225992..55d1587669a 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -7,6 +7,7 @@ import os import re import socket import subprocess +import sys import time import types from datetime import datetime, timedelta, timezone @@ -28,6 +29,7 @@ from fastapi.testclient import TestClient import litellm import litellm.proxy.proxy_server as proxy_server_module from litellm._internal_context import current_service_target +from litellm._logging import turn_on_json, verbose_proxy_logger from litellm.caching.caching import RedisCache from litellm.caching.dual_cache import DualCache from litellm.caching.redis_cluster_cache import RedisClusterCache @@ -46,6 +48,7 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.hooks.parallel_request_limiter_v3 import RequestRateLimiterStash from litellm.proxy.proxy_server import app, aws_error_type_exception_handler, initialize, openai_exception_handler from litellm.utils import _invalidate_model_cost_lowercase_map +from litellm.proxy.common_utils.banner import LITELLM_BANNER from tests._master_key import MASTER_KEY PUBLICLY_KNOWN_KEY: Final = "sk-" + "1234" @@ -14266,7 +14269,6 @@ async def test_moderations_response_carries_litellm_call_id_header(): async def test_moderations_failure_log_carries_the_callers_litellm_call_id(caplog): """LIT-7836: the /v1/moderations error line must carry the litellm_call_id the client sent, rendered in the message and as a structured log record field.""" - from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ProxyException call_id = "moderations-call-7836" @@ -15586,3 +15588,61 @@ async def test_spend_capture_rate_check_job_clears_the_gauge_once_the_setting_is call(api_provider="openai", capture_rate=0.97), call(api_provider="openai", capture_rate=None), ] + + +def test_feedback_box_is_one_json_record_in_json_mode( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] +) -> None: + """The feedback box is a dozen terminal lines; with JSON logs on it has to land as one parseable + record, not a plaintext line per row.""" + monkeypatch.setattr(litellm, "json_logs", True) + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + turn_on_json() + + proxy_server_module.generate_feedback_box() + + out, err = capfd.readouterr() + assert err == "" + lines: Final = [raw for raw in out.splitlines() if raw.strip()] + assert len(lines) == 1, lines + record: Final = json.loads(lines[0]) + assert "Give Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new" in record["message"] + assert "Thank you for using LiteLLM! - Krrish & Ishaan" in record["message"] + assert "\x1b[" not in record["message"] + + +@pytest.mark.asyncio +async def test_initialize_prints_the_banner_between_blank_lines_in_plain_mode( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] +) -> None: + """Operators without JSON logs see the banner exactly as before, through show_banner()'s flushing + click.echo.""" + monkeypatch.setattr(litellm, "json_logs", False) + monkeypatch.setenv("LITELLM_DONT_SHOW_FEEDBACK_BOX", "true") + + await proxy_server_module.initialize() + + out, err = capfd.readouterr() + assert err == "" + assert out.startswith(f"\n{LITELLM_BANNER}\n\n"), out + + +@pytest.mark.asyncio +async def test_initialize_prints_the_banner_as_one_json_record_in_json_mode( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] +) -> None: + """With JSON logs on, the six banner rows are one INFO record with their indentation kept, so the + first line a worker writes parses like every other.""" + monkeypatch.setattr(litellm, "json_logs", True) + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + monkeypatch.setenv("LITELLM_DONT_SHOW_FEEDBACK_BOX", "true") + turn_on_json() + + await proxy_server_module.initialize() + + out, err = capfd.readouterr() + assert err == "" + records: Final = [json.loads(raw) for raw in out.splitlines() if raw.strip()] + banner_records: Final = [record for record in records if record["message"] == LITELLM_BANNER] + assert len(banner_records) == 1, records + assert banner_records[0]["level"] == "INFO" diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py index cb4d3f82611..36be864a6f7 100644 --- a/tests/unit/test_logging.py +++ b/tests/unit/test_logging.py @@ -34,6 +34,7 @@ from litellm._logging import ( _parse_json_logs_env, _plain_log_format, _stdout_truncation_marker, + print_startup_message, turn_on_json, format_base64_size, session_id_var, @@ -1824,3 +1825,222 @@ def test_print_verbose_formats_args_only_when_set_verbose_is_on(monkeypatch, cap logging_module.print_verbose("key %s value %s", "k1", _StrCountingValue()) assert _StrCountingValue.str_calls == 1 assert capsys.readouterr().out == "key k1 value cached-vector\n" + + +def test_print_startup_message_prints_plain_mode_text_unchanged( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] +) -> None: + """Operators without JSON logs keep the colored, spaced terminal output exactly as before.""" + monkeypatch.setattr(litellm, "json_logs", False) + + print_startup_message("\n\033[1;32mLiteLLM: Proxy initialized with Config, Set models:\033[0m\n") + + out, err = capfd.readouterr() + assert out == "\n\033[1;32mLiteLLM: Proxy initialized with Config, Set models:\033[0m\n\n" + assert err == "" + + +def test_print_startup_message_is_one_json_record_in_json_mode( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] +) -> None: + """With JSON logs on, a multi-line colored startup message is one INFO record with the colors and the + surrounding blank lines removed, so a log pipeline parses it like every other line. The logger sits + at WARNING, as it does in the CLI process, and the record still goes out: a print has no level.""" + monkeypatch.setattr(litellm, "json_logs", True) + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + turn_on_json() + verbose_proxy_logger.setLevel(logging.WARNING) + + print_startup_message( + "\n\033[1;32mLiteLLM: Proxy initialized with Config, Set models:\033[0m\n\033[32m gpt-4o\033[0m\n\n" + ) + + out, err = capfd.readouterr() + assert err == "" + lines: Final = [raw for raw in out.splitlines() if raw.strip()] + assert len(lines) == 1, lines + record: Final = json.loads(lines[0]) + assert record["message"] == "LiteLLM: Proxy initialized with Config, Set models:\n gpt-4o" + assert record["level"] == "INFO" + assert record["component"] == "LiteLLM Proxy" + assert str(record["logger"]).startswith("test_logging.py:"), record["logger"] + + +def test_print_startup_message_yellow_notice_is_an_info_record_on_stdout( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] +) -> None: + """An operator notice printed in yellow (an ignored flag, a deprecation) stays on stdout in JSON mode, + where the print put it, as one INFO record with the color codes removed.""" + monkeypatch.setattr(litellm, "json_logs", True) + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + turn_on_json() + verbose_proxy_logger.setLevel(logging.WARNING) + + print_startup_message("\033[1;33mLiteLLM: --ciphers is not applied when using --run_granian.\033[0m\n") + + out, err = capfd.readouterr() + assert err == "" + lines: Final = [raw for raw in out.splitlines() if raw.strip()] + assert len(lines) == 1, lines + record: Final = json.loads(lines[0]) + assert record["message"] == "LiteLLM: --ciphers is not applied when using --run_granian." + assert record["level"] == "INFO" + + +def test_print_startup_message_shows_above_the_json_handler_level( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] +) -> None: + """`LITELLM_LOG=WARNING` sets the JSON handler to WARNING, which drops the INFO logger lines; the + startup notices still go out, as the prints they replace did at every level.""" + monkeypatch.setattr(litellm, "json_logs", True) + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + turn_on_json() + verbose_proxy_logger.setLevel(logging.INFO) + for json_handler in verbose_proxy_logger.handlers: + monkeypatch.setattr(json_handler, "level", logging.WARNING) + + print_startup_message("\033[94m\nSetting Cache on Proxy\033[0m") + verbose_proxy_logger.info("enable_redis_auth_cache is not set") + + out, _ = capfd.readouterr() + lines: Final = [raw for raw in out.splitlines() if raw.strip()] + assert [json.loads(raw)["message"] for raw in lines] == ["Setting Cache on Proxy"] + + +def test_print_startup_message_emits_json_after_an_operator_log_config_removed_the_handlers( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] +) -> None: + """uvicorn applies an operator `--log_config` after proxy_cli's turn_on_json() and before initialize() + re-installs the JSON handler through load_config(); a config that names "LiteLLM Proxy" leaves the + logger without handlers in that window, where the banner and the feedback box are emitted, and they + still go out as one JSON record each, on stdout.""" + import logging.config + + monkeypatch.setattr(litellm, "json_logs", True) + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + turn_on_json() + request.addfinalizer(turn_on_json) + logging.config.dictConfig( + {"version": 1, "disable_existing_loggers": False, "loggers": {"LiteLLM Proxy": {"level": "INFO"}}} + ) + assert verbose_proxy_logger.handlers == [] + + print_startup_message("\n\033[1;37m# Give Feedback / Get Help #\033[0m\n") + + out, err = capfd.readouterr() + assert err == "" + lines: Final = [raw for raw in out.splitlines() if raw.strip()] + assert len(lines) == 1, lines + record: Final = json.loads(lines[0]) + assert record["message"] == "# Give Feedback / Get Help #" + assert record["level"] == "INFO" + assert record["component"] == "LiteLLM Proxy" + + +OPERATOR_HANDLERS_ON_THE_PROXY_LOGGER: Final = { + "silenced": {"class": "logging.NullHandler"}, + "stderr_warning": { + "class": "logging.StreamHandler", + "stream": "ext://sys.stderr", + "level": "WARNING", + "formatter": "plain", + }, +} + + +@pytest.mark.parametrize("handler", sorted(OPERATOR_HANDLERS_ON_THE_PROXY_LOGGER)) +def test_print_startup_message_stays_on_stdout_under_an_operator_handler_on_the_proxy_logger( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str], handler: str +) -> None: + """An operator `--log_config` that gives "LiteLLM Proxy" a handler of its own, a NullHandler that silences + it or a plain WARNING-level handler on stderr, is in force when the banner and the feedback box are + emitted; the print they replace wrote to stdout whatever the logging config said, so the record still + goes out as one JSON record on stdout and the operator's handler gets nothing.""" + import logging.config + + monkeypatch.setattr(litellm, "json_logs", True) + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + turn_on_json() + request.addfinalizer(turn_on_json) + logging.config.dictConfig( + { + "version": 1, + "disable_existing_loggers": False, + "formatters": {"plain": {"format": "%(levelname)s %(message)s"}}, + "handlers": {"proxy": OPERATOR_HANDLERS_ON_THE_PROXY_LOGGER[handler]}, + "loggers": {"LiteLLM Proxy": {"level": "INFO", "handlers": ["proxy"], "propagate": False}}, + } + ) + assert len(verbose_proxy_logger.handlers) == 1 and not isinstance( + verbose_proxy_logger.handlers[0], LevelRoutingStreamHandler + ) + + print_startup_message("\n\033[1;37m# Give Feedback / Get Help #\033[0m\n") + + out, err = capfd.readouterr() + assert err == "", err + lines: Final = [raw for raw in out.splitlines() if raw.strip()] + assert len(lines) == 1, lines + record: Final = json.loads(lines[0]) + assert record["message"] == "# Give Feedback / Get Help #" + assert record["level"] == "INFO" + assert record["component"] == "LiteLLM Proxy" + + +def test_print_startup_message_keeps_a_listing_longer_than_the_stdout_cap_whole( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] +) -> None: + """The required migration SQL and the policy list run past MAX_STRING_LENGTH_STDOUT_LOG on a drifted + deployment. The prints they replace showed them whole, and that cap exists for echoed request payloads, + so the JSON record carries the whole listing too.""" + monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500") + monkeypatch.setattr(litellm, "json_logs", True) + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + turn_on_json() + commands: Final = tuple(f'ALTER TABLE "LiteLLM_Table{n}" ADD COLUMN "column_{n}" TEXT;' for n in range(40)) + listing: Final = "\n".join(("Changes to DB Schema detected", "Required SQL commands:", *commands)) + assert len(listing) > 500 + + print_startup_message(listing) + + out, err = capfd.readouterr() + assert err == "" + lines: Final = [raw for raw in out.splitlines() if raw.strip()] + assert len(lines) == 1, lines + record: Final = json.loads(lines[0]) + assert LITELLM_TRUNCATED_PAYLOAD_FIELD not in record["message"] + assert record["message"] == listing + + +def test_print_startup_message_keeps_a_model_name_the_secret_patterns_would_match( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] +) -> None: + """The model list printed `risk-assessment-gpt4o` as configured; the sk- key pattern matches inside + that name, so the JSON record has to carry the name as the print did, not `riREDACTED`.""" + monkeypatch.setattr(litellm, "json_logs", True) + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + turn_on_json() + assert secret_redaction.redact_string("risk-assessment-gpt4o") != "risk-assessment-gpt4o" + + print_startup_message("\033[32m risk-assessment-gpt4o\033[0m") + + out, err = capfd.readouterr() + assert err == "" + lines: Final = [raw for raw in out.splitlines() if raw.strip()] + assert len(lines) == 1, lines + assert json.loads(lines[0])["message"] == " risk-assessment-gpt4o" + + +def test_print_startup_message_blank_spacer_emits_no_json_record( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str] +) -> None: + """A spacer that only pads the terminal has nothing to say to a log pipeline.""" + monkeypatch.setattr(litellm, "json_logs", True) + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + turn_on_json() + verbose_proxy_logger.setLevel(logging.INFO) + + print_startup_message("") + print_startup_message("\n\033[0m\n") + + assert capfd.readouterr() == ("", "") From 8f31c7171dfbd28c53d294121a28c5d232d9137c Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 27 Oct 2026 15:12:53 -0700 Subject: [PATCH 032/102] fix(router): keep a lone deployment warm under router allowed_fails unless the policy names the exception (#48644) * fix(router): keep a lone deployment warm under router allowed_fails unless the policy names the exception Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): give cooldown fixtures a sibling deployment so allowed_fails=0 still forces a cooldown * fix(router): keep benching a lone deployment that has a fallback group or an empty deployment policy * fix(router): accept dictionary fallback targets in the lone deployment check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router_utils/cooldown_handlers.py | 44 ++++++- .../routing/test_payment_required_cooldown.py | 74 +++++++++++ .../router_utils/test_cooldown_handlers.py | 119 ++++++++++++++---- .../test_fallback_event_handlers.py | 14 ++- tests/unit/test_router/test_router.py | 14 ++- 5 files changed, 234 insertions(+), 31 deletions(-) diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index c362c6899db..330500789a4 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -185,7 +185,14 @@ def _should_cooldown_based_on_deployment_policy( override that safety net, matching `_has_explicit_allowed_fails_policy_for_exception`. """ allowed_fails_from_policy: Final = _resolve_allowed_fails_from_policy(dep_policy, original_exception) - if allowed_fails_from_policy is None and dep_allowed_fails is not None and is_single_deployment_model_group: + if ( + allowed_fails_from_policy is None + and is_single_deployment_model_group + and ( + dep_allowed_fails is not None + or litellm_router_instance.get_allowed_fails_from_policy(original_exception) is None + ) + ): return False allowed_fails_override: Final[int | None] = ( @@ -399,6 +406,34 @@ def _should_run_cooldown_logic( return True +_FALLBACK_ENTRIES_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, list[str | Mapping[str, object]]] | str, ...]) + + +def _fallback_target_name(target: str | Mapping[str, object]) -> str: + return target if isinstance(target, str) else str(target.get("model", "")) + + +def _fallback_chain_names( + entry: Mapping[str, Sequence[str | Mapping[str, object]]], +) -> Mapping[str, list[str]]: # mutable-ok: mirrors get_fallback_model_group's fallbacks shape + return {key: [_fallback_target_name(target) for target in targets] for key, targets in entry.items()} + + +def _has_fallback_group(litellm_router_instance: LitellmRouter, requested_model_group: str | None) -> bool: + from litellm.router_utils.fallback_event_handlers import get_fallback_model_group + + configured: Final[object] = getattr(litellm_router_instance, "fallbacks", None) + if requested_model_group is None or not configured: + return False + chains: Final = tuple( + _fallback_chain_names(entry) + for entry in _FALLBACK_ENTRIES_ADAPTER.validate_python(configured) + if isinstance(entry, Mapping) + ) + fallback_group, _ = get_fallback_model_group(fallbacks=chains, model_group=requested_model_group) + return bool(fallback_group) + + def _should_cooldown_deployment( litellm_router_instance: LitellmRouter, deployment: str, @@ -431,6 +466,9 @@ def _should_cooldown_deployment( litellm_router_instance.routing_group_has_alternatives(requested_model_group) or litellm_router_instance.team_model_has_alternatives(deployment) ) + is_lone_deployment: Final = is_single_deployment_model_group and not _has_fallback_group( + litellm_router_instance, requested_model_group + ) exception_status_int: Final = cast_exception_status_to_int(exception_status) dep_policy, dep_allowed_fails = _get_deployment_cooldown_policy(litellm_router_instance, deployment) @@ -450,7 +488,7 @@ def _should_cooldown_deployment( original_exception, dep_policy, dep_allowed_fails, - is_single_deployment_model_group, + is_lone_deployment, ) ## BASE CASE - single deployment @@ -495,6 +533,8 @@ def _should_cooldown_deployment( return False else: + if is_lone_deployment and litellm_router_instance.get_allowed_fails_from_policy(original_exception) is None: + return False return should_cooldown_based_on_allowed_fails_policy( litellm_router_instance=litellm_router_instance, deployment=deployment, diff --git a/tests/integration/routing/test_payment_required_cooldown.py b/tests/integration/routing/test_payment_required_cooldown.py index bc5cf1a5913..865edf31260 100644 --- a/tests/integration/routing/test_payment_required_cooldown.py +++ b/tests/integration/routing/test_payment_required_cooldown.py @@ -1,5 +1,6 @@ from __future__ import annotations +import itertools import json import re import uuid @@ -390,6 +391,79 @@ def test_a_router_allowed_fails_policy_cools_a_lone_402_deployment_past_its_coun assert markers_received(refusing) == (first.marker, second.marker) +def failing_reply(request: Request) -> Reply: + return model_list_reply() if is_model_info_probe(request) else anthropic_error(500, "upstream down") + + +def assert_failed(outcome: Outcome) -> None: + assert outcome.status == 500, (outcome.status, outcome.detail) + assert "upstream down" in outcome.detail, outcome.detail + + +@pytest.mark.timeout(900) +def test_a_router_allowed_fails_count_leaves_a_lone_deployment_warm_but_benches_one_with_a_sibling_or_fallback( + tmp_path: Path, +) -> None: + lone_group: Final = new_group() + pair_group: Final = new_group() + primary_group: Final = new_group() + backup_group: Final = new_group() + with ExitStack() as stack: + upstream: Final = stack.enter_context(gateway_from_environment()) + cache: Final = stack.enter_context(owned_redis(tmp_path)) + settings: Final = router_settings(cache, allowed_fails=1, fallbacks=[{primary_group: [backup_group]}]) + owned: Final = stack.enter_context( + owned_proxy_process( + upstream, + tmp_path, + {}, + config=cooldown_config(tmp_path, settings), + remove_environment=("PROMETHEUS_MULTIPROC_DIR",), + ) + ) + failing: Final = stack.enter_context(wire_server(failing_reply)) + healthy: Final = stack.enter_context(wire_server(healthy_reply)) + scenario: Final = stack.enter_context(owned.gateway.scenario()) + lone_id: Final = deployment(scenario, lone_group, anthropic_params(failing)) + failed: Final = tuple(raw_chat(owned.gateway, lone_group, new_marker()) for _ in range(4)) + for outcome in failed: + assert_failed(outcome) + assert not cooled(cache, lone_id) + assert markers_received(failing) == markers(failed) + + paired_id: Final = two_deployments(scenario, pair_group, failing, healthy) + history: Final = tuple( + itertools.takewhile( + lambda outcome: not cooled(cache, paired_id), + (raw_chat(owned.gateway, pair_group, new_marker()) for _ in range(40)), + ) + ) + assert cooled(cache, paired_id), [(outcome.status, outcome.detail[:80]) for outcome in history] + assert sum(outcome.status == 500 for outcome in history) == 1 + served: Final = tuple(raw_chat(owned.gateway, pair_group, new_marker()) for _ in range(10)) + assert_served(served, chat_identity) + assert markers_received(healthy) == (*markers(o for o in history if o.status == 200), *markers(served)) + + primary_id: Final = deployment(scenario, primary_group, anthropic_params(failing)) + deployment(scenario, backup_group, openai_params(healthy)) + failing.drain() + healthy.drain() + fell_back: Final = tuple( + itertools.takewhile( + lambda outcome: not cooled(cache, primary_id), + (raw_chat(owned.gateway, primary_group, new_marker()) for _ in range(40)), + ) + ) + assert cooled(cache, primary_id) and fell_back, [(o.status, o.detail[:80]) for o in fell_back] + assert_served(fell_back, chat_identity) + backed: Final = tuple(raw_chat(owned.gateway, primary_group, new_marker()) for _ in range(4)) + assert_served(backed, chat_identity) + received: Final = markers_received(failing) + assert received[: len(fell_back)] == markers(fell_back) + assert len(received) <= len(fell_back) + 1, received + assert {*markers(fell_back), *markers(backed)} <= set(markers_received(healthy)) + + def test_a_deployment_allowed_fails_policy_cools_a_lone_402_deployment(rig: CooldownRig) -> None: group: Final = new_group() policy: Final = {"allowed_fails_policy": {"BadRequestErrorAllowedFails": 0}} diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 0c9d7ab9f85..bab6d257579 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -2931,10 +2931,65 @@ async def test_bad_request_error_does_not_cool_down_a_deployment( assert tuple(cooldown_deployments) == () -@pytest.mark.parametrize("use_policy", [False, True], ids=["allowed_fails", "allowed_fails_policy"]) +TIMED_OUT_DEPLOYMENT: Final = { + "model_name": "test-model", + "litellm_params": {"model": "openai/test-model", "api_key": "test-key"}, + "model_info": {"id": "test-deployment"}, +} +BACKUP_DEPLOYMENT: Final = { + "model_name": "backup-model", + "litellm_params": {"model": "openai/backup-model", "api_key": "test-key", "api_base": "https://backup.test/v1"}, + "model_info": {"id": "backup-deployment"}, +} +BACKUP_REPLY: Final = { + "id": "chatcmpl-backup", + "object": "chat.completion", + "created": 1700000000, + "model": "backup-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +async def _lone_call_outcome(router: Router) -> type[Exception]: + try: + await router.acompletion(model="test-model", messages=[{"role": "user", "content": "timeout test"}]) + except (litellm.Timeout, RouterRateLimitError) as error: + return type(error) + pytest.fail("the timed-out deployment answered") + + +BENCHED: Final = (litellm.Timeout, litellm.Timeout, RouterRateLimitError, RouterRateLimitError, RouterRateLimitError) + + +@pytest.mark.parametrize( + ("router_kwargs", "model_info", "outcomes"), + [ + ({"allowed_fails": 1}, {}, (litellm.Timeout,) * 5), + ({"allowed_fails": 1}, {"allowed_fails_policy": {}}, (litellm.Timeout,) * 5), + ({"allowed_fails_policy": AllowedFailsPolicy(RateLimitErrorAllowedFails=1)}, {}, (litellm.Timeout,) * 5), + ({"allowed_fails_policy": AllowedFailsPolicy(TimeoutErrorAllowedFails=1)}, {}, BENCHED), + ({"allowed_fails": 1, "fallbacks": [{"test-model": ["backup-model"]}]}, {}, BENCHED), + ( + {"allowed_fails": 1, "fallbacks": [{"test-model": [{"model": "backup-model", "temperature": 0}]}]}, + {}, + BENCHED, + ), + ], + ids=[ + "allowed_fails", + "allowed_fails_with_an_empty_deployment_policy", + "policy_names_another_exception", + "policy_names_timeout", + "allowed_fails_with_a_fallback_group", + "allowed_fails_with_a_dict_fallback_target", + ], +) @pytest.mark.asyncio -async def test_allowed_fails_cools_a_single_deployment( - use_policy: bool, +async def test_router_allowed_fails_benches_a_lone_deployment_only_when_the_policy_names_the_exception( + router_kwargs: Mapping[str, object], + model_info: Mapping[str, object], + outcomes: tuple[type[Exception], ...], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -2942,38 +2997,52 @@ async def test_allowed_fails_cools_a_single_deployment( route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( side_effect=httpx.ReadTimeout("upstream timeout") ) - policy_kwargs: Final = ( - {"allowed_fails_policy": AllowedFailsPolicy(TimeoutErrorAllowedFails=1)} - if use_policy - else {"allowed_fails": 1} + deployment: Final = {**TIMED_OUT_DEPLOYMENT, "model_info": {**TIMED_OUT_DEPLOYMENT["model_info"], **model_info}} + router: Final = Router(model_list=[deployment], cooldown_time=300, num_retries=0, **router_kwargs) + + seen: Final = tuple([await _lone_call_outcome(router) for _ in range(5)]) + + assert seen == outcomes + assert len(route.calls) == outcomes.count(litellm.Timeout) + + +async def _group_call_outcome(router: Router) -> str: + try: + response: Final = await router.acompletion( + model="test-model", messages=[{"role": "user", "content": "timeout test"}] + ) + except litellm.Timeout: + return "timeout" + return response.id + + +@pytest.mark.asyncio +async def test_router_allowed_fails_still_benches_a_deployment_that_has_a_sibling_in_its_group( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + timed_out: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + side_effect=httpx.ReadTimeout("upstream timeout") ) + respx_mock.post("https://backup.test/v1/chat/completions").mock(return_value=httpx.Response(200, json=BACKUP_REPLY)) router: Final = Router( - model_list=[ - { - "model_name": "test-model", - "litellm_params": {"model": "openai/test-model", "api_key": "test-key"}, - "model_info": {"id": "test-deployment"}, - } - ], + model_list=[TIMED_OUT_DEPLOYMENT, {**BACKUP_DEPLOYMENT, "model_name": "test-model"}], cooldown_time=300, num_retries=0, - **policy_kwargs, + allowed_fails=1, ) - for _ in range(2): - with pytest.raises(litellm.Timeout): - await router.acompletion( - model="test-model", - messages=[{"role": "user", "content": "timeout test"}], - ) - + while len(timed_out.calls) < 2: + await _group_call_outcome(router) cooldown_deployments: Final = await async_get_cooldown_deployments( - litellm_router_instance=router, - parent_otel_span=None, + litellm_router_instance=router, parent_otel_span=None ) + served: Final = tuple([await _group_call_outcome(router) for _ in range(5)]) - assert len(route.calls) == 2 assert tuple(cooldown_deployments) == ("test-deployment",) + assert served == ("chatcmpl-backup",) * 5 + assert len(timed_out.calls) == 2 def test_per_user_session_upstream_errors_never_cool_down_the_shared_deployment(single_deployment_router): diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index 53375ee0ab4..c5a476c450b 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -1255,7 +1255,12 @@ class TestTriggerCooldownForFailedDeployment: "model_name": "fallback-model", "litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-fake"}, "model_info": {"id": "fallback-deployment"}, - } + }, + { + "model_name": "fallback-model", + "litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-fake"}, + "model_info": {"id": "fallback-sibling"}, + }, ], allowed_fails=0, cooldown_time=60, @@ -1345,7 +1350,12 @@ async def test_a_fallback_deployment_whose_stream_stalled_stays_out_of_cooldown( "model_name": "fallback-model", "litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-fake"}, "model_info": {"id": "fallback-deployment"}, - } + }, + { + "model_name": "fallback-model", + "litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-fake"}, + "model_info": {"id": "fallback-sibling"}, + }, ], allowed_fails=0, cooldown_time=60, diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index e2ee80b7874..d9b655a0aff 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -12212,7 +12212,12 @@ class TestBackgroundResponseCostPollCooldown: "model_name": "gpt-4.1", "litellm_params": {"model": "openai/gpt-4.1"}, "model_info": {"id": "dep-1"}, - } + }, + { + "model_name": "gpt-4.1", + "litellm_params": {"model": "openai/gpt-4.1"}, + "model_info": {"id": "dep-2"}, + }, ], allowed_fails=0, ) @@ -12340,7 +12345,12 @@ class TestCallerTimeoutCooldown: "model_name": "slow-model", "litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-fake"}, "model_info": {"id": "dep-1"}, - } + }, + { + "model_name": "slow-model", + "litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-fake"}, + "model_info": {"id": "dep-2"}, + }, ], allowed_fails=0, cooldown_time=120, From cb70bc560e1bbd8a1d619fb5dead7f2f0c333e74 Mon Sep 17 00:00:00 2001 From: Ambuj Upadhyay Date: Wed, 28 Oct 2026 03:51:11 +0530 Subject: [PATCH 033/102] fix(streaming): dict-usage arm drops prompt_tokens_details/completion_tokens_details its sibling arms preserve (#36678) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(streaming): dict-usage arm no longer drops token detail fields In _dispatch_provider_chunk's openai/azure else-branch, a plain-dict usage was rebuilt into litellm.Usage from only prompt_tokens / completion_tokens / total_tokens. prompt_tokens_details (cached_tokens), completion_tokens_details (reasoning_tokens) and cache token counts were silently dropped — while both sibling arms (Usage passthrough, BaseModel via Usage(**model_dump())) preserve them. Every in-repo chunk producer currently normalizes usage to Usage/BaseModel before this point, so the dict arm is reachable today only via chunks litellm did not construct (e.g. a custom streaming_decoder feeding CustomStreamWrapper). When it does fire on a usage-only final chunk (stream_options.include_usage), the stripped Usage is exactly what the stream consumer receives — the early-return path has no later aggregation to repair it. Fix: pass the full dict through Usage(**usage_dict). Usage.__init__ coerces nested detail dicts into their wrapper types and tolerates extra keys. Side effect: a legitimate 0 count is no longer coerced to None by the old 'or None' expressions. Co-Authored-By: Claude Fable 5 * chore: drop explanatory source comment per repo comment policy Rationale lives in the PR body and the regression tests. Co-Authored-By: Claude Fable 5 * fix(streaming): type the dict-usage unpack for the basedpyright gate Usage(**response_obj["usage"]) unpacks a dict whose value type pyright cannot see, and the delta gate reports that once per typed constructor parameter: five, against the three .get() calls the line replaced. Moving the unpack behind _usage_from_payload keeps the same Usage(**payload) call and is reported once, at the call site, so the rule count ends below its merge base --------- Co-authored-by: Claude Fable 5 --- .../litellm_core_utils/streaming_handler.py | 11 ++- .../test_streaming_handler.py | 99 +++++++++++++++++++ 2 files changed, 105 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 8fb602ebeb0..b64e146ca08 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1589,11 +1589,7 @@ class CustomStreamWrapper: setattr( model_response, "usage", - litellm.Usage( - prompt_tokens=response_obj["usage"].get("prompt_tokens", None) or None, - completion_tokens=response_obj["usage"].get("completion_tokens", None) or None, - total_tokens=response_obj["usage"].get("total_tokens", None) or None, - ), + _usage_from_payload(response_obj["usage"]), ) elif isinstance(response_obj["usage"], Usage): setattr( @@ -2662,6 +2658,11 @@ class CustomStreamWrapper: _strip_sse_data_from_chunk = strip_sse_data_from_chunk +def _usage_from_payload(payload: Mapping[str, Any]) -> Usage: + """Build ``Usage`` from a wire-shaped usage dict, forwarding every key.""" + return litellm.Usage(**payload) + + def _cache_token_count(details: PromptTokensDetailsWrapper | None, keys: tuple[str, ...]) -> int: for key in keys: value = getattr(details, key, None) diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index cc1fa1b8803..6d700bae514 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -7844,3 +7844,102 @@ async def test_leaving_an_async_with_block_closes_the_provider_stream(): async with _wrapper_over(stream): pass assert stream.close_calls == 1 + + +def _final_chunk_with_dict_usage(choices) -> ModelResponseStream: + """A stream chunk whose ``usage`` is a plain dict. + + ``ModelResponseStream.__init__`` coerces a dict ``usage`` kwarg into + ``litellm.Usage``, so the dict is attached after construction — the same + shape a custom ``streaming_decoder`` (or any third-party iterator that + bypasses litellm chunk construction) delivers to ``chunk_creator``. + """ + chunk: Final = ModelResponseStream( + id="chatcmpl-dict-usage-test", + created=1754900000, + model="gpt-4o", + object="chat.completion.chunk", + choices=choices, + ) + chunk.usage = { # mutable-ok: wire-shaped usage payload under test + "prompt_tokens": 100, + "completion_tokens": 10, + "total_tokens": 110, + "prompt_tokens_details": {"cached_tokens": 80}, # mutable-ok: wire payload + "completion_tokens_details": {"reasoning_tokens": 5}, # mutable-ok: wire payload + "cache_creation_input_tokens": 42, + } + return chunk + + +def test_dispatch_provider_chunk_dict_usage_preserves_token_details( + logging_obj: Logging, +): + """The openai else-branch of ``_dispatch_provider_chunk`` handles three + usage shapes: ``Usage`` (passed through), ``BaseModel`` + (``Usage(**model_dump())``), and plain ``dict``. The dict arm used to + rebuild ``litellm.Usage`` from only prompt/completion/total tokens, + silently dropping ``prompt_tokens_details`` (cached_tokens), + ``completion_tokens_details`` (reasoning_tokens), and cache token counts + that both sibling arms preserve. + """ + wrapper: Final = CustomStreamWrapper( + completion_stream=None, + model="gpt-4o", + logging_obj=logging_obj, + custom_llm_provider="openai", + stream_options={"include_usage": True}, # mutable-ok: constructor payload + ) + chunk: Final = _final_chunk_with_dict_usage( + choices=[ + StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=None)) + ] # mutable-ok: chunk payload + ) + model_response: Final = wrapper.model_response_creator() + wrapper._dispatch_provider_chunk( + chunk=chunk, + model_response=model_response, + completion_obj={"content": ""}, # mutable-ok: dispatch scratch dict + ) + + usage: Final = model_response.usage + assert isinstance(usage, Usage) + assert usage.prompt_tokens == 100 + assert usage.completion_tokens == 10 + assert usage.total_tokens == 110 + assert usage.prompt_tokens_details is not None, ( + "dict-usage arm dropped prompt_tokens_details (its Usage/BaseModel sibling arms preserve it)" + ) + assert usage.prompt_tokens_details.cached_tokens == 80 + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 5 + assert getattr(usage, "cache_creation_input_tokens", None) == 42 + + +def test_chunk_creator_usage_only_chunk_with_dict_usage_keeps_details( + logging_obj: Logging, +): + """End-to-end through ``chunk_creator``: a usage-only final chunk (empty + ``choices``, ``stream_options.include_usage`` set) takes the early-return + path, so whatever the dict-usage arm wrote is exactly what the consumer + receives — there is no later aggregation to repair it. + """ + wrapper: Final = CustomStreamWrapper( + completion_stream=None, + model="gpt-4o", + logging_obj=logging_obj, + custom_llm_provider="openai", + stream_options={"include_usage": True}, # mutable-ok: constructor payload + ) + chunk: Final = _final_chunk_with_dict_usage(choices=[]) # mutable-ok: empty choices list + result: Final = wrapper.chunk_creator(chunk=chunk) + + assert result is not None + usage: Final = result.usage + assert isinstance(usage, Usage) + assert usage.prompt_tokens_details is not None, ( + "usage-only early return delivered a Usage stripped of prompt_tokens_details to the stream consumer" + ) + assert usage.prompt_tokens_details.cached_tokens == 80 + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 5 From 9b2fc9db40f8f976597388b4450ff002ee0aa732 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 27 Oct 2026 15:26:15 -0700 Subject: [PATCH 034/102] test: remove checks that pin implementation details and add a shadcn testing rule (#48663) * test: remove e2e checks and a CI line count that pin implementation details Deletes the Auto Router template select anchoring spec, the 390px logs pagination variant, and the call id tooltip and clipboard step in the logs spec. Each one tests library or test helper behavior rather than what users see, and each has been red or flaky on main without a product regression Also drops the exact line count on the fan-in requirements export in the integration coverage job. It went from 6 to 10 when uv.lock gained a Python 3.15 fork, while the install was still correct. A package the filter drops still fails the job, either at the hash-checked install or at the next step's import * docs(agents): do not test what shadcn and Base UI already guarantee Adds the boundary to the dashboard AGENTS.md next to the test tiers, and points the Playwright suite's entry in tests/e2e/AGENTS.md at it --- .circleci/config.yml | 1 - tests/e2e/AGENTS.md | 2 +- tests/e2e/ui/tests/logs/logs.spec.ts | 11 --- .../e2e/ui/tests/logs/logsPagination.spec.ts | 75 +++++++++---------- .../autoRouterTemplateSelect.spec.ts | 52 ------------- ui/litellm-dashboard/AGENTS.md | 2 + 6 files changed, 38 insertions(+), 105 deletions(-) delete mode 100644 tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts diff --git a/.circleci/config.yml b/.circleci/config.yml index 1edf22578ff..4ce5ea8b6fe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1379,7 +1379,6 @@ jobs: uv export --frozen --all-groups --all-extras --no-emit-project --no-emit-workspace --no-header --no-annotate \ | awk '/^[^ ]/ { keep = $0 ~ /^(annotated-types|coverage|pydantic|pydantic-core|typing-extensions|typing-inspection)==/ } keep' \ > /tmp/fan-in-requirements.txt - test "$(grep -c '^[^ ]' /tmp/fan-in-requirements.txt)" -eq 6 uv venv .venv --python 3.12 uv pip install --no-config --python .venv --require-hashes -r /tmp/fan-in-requirements.txt - attach_workspace: diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index ef57135a4d2..cf1e41a59a6 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -49,7 +49,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `secret_manager/` - the gateway's `key_management_system` against a real secret manager: deployment keys resolved from it (`os.environ/` where the name exists only in the manager) and virtual keys written to and deleted from it. The tests are backend-agnostic and each backend is its own lane, because the setting is global to the proxy: `E2E_SECRET_MANAGER=` opts in and picks the backend from `secret_backends.BACKENDS`, the proxy is booted from `gateway/secret_manager__ci_config.yml` against the live manager, and the tests reach that manager through the backend's `SecretStore` (`secret_store_.py`). A test needing something not every backend does carries `requires_capability(...)` and is deselected on lanes that lack it. `secret_manager/backend.sh up ` runs a backend in Docker and writes the proxy's and the tests' env. Marked `secret_manager`, deselected unless `E2E_SECRET_MANAGER` is set, and kept out of the per-PR selector. Backends today: `hashicorp_vault` and `cyberark` (CyberArk Conjur, which cannot delete, so the delete test is Vault-only) - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke -- `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream, see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport. The Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` +- `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream, see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport. The Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`. Its specs follow the dashboard's rule against testing what shadcn and Base UI already guarantee, in `ui/litellm-dashboard/AGENTS.md` ## MCP suite: real Datadog only diff --git a/tests/e2e/ui/tests/logs/logs.spec.ts b/tests/e2e/ui/tests/logs/logs.spec.ts index 688bfe8e3b4..49400f1509a 100644 --- a/tests/e2e/ui/tests/logs/logs.spec.ts +++ b/tests/e2e/ui/tests/logs/logs.spec.ts @@ -194,17 +194,6 @@ test.describe("Logs page", () => { await expect(row, `no logs row for call id ${callId}`).toHaveCount(1, { timeout: 30_000 }); await expect(row, "the row itself shows only the request id").not.toContainText(callId); - await row.getByText(requestId).hover(); - const tooltip = page.locator("[data-slot='tooltip-content']"); - await expect(tooltip, "hovering the Request ID cell does not list the x-litellm-call-id").toContainText( - `x-litellm-call-id: ${callId}`, - { timeout: 10_000 }, - ); - await tooltip.getByRole("button", { name: "Copy x-litellm-call-id" }).click(); - if (await page.evaluate(() => window.isSecureContext)) { - await expect.poll(() => page.evaluate(() => navigator.clipboard.readText())).toBe(callId); - } - await row.click(); const drawer = page.getByRole("dialog").first(); await expect(drawer.getByText("Request & Response")).toBeVisible({ timeout: 20_000 }); diff --git a/tests/e2e/ui/tests/logs/logsPagination.spec.ts b/tests/e2e/ui/tests/logs/logsPagination.spec.ts index 838a829a3e2..585eb31c37c 100644 --- a/tests/e2e/ui/tests/logs/logsPagination.spec.ts +++ b/tests/e2e/ui/tests/logs/logsPagination.spec.ts @@ -54,52 +54,47 @@ async function setRowsPerPage(page: PlaywrightPage, size: "25" | "50" | "100"): test.describe("Logs pagination with Aria in the header", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); - for (const viewport of [ - { width: 1440, height: 900 }, - { width: 390, height: 844 }, - ]) { - test(`keeps pagination reachable and Aria available at ${viewport.width}px`, async ({ page, request }) => { - await page.setViewportSize(viewport); - const alias = `e2e-aria-header-${uniqueSuffix()}`; - const key = await createVirtualKey(request, { key_alias: alias }); - try { - const ids = await Promise.all( - Array.from({ length: 26 }, (_, index) => - sendChatCompletion(request, { - model: CHAT_MODEL_A, - prompt: `${alias}-${index}`, - apiKey: key.key, - }), - ), - ); - await Promise.all(ids.map((id) => waitForSpendLog(request, id))); - await openLogs(page); - await setAriaHidden(page, false); - await applyKeyAliasFilter(page, await openFilterDrawer(page), alias); - await setRowsPerPage(page, "25"); - await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 1-25 of 26"); - await visibleTestId(page, "pagination-next").click(); - await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 26-26 of 26"); + test("keeps pagination reachable and Aria available at 1440px", async ({ page, request }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + const alias = `e2e-aria-header-${uniqueSuffix()}`; + const key = await createVirtualKey(request, { key_alias: alias }); + try { + const ids = await Promise.all( + Array.from({ length: 26 }, (_, index) => + sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt: `${alias}-${index}`, + apiKey: key.key, + }), + ), + ); + await Promise.all(ids.map((id) => waitForSpendLog(request, id))); + await openLogs(page); + await setAriaHidden(page, false); + await applyKeyAliasFilter(page, await openFilterDrawer(page), alias); + await setRowsPerPage(page, "25"); + await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 1-25 of 26"); + await visibleTestId(page, "pagination-next").click(); + await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 26-26 of 26"); - const aria = page.getByRole("banner").getByRole("button", { name: "Aria Agent", exact: true }); + const aria = page.getByRole("banner").getByRole("button", { name: "Aria Agent", exact: true }); + await aria.click(); + await expect(page.getByRole("dialog", { name: "Aria Agent", exact: true })).toBeVisible(); + await page.keyboard.press("Escape"); + await expect(aria).toBeFocused(); + await visibleTestId(page, "pagination-first").click(); + await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 1-25 of 26"); + for (const destination of [Page.Usage, Page.LlmPlayground, Page.ApiKeys]) { + await navigateToPage(page, destination); await aria.click(); await expect(page.getByRole("dialog", { name: "Aria Agent", exact: true })).toBeVisible(); await page.keyboard.press("Escape"); await expect(aria).toBeFocused(); - await visibleTestId(page, "pagination-first").click(); - await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 1-25 of 26"); - for (const destination of [Page.Usage, Page.LlmPlayground, Page.ApiKeys]) { - await navigateToPage(page, destination); - await aria.click(); - await expect(page.getByRole("dialog", { name: "Aria Agent", exact: true })).toBeVisible(); - await page.keyboard.press("Escape"); - await expect(aria).toBeFocused(); - } - } finally { - await deleteVirtualKey(request, key.token); } - }); - } + } finally { + await deleteVirtualKey(request, key.token); + } + }); }); test.describe("Logs page session-grouped pagination", () => { diff --git a/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts b/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts deleted file mode 100644 index 5f05953cc80..00000000000 --- a/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { expect, test, type Locator, type Page as PlaywrightPage } from "@playwright/test"; -import { ADMIN_STORAGE_PATH } from "../../constants"; -import { navigateToPage } from "../../helpers/navigation"; -import { Page } from "../../fixtures/pages"; - -/** - * Opens Add Auto Router and returns the Template select's trigger, which is the - * shallowest real page that renders SelectContent with tall multi-line options. - */ -async function openTemplateSelect(page: PlaywrightPage) { - await navigateToPage(page, Page.Models); - await page.getByRole("tab", { name: "Auto-Routers" }).click(); - await page.getByRole("button", { name: "Add Auto Router" }).click(); - - const trigger = page.getByTestId("template-selector"); - await expect(trigger).toBeVisible(); - return trigger; -} - -async function boxes(trigger: Locator, options: Locator) { - const triggerBox = await trigger.boundingBox(); - const optionsBox = await options.boundingBox(); - return triggerBox && optionsBox ? { triggerBox, optionsBox } : null; -} - -const clippedPopup = (page: PlaywrightPage) => page.locator('[data-slot="select-content"]'); - -function pollOptionsCoverTrigger(trigger: Locator, options: Locator) { - return expect.poll(async () => { - const box = await boxes(trigger, options); - return ( - box && - box.optionsBox.y < box.triggerBox.y + box.triggerBox.height && - box.optionsBox.y + box.optionsBox.height > box.triggerBox.y - ); - }); -} - -test.describe("Auto Router template select anchoring", () => { - test.use({ storageState: ADMIN_STORAGE_PATH }); - - test("keeps the trigger uncovered when the options open with no room below it", async ({ page }) => { - await page.setViewportSize({ width: 1280, height: 560 }); - const trigger = await openTemplateSelect(page); - await trigger.scrollIntoViewIfNeeded(); - - await trigger.click(); - await expect(page.getByRole("listbox")).toBeVisible(); - - await pollOptionsCoverTrigger(trigger, clippedPopup(page)).toBe(false); - }); -}); diff --git a/ui/litellm-dashboard/AGENTS.md b/ui/litellm-dashboard/AGENTS.md index b6fe5219022..f5576b1228b 100644 --- a/ui/litellm-dashboard/AGENTS.md +++ b/ui/litellm-dashboard/AGENTS.md @@ -8,6 +8,8 @@ When you fix lint violations that are grandfathered in `eslint-suppressions.json Tests come in three tiers, named by the standard definitions. `Foo.test.tsx` is a unit test: one module, collaborators replaced by doubles, no multi-component tree, and it should run in milliseconds. `Foo.integration.test.tsx` renders a real component tree with real children and only stubs the network boundary. It costs seconds per case, so it earns its place by proving wiring that a unit test cannot reach. Browser-level tests live in `tests/e2e/ui/` as Playwright specs against a live proxy +Do not test what shadcn and Base UI already guarantee, such as a tooltip opening on hover, a popup positioning itself against its trigger, focus trapping and Escape in a dialog, keyboard movement through a menu or select, and open and close animations. Those are the library's contract, and a test of them only fails when we restyle or move the page. Test what we put inside the primitive and what happens because of it, like the text a tooltip carries, the options a select offers, or the request a confirm button sends. Reach that content the most direct way, by a click, the keyboard, or the data the page renders, rather than through a hover race. If we change a primitive's behavior in `src/components/ui/`, its test lives next to that primitive as `*.test.tsx`, never in a page spec that borrows the page as a host + When a component holds logic worth asserting, extract the logic and unit-test it there rather than driving it through a render. `CreateMCPServer` is the worked example: its payload building lives in `createServerPayload.ts` with 46 unit tests that run in single-digit milliseconds, while `CreateMCPServer.integration.test.tsx` keeps only the cases that prove a form field reaches the right payload key. A test that renders a whole modal to assert the shape of one object belongs in the first category, not the second Most of the suite predates this split and is not yet classified, so an unsuffixed `*.test.tsx` is not evidence that a file is really a unit test. Classify what you touch From c4c2fe6ba12c9b76f43e8bf3cde5ce63efb91a2b Mon Sep 17 00:00:00 2001 From: tin-berri Date: Tue, 27 Oct 2026 18:38:07 -0400 Subject: [PATCH 035/102] docs(router): clarify heuristic v2 keyword settings (#48666) --- .../complexity_router/README.md | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index 08d39841b61..3df1803d264 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -13,7 +13,7 @@ Unlike the semantic `auto_router` which uses embedding-based matching, the `comp ## How It Works -The router scores each request across 7 dimensions: +The default `classifier_type: heuristic` scores each request across 7 dimensions. These weights, keyword lists and score boundaries apply to the weighted heuristic scorer. [`heuristic_v2`](#heuristic-v2) uses a separate success-probability predictor | Dimension | Description | Weight | |-----------|-------------|--------| @@ -254,11 +254,23 @@ model_list: No classifier model call or per-model training data is required. The classifier uses global tier quality, request-type quality, and similar-request cohorts from -the bundled UltraFeedback artifact. It estimates success at every tier, enforces +the bundled UltraFeedback artifact. Cohorts group prompts by request type, length, +code and math markers, multiple-choice formatting and non-ASCII character share. +Each tier's global estimate is `(successes + 1) / (observations + 2)`. Request-type +and cohort estimates combine their successes and observations with the broader +estimate as a prior, so sparse or missing statistics fall back toward broader data. +The predictor estimates success at every tier, enforces monotonic probabilities, and returns the first tier meeting the success threshold, or REASONING if no tier meets it. The existing complexity-router tier pool then selects and dispatches a model from that tier +`heuristic_v2` does not read `reasoning_keywords` or apply `reasoning_override_min_score`, +even when the floor is `0`. The weighted scorer's keyword lists, `dimension_weights`, +`token_thresholds` and `tier_boundaries` do not affect v2's probabilities or tier selection. +This also applies when a chain selects `local_heuristic: heuristic_v2`. Use +`classifier_type: heuristic` for the [reasoning-marker override](#reasoning-override). +The separate `keyword_tier_rules` routing overrides still run before classification + Set `heuristic_v2_success_threshold` to a value from 0 to 1 to override the artifact's threshold. For example, `0.9` requires a predicted success probability of at least 90%. Higher thresholds favor more capable tiers. Omit the setting or @@ -646,7 +658,16 @@ of that threshold, or when no tier meets the threshold. A zero margin still defe ### Reasoning Override -If 2+ reasoning markers are detected in the user message, the request is promoted to the REASONING tier even when the weighted score maps lower, so complex reasoning tasks get the appropriate model. The promotion requires the score to reach `reasoning_override_min_score`, which tracks `tier_boundaries.simple_medium` unless set, so stock phrases on an otherwise trivial prompt cannot buy the top tier. Set it to `0` to promote on the markers alone. +The weighted heuristic scorer promotes its result to REASONING when the user message matches 2+ entries in `reasoning_keywords` and the weighted score reaches `reasoning_override_min_score`. The floor tracks `tier_boundaries.simple_medium` unless set. An explicit `0` lowers the floor to zero, so a negative weighted score still does not qualify. `heuristic_v2` bypasses this scorer and does not apply the override + +For routing based on these reasoning keywords, set the following under your existing `complexity_router_config` and keep your tier mapping + +```yaml +classifier_type: heuristic +reasoning_override_min_score: 0 +``` + +Omit `reasoning_keywords` to use the built-in list, or supply a nonempty list to replace it. These scoring keywords are separate from `keyword_tier_rules`, which can select a tier before either classifier runs ### System Prompt Handling From 969161501acfca278642ea645130754e66dee846 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 27 Oct 2026 15:54:39 -0700 Subject: [PATCH 036/102] feat(ui): add LITELLM_UI_USE_SI_TERMINOLOGY to show SI instead of AI in the Admin UI (#47440) * feat(ui): add LITELLM_UI_USE_SI_TERMINOLOGY to show SI instead of AI in the Admin UI Setting LITELLM_UI_USE_SI_TERMINOLOGY=true on the proxy makes the Admin UI say "SI" where its own copy says "AI" and "Super Intelligence" where it says "Artificial Intelligence". This covers the nav, page headings, settings text, guardrail garden blurbs, the compliance playground prompts and the playground sample prompts. Third-party product, provider and law names keep their spelling (Vertex AI, Cisco AI Defense, EU AI Act, ...), and user data such as guardrail names, gateway URLs and keys is never rewritten. The proxy exposes the flag on /.well-known/litellm-ui-config, a TerminologyProvider at the app root reads it, and components call useTerminology() on their own strings. The context defaults to a pass-through, so nothing changes when the flag is unset. * fix(ui): apply SI terminology to the command palette, page visibility groups and guardrail garden search * test(ui): cover the flag-off wording on the palette, page visibility and garden search tests * fix(ui): apply SI terminology to the Aria usage assistant copy * test(ui): keep the Aria ui-config stub within the inline object lint budget * fix(ui): keep AI wording in prompts sent to models --- .../ui_discovery_endpoints.py | 2 + .../ui_discovery_endpoints.py | 1 + .../test_ui_discovery_endpoints.py | 22 ++++++ .../agents/_components/AgentsPanel.tsx | 7 +- .../_components/EvaluationSettingsModal.tsx | 6 +- .../_components/guardrail_garden.test.tsx | 30 +++++++- .../_components/guardrail_garden.tsx | 6 +- .../_components/guardrail_garden_card.tsx | 4 +- .../_components/guardrail_garden_detail.tsx | 6 +- .../_components/MCPGatewaySessionsTab.tsx | 17 +++-- .../mcp-servers/_components/mcp_connect.tsx | 12 ++-- .../components/chat_ui/RealtimePlayground.tsx | 8 ++- .../components/complianceUI/ComplianceUI.tsx | 4 +- .../_components/ai_suggestion_modal.tsx | 4 +- .../_components/guardrail_selection_modal.tsx | 4 +- .../policies/_components/policy_templates.tsx | 4 +- .../_components/template_parameter_modal.tsx | 4 +- .../UsageViewSelect/UsageViewSelect.tsx | 4 +- .../components/overview/TopAgents.tsx | 4 +- ui/litellm-dashboard/src/app/layout.tsx | 9 ++- .../src/components/AIHub/ModelHubTable.tsx | 4 +- .../CommandPalette.integration.test.tsx | 28 ++++++++ .../CommandPalette/CommandPalette.tsx | 4 +- .../components/CommandPalette/utils.test.ts | 11 +++ .../src/components/CommandPalette/utils.ts | 7 +- .../src/components/DashboardHeader.tsx | 4 +- .../Navbar/ViewSwitcher.integration.test.tsx | 24 +++++-- .../src/components/Navbar/ViewSwitcher.tsx | 9 ++- .../PageVisibilitySettings.test.tsx | 32 ++++++++- .../UISettings/PageVisibilitySettings.tsx | 8 ++- .../AdminSettings/UISettings/UISettings.tsx | 12 +++- .../src/components/leftnav.test.tsx | 19 ++++- .../src/components/leftnav.tsx | 15 ++-- .../liteadmin/LiteAdmin.integration.test.tsx | 69 ++++++++++++++----- .../src/components/liteadmin/LiteAdmin.tsx | 7 +- .../components/model_add/CredentialsPanel.tsx | 4 +- .../components/model_add/CredentialsTable.tsx | 4 +- .../src/components/moyai/MoyaiConnected.tsx | 4 +- .../src/components/networking.tsx | 1 + .../organisms/create_key_button.tsx | 11 ++- .../shared/CreatedKeyDisplay.test.tsx | 8 +++ .../components/shared/CreatedKeyDisplay.tsx | 9 ++- .../templates/KeyEditViewControls.tsx | 51 ++++++++------ .../src/contexts/TerminologyContext.test.tsx | 37 ++++++++++ .../src/contexts/TerminologyContext.tsx | 18 +++++ .../src/data/compliancePrompts.test.ts | 33 +++++++++ .../src/data/compliancePrompts.ts | 11 ++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 ++ .../src/lib/terminology.test.ts | 45 ++++++++++++ ui/litellm-dashboard/src/lib/terminology.ts | 28 ++++++++ 50 files changed, 567 insertions(+), 113 deletions(-) create mode 100644 ui/litellm-dashboard/src/contexts/TerminologyContext.test.tsx create mode 100644 ui/litellm-dashboard/src/contexts/TerminologyContext.tsx create mode 100644 ui/litellm-dashboard/src/data/compliancePrompts.test.ts create mode 100644 ui/litellm-dashboard/src/lib/terminology.test.ts create mode 100644 ui/litellm-dashboard/src/lib/terminology.ts diff --git a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py index 8225984a964..792ef8a3f8c 100644 --- a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py +++ b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py @@ -25,6 +25,7 @@ async def get_ui_config(): or general_settings.get("auto_redirect_ui_login_to_sso", False) is True ) admin_ui_disabled: Final = os.getenv("DISABLE_ADMIN_UI", "false").lower() == "true" + use_si_terminology: Final = os.getenv("LITELLM_UI_USE_SI_TERMINOLOGY", "false").lower() == "true" hide_default_credentials_hint: Final = should_hide_default_credentials_hint(general_settings) sso_configured: Final = has_user_setup_sso() @@ -49,4 +50,5 @@ async def get_ui_config(): is_control_plane=is_control_plane, workers=proxy_config.worker_registry if is_control_plane else [], mcp_stdio_enabled=is_mcp_stdio_enabled(), + use_si_terminology=use_si_terminology, ) diff --git a/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py b/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py index e30ad26bfe5..d396c79b844 100644 --- a/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py +++ b/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py @@ -14,4 +14,5 @@ class UiDiscoveryEndpoints(LiteLLMBaseModel): hide_default_credentials_hint: bool = False is_control_plane: bool = False mcp_stdio_enabled: bool = False + use_si_terminology: bool = False workers: list[WorkerRegistryEntry] = Field(default=[]) diff --git a/tests/unit/proxy/discovery_endpoints/test_ui_discovery_endpoints.py b/tests/unit/proxy/discovery_endpoints/test_ui_discovery_endpoints.py index 1ef0eee761a..d338f38d862 100644 --- a/tests/unit/proxy/discovery_endpoints/test_ui_discovery_endpoints.py +++ b/tests/unit/proxy/discovery_endpoints/test_ui_discovery_endpoints.py @@ -501,3 +501,25 @@ def test_ui_config_tells_the_dashboard_whether_stdio_mcp_servers_are_enabled(mon assert response.status_code == 200 assert response.json()["mcp_stdio_enabled"] is expected + + +@pytest.mark.parametrize(("flag", "expected"), [(None, False), ("false", False), ("true", True), ("TRUE", True)]) +def test_ui_config_tells_the_dashboard_whether_to_use_si_terminology( + monkeypatch: pytest.MonkeyPatch, flag: str | None, expected: bool +) -> None: + if flag is None: + monkeypatch.delenv("LITELLM_UI_USE_SI_TERMINOLOGY", raising=False) + else: + monkeypatch.setenv("LITELLM_UI_USE_SI_TERMINOLOGY", flag) + app: Final[FastAPI] = FastAPI() + app.include_router(router) + + with ( + patch("litellm.proxy.utils.get_server_root_path", return_value="/"), + patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), + ): + response: Final = TestClient(app).get("/.well-known/litellm-ui-config") + + assert response.status_code == 200 + assert response.json()["use_si_terminology"] is expected diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.tsx index b392e270d33..3f599c38976 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.tsx @@ -1,3 +1,4 @@ +import { useTerminology } from "@/contexts/TerminologyContext"; import React, { useState, useEffect } from "react"; import { Info, Plus } from "lucide-react"; import { getAgentsList, deleteAgentCall } from "@/components/networking"; @@ -31,6 +32,7 @@ interface AgentsResponse { } const AgentsPanel: React.FC = ({ accessToken, userRole, teams }) => { + const term = useTerminology(); const [agentsList, setAgentsList] = useState([]); const [isAddModalVisible, setIsAddModalVisible] = useState(false); const [isLoading, setIsLoading] = useState(true); @@ -140,8 +142,9 @@ const AgentsPanel: React.FC = ({ accessToken, userRole, teams

Agents

- List of A2A-spec agents that are available to be used in your organization. Go to AI Hub, to make agents - public. + {term( + "List of A2A-spec agents that are available to be used in your organization. Go to AI Hub, to make agents public.", + )}

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx index 900a04e480d..78091584cde 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx @@ -1,3 +1,4 @@ +import { useTerminology } from "@/contexts/TerminologyContext"; import { Play } from "lucide-react"; import React, { useEffect, useMemo, useState } from "react"; import { fetchAvailableModels, type ModelGroup } from "@/components/llm_calls/fetch_models"; @@ -47,6 +48,7 @@ export function EvaluationSettingsModal({ accessToken, onRunEvaluation, }: EvaluationSettingsModalProps) { + const term = useTerminology(); const [prompt, setPrompt] = useState(DEFAULT_PROMPT); const [schema, setSchema] = useState(DEFAULT_SCHEMA); const [model, setModel] = useState(null); @@ -95,8 +97,8 @@ export function EvaluationSettingsModal({ Evaluation Settings {guardrailName - ? `Configure AI evaluation for ${guardrailName}` - : "Configure AI evaluation for re-running on logs"} + ? `${term("Configure AI evaluation for")} ${guardrailName}` + : term("Configure AI evaluation for re-running on logs")} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden.test.tsx index 613fee04f68..ee214b451fa 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden.test.tsx @@ -1,9 +1,16 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; -import { render, screen, within } from "@testing-library/react"; +import { fireEvent, render, screen, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; +import { TerminologyProvider } from "@/contexts/TerminologyContext"; import GuardrailGarden from "./guardrail_garden"; import { ALL_CARDS } from "./guardrail_garden_data"; +const uiConfig = vi.hoisted(() => ({ data: undefined as { use_si_terminology?: boolean } | undefined })); + +vi.mock("@/app/(dashboard)/hooks/uiConfig/useUIConfig", () => ({ + useUIConfig: () => ({ data: uiConfig.data }), +})); + vi.mock("./guardrail_garden_detail", () => ({ __esModule: true, default: ({ card, onBack }: { card: { name: string }; onBack: () => void }) => ( @@ -21,6 +28,7 @@ const DECISION_MODEL_CARDS = ALL_CARDS.filter((c) => c.category === "decision_mo describe("GuardrailGarden", () => { beforeEach(() => { vi.clearAllMocks(); + uiConfig.data = undefined; }); const renderGarden = () => render(); @@ -114,6 +122,26 @@ describe("GuardrailGarden", () => { } }); + it.each([ + [true, "SI"], + [false, "AI"], + ])("should match the wording a card shows when SI terminology is %s", (useSi, shownTerm) => { + uiConfig.data = { use_si_terminology: useSi }; + const card = ALL_CARDS.find((c) => c.description.startsWith("AI security platform protecting")); + expect(card).toBeDefined(); + render( + + + , + ); + + fireEvent.change(screen.getByPlaceholderText("Search guardrails"), { + target: { value: `${shownTerm} security platform protecting` }, + }); + + expect(screen.getByText(card!.name)).toBeInTheDocument(); + }); + it("should show an empty result set for a query that matches nothing", async () => { const user = userEvent.setup(); renderGarden(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden.tsx index f8e1ab1ef66..d430a5a9f19 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden.tsx @@ -1,3 +1,4 @@ +import { useTerminology } from "@/contexts/TerminologyContext"; import React, { useState } from "react"; import { ArrowRight, Search } from "lucide-react"; import { InputGroup, InputGroupAddon, InputGroupInput } from "@/components/ui/input-group"; @@ -11,6 +12,7 @@ interface GuardrailGardenProps { } const GuardrailGarden: React.FC = ({ accessToken, onGuardrailCreated }) => { + const term = useTerminology(); const [searchQuery, setSearchQuery] = useState(""); const [selectedCard, setSelectedCard] = useState(null); const [showAllLitellm, setShowAllLitellm] = useState(false); @@ -22,7 +24,7 @@ const GuardrailGarden: React.FC = ({ accessToken, onGuardr const q = searchQuery.toLowerCase(); return ( card.name.toLowerCase().includes(q) || - card.description.toLowerCase().includes(q) || + term(card.description).toLowerCase().includes(q) || card.tags.some((t) => t.toLowerCase().includes(q)) ); }); @@ -106,7 +108,7 @@ const GuardrailGarden: React.FC = ({ accessToken, onGuardr Partner Guardrails

- Third-party guardrail integrations from leading AI security providers. + {term("Third-party guardrail integrations from leading AI security providers.")}

{partnerCards.map((card) => ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_card.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_card.tsx index 2b7cedd6d85..8a6375d0d68 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_card.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_card.tsx @@ -1,9 +1,11 @@ +import { useTerminology } from "@/contexts/TerminologyContext"; import React from "react"; import { CircleCheck } from "lucide-react"; import { GuardrailCardInfo } from "./guardrail_garden_data"; import { Logo } from "@/components/molecules/logo/Logo"; const GuardrailCard: React.FC<{ card: GuardrailCardInfo; onClick: () => void }> = ({ card, onClick }) => { + const term = useTerminology(); return (
void }> {card.name}
-

{card.description}

+

{term(card.description)}

{card.eval && (
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_detail.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_detail.tsx index 8ac0493b012..e549ebca50a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_detail.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_detail.tsx @@ -1,3 +1,4 @@ +import { useTerminology } from "@/contexts/TerminologyContext"; import React, { useState } from "react"; import { ArrowLeft } from "lucide-react"; import { Button } from "@/components/ui/button"; @@ -21,6 +22,7 @@ interface GuardrailDetailViewProps { } const GuardrailDetailView: React.FC = ({ card, onBack, accessToken, onGuardrailCreated }) => { + const term = useTerminology(); const [isAddFormVisible, setIsAddFormVisible] = useState(false); const [activeTab, setActiveTab] = useState("overview"); @@ -66,7 +68,7 @@ const GuardrailDetailView: React.FC = ({ card, onBack,

{card.name}

-

{card.description}

+

{term(card.description)}

{/* Action buttons — outlined style like Vertex */}
@@ -101,7 +103,7 @@ const GuardrailDetailView: React.FC = ({ card, onBack, {/* Left column — overview + details table */}

Overview

-

{card.description}

+

{term(card.description)}

Guardrail Details

Details are as follows

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPGatewaySessionsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPGatewaySessionsTab.tsx index 04a095f8792..46c1d2c6848 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPGatewaySessionsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPGatewaySessionsTab.tsx @@ -1,5 +1,6 @@ "use client"; +import { useTerminology } from "@/contexts/TerminologyContext"; import React, { useState } from "react"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { RefreshCw, Unplug } from "lucide-react"; @@ -132,6 +133,7 @@ function SessionsBody({ isLoading: boolean; onDisconnect: ((selector: MCPGatewaySessionSelector) => void) | null; }) { + const term = useTerminology(); if (isLoading) { return (

- No live MCP connections on this worker (pid {data.worker_pid}). Connect an AI client to the gateway to see it - here. + No live MCP connections on this worker (pid {data.worker_pid}).{" "} + {term("Connect an AI client to the gateway to see it here.")}

); @@ -166,11 +168,11 @@ function SessionsBody({ <>
- +
- + (null); const queryOptions = { @@ -282,9 +285,9 @@ export function MCPGatewaySessionsTab({ accessToken, canTerminate }: MCPGatewayS

Live Connections

- Stateful Streamable HTTP sessions currently open on this proxy worker, grouped by the AI client that sent - the MCP initialize request and by the authenticated LiteLLM user. Stateless requests and SSE connections are - not counted. + {term( + "Stateful Streamable HTTP sessions currently open on this proxy worker, grouped by the AI client that sent the MCP initialize request and by the authenticated LiteLLM user. Stateless requests and SSE connections are not counted.", + )}

- Use tools directly from Cursor IDE with LiteLLM MCP. Enable your AI assistant to perform real-world tasks - without leaving your coding environment. + {term( + "Use tools directly from Cursor IDE with LiteLLM MCP. Enable your AI assistant to perform real-world tasks without leaving your coding environment.", + )}
@@ -475,8 +478,9 @@ const MCPConnect: React.FC = ({ currentServerAccessGroups = []

Connect to your MCP client

- Use tools directly from any MCP client with LiteLLM MCP. Enable your AI assistant to perform real-world - tasks through a simple, secure connection. + {term( + "Use tools directly from any MCP client with LiteLLM MCP. Enable your AI assistant to perform real-world tasks through a simple, secure connection.", + )}

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx index 8acc54bee8d..01151203e65 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx @@ -1,5 +1,6 @@ "use client"; +import { useTerminology } from "@/contexts/TerminologyContext"; import { CircleX, Mic, MicOff, Send, Volume2 } from "lucide-react"; import React, { useCallback, useEffect, useRef, useState } from "react"; import { Button } from "@/components/ui/button"; @@ -27,6 +28,7 @@ const RealtimePlayground: React.FC = ({ customProxyBaseUrl, selectedGuardrails, }) => { + const term = useTerminology(); const [messages, setMessages] = useState([]); const [inputText, setInputText] = useState(""); const [isConnected, setIsConnected] = useState(false); @@ -408,8 +410,8 @@ const RealtimePlayground: React.FC = ({ Realtime Voice Playground

- Click Connect to start a realtime session. You can speak using your microphone or type messages. - The AI will respond with voice and text. + Click Connect to start a realtime session. You can speak using your microphone or type messages.{" "} + {term("The AI will respond with voice and text.")}

)} @@ -428,7 +430,7 @@ const RealtimePlayground: React.FC = ({ : "bg-muted text-foreground rounded-bl-md" }`} > -
{msg.role === "user" ? "You" : "AI"}
+
{msg.role === "user" ? "You" : term("AI")}
{msg.content}
)} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/complianceUI/ComplianceUI.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/complianceUI/ComplianceUI.tsx index 54ea3702cb4..db576ed03a5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/complianceUI/ComplianceUI.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/complianceUI/ComplianceUI.tsx @@ -7,6 +7,7 @@ import { type CompliancePrompt, } from "@/data/compliancePrompts"; import useCan from "@/app/(dashboard)/hooks/useCan"; +import { useTerminology } from "@/contexts/TerminologyContext"; import { getGuardrailsList, testPoliciesAndGuardrails } from "@/components/networking"; import PolicySelector, { getPolicyOptionEntries } from "@/components/policies/PolicySelector"; import { Policy } from "@/components/policies/types"; @@ -131,7 +132,8 @@ export default function ComplianceUI({ proxySettings, }: ComplianceUIProps) { const canViewPolicies = useCan("viewPolicies"); - const frameworks = getFrameworks(); + const term = useTerminology(); + const frameworks = getFrameworks(term); const [policyValueToLabel, setPolicyValueToLabel] = useState>(new Map()); const [guardrailOptions, setGuardrailOptions] = useState([]); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/ai_suggestion_modal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/ai_suggestion_modal.tsx index c4f76b07a4e..50ffb89ae83 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/ai_suggestion_modal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/ai_suggestion_modal.tsx @@ -1,3 +1,4 @@ +import { useTerminology } from "@/contexts/TerminologyContext"; import React, { useEffect, useMemo, useState } from "react"; import { Button } from "@/components/ui/button"; import { Card, CardContent } from "@/components/ui/card"; @@ -62,6 +63,7 @@ const AiSuggestionModal: React.FC = ({ accessToken, allTemplates, }) => { + const term = useTerminology(); const [attackExamples, setAttackExamples] = useState([""]); const [description, setDescription] = useState(""); const [isLoading, setIsLoading] = useState(false); @@ -815,7 +817,7 @@ const AiSuggestionModal: React.FC = ({ {/* Header */}
- AI Policy Suggestion + {term("AI Policy Suggestion")}

{showResults ? `${suggestions?.length || 0} template${(suggestions?.length || 0) !== 1 ? "s" : ""} matched your requirements` diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/guardrail_selection_modal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/guardrail_selection_modal.tsx index c4a45d88d1f..25a4b89ed2a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/guardrail_selection_modal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/guardrail_selection_modal.tsx @@ -1,3 +1,4 @@ +import { useTerminology } from "@/contexts/TerminologyContext"; import React, { useState, useEffect } from "react"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -40,6 +41,7 @@ const GuardrailSelectionModal: React.FC = ({ isLoading = false, progressInfo, }) => { + const term = useTerminology(); const [selectedGuardrails, setSelectedGuardrails] = useState>(new Set()); // Prepare guardrail info with existence status @@ -199,7 +201,7 @@ const GuardrailSelectionModal: React.FC = ({

✨ - AI-Discovered Competitors ({template.discoveredCompetitors.length}) + {term("AI-Discovered Competitors")} ({template.discoveredCompetitors.length})
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/policy_templates.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/policy_templates.tsx index 46dda1dd11c..0db1f74f540 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/policy_templates.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/policy_templates.tsx @@ -1,3 +1,4 @@ +import { useTerminology } from "@/contexts/TerminologyContext"; import React, { useState, useEffect, useMemo } from "react"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -106,6 +107,7 @@ const PolicyTemplates: React.FC = ({ onTemplatesLoaded, accessToken, }) => { + const term = useTerminology(); const [templates, setTemplates] = useState([]); const [isLoading, setIsLoading] = useState(false); const [selectedTags, setSelectedTags] = useState>(new Set()); @@ -191,7 +193,7 @@ const PolicyTemplates: React.FC = ({ - Use AI to find templates + {term("Use AI to find templates")}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/template_parameter_modal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/template_parameter_modal.tsx index b90379b149d..6aa26088756 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/template_parameter_modal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/template_parameter_modal.tsx @@ -1,3 +1,4 @@ +import { useTerminology } from "@/contexts/TerminologyContext"; import React, { useState, useEffect } from "react"; import { Button } from "@/components/ui/button"; import { Badge } from "@/components/ui/badge"; @@ -44,6 +45,7 @@ const TemplateParameterModal: React.FC = ({ isLoading = false, accessToken, }) => { + const term = useTerminology(); const [parameterValues, setParameterValues] = useState>({}); const [competitorMode, setCompetitorMode] = useState<"ai" | "manual">("ai"); const [selectedModel, setSelectedModel] = useState(null); @@ -260,7 +262,7 @@ const TemplateParameterModal: React.FC = ({ className="grid-cols-2" >