From fa08f157d8255f5f96299002c45aaceb7c47a589 Mon Sep 17 00:00:00 2001 From: Akiva Kraines Date: Mon, 5 Jan 2026 00:14:14 +0200 Subject: [PATCH 01/56] fix: Improve error messages and validation for wildcard routing with multiple credentials - Enhanced error messages to show which deployment/credential was used when routing fails - Added debug logging for pattern-matched deployments to track which deployment was selected - Added validation warning at startup when multiple wildcard patterns exist for same provider with different credentials - Helps diagnose intermittent authentication failures caused by non-deterministic wildcard routing Addresses issue where multiple wildcard deployments (e.g., openai/*) with different credentials cause intermittent 403 errors due to random credential selection. --- litellm/router.py | 56 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 6821ab9e6c6..a040b3bfeaf 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4458,9 +4458,21 @@ class Router: if hasattr(original_exception, "message"): # add the available fallbacks to the exception - original_exception.message += ". Received Model Group={}\nAvailable Model Group Fallbacks={}".format( # type: ignore - model_group, - fallback_model_group, + deployment_info = "" + if kwargs is not None: + metadata = kwargs.get('metadata', {}) + if metadata and 'deployment' in metadata: + deployment_info = f"\nUsed Deployment: {metadata['deployment']}" + if 'model_info' in metadata: + model_info = metadata['model_info'] + if isinstance(model_info, dict): + deployment_info += f"\nDeployment ID: {model_info.get('id', 'unknown')}" + + original_exception.message += ( # type: ignore + f". Received Model Group={model_group}" + f"\nAvailable Model Group Fallbacks={fallback_model_group}" + f"{deployment_info}" + f"\n\n💡 Tip: If using wildcard patterns (e.g., 'openai/*'), ensure all matching deployments have credentials with access to this model." ) if len(fallback_failure_exception_str) > 0: original_exception.message += ( # type: ignore @@ -5713,6 +5725,37 @@ class Router: return True return False + def _validate_wildcard_deployments(self): + """Warn if multiple wildcard patterns exist for same provider with different credentials""" + provider_wildcards: Dict[str, List[Tuple[str, str]]] = {} # provider -> [(credential, deployment_id)] + + for deployment in self.model_list: + model_name = deployment.get('model_name', '') + if '*' in model_name: + # Extract provider from pattern (e.g., "openai/*" -> "openai") + provider = model_name.split('/')[0] if '/' in model_name else model_name.split('*')[0] + + litellm_params = deployment.get('litellm_params', {}) + # Get credential identifier - use litellm_credential_name or first 10 chars of api_key + credential = litellm_params.get('litellm_credential_name') or \ + (litellm_params.get('api_key', '')[:10] if litellm_params.get('api_key') else 'none') + deployment_id = deployment.get('model_info', {}).get('id', 'unknown') + + if provider not in provider_wildcards: + provider_wildcards[provider] = [] + provider_wildcards[provider].append((credential, deployment_id)) + + for provider, cred_deployment_pairs in provider_wildcards.items(): + unique_credentials = set(cred for cred, _ in cred_deployment_pairs) + if len(unique_credentials) > 1: + deployment_ids = [dep_id for _, dep_id in cred_deployment_pairs] + verbose_router_logger.warning( + f"⚠️ Multiple wildcard deployments found for '{provider}/*' with different credentials ({len(unique_credentials)} credentials). " + f"This may cause non-deterministic authentication failures. " + f"Deployments: {len(cred_deployment_pairs)} ({deployment_ids[:3]}{'...' if len(deployment_ids) > 3 else ''}). " + f"Consider: 1) Using concrete model names, 2) Using one credential per provider, or 3) Using tag-based routing." + ) + def set_model_list(self, model_list: list): original_model_list = copy.deepcopy(model_list) self.model_list = [] @@ -5762,6 +5805,9 @@ class Router: # Note: model_name_to_deployment_indices is already built incrementally # by _create_deployment -> _add_model_to_list_and_index_map + + # Validate wildcard deployments after all models are loaded + self._validate_wildcard_deployments() def _add_deployment(self, deployment: Deployment) -> Deployment: import os @@ -7629,6 +7675,10 @@ class Router: ) if pattern_deployments: + verbose_router_logger.debug( + f"Pattern match for model='{model}': Found {len(pattern_deployments)} deployments. " + f"Deployment IDs: {[d.get('model_info', {}).get('id', 'unknown') for d in pattern_deployments]}" + ) return model, pattern_deployments if ( From 31430156c7e9654352ab4a133a2dce36dfd66c63 Mon Sep 17 00:00:00 2001 From: Akiva Kraines Date: Mon, 5 Jan 2026 22:26:46 +0200 Subject: [PATCH 02/56] refactor: Remove incomplete wildcard validation per maintainer feedback Per maintainer feedback, removed the wildcard validation logic as it doesn't cover all auth mechanisms (Google, AWS, etc.). Keeping only the core improvements: - Enhanced error messages showing deployment/credential used - Debug logging for pattern-matched deployments The validation logic needs more work to handle all provider auth mechanisms properly. --- litellm/router.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index a040b3bfeaf..b709dc764c5 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5725,37 +5725,6 @@ class Router: return True return False - def _validate_wildcard_deployments(self): - """Warn if multiple wildcard patterns exist for same provider with different credentials""" - provider_wildcards: Dict[str, List[Tuple[str, str]]] = {} # provider -> [(credential, deployment_id)] - - for deployment in self.model_list: - model_name = deployment.get('model_name', '') - if '*' in model_name: - # Extract provider from pattern (e.g., "openai/*" -> "openai") - provider = model_name.split('/')[0] if '/' in model_name else model_name.split('*')[0] - - litellm_params = deployment.get('litellm_params', {}) - # Get credential identifier - use litellm_credential_name or first 10 chars of api_key - credential = litellm_params.get('litellm_credential_name') or \ - (litellm_params.get('api_key', '')[:10] if litellm_params.get('api_key') else 'none') - deployment_id = deployment.get('model_info', {}).get('id', 'unknown') - - if provider not in provider_wildcards: - provider_wildcards[provider] = [] - provider_wildcards[provider].append((credential, deployment_id)) - - for provider, cred_deployment_pairs in provider_wildcards.items(): - unique_credentials = set(cred for cred, _ in cred_deployment_pairs) - if len(unique_credentials) > 1: - deployment_ids = [dep_id for _, dep_id in cred_deployment_pairs] - verbose_router_logger.warning( - f"⚠️ Multiple wildcard deployments found for '{provider}/*' with different credentials ({len(unique_credentials)} credentials). " - f"This may cause non-deterministic authentication failures. " - f"Deployments: {len(cred_deployment_pairs)} ({deployment_ids[:3]}{'...' if len(deployment_ids) > 3 else ''}). " - f"Consider: 1) Using concrete model names, 2) Using one credential per provider, or 3) Using tag-based routing." - ) - def set_model_list(self, model_list: list): original_model_list = copy.deepcopy(model_list) self.model_list = [] @@ -5805,9 +5774,6 @@ class Router: # Note: model_name_to_deployment_indices is already built incrementally # by _create_deployment -> _add_model_to_list_and_index_map - - # Validate wildcard deployments after all models are loaded - self._validate_wildcard_deployments() def _add_deployment(self, deployment: Deployment) -> Deployment: import os From 9e6714fe1b9344b06fe819c486d9869e588480ce Mon Sep 17 00:00:00 2001 From: Wiley Kestner Date: Tue, 6 Jan 2026 16:04:02 +0100 Subject: [PATCH 03/56] Tally total_tokens in response if missing (#18468) (#18445) Calculate `total_tokens` in usage data in Response manually if: - `total_tokens` is missing - `total_tokens` can be calculated from input and output tokens Run the test for this feature with: `poetry run pytest tests/test_litellm/responses/test_responses_utils.py -k "test_transform_response_api_usage_calculates_total_from_input_and_output_tokens_if_available" -v` --- litellm/responses/utils.py | 17 ++++++++++++----- .../responses/test_responses_utils.py | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index a92b5d25a37..7667d1bad84 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -443,11 +443,18 @@ class ResponseAPILoggingUtils: completion_tokens=0, total_tokens=0, ) - response_api_usage: ResponseAPIUsage = ( - ResponseAPIUsage(**usage_input) - if isinstance(usage_input, dict) - else usage_input - ) + response_api_usage: ResponseAPIUsage + if isinstance(usage_input, dict): + total_tokens = usage_input.get("total_tokens") + if total_tokens is None: + input_tokens = usage_input.get("input_tokens") + output_tokens = usage_input.get("output_tokens") + if input_tokens is not None and output_tokens is not None: + total_tokens = input_tokens + output_tokens + usage_input["total_tokens"] = total_tokens + response_api_usage = ResponseAPIUsage(**usage_input) + else: + response_api_usage = usage_input prompt_tokens: int = response_api_usage.input_tokens or 0 completion_tokens: int = response_api_usage.output_tokens or 0 prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index 96ac2e2c345..09628cd4a76 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -203,3 +203,22 @@ class TestResponseAPILoggingUtils: assert result.prompt_tokens == 0 assert result.completion_tokens == 20 assert result.total_tokens == 20 + + def test_transform_response_api_usage_calculates_total_from_input_and_output_tokens_if_available(self): + """Test transformation calculates total_tokens when it's None and input / output tokens are present""" + # Setup + usage = { + "input_tokens": 15, + "output_tokens": 25, + "total_tokens": None, + } + + # Execute + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + usage + ) + + # Assert + assert result.prompt_tokens == 15 + assert result.completion_tokens == 25 + assert result.total_tokens == 40 # 15 + 25 From 9f68081f6d044faa018284fea99532c3b9374c6d Mon Sep 17 00:00:00 2001 From: minijeong-log Date: Wed, 7 Jan 2026 03:16:24 +0900 Subject: [PATCH 04/56] feat: Add built-in migration lock to prevent concurrent Prisma migrate deploy (#14440) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: prisma migrate deploy with lock Author: Mini Jeong * fix: use redis cache from proxy server Author: Mini Jeong * fix: add type checks and fix unit tests for migration lock - Add DATABASE_URL validation in _create_baseline_migration() and _resolve_all_migrations() - Fix MyPy type errors by adding None checks before using database_url in subprocess calls - Add _resolve_all_migrations mock to failing unit tests to prevent filesystem errors - Apply Black formatting to modified files Fixes: - MyPy type errors: database_url could be None when passed to subprocess - Unit test failures: _resolve_all_migrations tried to create directories in read-only /test path * fix: resolve MyPy type error in vertex_ai vertex_llm_base Fix MyPy type checking error where vertex_api_version parameter type was incompatible with function signature expectation. * fix: Return 403 exception when calling GET responses api * fix: added new step into rotate master key function for processing credentials table * Add redisvl in requirements.txt * fix: fixed the issue of handling root paths when processing Discovery protected resource metadata and authorization server metadata URLs. * fix: added additional grant type into oauth_authorization_server response for fixing mcp auth register bad request issue * fix: added RFC RECOMMENDED property(scopes_supported) to protected resource and authorization server metadata * fix: removed initialize the tool name to MCP server name mapping(oauth2) on startup for avoiding 401 error * fix: upgraded mcp sdk depency version for fixing ClosedResourceError * Use already configured opentelemetry providers Users that instrument using opentelemetry-instrument can now setup exporters as per their environment. * Handle all protocols for all telemetry * Add more tests * feat(mcp): parallelize tool fetching from multiple MCP servers (#18627) * feat(mcp): parallelize tool fetching from multiple MCP servers Replace sequential tool fetching with asyncio.gather() to reduce client timeouts when using multiple MCP servers. Changes: - mcp_server_manager.py: list_tools() now fetches tools in parallel - server.py: _get_tools_from_mcp_servers() now fetches tools in parallel Real-world impact (7 MCP servers example): - Sequential: ~4.5+ seconds (exceeds typical 5-second client timeouts) - Parallel: ~1.2 seconds (max of all servers) Fixes #18626 * fix: copy oauth2_headers to avoid shared dict mutation in parallel tasks * feat: add display_name, model_vendor, and model_version metadata * added the option of adding langsmith tenant id in the env (#18623) * fix(router): Validate routing_strategy at startup to fail fast with helpful error. (#18624) Invalid routing_strategy values (e.g., "simple" instead of "simple-shuffle") previously failed silently, causing confusing "No deployments available" errors downstream. This change adds upfront validation in routing_strategy_init() to: - Check if the provided strategy matches valid string values or RoutingStrategy enum - Raise a clear ValueError listing valid options if invalid - Fail fast at startup instead of at request time Fixes behavior reported in #11330 where users had to debug cryptic errors. Valid strategies: simple-shuffle, least-busy, usage-based-routing, latency-based-routing, cost-based-routing, usage-based-routing-v2 Co-authored-by: Flibbert E. Gibbitz * Add libsndfile to database Docker image for audio processing (#18612) The litellm-database Docker image was missing the libsndfile system library, which is required by the soundfile Python package for audio file processing. This caused failures when using audio transcription endpoints that attempt to calculate audio duration. This adds libsndfile to the runtime dependencies in Dockerfile.database, consistent with Dockerfile.alpine which already includes this library. * Fix: Map Gemini cached_tokens to Langfuse cache_read_input_tokens (#18614) * Fix: Map Gemini cached_tokens to Langfuse cache_read_input_tokens Fixes #18520 ## Problem Langfuse integration was not capturing cached tokens from Gemini models. Gemini returns cached tokens in `usage.prompt_tokens_details.cached_tokens`, but Langfuse only read from top-level `usage.cache_read_input_tokens` (which only Anthropic populates). ## Solution Updated langfuse.py to check both locations: 1. First check top-level cache_read_input_tokens (for Anthropic) 2. Then check prompt_tokens_details.cached_tokens (for Gemini, OpenAI, others) This ensures all providers' cached tokens are properly reported to Langfuse. ## Changes - Modified litellm/integrations/langfuse/langfuse.py (lines 742-761) - Added 3 unit tests in tests/test_litellm/integrations/langfuse/test_gemini_cached_tokens.py - All existing Langfuse tests still pass (11/11) ## Testing - test_cached_tokens_extraction: Verifies Gemini cached_tokens extraction - test_cached_tokens_not_present: Backward compatibility (no cached_tokens) - test_cached_tokens_is_zero: Edge case when cached_tokens = 0 * Refactor: Extract cache token logic into helper function Address review feedback from @officer47p - Created _extract_cache_read_input_tokens() helper function - Reduces code bloat in _log_langfuse_v2 method - Improves testability and reusability - All tests still passing (11/11) * Adding Role Mappings * Fixing Edit SSO Settings Modal * feat: add user_mcp_management_mode for view_all visibility * Fixing tests * fix: missing mcp_allow_all_ui.png * docs: add user_mcp_management_mode * Align responses API streaming hooks with chat pipeline * Clarify responses API streaming context * Address review comments * feat: Add GigaChat provider support (#18564) * feat: Add GigaChat provider support Add native support for GigaChat API (Sber AI, Russia's leading LLM). Supported features: - Chat completions (sync/async) - Streaming (sync/async) - Function calling / Tools - Structured output via JSON schema (emulated through function calls) - Image input (base64 and URL) - Embeddings Closes #18515 * fix: resolve mypy type errors in GigaChat handler - Fix _prepare_file_data return type (use 3-tuple for cleaner type flow) - Add type annotations for lists in _process_content_parts methods - Add type annotations in _collapse_user_messages - Use ChatCompletionToolCallChunk for proper tool_use typing - Add type: ignore[override] for astreaming async generator * refactor(gigachat): migrate to BaseConfig pattern * fix: remove unused imports * fix: resolve mypy type errors * fix: mypy type errors * refactor: address review feedback for GigaChat provider - Remove singleton pattern, reuse litellm HTTPHandler - Move constants/errors to transformation files, delete common_utils.py - Add models to model_prices_and_context_window.json - Fix ssl_verify not passed to HTTP client for embeddings * docs: update GigaChat documentation with ssl_verify requirement * Revert "Add redisvl in requirements.txt" * Put reasoning summary behind feat flag * fix: model eol * fix: anthropic claude-3-opus-20240229 EOL * Revert "fix: model eol" This reverts commit 5aa1665d79d75e0842ec44a8dc23d2e15fdfadd8. * Fix: ImportError: qualifire package is required for QualifireGuardrail. Install it with: pip install qualifire * fix: test_secret_manager_failure_does_not_block_email * fix: test_update_ui_settings_allowlisted_value * fix: test_aaamodel_prices_and_context_window_json_is_valid * fix: test_all_models_have_display_name * fix: async def test_bedrock_apply_guardrail_blocked() * fix: test_databricks_embeddings[True] * fix:test_anthropic_beta_header * fix:test_api_error_handling * fix:mypy mcp management * Revert "feat(model_cost): add display_name, model_vendor, and model_version metadata to model entries" * [Feat] New API Endpoint - Responses API (v1/responses/compact) (#18697) * init transform_compact_response_api_request * init acompact_responses * init async_compact_response_api_handler in llm http handler * init transform_compact_response_api_request for openai * init acompact_responses * fix acompact_responses * add OAI Compact API * docs responses API Compact * code qa checks * test_openai_compact_responses_api * fix mypy linting * fix: remove display name * Add the LITELLM_REASONING_AUTO_SUMMARY in doc * fix model map * [UI] - Feat add request provider form on UI (#18704) * add request provider form * fix link to github * add button * fix link * fix(streaming): normalize status code extraction to prevent 4xx errors from triggering mid-stream fallback (#18698) 在流式处理错误时，添加状态码标准化逻辑，确保 4xx 客户端错误直接抛出而不是被包装成 MidStreamFallbackError。 - 新增 _normalize_status_code 函数用于从异常对象提取状态码 - 优先从异常的 status_code 属性获取，其次从 response.status_code 获取 - 当映射异常或原始异常的状态码在 400-499 范围内时，直接抛出映射异常 - 添加单元测试验证 Vertex AI 400 错误正确抛出为 BadRequestError - 确保流式处理中的客户端错误能够正确传播，而不会触发回退机制 --------- Co-authored-by: Eric84626 Co-authored-by: Eric84626 <97266539+Eric84626@users.noreply.github.com> Co-authored-by: Sameer Kankute Co-authored-by: mangabits <1457532+mangabits@users.noreply.github.com> Co-authored-by: Costa Tsaousis Co-authored-by: Nik Co-authored-by: Shivam Rawat <161387515+shivamrawat1@users.noreply.github.com> Co-authored-by: FlibbertyGibbitz Co-authored-by: Flibbert E. Gibbitz Co-authored-by: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Co-authored-by: yuneng-jiang Co-authored-by: Yuta Saito Co-authored-by: LingXuanYin <3546599908@qq.com> Co-authored-by: YutaSaito <36355491+uc4w6c@users.noreply.github.com> Co-authored-by: 0717376 <103773680+0717376@users.noreply.github.com> Co-authored-by: Ishaan Jaff Co-authored-by: Kris Xia Co-authored-by: Krish Dholakia --- docker/Dockerfile.database | 2 +- docs/my-website/docs/mcp_control.md | 17 +- docs/my-website/docs/providers/gigachat.md | 283 +++++++++++ docs/my-website/docs/proxy/config_settings.md | 7 +- docs/my-website/docs/reasoning_content.md | 65 +++ docs/my-website/docs/response_api_compact.md | 104 ++++ docs/my-website/img/mcp_allow_all_ui.png | Bin 0 -> 137849 bytes docs/my-website/sidebars.js | 9 +- .../litellm_proxy_extras/utils.py | 133 ++++- litellm/__init__.py | 4 + litellm/_lazy_imports_registry.py | 4 + .../transformation.py | 23 +- litellm/constants.py | 1 + litellm/integrations/callback_configs.json | 6 + litellm/integrations/langfuse/langfuse.py | 40 +- litellm/integrations/langsmith.py | 19 +- litellm/integrations/opentelemetry.py | 256 +++++++--- .../litellm_core_utils/streaming_handler.py | 54 +- .../llms/base_llm/responses/transformation.py | 27 + litellm/llms/custom_httpx/llm_http_handler.py | 214 +++++++- litellm/llms/gigachat/__init__.py | 23 + litellm/llms/gigachat/authenticator.py | 241 +++++++++ litellm/llms/gigachat/chat/__init__.py | 12 + litellm/llms/gigachat/chat/streaming.py | 134 +++++ litellm/llms/gigachat/chat/transformation.py | 473 ++++++++++++++++++ litellm/llms/gigachat/embedding/__init__.py | 7 + .../llms/gigachat/embedding/transformation.py | 212 ++++++++ litellm/llms/gigachat/file_handler.py | 211 ++++++++ .../llms/openai/responses/transformation.py | 66 +++ litellm/llms/vertex_ai/vertex_llm_base.py | 7 +- litellm/main.py | 65 +++ ...odel_prices_and_context_window_backup.json | 63 +++ .../mcp_server/discoverable_endpoints.py | 44 +- .../mcp_server/mcp_server_manager.py | 141 ++++-- .../proxy/_experimental/mcp_server/server.py | 24 +- litellm/proxy/_types.py | 7 + litellm/proxy/common_request_processing.py | 2 + litellm/proxy/db/prisma_client.py | 6 +- .../guardrails/guardrail_hooks/lasso/lasso.py | 2 +- .../mcp_management_endpoints.py | 58 ++- .../proxy/response_api_endpoints/endpoints.py | 82 +++ litellm/proxy/route_llm_request.py | 2 + litellm/responses/main.py | 202 ++++++++ litellm/responses/streaming_iterator.py | 217 +++++++- litellm/router.py | 22 + litellm/types/integrations/langsmith.py | 2 + litellm/types/utils.py | 2 + litellm/utils.py | 4 + model_prices_and_context_window.json | 63 +++ provider_endpoints_support.json | 4 +- requirements.txt | 2 +- .../test_bedrock_apply_guardrail.py | 34 +- .../test_litellm_proxy_extras_utils.py | 337 ++++++++++--- .../test_openai_responses_api.py | 46 ++ .../test_responses_hooks.py | 165 ++++++ .../test_anthropic_completion.py | 2 +- tests/llm_translation/test_databricks.py | 6 + tests/llm_translation/test_gigachat.py | 349 +++++++++++++ tests/local_testing/test_completion.py | 16 +- tests/local_testing/test_streaming.py | 6 +- .../test_langsmith_unit_test.py | 111 +++- .../test_opentelemetry_unit_tests.py | 25 - .../test_router_helper_utils.py | 67 +++ ...responses_transformation_transformation.py | 94 +++- .../langfuse/test_gemini_cached_tokens.py | 90 ++++ .../integrations/test_opentelemetry.py | 103 +++- .../test_streaming_handler.py | 23 + .../test_function_call_args_serialization.py | 355 +++++++++++++ ...test_vertex_and_google_ai_studio_gemini.py | 35 ++ .../mcp_server/test_discoverable_endpoints.py | 57 ++- .../mcp_server/test_mcp_server_manager.py | 19 + .../guardrail_hooks/test_qualifire.py | 135 ++--- .../hooks/test_key_management_event_hooks.py | 10 + .../test_mcp_management_endpoints.py | 79 +++ .../test_proxy_setting_endpoints.py | 48 +- .../(dashboard)/hooks/sso/useSSOSettings.ts | 20 +- .../ModelsAndEndpointsView.tsx | 25 + .../Modals/BaseSSOSettingsForm.tsx | 18 +- .../Modals/DeleteSSOSettingsModal.test.tsx | 46 +- .../Modals/DeleteSSOSettingsModal.tsx | 103 ++-- .../SSOSettings/RoleMappings.test.tsx | 92 ++++ .../SSOSettings/RoleMappings.tsx | 74 +++ .../AdminSettings/SSOSettings/SSOSettings.tsx | 92 ++-- .../AdminSettings/SSOSettings/constants.ts | 7 + .../AdminSettings/SSOSettings/utils.test.ts | 12 + .../AdminSettings/SSOSettings/utils.ts | 21 +- .../src/hooks/useMcpOAuthFlow.tsx | 2 +- 87 files changed, 5826 insertions(+), 566 deletions(-) create mode 100644 docs/my-website/docs/providers/gigachat.md create mode 100644 docs/my-website/docs/response_api_compact.md create mode 100644 docs/my-website/img/mcp_allow_all_ui.png create mode 100644 litellm/llms/gigachat/__init__.py create mode 100644 litellm/llms/gigachat/authenticator.py create mode 100644 litellm/llms/gigachat/chat/__init__.py create mode 100644 litellm/llms/gigachat/chat/streaming.py create mode 100644 litellm/llms/gigachat/chat/transformation.py create mode 100644 litellm/llms/gigachat/embedding/__init__.py create mode 100644 litellm/llms/gigachat/embedding/transformation.py create mode 100644 litellm/llms/gigachat/file_handler.py create mode 100644 tests/llm_responses_api_testing/test_responses_hooks.py create mode 100644 tests/llm_translation/test_gigachat.py create mode 100644 tests/test_litellm/integrations/langfuse/test_gemini_cached_tokens.py create mode 100644 tests/test_litellm/llms/vertex_ai/gemini/test_function_call_args_serialization.py create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RoleMappings.test.tsx create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RoleMappings.tsx diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 0e804cbfd12..9a4e9a315ea 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -48,7 +48,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root # Install runtime dependencies -RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip +RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile WORKDIR /app # Copy the current directory contents into the container at /app diff --git a/docs/my-website/docs/mcp_control.md b/docs/my-website/docs/mcp_control.md index f6ab4a73087..a7d66a6b7fc 100644 --- a/docs/my-website/docs/mcp_control.md +++ b/docs/my-website/docs/mcp_control.md @@ -108,7 +108,7 @@ Some MCP servers are meant to be shared broadly—think internal knowledge bases 3. Toggle **Allow All LiteLLM Keys** on. MCP server configuration in Admin UI @@ -634,3 +634,18 @@ Control which tools different teams can access from the same MCP server. For exa This video shows how to set allowed tools for a Key, Team, or Organization. + + +## Dashboard View Modes + +Proxy admins can also control what non-admins see inside the MCP dashboard via `general_settings.user_mcp_management_mode`: + +- `restricted` *(default)* – users only see servers that their team explicitly has access to. +- `view_all` – every dashboard user can see the full MCP server list. + +```yaml title="Config example" +general_settings: + user_mcp_management_mode: view_all +``` + +This is useful when you want discoverability for MCP offerings without granting additional execution privileges. diff --git a/docs/my-website/docs/providers/gigachat.md b/docs/my-website/docs/providers/gigachat.md new file mode 100644 index 00000000000..13eec298c25 --- /dev/null +++ b/docs/my-website/docs/providers/gigachat.md @@ -0,0 +1,283 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# GigaChat +https://developers.sber.ru/docs/ru/gigachat/api/overview + +GigaChat is Sber AI's large language model, Russia's leading LLM provider. + +:::tip + +**We support ALL GigaChat models, just set `model=gigachat/` as a prefix when sending litellm requests** + +::: + +:::warning + +GigaChat API uses self-signed SSL certificates. You must pass `ssl_verify=False` in your requests. + +::: + +## Supported Features + +| Feature | Supported | +|---------|-----------| +| Chat Completion | Yes | +| Streaming | Yes | +| Async | Yes | +| Function Calling / Tools | Yes | +| Structured Output (JSON Schema) | Yes (via function call emulation) | +| Image Input | Yes (base64 and URL) - GigaChat-2-Max, GigaChat-2-Pro only | +| Embeddings | Yes | + +## API Key + +GigaChat uses OAuth authentication. Set your credentials as environment variables: + +```python +import os + +# Required: Set credentials (base64-encoded client_id:client_secret) +os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here" + +# Optional: Set scope (default is GIGACHAT_API_PERS for personal use) +os.environ['GIGACHAT_SCOPE'] = "GIGACHAT_API_PERS" # or GIGACHAT_API_B2B for business +``` + +Get your credentials at: https://developers.sber.ru/studio/ + +## Sample Usage + +```python +from litellm import completion +import os + +os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here" + +response = completion( + model="gigachat/GigaChat-2-Max", + messages=[ + {"role": "user", "content": "Hello from LiteLLM!"} + ], + ssl_verify=False, # Required for GigaChat +) +print(response) +``` + +## Sample Usage - Streaming + +```python +from litellm import completion +import os + +os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here" + +response = completion( + model="gigachat/GigaChat-2-Max", + messages=[ + {"role": "user", "content": "Hello from LiteLLM!"} + ], + stream=True, + ssl_verify=False, # Required for GigaChat +) + +for chunk in response: + print(chunk) +``` + +## Sample Usage - Function Calling + +```python +from litellm import completion +import os + +os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here" + +tools = [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a city", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name"} + }, + "required": ["city"] + } + } +}] + +response = completion( + model="gigachat/GigaChat-2-Max", + messages=[{"role": "user", "content": "What's the weather in Moscow?"}], + tools=tools, + ssl_verify=False, # Required for GigaChat +) +print(response) +``` + +## Sample Usage - Structured Output + +GigaChat supports structured output via JSON schema (emulated through function calling): + +```python +from litellm import completion +import os + +os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here" + +response = completion( + model="gigachat/GigaChat-2-Max", + messages=[{"role": "user", "content": "Extract info: John is 30 years old"}], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "person", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"} + } + } + } + }, + ssl_verify=False, # Required for GigaChat +) +print(response) # Returns JSON: {"name": "John", "age": 30} +``` + +## Sample Usage - Image Input + +GigaChat supports image input via base64 or URL (GigaChat-2-Max and GigaChat-2-Pro only): + +```python +from litellm import completion +import os + +os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here" + +response = completion( + model="gigachat/GigaChat-2-Max", # Vision requires GigaChat-2-Max or GigaChat-2-Pro + messages=[{ + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}} + ] + }], + ssl_verify=False, # Required for GigaChat +) +print(response) +``` + +## Sample Usage - Embeddings + +```python +from litellm import embedding +import os + +os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here" + +response = embedding( + model="gigachat/Embeddings", + input=["Hello world", "How are you?"], + ssl_verify=False, # Required for GigaChat +) +print(response) +``` + +## Usage with LiteLLM Proxy + +### 1. Set GigaChat Models on config.yaml + +```yaml +model_list: + - model_name: gigachat + litellm_params: + model: gigachat/GigaChat-2-Max + api_key: "os.environ/GIGACHAT_CREDENTIALS" + ssl_verify: false + - model_name: gigachat-lite + litellm_params: + model: gigachat/GigaChat-2-Lite + api_key: "os.environ/GIGACHAT_CREDENTIALS" + ssl_verify: false + - model_name: gigachat-embeddings + litellm_params: + model: gigachat/Embeddings + api_key: "os.environ/GIGACHAT_CREDENTIALS" + ssl_verify: false +``` + +### 2. Start Proxy + +```bash +litellm --config config.yaml +``` + +### 3. Test it + + + + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data '{ + "model": "gigachat", + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] +}' +``` + + + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="gigachat", + messages=[{"role": "user", "content": "Hello!"}] +) +print(response) +``` + + + +## Supported Models + +### Chat Models + +| Model Name | Context Window | Vision | Description | +|------------|----------------|--------|-------------| +| gigachat/GigaChat-2-Lite | 128K | No | Fast, lightweight model | +| gigachat/GigaChat-2-Pro | 128K | Yes | Professional model with vision | +| gigachat/GigaChat-2-Max | 128K | Yes | Maximum capability model | + +### Embedding Models + +| Model Name | Max Input | Dimensions | Description | +|------------|-----------|------------|-------------| +| gigachat/Embeddings | 512 | 1024 | Standard embeddings | +| gigachat/Embeddings-2 | 512 | 1024 | Updated embeddings | +| gigachat/EmbeddingsGigaR | 4096 | 2560 | High-dimensional embeddings | + +:::note +Available models may vary depending on your API access level (personal or business). +::: + +## Limitations + +- Only one function call per request (GigaChat API limitation) +- Maximum 1 image per message, 10 images total per conversation +- GigaChat API uses self-signed SSL certificates - `ssl_verify=False` is required diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 87064d442ae..f4359a86ba9 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -111,6 +111,7 @@ general_settings: master_key: string maximum_spend_logs_retention_period: 30d # The maximum time to retain spend logs before deletion. maximum_spend_logs_retention_interval: 1d # interval in which the spend log cleanup task should run in. + user_mcp_management_mode: restricted # or "view_all" # Database Settings database_url: string @@ -230,6 +231,7 @@ router_settings: | image_generation_model | str | The default model to use for image generation - ignores model set in request | | store_model_in_db | boolean | If true, enables storing model + credential information in the DB. | | supported_db_objects | List[str] | Fine-grained control over which object types to load from the database when `store_model_in_db` is True. Available types: `"models"`, `"mcp"`, `"guardrails"`, `"vector_stores"`, `"pass_through_endpoints"`, `"prompts"`, `"model_cost_map"`. If not set, all object types are loaded (default behavior). Example: `supported_db_objects: ["mcp"]` to only load MCP servers from DB. | +| user_mcp_management_mode | string | Controls what non-admins can see on the MCP dashboard. `restricted` (default) only lists MCP servers that the user’s teams are explicitly allowed to access. `view_all` lets every user see the full MCP server list. Tool list/call always respects per-key permissions, so users still cannot run MCP calls without access. | | store_prompts_in_spend_logs | boolean | If true, allows prompts and responses to be stored in the spend logs table. | | max_request_size_mb | int | The maximum size for requests in MB. Requests above this size will be rejected. | | max_response_size_mb | int | The maximum size for responses in MB. LLM Responses above this size will not be sent. | @@ -669,6 +671,7 @@ router_settings: | LANGSMITH_DEFAULT_RUN_NAME | Default name for Langsmith run | LANGSMITH_PROJECT | Project name for Langsmith integration | LANGSMITH_SAMPLING_RATE | Sampling rate for Langsmith logging +| LANGSMITH_TENANT_ID | Tenant ID for Langsmith multi-tenant deployments | LANGTRACE_API_KEY | API key for Langtrace service | LASSO_API_BASE | Base URL for Lasso API | LASSO_API_KEY | API key for Lasso service @@ -707,6 +710,7 @@ router_settings: | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development) | LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers | LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60 +| LITELLM_REASONING_AUTO_SUMMARY | If set to "true", automatically enables detailed reasoning summaries for reasoning models (e.g., o1, o3-mini, deepseek-reasoner). When enabled, adds `summary: "detailed"` to reasoning effort configurations. Default is "false" | LITELLM_SALT_KEY | Salt key for encryption in LiteLLM | LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections. | LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM @@ -774,6 +778,7 @@ router_settings: | OTEL_EXPORTER_OTLP_HEADERS | Headers for OpenTelemetry requests | OTEL_SERVICE_NAME | Service name identifier for OpenTelemetry | OTEL_TRACER_NAME | Tracer name for OpenTelemetry tracing +| OTEL_LOGS_EXPORTER | Exporter type for OpenTelemetry logs (e.g., console) | PAGERDUTY_API_KEY | API key for PagerDuty Alerting | PANW_PRISMA_AIRS_API_KEY | API key for PANW Prisma AIRS service | PANW_PRISMA_AIRS_API_BASE | Base URL for PANW Prisma AIRS service @@ -888,4 +893,4 @@ router_settings: | DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL | Time-to-live in seconds for health check lock in shared health check mode. Default is 60 (1 minute) | ZSCALER_AI_GUARD_API_KEY | API key for Zscaler AI Guard service | ZSCALER_AI_GUARD_POLICY_ID | Policy ID for Zscaler AI Guard guardrails -| ZSCALER_AI_GUARD_URL | Base URL for Zscaler AI Guard API. Default is https://api.us1.zseclipse.net/v1/detection/execute-policy \ No newline at end of file +| ZSCALER_AI_GUARD_URL | Base URL for Zscaler AI Guard API. Default is https://api.us1.zseclipse.net/v1/detection/execute-policy diff --git a/docs/my-website/docs/reasoning_content.md b/docs/my-website/docs/reasoning_content.md index fca3df638c7..04c6d7ee6cc 100644 --- a/docs/my-website/docs/reasoning_content.md +++ b/docs/my-website/docs/reasoning_content.md @@ -591,3 +591,68 @@ Expected Response + +## OpenAI Responses API - Auto-Summary Control + +When using OpenAI Responses API models (like `gpt-5`) via `/chat/completions` with `reasoning_effort`, you can control whether `summary="detailed"` is automatically added to the reasoning parameter. + +### Enabling Auto-Summary + +You can enable automatic `summary="detailed"` in two ways: + + + + +```python +import litellm + +# Enable auto-summary globally +litellm.reasoning_auto_summary = True + +response = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=[{"role": "user", "content": "What is the capital of France?"}], + reasoning_effort="low", # Will automatically add summary="detailed" +) +``` + + + + + +```bash +# Set environment variable +export LITELLM_REASONING_AUTO_SUMMARY=true + +# Or in your .env file +LITELLM_REASONING_AUTO_SUMMARY=true +``` + + + + + +```yaml +litellm_settings: + reasoning_auto_summary: true # Enable auto-summary for all requests + +model_list: + - model_name: gpt-5-mini + litellm_params: + model: openai/responses/gpt-5-mini +``` + + + + +### Manual Control (Recommended) + +For fine-grained control, pass `reasoning_effort` as a dictionary: + +```python +response = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=[{"role": "user", "content": "What is the capital of France?"}], + reasoning_effort={"effort": "low", "summary": "detailed"}, # Explicit control +) +``` diff --git a/docs/my-website/docs/response_api_compact.md b/docs/my-website/docs/response_api_compact.md new file mode 100644 index 00000000000..f5caa32ea33 --- /dev/null +++ b/docs/my-website/docs/response_api_compact.md @@ -0,0 +1,104 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# /responses/compact + +Compress conversation history using OpenAI's `/responses/compact` endpoint. + +| Feature | Supported | +|---------|-----------| +| Supported LiteLLM Versions | 1.72.0+ | +| Supported Providers | `openai` | + +## Usage + +### LiteLLM Python SDK + +```python showLineNumbers title="Compact Response" +import litellm + +response = litellm.compact_responses( + model="openai/gpt-4o", + input=[{"role": "user", "content": "Hello, how are you?"}], + instructions="Be helpful", + previous_response_id="resp_abc123" # optional +) + +print(response.id) +print(response.object) # "response.compaction" +print(response.output) +``` + +### LiteLLM Proxy + + + + +```bash showLineNumbers title="Compact Request" +curl http://localhost:4000/v1/responses/compact \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "openai/gpt-4o", + "input": [{"role": "user", "content": "Hello"}], + "instructions": "Be helpful" + }' +``` + + + + +```python showLineNumbers title="Compact with OpenAI SDK" +import httpx + +response = httpx.post( + "http://localhost:4000/v1/responses/compact", + headers={"Authorization": "Bearer sk-1234"}, + json={ + "model": "openai/gpt-4o", + "input": [{"role": "user", "content": "Hello"}], + "instructions": "Be helpful" + } +) + +print(response.json()) +``` + + + + +## Request Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `model` | string | Yes | Model to use for compaction | +| `input` | string or array | Yes | Input messages to compact | +| `instructions` | string | No | System instructions | +| `previous_response_id` | string | No | ID of previous response to continue from | + +## Response Format + +```json +{ + "id": "resp_abc123", + "object": "response.compaction", + "created_at": 1734366691, + "output": [ + { + "type": "message", + "role": "assistant", + "content": [...] + }, + { + "type": "compaction", + "encrypted_content": "..." + } + ], + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "total_tokens": 150 + } +} +``` + diff --git a/docs/my-website/img/mcp_allow_all_ui.png b/docs/my-website/img/mcp_allow_all_ui.png new file mode 100644 index 0000000000000000000000000000000000000000..f074deb801eb4ce4c9ad2a48d80f66007a8cb6d2 GIT binary patch literal 137849 zcmeEubyVC>vM=rgg1fs*aCdhP4#C~s-3d-`cX#(-L4$j+!3pm0CcAs@{@#7Nd-(UA zb7#JDrl-5BrK-BVpQ`TfL@3HjAj09mfq;M@N=b?;gMdIsf`EWW!$1LZIuQBIKtMqC zEkr~Vr9?!C6dmnMEv!vIK&ZaD#0$xQDWQc-6_O;0!Uti{N3TF&u#`dCvP8k?M1THr z`hDip^aV33`sw4xnSMfKdTnwsp;cEgMP&MwxqLOWAH};aIqC&scMh$DVNQgp@Xima z-^Xl0(AQaJuUG?4#w>fE5MT-Jkq5%d)9I?Iq$0OE+){i*EXt3WFBX5z%&F1rZ)Q#| z=wQ|`-uYeO&u1~F;oKl8YZ`|_N)j8Hj2i{Ig}*@2i*?erE^UMNwPED2vql|4T#!5D zs4HI$A`SNGWawM(NPCDptq574(;moRAAuJN{V@6<Sf~R;gh{ZZlEd*Phcj;jlqex=D851$xIBqq&$hb=SMD7w zCy!})-m8v!{rdh&@Y;4Z_tx^xvkms|Sg;uF$<5g;(;F^^*_GB6GB0tfxl3IpWs}u|EdignG613Y4B)Z8HkXIh?ErYTgBMX#KhLg z+|K!a>X90l0c$U*k@7LeQY2t42&z@|Z z{xL1!1R4L7FfubRG5#wuXA9GRL-wcS?___Q*WbJ2{nHq?qJ_JOwWg?r4UpBqq46_w zu`==gt)Ks==$|9~JE@wJiKB>}4Up2A|DVG82k~DE|0m(!`qcVopIl6wf9>)wC4Zs( zGY4*E6DK=smp?pIv$b&MXW?c1SK0rQO6#9w{LG&|{e$SQx&Mu%lO|~%MV9Ga^(gBA_yWSDx~TT zdXfd5HZO`9D)F(-ueTu1itr2fH8`+50XBasS9|&W>N5 zI`18SZho50JGtWzu^kP9KIGs!;L-NZ@sJ6MF|WRFo_YMht$Qt~;Q$;odszPi7?&SEpwTC8h8S34-ISHevglEQ) znJV*MZBRvPKzyf`#S3PB%+2SVwB=+>ecxK~|D*W0|Eu`_P8k2I`2WEK{jcKxhjsb? zx5c*y{YZv#f4BKIm*e+R_4@i0%c+*sOZ_)tw{-%e%8df@?I9GX}oT4;n6)gVZs5+fl5gG7N2!v|)Ul z%n|RixkQMl{#&gkIqKE zzSOwk3gCt;@4^|Gau6u}ldpO5S`oux>x!B6=KY zBBma+C*aB6!RK&eFea=JdE|DX0~qQ6h;3=5=kc0h7g?j$jAC*J1CcF^_s!M{A{RB- z_V;dt3S7IeG)X|K5}K9%?(kbl4IXm@^}F;)5DdT%T7b*T&@Oy{Kn)AAEw1GKd`yUM8Ka0Xq!47}w=jM_IV`Uj;>UfTfN5hL<^lZC5#|ga75izvw=En=R zCcaNWw$?}toP(#ah212TXc*9j{9bQbx)(9^()5ae%6ymgzE+50VFV>$bcqKKKI|M^ z@-edgY3a>JSiiIFA)72s^8Xg^zeA?>lPFB>!Ua6MW{jVHz()Zm6_H5E_TiqezzY9F{KMRp%@$fM#1+7O?4XC8d_t2VKBjp0JL_W&;Dj7hJ0^_+W0Sm}B5 z(0>k)O{mxHV>m_(d!!Q%~K`PrC9k7u!x zOq0*>*&JU0RopGmjFnNV=qrX0P#4jtdJEHk?$mj?+w@af_-^z^Ly&m_{9Yd?_0QwwBatG?qdMU&RP&T2dcp=1h z{%~hLBQe*>HN~^4LmZ$9V-PsEsP9+M{cx*&v+lN8A;Z`us_VDza`YXyMuTI`-iJ=D z!5R*^wWqh>D8mk=`&fyuyW+Qx*uO?#0g^CyJhzFH**6riZ#u}ZocnwnFwc+2cI83? z7>`D-XDi1#z=2)y9GmV1USpQx*7WF^v{5(EJWCQQVeM-Gauh_I2ZpUYPM!z@lLklb zj;xMVZ!D9$A`8S>rPBsaXw?+gTmArG^8;<$iB39J#uSH5xK%<(mBAVw|w+ z?RRR>9|&f(c6#0ja>>LWfm5|`9d5K<=*$xKivc*_ zW&5;OlZ3#REwh%CULXD(R^@;%u>q@H1alDd{~X2HI(jDad|cC$R7J%8)HVBqpKdrNC!gLr z6!EY`8jaENe7jwYXvZYg0zP%q?`=R@%_!Yjx7YRaS}WJF+|d?g4piof9E;SJPV%{! z)b9M*+g*=9fifB)|8vndKDRu|YLyasC8ZOo&Z;va8BC&73`@#sz)GGAhX_`I8*+T@ncLJv~Sz!tNycNTOIiL45r1VhfmYS27N<$Hj3d;HQm`7R6L_+U2-V}emn2nPn2^6w?88M)A$b#U7HOL?OgZsrWA)8b6 z{dlS=*G*)1Hk^1FBO}O=!yx2G1~c^i9^`brPUp>@kACf*SzTa!CBH&6d)9FWBa6p@ z=;pZ%YUlo$j7KO*ymX?X3}cN`#!lbWGYf-~4Fy{W7r?EsxG?NdD+oH@?vha&%VpT_ z-ed)k*L9s-^H7@a6Q-5`8pHE>2fYP5T(x2`Suwdf6a2-r8?40-}dsNJeNNZZ%Q$JND? za?+)?tSNq91Oz?$80Zh|s1c}4#Sj`&4mEbWZFUIlUG29~Qqj)_C(|z8V`z16qtitC z9A>Wbuv;%xW>+tM+K%ci3}3L}so?XvD-#0uEW4WIVch1PRR(pRU1~k7@>f}DG;C_; z^J@10WJ`twilFZ6y%ql`t&{2Zbb8l5YgIE^c-GF^(SEv+kq987)`747HN*u!e~a%m zzqm=!`Z`IT7#NCllo$7Wbc{O}OwD+7W46ks5r;vY)^9d=1ibqh5&wR`)Gu$&@~rS1 ztdMS-9a60@MKmg7D6UVFU9I(?ycviSDEonOX}B`@%k0IAor1`s^!L^;)VZ^+^8?n; zx3kV0sbAVT1|$@eGd5Q87|@W!QHMXMr<1S2l0U%IDo7c??u31(&r#D{C{xo_tI}a0 z+s*NP6^j<<|C#6Rdw*!=GM}tbid)^IZZhRs5xmz%@nv~#Ej?vJf7D~$FH%UO%9GwD zSqxsFaHYwza`CGkAbsH|$M01Q@pw@+E}1qxGt5Mn4EZA{RXAldVU25KeoJoU=c)8Qrfj{Bw^( z!mdP?OIw?oqOao*JgUcU`%1TVVT9IqlS7S>}Z?!{<&} zv>{mV++UeL%2eJ;mz^+z6O%oFj&na*?YK!@bZZ?)qJo!k}k z-pgdp;RZyr-6nc+!cw)(!U^QjWi+C~djcQ+A`p@msFC>Qgon(pL z#x7^m!t05&VT)&<64|ZiM4L=jtNpLIJT8M9x~%s;o4uGkgcjy6_4T2nNG3w)X}179 zKnF)FlYEKs;+v^76#LB%LUVHFNk~+sTfgUsv1D08aYagUPp}?*fog*`2`PRs#YaIY z1klwR+)ZVL#ps@hhf#@7trWM^u(En-FA1-&gTcY|MFfaYF5G*GA~_XpjPz<=8@^K< z3X^zl<|bENZKSM)zpI!+@huKt&<9K)&|fsr`igr(QVB>g^_{=cXmy5vZrVp=v-z&0 z)@UyzcFq_2A+<~nM14^47Kr#8H8BaF$%s#kS^)tp$W##pO_Cq(G=E^?1MynHj;nmi zx9hc1|Kmp`(;MI02TaR(TB~^^0Wyf&W>#nK<`NnPhSHi2_R}8>4O8c`HGpxqoc_&a z3VVU|e2;>L)5gRXZ%l^I)$J~aRB%^O{pDWcDOK51Qko&@K$o5<90aYs!?g!LUaxd3 z2y}JEI#<5N?VIjmI|s*61dA)q;*=g3@FtylbYFQlm}50q-(YjRUtars&@Dul-p%c( z+Q)W9TcLjwtEH%Pp~jdu;O3LSPMzNiEMYJJ4JegQy=j{gKG$PRw`faA;R^~$(RpvM zL)%fDK(VDN4@ZA?u#jfJLP1B!}6|sA}P5Hu{?|(C7= zAa@wsDA2Sl$>+rIWKQ9GUiY&+ja517Qg~C^e7?`AKK))XrE+JgP%_5n9MHb^IOk*o zLt8bLJm#gR6la??7;`Dg=HU1ndT_Rgv0{G$$1}y41U7R5V$1TnX>k)lTkDfu-*&@; zqsJLp{5}jG_jS7Cu&3YFt2c8FU@W;JvC{akK6SwNHnB1qdy*5>d%%5&j6u0I{@cZ~ z6KD`+cwI~njskCWk5?!HKECrSICPBScxBc(<;K!x>595LaU%nwcdSC2)oNqQ+$tZ>92o@(uhDD!hwzFl= zGAV#+QhoKlI}%#wz1w%3Z4(CG``BUEa^CpaeCI%JCR&D71D>W%1}QbgC(`-lPZJ~9 z9>fApWBsz(VJ|Bsi9ErKC9X9(28-^hI5rDyQ_jJ(nc6Op9MBQ!s^Cs}0Y{HW0?!9J zJe~FKu(N(VuN@m9&wBm*Vq9bTvfb1>#&XXq?J<_mg_wB3wY zzONGDnLsH9qxRlf>he`;DKK0X>EbY^*%$Wn_2=xhF|t z<+9B7)09eTZ&h^#uvduOgY`C6A+6}N0V!(Vuh5YuBRb#7Z=o}wZWWchLv8y{G!)AA zwB@1j@!@vte0huXR$Zj|^f)X2Wan#?|BS1VJ7bQsl<)wK_}V|yjH^?*XcFM569xFG#l=X<%r&_6~esj34#+t=)19>w)Q-j}qxQEr{>AK)jrFZqs)8dzM z(_2AYlbV*igN@61SDRYUqXP6$;ftoJ^CXH}KcRHzN|#*^ErUXhMhGEwA%Qnh4vk)A zc>C+)ai;MSEoX~J4#kH`-FSwqqnWNmegz)F1!h{uaqZO`=G*by?Nz&;0_aAEs=IZ! zxON_i{s(mL15me_TpKDib{k)rvw5MLK4`)+pYzU&nhQ{DkdbuOu<8oI0BE$M+XT6c zmU4~-@*ljPo^a{S56#tVjppV(MZO`=7wPV}Z zL{;2j@sP5nr%umiqn_Tt?I#7I>Ow~Sfo30&IVwDEr}HlQ6hW|e8;LSdm*=Vqi@sxb zpTTIN9E~@g#SE#VV)38!s(f5@lO&Xm3N;CYsAB?l;tCN@Hs9)O!Ob(;oB+ygjucc3 z1PNZL@+%3c_e~zVJ}TStQyD#)nr9l}tg)_{JR0&YQ+!-%1!lmzX?5#b#h8C1q3^R& zRVsFgJ^kg3#B6sVgQiKo&E8~|%3dc=7_UTIV0NCrbY%<>saIn#>g8>$wCno3J{aEo zoDvxC>eQ`f3oHNJl+Zmq!te8)ZoOJoiC2ct%Q?k&AS1{lJNJOc{kO`^17=uLJ+pSR z_I%YPvu^%<0i1)gh;9R+%(YnAj9;475DQl|C07n(zO*A z7n1Yrl_a5h*Jl$mo`htdbBKU(1Ca@p6r%OT^bO3R&E(5VDE*Na zws-68w+cDsgmJs>2X$ikhi02gy#%*xCE5JbUs)Q>vao8^)@AWFQ~WU7g^XJD!rCpq z9=o(JC;4yVbk!{vwp9AnNi?_ShEsQux=0I&1^CM@1n9KkRjaNCuW$fZcpaEZYbR;H zfruF+hADXr8W_u)Ce*=;z!6 zy~v6onUsooEB(cDv(9nbu>`FuJ#=0N8m_d~DXuEp9%@@M8*+yziW}9CdU7y%Ok1@I zRl15T4Eu*MjhpVp%{4lP4EVCA-Sx~A2z6?4IE)cHb?_8-y-RYD=X-zYHehf&J|DuM zEknK1jn$-GI@$gDD_WPHx;&Ug2ndA;+Q%upq$E=0DW5U7VP8e)FDK~ATLGQi7|j`< zzW=6>Pd_9ZXc)}()kQ-~5Z~gWD2Wy$X}*~LI#p)Fb9bc6NbjM=JI-4=tS-ATr&nBb zr(vnoBQf`hfc^b9idIWC5Mz@NA z9G_e$M1s6xxg7L(TRZ3T1)(PE4b6^vYz(alo>e^927}px;rgdqKLwY=qSUlPe{v}Qj88T@RFq(@?d*>? zhvhhtfzzj^*4JqfmUC$UFvqEEF0mx5fKy0p1dNYI>;Kp^T95mvUtT zJvJXUE|RCpe1?#=Z&YbZG67pFH7p7pFb$+ocTeytxP4iZ7G;lGGd*i^a6vB+Z|BV%sC zdtnhGwwXp$n2!{cd6z1O-CJ*7BOP|6p{F4yCds)+h(TvCddfl4vH z?z4Dmxf{RHUGg%{*mH!HE%5SO(JoEl@#_iSA@*fAsnzsZhQe0(_ltS=)7Ir5xWEFPPwr4!pZJMgTE};?pNZAi zkP#*`V9V{x5TR1_vuD~UIyQjumeDrfg6@|yFPC2-hePwXH=t+Dr=4G`cQU(EBD^HJ zA3tvft=e`i8)D1rnnOzzJ=dT-;h@?#eCh6MYmrJG_IWv~u)F?3rHNqDacUYSrEKD` zQ7iZ&zM*oDZ$EIecod-YEze#u(#&&)D*HwhCxK())&dx zz^lSBy35?-x?hcIgL(>adI)DB(~7Afy?TpU;~ERN-T@Con*?_tQrbAz@k{aY6QmH< z^IK=TJTX!Mkuod#;{YvF+7k#9p}BuDKCNS-!L?E}!2NX6woxfQv}BPE}1>zhItrMHSi&^5C4d`vAHhaiPO< z1f22%+2|mmYYAOmNjD*yjwqi~#_7)YmCP#9NVBvqKmS+rjsuI#CRL8T;1<868k0ikz^h#OWcW_T=kJq>~DVaR;U$AqV1xg31KnQtb#Hdf%1Wz<1a=Jx=kD)EEM~E*k z(yskc&`O(4^7Xr3pg%08tgF#z{2oV9c{H-%$l__bsRAdQXc#);#Sv<=<$Da%MT7fE znO#{?)nw3=oYY&+o|J&KHxrvvy(((6dikvxq~&;!$Y5B0y(zgci@?&Std$~g{t_4m zAMdsNO@^ttdUwD^q>Apj8N~`1C0h58+GoXnP7*Th@rr1^xfq6?pC*X2{YO6!rp56Nvfsa?5(DFn?N4vDQ~Swv(P6+9hvvT>vNLp*(ib# zG*^~Cph~u_+-USMlSZXoRFPeCf1zxV?BuS$`LshdBvPStxuLjxwZRO&oRMqw<73b( zcvv|$;}1mcSPigl#O(5$=%cy3Cd9)*le&wsqs2gFLi$HZO=%;v!CpLy z_?tvnx#lgOeRVW1Inmnj*2Q0Cg`#EB?L7z_`wlGxFTykt4hW6IkBHi(h{x(f&p%qH zmaUB)H8?jx%K8Q~RX%GPCFCoi=+j6?mt>b_?pn-OyaX3ey-FsDVa>VRx#ZwI>d5)} zHX9QgX~0!P?#CQQE~X7~=9s?Kqg%~YdPLpq=XUGPF)K}XbDHJtG|%jcto>S;$br4j zu2HKgiVZQnG~hBY-yS5?d~-4XDSGaE6BA=O+o`YBpBA}0cDW+#PS#87QtawH zQsW!3nysVL%@4O!?MbO#MG#zpi=e;k;`aXWMYqrP+kqwE zN_%)~BMtWFTy_iU%f%lW7OpxvF8PQy zo>pPHsx`*(trx6KB6^Fa)OXKOS508E)mui`2-Qke$|IcL?0T-caX1`838t?j z?;^sx07Ypr&6z$3UhC@^1hg8;a%s(r8|T3c2eD#*Ql` zf_hKu6REeQimFskvR1ilMgR=<8<~aXg(G5{?a**tp{P7&;yiBZRm9rmoHd)7xu-JTpQO9qvYnq3W66avu20s~b-iqI-BNbjDe%B;zogQ6 z4K?Wd7VF7I^kF(wPR@MuyglJ5AjtR@+|uQ$B5h^@3X6HWK-DNWzh= zl=yA%LJu*=meZevJR;7KdCjoJ3@vu0lpKY>=EzZ?Arh{sOk}r+R_VO7FIT0ypKDYq ze(rvqoB>x;_t=9outXGf zsU3%f$hin(*%l8P{8XKTt|9OoKW(E_Ygo=2Z}B^5=6 znMRj^+PRqs5>nPQ~|RLne0c^JL%$8 zWU_)=2FPd?wU2|W1QGr!A(3ozb?1ICR4Hqx2PP4xCnaRxdEirG^jI9PA%Hu;wq_cR z-EW!FB9$j-ZU?qPOHq=AN(1djS0d+7&xkl!&kU~G6=6;YYj>eB+7|HJg_o4qRS^uc zB74JkU>o|>R+0{#8r4{!kkrVN-kuSXDpf`V?gszAsCZ*t% zHuB2R3S+VSm5?kXZFDl+QMS1?6PrJk1ZxI+yay$>;zbi7x6b$YLB(G8j2sK=bg9WN zJ^{<458aj`qv>p0O8{-doK#QRC~4S$r`oD7AAJb{r7_De{|5^kG!Hn1xB*K)7%b&B zxxsS9TQ~@0$!OS>UcFJ1<$Slq^h;cGW*JY}-jq>t+)jJ)AQklZ_DuftX5F#Pl%>JL zoE#l}SJEIA$l?`#`?Z>)f9uePvQMAUdh~|fQu`o&@Pms(F%SBL?ET8_Ixi>i35BGN zP(nIGI?Jj;6KsF^(Vj#5e0DhR*VJmB_p|S#5A{lic5bQ7Msr4>ORlzZIeoS7jof^(F{1)-#|0Gb*S<}M z=k4h}UO^W{Qdk1k;{GC|+sTT;FbUgmL@#8(}XuEubve};mH|j_P%vYA4*4z!PUE<#Uc><263q< z)-PjaV_Bxifv%Q^1iKr?l#RQ+KgtpeMSweOWu(`**Jc%&xE`Kch%8Pp$~;Dot2Kc6 z*j4Yz_uGrvV+S^>ka%>~$wODnE9{F=hgsrnw?*}^-i8a{bh4Pxj}lx%k|-Gl3z8{m z$R%6;dIg4@d6vitE0r_(O#CD)Pt4&2VW8R&81**4Pod-ruVLMjsd{=-SdEBn_DA!#Tl&SGYAixc$=CN|Eg`_`|n4qjEBM~D| zr&P$Ci`-3b06*5m))@=!6E&DR)O#1Mw9^#0F-k<+YKF^h zJHyO#dTqWSlghwgTdVK9Mv~F)FsXMr+iwCo%9c}3u|O>~wUEVOXDG;J?Qht^fbU$n z3EWiG(;4+}-VzMdf9N8qL`x>}Qy{nZZBQ+6fm>m`bS zdmF9qMY+a8`9jsVZ*-tCjpQxc*w{>5CHpG#YD$OjAK`H?44{#)(dPh2c%O98sXO~U z05R7GtBQpEz(Y5cnKWeTOeJ6JaV~yH#AO{^D~*7g>?BdMs-ElV)|QdCt~U&+l+ zqMWUIIUnzOLklu5RZR8f0`ADuIgNy6$pBANTwjrC&)n$i`LCygh@~OU1B)_J!ytGl z*RH%^`^A$O)O0HO&Xc4d&nwtarrd76^)BDIPhFpKh{oXZ{HTb`{Y15AC4;y9WRgEX zjcxCK)5L*J(cL<97WQ;=TD{V2QQhF|*(@>Lg4B&|i)w{V+OkQok^Av+C=ik${!ZQ2 z=3_YLzPNH?1lNALo{_=k$SFa#`w#v*mW-i|jW8iSs!XO@h&~YAnSRJ2np;4tJgL=xF@iJ`fD{qLTHkvsGc^U2@e~xo@QBG z!Zay1a|9gvX(}C|5EW{n1dAn#UeM#8QiVFd_l`MMH6r1;odud~yFPsPZ_OCqTZb@7 z`U!i>NI8> zY6GDjDdjku)3w$$+2ROdN@!r1i&v@J`cOj6wvHrpcI(C^V-E9@9^S;#OleYp!!#Zs z$afU9jXFhUlsHi2<>_!5eZObR>_AML!dO&w(`Wsa>YhmoV%BV$sw?>Bw~B)5jElG1 z2{fC+S&35QWM&7|$gDo*LJ<2M+BxB zFdOfK0hTj4ZHkXp3ksaLspl~aLHX4vrm?7X4ulRducBNuF+#jJ{C+w5_~jksaJm%R zNd}r%!wv@rB1jvN)+&Rew$f}i0q~n_UrfW=Mo~u{M=$3m2Sc?@P`smxZ{{bq9WiWN zgES|q6MYH67@=!rTh^_IXOV_iHpLuc@-DvthR=0()SBh1j>{FxfP_j81kvhT8MuDC}@^O`v#NK!k%*uHxGBB*uY&Xctr$aF&!~{A-)dGEY ze5BcCI_@eHm|VUa#Afa00WE^eUtl$y;<@f?b`;F?8g<*Qm`@8sz2yY=!^TMQlM@*b)py03m1K>cn`RD&ZEKBW&p{}#-;F$> z`q284V%6~9%WDEpNT@Z4qd#3I>a?6w$BGdLGzx;875{R@VJMC>W7^0A&&<_l4vL+K?enI zlmiE7(7?CEnGSMT%|58r2=xnANQDI=Tzp~L!l)Mfb(n%Nfx`vFftu*yX%+g{RV`y%p$nzu4hZ5O8`ne2hKj% z3mI1ONwa)0ExiX&eRbcYBs$dRiw64{7N1z(VWz{{v(st*Ups^rNNYI?M|5S8XtbEq zuC?dCe@&uYL5r>n)uWU**$?*0_A#o)3JpQzR+Aq~mED3o9^5L+nzQ@qBVxZ}g!n zObFWj+)vt_=IXsXNH6YZI$ZMuVq*}hNSIAw{4cUbFs7;9_l$dcSzrta7v9U+Jm1(Y zh$3^XRF9jpz>hLo4C;=^9lDs8QczAu_^=_M&q*UKuA+@`Ulv=L2UPdUn+AtY?<8k0 zn+DHV5;D&PR7y5yU*wD~Ln6)0FGcHvTY}fUZXwYv*6SmHI|N;cANXMsr^_cAgc4Cw zB%gxw1Z(1>qtEFKMFH1fi0xX$K5906w^H-J=JuawG)Xs*kXUV zcGS0B1n5b~KF*LrL$^tZch7y<=C5IGdpWKd&Ke-a(5eK(Ec>311h=9j;>81g)FxV| zDwfoZ#T1f{1pXadEP%~c)Nga(_H)afl#{4Y7f_E%wqfl!WOm|^LB=CG|I84Qola@|Qf!pjSlN?Exgy^mw|z4Dsw ztLp6bO^D;>2q!|!+@8;>`g<`Qj+#is)zlEu8y|zbAO{Zyh)AH8^Wr7e(;(sp>Tv?B zJ{!Y+D1u*$`*sI5W|aF874lZ*hZ;=M6I;33-0`?Zwc2@SkA98uwCi(ez-3jclwFYD z1Iy67OqBk@S@TyIc>MyZ(1dD<&0tWE6RTOlGyS2#CukF|jVDJQhEa*gD0^85Ch*(l zCK-CJ;iEW8&olpgJex|DmS9jfD)X3p&J{<_VMz$MNa~$1?!>HKlvC7{^>axqk~MhV zAXITA=kgFE!xqio!iofuThW(@NCEZ5!pkV-Rjs%~L`+E;G9hMiN*kWAI-GF?3yVL+ znt1$^`z0WR9YWgbx45?x zsUi*|#^R`Aum}_ab_K>%MzL|*H{RPzDG!FD$P7zT^Ptvq`W2T&DZ6v{XP*>hn+y7M zGXGmhYVt@ls+}0351+F8LMH~F(x?ExVnC+yJ*s8-8=|kjO)vItPmG|(X~c83M)bg6 z#0m;5cgkr1@Z>h^Q{A(P(_jKzEjq1or}_x{HHGp*eGwCtWc6{a1n^}1FwdGoGNyoc zBn<+G;ft_Q5pkCBLcba}Ti~`e{`tqW>+K?{4Wep0;#O#1+wBR56O+a}hL!G*J=#Cm z@)_A8jbyW#&$*m@N5alOlaAPpeQT6V{pyXGXWP2K0SM=JNRXvqaVVZi-CAciUXWRd zirj1y9bc;9=fOAp8h!79@d*&;?H0_%JU8c&s!7=Mx=+#dP|ymFM1F7DL@8{9vR0qE zXwFXsJ%t}PLWUjPBR2agf`29J%dykR0Z+yUX%=qqz#xNh6pjjUab=H4qm(|U98GerhDfB@IiZB=9I|LUJij*OgUzzQ6=EN zV*LHT0CXu*>Xituo%{Zr-O$0kL#W3ZJfke9eEoeB`2qv4r7dMWNgnkaKD3putsbRj ztUOy|*AkvQSlVEACm-uLkGCWJkOyl;*djMk)~Nt2X>h;Q zZT#dhFX$;v&Mac7?N*|@Vc>IsS|d+2x$A&%jm@#33Eynu4nZz~&zGvO z3>GRZn*bB*S8tCKvdtTfMLIQ!p68K=fv(!*!D6B_+?6bCYw~EXoaNAj^@^(9QKyYc!HXYH?0Ch@^@M*_;no|C&*dqay^%}$Hi{$#w` z2Q>xB=iNLQpYlMeAMCohDpX8>Pej6dGUS>MKv1+ilLhpItL<%pz*4<~O>c+k zMZcD53xI`d)9)74H|ZRcu}${O#c_<*DzHb_-jCbG`qU$h_)@RY?;u0A7C~Ox;K7PE zLhjYm#flpvm;!O!u7}Umj$+l53%d}H)iN9yih`DsqZoBn8daQ`>LL3uZCERS5ilPa zu?$bHD4=w?q7FRzyV}*>{1Xn%df;=P-BxL?x5l!x(Qa50ZN@#HLFFyV?cnQMNsiwy zRWNu#pf(axQR>C&HrTi#r(RNc;>{;6Xi#-d@}|Z~5RB zL!0_TH9QAKW;aUo*B*u!5g~?=`1FV9d`$2-aPoyluitnc9j=x|xuT&YE7_S>N4U6h zYUO4c5pG8pJo~_Njg28VV04VWjdD5Rr^jp3%Kv^COUl zL4XpG!*I^7Ef&ej;b>9q$s{BOVaHONf3zlxigRUVe_KZNa>@)k31P%)O@T)o)GBfZyo<}q`df2k0YDi@c%akcvwiFdcEiIFE)`y!rSZT~6^+^mS_;yC=BofMn&>8E?$=2wF}`O+H-m zI-l67IUi--&^1#^uH!l;R~p#5OW+PamsicCb5VnB&GWr+ejsMH9LdMdmY5|tvqaOB z3EV|QMU)KZL90n81^EW-*{1G9>L1CCzt`8wbXnJHe&JitMof^exevI|Tv_bk$bvJOacJd^GF+;}%1XMv|KC^4*!&Sz+#Jq)`@nj5~F`;uPOP zSY%mCZfnssPH>+uqu7>I#2#A2Zc` z@4ApO`v}~!&6_uqmv3wX=G`N<#nwkBZxh+dxsOdDtNRm_Px?)2y#(iy!uAriyBjJO z*Pck$L$$3n+gHG>7+Zd1r80S~7cQd8eCcsw_sG}k=PS@oP;Mj@NDA=XzIpax#9*8m zo7d1sSDix;HW&^7?*lgh9h4;bye=Ac!e%-&c;id^v-+}?PanpvCUJdW9Vr8{*T^E6iZ zX_LjeY*oT2#%)eI#AWli3r}p26Enu^eygbk#p{xbA)ibN9k+~Q3+c5LIp5<$J$iYW z#nhBD+T8xLFn!|H3+55$65vjU(oLYtr~ z0@YjJqE$Y~*mnD41j+{!r7(+OFZ$!LXOD9sb%g+JBU1Z4t3ml3KD!9byh*po;oXbR z;@XoH3_YKy^zA0U{ho9N1YF+pAKww9Ss{CS;ihIN&fX_W&?#QY;&Y~r4ESU!p3Y3d z)3PxJ9DgBYXH5*#$8#+hjk84U{|CDpk1d=vuNxb8dA07xcOw-Ddp zvk|gup15Qn+IqCH^y)0B&<>hW85`TIh>UC*IvH)CU|E}D`VO{=i*#^5^Fv& z4FhIFug!d$)g#G-46iSP*;vqVV58&#n=%$x9oR2P*gVa{VH9$g?5PUT*l!ZS!W+;2Q+K93tD$ypr!bdwYfOJi|pG>)+} zrE909`rH}FL%qw6wvY!wX#8ss<))mpqjmf$A&Q}$it52M0%@1BHpNk*(o&l)i1 z($f6wfHXoZPxnfssM@c_u+7bXDcJ{dm(4wJ_=fDYAhX4d?WcGka4*Pz+Khb>^s5`4 zuyiIo72T@3qYIi*_QjqW!Ah{DLkU6eX&p6bfM*{4>4hCxO9^uQk={zU34F< z7Aqz7l9-%}P9suw*GC3k2?pARAX??rtF??z|?*zMe z<$DHD$;bc2*J%4v?C;c9nB$95fah21&(5$RBe`bR%6*#vJhI?1Z&&g#q!=R8rbf#0mRUI}!skb0t?$rYDo={4 zBGqzaqU6+hNsg~WR* z#`w=Mhy0@>42)Y1WjX!&6e|s9U0J#jqU?Yyny&enMCek9#8@q9RqQkqb;&hd(VIeE zVa(VQ)_n7>%ab}W$552g!;ND9)%EUC5CGoAzAl~+b3J>v|Kddf7OwLM{U(9f@+R`; ztX5S=FH7zU=?aL?fPWL?_f%cOP|#{0!$?8@@hR}_QMl62>pZgsSOIYPmEqNZ=;AbG&90BYS<)90j)Va=3(V&@8g_I z^8{J8fT5sb0H?F=UGOGJ!IjjFiGanmy25_T2MO2RnpW@5qoeICRD;Ov$N6Jc+E`Oh z)!-4knqi&M`Q_!oP@r9ZX!Et4FI^85%R&Q{k$ccZY6pR2#LI zCjde7r?cAcy3tmKaHUS1BhreszC$Rh#yC+WkT| zY=qCGR2*wUyt$H7{pP8SXQB!Ep_MJ|54*NE1FLn!o9CnQZA&$VWu>Q@0iu}V zI-MMvsf9+)- zkPzE0Q0RdG?fR`}Z6LL5p#oHMO7%`p6VpPqpBt)?r7PsInQ0r|dPA#HYxg2j;D;=` zB=THKeX@AN{98308DnsaN8mev3Rl?a!;qv0MkA4$WMy)9NuN=3kF*nw2UB&i0O@rI)2$-5x-%xw)hr2UP7Rd+SryJM z_rm*ZuKxHmv%ffA_BWfyYuxT)mZ62T$e!WXeB9A;=3CloO!jao@vBnxl90R zBMNF9-xo)siN0aweO>QlX)SgmH6H>;%&vVQoh~2T3b&35KL~i_^moU-4Q@0#E_4zy zIz%v5&93MAUyWiubWu?8XHzS9H$ac|2{KnxLs$EDT)4|_M40?ty~n0KC^1+cYI^{w zCC#;Oy+~-M>k(1DLXgA^C}}vs%bJ;&Qy9}OX7)X2;KqQQ={|3%usBIQW#9dIO|YZC ze@siZ=RMGmBOiJ{WOGedSe6gH|IIJ@Fi|zv9AA^d*=}2`#QskXv3o}**^TNJdVmlu z|LhoB&6gmra|i~m|64oHGm(2N*z-ROYsF0Lk4;{9N%>xUtHLDlM0&E~vZ$}SPBv&S zLcR0oXqyBvQl6l4%yYNn*?kKQul;bB>BZ7fy>g|pGU8MZxOT>@aN2}gSNqnD3|}H( zPEZ;~Bv(UT>)q|OAk(<|(Usk!C^5ZvqI|}wLJIq&8f5ZM)j4RwYzequb*`&hge1-* zHNNn!A1dLljy(d<3OQwyim+s#+0gywD|n-XA|p^80BXPraAzEWg<51qP6i*N1|r`a z4vIvDtfzG3 z`s5SN9@n*;(cPZ1^IObdG<#h>Z?@}=eNHl>$tD2aAWt92+2zf=*h%Y|_RH!WJ?&7F zEVr33-pdyG)K`~rt+gD|&PnDdp$#6U9RDG>8d&)`rt+kVj?1?1Sst~&4|JuHTF{xa zergu792ff-U@F9=0oC*M(XYBYQ`@7rgWVLg#N+0x`)dZQw)==ro=O87F((SsHOIzD z9fDHzxcwFG!uqoA(!LMK}T^edyZ+!dP{*#Luyv|Kzy ze3km16imYH1Ij*&4c3*^k}iHy9H%IIiL?ms<#{LX#%|{BGF-+6*9kWE7-lX|RWWwZ z=gy2n^(U6;i&v)K-O}_Zqc-OoAlj?L*bEqBRZ4td&bz^!qEi*~e%J7K3wfWF`sKk- zA8J0Rk~`G<1oxM+4?mjL+gg3pgL5p7i>vs6%q(~%+^j3C8xwuh_Z`&|VSQ=5EbW23 z(NDlpyCKIx4T#6(B)9KErr>~L>FM6kWGxsd<6>$a6hbfld0tMMqQPg;x%`KvHQEN) zVB!4viX~Nl?8l{V)G4<_lpVu-&9h5&bqk1G>7hi%GO)1BLJZ{nE_MHBeiCP4Es63H zcl*QiiG(veM~e@J3ri{+($~@JVMss19A~r=ZVQ7=!HE`Hz>Gw@Ens^k(-wn5^LIO9 zA2wXQuHzq&&~ly{0E!{0X3c zqgRC-e@CpR)#|V+qe7mJ*fII9yww7T)+y;GN82j;TAt7>v-kLh(+vOvbs68I+y16f&}ZM#?7{+lR2muymcN`W?kpY z-0m_sdvWHGJi{acl$>~u)bLO+_~indbj{#ZCO$>a)iP@j-cC&)L9AF`Wz|l6O}fgA zhj)wt9$id-7BYz8kVj)$lV@ahDv$@1VeVQU9dFm0vk*|Rg( zY?blyxeol8J%|?+@H6OMNZ0I=Gj4WAvhxb1f>m|eLS`$qbpUXf)`%=lr~TR7#|W3Y z<7&O_XU&$7tovNr=+C{K{q%|2IU|i_&v%}%QzS3sQFCRE))yMj`4u{Z7W$Qs2E#yQ zKpRIO{@}fzj3ipZMz8R6`r+pd0kX!vyKP!n4{Ijf_m2DgVK=g`ViMdE$(_3sB_+Pl z)xw)>-PxdK3k0#}>;tEwMcIrNpQA_+U!R=9P`zqp*QQ#8RhiD?EMSg;wr^%JqH3UV zDAD5~D_n!Zl}bKqP0BZSHW)u@`8anMFJ6z3rd_v)ahyam=HR>jkh!e$8}?|Vx*&cs=o)g(dgdrb)2Fvl1kS*s`^z<+aNQWHhZB_*T~49>V;) zG_|&*8Y$vy$hqK6OKSa^BfM9e@0U+jTd%;6UcG~t)~zn9&N{P}+lm#aGGCD;n}L?c z0`_{}s(X^u#JbDIBx<))Jyhh(_6Z#C-f(<5T=lK1Yu^0&IiSyYqYbg!qjpz-$Ab2B zr?Otp7=HTso@{-&;G#!q=PGact6)IkAaucXc6MIuB<}mSzN5}51CehwpV;6ZZu{ET z^>YL)Yl`-wAclq-PUDiOZ;7R`jPuQ1!GKlo zMO_YL9WqdEdbMN@$fA?$bD2Ew&{39WD(-q;P>_W!hHvC`vEHiX`if{4vVBExBPi7E zpG0O)gOD0*nP4W#3;TzzOHTGP-;cp1dkOZra3ZlW>Az}A1 z7uA}=i7<%}81Z%xOX1&3Y8?(!Lh@3%ks1%0_lQY;_w}V|Gc+RPn}}S&@;=#*;wCBK zZMy#Wtqf2A-e=35w~#cIr{ydzXHReK2jSJ{^c%W?$Lqn(&P%G!7m$(%A%kP{7d04oAM3JyLNF*n! zwSLvsG@xYTMbT*hq;8Da;&f{LIoD?LYP1;(7XoX$;WnshK=d$FY1OzE4~B11aLoX^ zDif8U%~&q|Tf?Mpku%5UNO8-=ZtnV2-CBi5j|rc%;pmr0x{cV_r#l-6p3yc1IzoJW zlk%ZijW#$QD)Bg^NqRr9wF9t7$D1M1WZUVy{zKwn-_C85<|1#7hC*$sbR525rw|J` z1uCVo82(&s|7BnQCMdNJ~cuzo6oqya? zg6gpx2hPPJZ1X~hH8l?T>(`v1txjAv;wG8P-nKrpA>=jQJP%&l_L9tg3FeF#@WiR) z;X8(-dPC0FXnDU;gN6pNB{Wbzd!y|TkzQOU;#Que@`tlNLO9yiZXh89M+xH3&8gL# zS3&J2U>a?0-}9;ned-W@NwzMJoNYPvyT#@E&24hs@*Ro8Wn|0IJyEhNn`3eqd0+ib zsK?lwcn9(^hp!${ZRT(iR2fbD*LY?YywF!^qkm)b;Tk5cIrqLDx{8)b&C}b*GYB9#qE3 z)w^>1o@T>vt8-G-ZLIFYJMXsi8@35{GSuRw(7t~DJcMOB&Vz`wjXN1`hHF#`CmXPt zo-_7ph?b0xw;NfwSb~`OR^9~vlCFYI-ufN|4QiD~AAO9^Y894sPZ(x?Hc4@p>>IH^ zVkUf8!u0*}=;QprvnMSUSU>FaPA&0zIcYdbVL=5gYB?g?ZKD%xqybO~61fK-%n7hQ z00_d8AEpp9)SvgYy+;MUX97TbxcFy1rYY((GQ?3trhYx&kS~pX(X>u)k5CYvu|+Qw zcKJS=6qF9LHymWiwchxgH{fKEuw^=u!xyDPR^%ab9sASL3wnVv{@TdqAmf>%f; zcVzs7J-G7$MhJ1hwuM0hiC@c3(G&NWL#iVoTfA}kD-TJU8vc@r zlVl~XxrhnqVv9P>Tvs8{^&IC?3+uSF&rxG5F<f1~Bs>i``pJ7F4(`fO(^K9@p|vs$i`B1U38NQd>_T5v8Z`?J&% zR3-|DaqVG^F>O_TB)P1mfae=lXGX}JNTKrRUftQ@r#9P7R;JsvIt1;n3b@&|xXF~WL=0J1|A6MB;5GanG^Ls$kpo~#-kZzd? z=|v-6pp0fC?W;_;5l+F%=c(*pcE2a8*VgDC`2RpL6~GY7RH#t4?ZJ1^pGsJSbo;4~ zKl!Y!6)-UK@QPcF6(Tmeg5(P^&DBei*Z7Hd#zkEbKR!RV&%n&f_TYLE{T@3PyC3Hz z8uL#tJw}2HB(0mve%F07#|t+zf&}P zVJ9GKQT{##+in+O$gSU5*N9sq=Dc0AaKA)@PvaxM%VpJV1n-N-!%ppV?|J7q)gL&Y z)=PxXs`a8>dJ`L2A|zyfcebG8T@lh1S%!QoY1D9fs&>g6{Ue-*G#8ZCiP|PjWCPzA zUKDVU>3)xu&jomv$EvYET)kdnfDe=uGHkd%y%aIHub?6-iD>mr%=BT#!T7M)N*)R`^8dbGb4j+zF6Cq z-(>@@`o@T1Rmd3?D;0ue1*tuw_}-mrAnLT;0cBbIQK(BZS6j|nV7|FzxmuZEeB}?9 zolgv|kY9bB{^N}!_=buj5OY-{17m|C@a!oW4hCuc;VyxFQcK7u9uJgz0d*Cx9gaoO zkETqS6LW4U`Ee|fK8qq5bmqNZ!i+iYlC3H%;FG*y^e3{<) zBA)EIo|O3Xnth}HI2Sg4QXAP^K1c8ipM#%^?v4&zk6|>BH`&YDwVe*h=To1MI?F@i zg8P@DK?^Rx*_0=rE}L@$q~obcf4QH^W5m$1@hwxd2>D$TfM*Ufwa{$v7-@GG8oXJn zu-=p>4_GnqSxOlgfWM%Ms}B|c`JMsaExs|4QKqXUb#UHpr(;+rl%tQtQALnzy@|@& zQcvp`N+GczQ}qF-n`{InWC=w1h{uYY?zHQwvACbfbUKZ` z9fg=id{nVB`kC4)^SQA_m&0G!_sA=;>GaoS{l;|zD>L2zZf1+A?96(ns(hFOrEbC9^ zx;w_5L`E8{6-6NBT@A)8`Rgmp>Djo-h1U1k<8L;SFiymtH12TfxDMr;H$UHIey2oH z{U{=GSI6At6U|VRpLa_M<&;RXV~s%*wA73ojs3ganCP>%*7vhySf}gbR9pg2)SJAI zmJMplO76^_y$`Eqwh+b-+fIqzbQ&ks$y6tirrM{_%LluGn*$g~V>H>AAhW^JV7ET+ z)A;kQ-<(s<3;SG@`vS``s>%B1`jtx@(^*g}L-uNQPV->g&-`h}g@_e<%+tQgluGdJf@TJ2OX#TN`g?O&a z;!oRCaMO}IW?Y!AVf*`DfL^vMnWM$ABl{1-rb#rytyymq?`r#cN=rwzmT6bch8$xmSF&z!tS^krKCP1y5W^;*t*nUMep>a3 z=OeyuZynpqhqv;v{U+nWZW-)0AL)58Ji$R_;f5+~hw;sW{graV5N+9ItW{sN zZ`bQN2#L4qqtsy#LsY%9vUu>w;E(i~&H|&}%X~==j`ccN>RGSg0xArHew7jygbJ6~ zo#MO8F1<(A3>NREtjZd0FDaU80bvzh(N7mi?=3QL$T*L@)M9B*d(^r=T@;~nldsxF zc&A*oNze5wv{RnfVY?X0B{{Vz!4`x|wytjUZNN!4Q$0+Au;78&{fF#3&9$y>6$9?^ z{fmTyQ@^)m-;F9)c~NwRImYVme26ejc=qW}kY>3utPxdYpQ2;;Du1R^V;a%Qm&LMnnqF zo~z3xJf2cgqn4o=1l$Yi1|Mx=97r0=sD1OuQ|5>Rd;2@vrVOjdNr<=&Rbg?O$1j?6 zbIK)TRUp>&2e$H0t(Eo==j0>?c&luCu|%2!%DK`z?=NzvIwPKYGkZaW^~eMEf{`kC z)_hlDKv&3ey|sD>t^Ktx?P*-FAmfT?cf?PY zieOc(FY}b4SO*0!cCO-I`8WFO;b2=gkmOcrXyX&fW^;Z33wix|G~1>TWc%}QNLPM# zC}|UYSs;s7TYULvMv%QT4)X?z9}!nm?($i@!zjQbe5M(MsqnPR!?@ zyOEfcA0}u8s@s$c*w3k|Wbw{vWEa%@<_LN}?7lpqtWYo1R4w~W*hO($II9Mm4!6@P zhHc0}T-(?<$eX1W{A4=6s)0EXk8O!B78;~b-S-p(9lPhuY`Sut{JetG%JWA)j~L7R z=_15cB#8Y=&)3NXOsUuj`yLMwZ5&ICY5u-d&KG`lgH|Z?B1qUst*=>hxqd19s&VC$ zEc;_~=h$e1<2P&DG2Ox7#RSLQEf;yo5ex!^X9QHxG(>;fZJ;TfWT~3zJ3rXNQzyuN z_~N!vn7MgHt)BZZ*%6#kzp4x_$C0QxO(y*w!!%s`GQwepqDk6z8KywBk*P z4l6B{bY4-?<7`II&N>1b=F6!Hnhq@QyqTCaStVT&t~0mWN6R&uMKRQ3ft^99 zhq`9Zk;vmF;W8B$ihPt2L0(<$M>s^ZBbORuJ=NXtakr8W zY%My&QtU>17Ihb?&5-jvW4ij-BvdD12Az9=88!OH4553D-kTJQpI(XXfhx5=0@o6p zvmBI^+i@8i*Omztq-0GROYWUH;b zcaid&I?UBrabQ!*{k3ibocGuDsIKFyiSa0JHtw5cZ{S&JiE~o~IyUw}uZyyBhJ z?tUt}v2u4g%pERkI9~qM60;NFWzU{MeziYLqCWc7U}D6g*04+`_as@o0$=o5$)W|S zRqO-rtvHxIoTrnM?IwppOf6%yejvu$3C=u7vV!K;V>7DQ)~e3do~r1pWP0zm1#se) zfG^lz{aL(7E-|kn5AX?5S!7%(_i(kf<^RBCw+TXo%*zep$fEk{GFDRg3$^5Pw`8^- z(!KbY<>0dPMlppp2w^krNS9JVcYV0>V?olg<t+kG6sSK4b0X(eSWU{a;h9YPf z=M3~ExNyuuiP3k}36^+$maQTs*(_bs7OvDkTD$;rKKdF(!b^uS2XYxPaRbN)zpx0E zY`qZ*O~v47Gt=Io*i4{K!!t=UL?v8xW7DhD>wbPeoJJAOX*TYR`ffyJUVXhuiek+2 z*ROTq5 ziOaVx)Zdy<^@H1SzeHKT-~qHOkr^F-ONC}8WA_`3iziubZ0$b`As+~UZ76YRi2A|^ zG+ClVOqE&m8FeyYdU?Ut`Ct01aM*&87>fPBsbgC#hV605T!cNMwbRd){F!tXKqS+y zD3-YhGFi6iDtkMSf=}18u3!P<2^B%>lm7eLEVuc;P;3+L4j5f>`-s_2!7&TrQ>L%k z*-cdIOd3aYD-E7GG3%8ml82Am4mYIJ%DVSqdC#Ej31yJ&O|A&YRUzH6v z>A*9Kxd}h>njG2MB2&H}!&?D&n6yl<(O7oP3Lmb-ySy?rCp$mRIJpp94SyYO?echkALK>@lc{buJ0y+qW?f6}5?r=hcfJOl9Stw*W6; znTN_O3km`YV=$JjkX1w8i}9RQ1dthzNvlj%EmX{7aoA$zc>IkF6tT$o`pM{1ibm$l?PZ{5o|Nv6M|WGv`*Ih zdvhp5D}14g-a2yXIU>#3*DEm2!LCH{>K-+^pkr14lhoF#wd^h<(+ga0DD1lAu6vuq zr*%4S+a+safn~8@JKdCxe@rHPQM@G~X^)Hg<6J(xoHdW=B32gxhc{tKpHoYm-Gr#H z_`it#X;PM{u^3ro^KL0@gF`n9ytIG_!x93^MksQQ>3tW>aOrjgJ-_*w7J%2etK)x` zL%79vRssyBa-!@qiKpSOgUK|XKNoplffc*zcOAs)xv-*&kQ~Oqn-Jt$jEDg4QO-@^ z(b-vB0Q?Wy9qTky_=n8WH(<6$^b?ivuP5L55{KApyBSkdu*!gTL{di)c9!6SbQpD; zAK1^%^}dT*wLHB++u??%_+aiuM+?kyY5HWj6^3wZLKvq>ZXML{9g1&UW&hkfvb_Z$ z#|kEf&bWKbzYph7!QG;25`BUPz-o}DrA=?TtIPAfucER*OS-e|e(9B~f$M?N^!sB* zu^cc@AH&x!lw%D|Fxj!}pp%7HJNGkfABwLP9$6!me@;9U{c3S-2UH4Bg|{Pwp;ntP zzCQZEw|mri{sDx-ciAhYO4qE=F;pNiz)OA_S6_==uL``{vC0?_?+v~H84-JP{7Mca zNZ&fEsR{t#N* z*p^B%?>k^^7GSZaPVCNQZVf2Gaktf2OZJ0qE|-yuk8p4K?kA~_R-XxY*J3rqoF;<` zI`%sDcI*p+0rNfv(VLdg39B-UWMlh^*;X(8I;9p7?}v9$o)yi_9E(N93v?^D8FezV zg2v6G83H^``tCL_9WRQt#@WsX{KdS3eucOy3^K|X6WW7Xs|j!jEeVdVtX1!bUgD^xlG}sWj?E8OF|Jx+dC2Yjy$8zd)HcO zVuu*wP2X7a0=m`j83I2gJl@n8tQBvU-^gsD) zowY7=X|fnlv=6art%Ej;LIh0!Pn`5M_=@C0%z3+A@Rpo5erf1X4#6V*ZIr(f$kW<- z*YBDHd-sfN*ND2fb2pUc(VTh|;4Mh*AlJM&Yp5K>)kIlsettDB?~t|mCXN`joGSBG zvu%70^9S!w6Xy-jg5KLA4l=5PYTNoNJWFuWyu@KM(u3Xvk9gK+zuT|Iv(Iq0Vg;kJ zf-BNT1s@KoPMV@ZekGg>X8I;AMC0z1poe&@C`z-ApRl6-h2}{fv{`v5Ne~GWd@rFm^(bDYOIpV@8Vj~ zMRo@4N3!NjU;re?+-U7OWK^5*+DdwiR(nhfz{&pf_y7btmD{y=WP-*F8q(?4R+0Hs z;Xv42>F^CwccvZ;xtK6UI?Rk8E7h!p14o&JnTKX2UNhQKsLiHQz@P5Hf~zet-S( znQ*l+H#c7wuH6yO0(pZ~$eEO03X(d?+VuPMF|DXiOacJKxKEK%-4L~xZcaV)X0|Zo zXwTnq@(Iq7jiw@hdS#E0;!PFA(+eMbK zB~r59bD zHe;saw&4<9TLgMcn;9Z=7a}Q_<$c9_5syyNl7k6E5lTD~M?VgHRXA|PLy)7h2qWvW zuNqBjlb_1@whF>cgcAh&#Fy>ps1}o_nRdx(H?nP-5 z2fDBL6>yqv$>h$Y9=#9d>=m%J$Gu&myV^qG=!z|}CDWv7wE5*Cb5$YTg#0F&KnzhJ{2Xy!H>csPk`arYJp}?+iBx#110v zb9Wto^6xl);j8Vg)@mPA7s$Uu+TMB8?2Zj=RvIg)6*FXe^BD1J#9any3E#k-;3P8> zf{d;VIHcZUCt_7K0k#TPSXr~UG7_MR0~mJUrcUIL(1Cdcbn_y)PFL@sYEy!N=ZIAR zkm?SmdpRV=Z}4`~z4hW1_Sn>1zMfTDteW*Bh0i*iUyt)?T-It1!|N#k3>>+OZJW;vS|m4D~LeTMNB&Z8@K{>0BqV zy=vNSHv9zDe)wF8-h8L2Nh`}$JzTHYcr0eD9OLN{eORQ1WZfJN!Ixq#%6+|PlI(n; zBmphd1;);LfI@F`^+RgAF1j1zZEC+izY@%}*-Z7BoWL97n9H-G#Fvx9{XFPdmAs67 zP}XaWe5N{?Z{&{dx#$9w;Gfi7MsA2F_@sladV)5Iv(n0>VF|5UIeQ-))Lis?9{ZF4FD|xu zvHz(3^FB2dQwzk=qTB28d=pW1_y!7lX)tL9O=Yq4TAnA)ePw~aA)97L@tGDrU`s!Y zM1a@51ZmrA$00oQ^yi_ek>r^QPOe9E6Qd>+klgd-Zi{t@9f%w{3y~=Q2sNId3i<{d zoaBn<|FWG9!#3uHmXTFpk1i}u$0;5vTH%@e$8Vh6ca&# zUvI~hi4O0!l&6hryG(SyYpq5oZAri`W4XoTOVwyyG2@p`#)(zW>krpsD>{7av!^I0g#!9q>=dP*`xG22r^xe6^Q;5F z*YRb1L*l_W&@on7c3@%ZB*h?XoTcnSWFWJJ^?N)aQvNfS=uF(c$a)i4OO>7mY>m z?Y>4Xg9IQ)!T6!wXLIc4uPmx%lz2n#FdgIWOlhyo=FSy7MpQul!8NjT#bBGB+k-Qm zMDj`AXlT%kVeJuXUzC1c7Tlv26w585+kLZPSxJeFe9F}D`gZ*1MZJWfS5V8~^&*5% zzIFBtbaoJI(JFmpXq>lw1z<>;cnt;_C)*cZ1O%CMBo2Bx;OtRCZ8O~W4hyN&5Sd&E zi5tI9c33lIjDtg{pKvkvK67&QRNiHTD*V=Dlv6DARmQBHN=G5;$PL9RV_|_@zV3I9UcGmR6*0OL2pxrlkv)YWMi{*BC;~xWnjxpFT#KL^9H5}| zteRk*8UxO&;^u^`vYG&~+Wfdi%NW(-=}S9Gv&*o+GygnVM93ouzr#X@jGj${ME>#U zbcE)4jGy4h?cprI7+5faXiU&5CLEsO18cAtG>tS{??}vDwYV(VfPtUEb4~lfQ@X|5yx?5t#hxjzM4+^Q^Rkt9 znZcmdqP?!8{Abvr!))9wmG(^i%V`tmCR{=Bv|x>5#H&6m{Ec8&GVeI6>Q$HS(0S_) z)3J#WLH$*LAHi%=oMlOh0`GXbNI!MYQLlV%8hywmCreZU&6mx+gNzX}3`H3R1JHDd zJIBFml$I!J<;&+Cuh_lxyIf8oBw^!HIS=o+c-1P8O^8DS072@Y1g?y}%O1v=UeQqR znqQ2c;q6ND1RAN`hT^>pMr#a#GP+$Idi?+!U$Vi{@*bt0j{fipjTsA z!vmDr%pw9Tm|;^SKe4l19=qjfBZY0@$(|Drk2t?HB`9_MzI6lUa@a;`DdYy|+VkpQ zPN55XEG1~ATr)30`(}gPb(p(nJSpDhu|3f4Yq$P%P>I$cdf`q8@`*h7dL_o%r?S@WJONf!3=^B`Oe|xs1AYmLliw z;dddo0|lqh*6D98vsBQU>tCB46b9qr=M;WN$i%H3!y^Jw9H@@|pjp{(BYbdZg0`qq z8XTrS5jR`eg|9*MUogNB9F|%+r)Rn8pe8f0Kg@c|SZ}R?CW3W*!0)J%D5h=2JsnH+ zdlxOm(1r1UEUJbwY~v%Q7D`z6&`Q{j!87|FI}7VzD`pR&F&5N9n2sBd$bLiXl=T_S zWMMOw@>bIUyj@m`RxSMx)cN<9um&Oq^~ajD_rO7Z{Z0?7QYlk?9pL_VKLm_9mv z_qMu2{u{P%#U?Q?YHx=_!pf0OC#sxDf?`=K*TN3(-*?})WB#o@za{stFU9ru!Zu!Z z`@N+79SaZ)n-0i%YN~c69RS~>)17X1Jm^gnXE^G*{`|IFQ@AE*o~;XG?zUN#f?JG_aa19N5`a(?1@R)^FV5PLh&x zn5|*HfPhj}jlMeT(SIBL|GXV0 z%7^TKeENT+|L^nspEDmY)Y0IgbjjGLBlo|R>0d?iPpAI=!0H`JLUC=Q*-ZbpdH#=7 ziAYe7tHJEyZ-w~xXZbJ3h92BU8=!K}6ju1VGo=`qr2fbF|K~=38t_MAG>jO&4PMXW zznJ@feNpO7sFeFq__M#iIU`D|ZTqcvnEv;=_tKa7kj zJpI!&{^i&cl$f62mi~=2QO;)^%22Y2(=$_k1M^>>E{zhC42ANOf4R}022Aw?C3#c? zjWzl&<^KCR?|vIUN5-G6_@4*FD~*!8oRN(x$v+?c)SJg^m9PSBRuSrFam3H+hs7X8s$e55qk>{scPzdMwNW_IKhzxgLGLL3$o@*XwUV z8a0(=^nbm@AH*c}9tAz)b-3gEfA{e3p{CM~{YUuv=LIkfj}!&Z=9JmfvVSY^2V|av z|F#1E;Vl&l3Y+`5gvw(7R^UWzJ9Pg%m_G}YR~ZG&?~UD5DgXTWKh2RreN_BEiueaH z{ePi|R<`q(9)DvgJQ%0ld2IgEtb_C6+7F6rrMBY@tG*vqb<1n2KCMJfCVRgI70+nT zr4q4mIl3oA{guiT*M(w5pRES)M;gAFCSvQBnhiHTi#3%k5TE8(R|ZW-axvrnlP>&O zvBJQHp;9NXpdew#d8=^ zfQ9ix-)!?crw(p~?keMY+SbI4+!m-E=P12Gl!#4rmR$z;SCa`NBmT+YKHvybNSgbJ zhhk!?b6#+8+%Ji8zpO^mjk7%9|lZd2S-0@U%ZT zVgKYv7AJ*bMNr<5e~g5nH3Z5~^NWp_I)sY)mJLyyH3pM#AmH$h>t5cJa#MSP;NU_+(KNa{zX-A@=yx$!bW>)$}G zBk1qx_W}=JBCTj>=!#n9{7< zxx+RZW_I*Y)02AtaMU=7`%GtNWSQJQEp*&bxGs!p?OVWD^3DZvytDaZ$e&Odb?o=j z$yM58Y}LJMl=T!}anfd9|NEFDcCi+zq_Jk%o_9l*Z2o-O8u@R8XV^!hCZZGeOvfR@ zM(<0)u1jC4?daA8KAdXg?8$vvogx26@r0ccXLLPhAf?23JqRi3XU)HV5eYdhf!s>i zYbV0KY-+JzSN~$^J{%K%il%{idWOy$w3FBXul1gSW~geGC~kjQYCsEnOYwn%VGeHt zoyLmX;pzhwFxwjbvezQ&9nPP97dFp~n-j$cdHsnbMsezg$ z;cp_pnZ|xMv~G849q}S*pA8HGx&x%o+PCFWtw8 zjiq3}=6pbw-qpTKj2;&1wAw9mdUnRl%{}pu$i|j3$Uh*@vz{+?v5KbunX$mNSuW^hF1oM)QK+X?dlu&i2Ce# zcg$92kvNQg1Hy2A=h7|BMVoR;BeUHHBw@Dn5Spfk_^%Qy)h`R!Y50cnu$(~ z_gps|D+Tj26< z@2!`FUK#_Xk4Be2c*6yNeGo_2E^co8pb_CzQI{^&@k4N5W}<+GfzKd9NVmlWY(wj_ zCf=%;n351X_fC3bQ03;kK71+LU&qEB#f_h1ZOyNd?5TIYOCo%B^HTbZ2MrnW&7TB;wfVciRv6aCRc4DL_CZyf992_7x!4^X?Y=w(OylI+I)$*-5X#Bo}9 z#=8LuOcAF6;(r!jZmCxza3*?4;L2y7MdgXk3ow_W;NV+mbwqCnE*kzZ>!x}WJzP(i zJ=4rXZLL}vBRtqunZ>ccD6!C!v@h3DuC^$E(7%5YF#9?|;7qdS!O&<)Scd+0YB%~@ zrS5Z)d2Ya4NlArPh8Ap`~DQ0qD`3<#ur` z?oc%@=O`F0->b!n%>p4r^@lNH`F7yzum8Awu=(B}V@hCsvusYE=e5jD#&CPJ8yc6H z#!YI&OhL((N1A{tiD;}YMb|*;x3Dm1h2TSd57#C3S9-Z~46)I-sX6Xo6QRn6B#SJk z%P)^>T7nEx^dH-6nVez7j3}*m8;v>9-rBFP&Gip!73(IF8XNLbJ$rEtYHtV}sHhw0 zxw!()(jp|{Y2I_HMP=enh{_l|_6xmfG%xZxnD16kX?0q79&+a_awlkwyjpWCi1%s4 z{G20hKGo{sfpC${UHIG_lIdJV)^4mI`0J$1bh`ye?@?jjuqwg(YZ6dwF)$j8rwB}F zQF4#yhu%VovRh&wn-K#W8~Bf`#DN>kp_ zg8avJezw4#HEUbqQ)vA#4Xsqa1xa-yT{>*p<$~a;yB-wpF(eX)`ddJ8mk!C|nL4!+ zeBV8QEm1=!;Ep$ime-}T(Jb@xgzpn$91|P-CT`mO%8h)_z^VZFqNAGsdS(=##I*yQ9wgC6o3`4H)*zh%%r)0Cd0-~|hxpHLJ0hwL#S*fNS6D8r8M=(R zu4Xu)F9mLAMQsfuYqv{YebhtmA?`KBC(6n@zS~DMUs<}k_UvRaUy)wr!73%G8R&ln zxrYNGnV@_50kt!e7+0cYESk*LOa^dv*(w*&W zIatHgec!*(yG(1L(?7P#&{g7zO1Wl9!}P`Z!K|b8bpmaJ4>(}g>m|_dB-wf{(8ot- z#ywQfWs$>km-C}`p_<9l{@AJ!DyENj3O9+6X3G4 z_7!bv4UY`rN|U_Rb22_ERe>jckEbSE9Pd(dFM~4FWBE2i_Vfm5&!Gzfb7>Z{{Jyfe zK)O`vpaLpQR5~FP=^ac$M?^(IK&5vD>4DH& z2#ECF6CebX4rx*nN(k)ue(T+Puk}99`u^}E97pENoOc=H8s|0cdJd1joy}Tb_D|QC zGc2zqk42?MXKn?7)NCcMxAvuKcjSq*UvA?ls4Bu{$v1k$ybX!Ex!(*_|3sFo{swy5 zCVw070EIz|n2IxvfOhI%LTv><>Ilmm5BvyC*AOv2mIoR#VP(*1DhdmMn>ahvB>k|` zS*>j9I903dS;d8qmd;tuuskHlAh5&~u^Y1#n60lO9ZU5JSIM=^sHGAk6nnqH+5)UX zx;NCH|H`|$J(NT&cXBjzhB9jDm(ggUPd(AE>)OMh8b#R>O289hr53S9x>XFky6UD5 zA5e$Pt0Q$5O1&A!kP~z{ zp%x=~j5!lQ$qIQ>xuDMpKjs_MKst8S7`yuXQ`IbT`ZVw0l@@lcKqh&RG34WZm+|&W_U(-e zH{$OIvX8Q`U10jA*CVkpqij7ZAA1a_V*F61xcg8}Zgy=8PogZG+ujj8kloI-;ihH_ zs~==l){O9NS3!21nhr`^7dFB-#_y9tO`^Ko=L3y%CyT$KP5SQ%9VSaBAfNxd-D9}l zt&EMzAWCO&1TJp%+!@f|SLjYRBENQ4b_ah`eas(po?rD~I7AR6%LfXJ21l!A4y7F= z%6S>PzC;4#UvgQ>-(-p@7w9V8{~Yb8c>yrt@m~`KhPR6srV)^az#X%Hz(bbCLFYRf{vJUjLLXCjpXtcLn<(O5pcBmnJN3*EGcASkL56QY}0zxy*e%`^Hyf z*Y*w``V=tEc{i>ISki*DA_9#E{Dg%y*;KUP?v8z_1IT|o8g(4W7`eDZ=kSOEj-D>N zF9$jC2Sr203k^@&En(~`j5V0#3dCYht_wQRfASMPYa>uZzL&bv*f+e|1HwNy6oh!X zxXkp9v|h&N1VcnKB*O++PB>Zqn$@W(2Bja*`C&n12P|maqr;$K4yZB^uTqUT-?-uK zVa+m#UKz3%w6L5ehn6Cc(5BR^;QlSM8d~5fyd)Biow+}BBS*$9rp^fNp>aGdm4oJh z{BksI6-{i*Ud4q|BvFr0`3e532;rCzmS(|)sS=}+gPz=Q&?^2c0;b6;H=m=RD{^Oo zc)TEDq+2aDi9JR}7{p53)vBwdKE(LT*St5C}sqW0?yI4`kDc@rR4F5p1Qqt>}I#)GJ^kI24r;_N(^~c-rFQKyt zYd+Jc?`=crYgzgYj;|s`_;>c!jrg-xfJt|qacE+=C;>We?Sm76shy;4#FHhOHe;Bb zrg`t(ha>6+XsFwr^q!2SX&Q}+GnJ*F7P8&4(;QBB5c1k9ns!cnIak)0e!S|V&8$^u zEW}}P+3xY<2GmQp0nVXv0J=eSk_T<>vfKEyXs9-&h@x4PHEB4kN_jD6*9v$6wFDGu zx{wL>()AVC0a)wOAkhEU%r=dR>ZMUc)%9VtU}WVyiX4>_Xj{v?{Y(-Ol8uwccO>X3 zPY}++#a{plIXT2LfR7%EsSxTYjfWSyEW?cAEa9yUv^Q8Qj;&-;Xic|Tx2PH1Um=Xw z2}M~6_U_MdXoUy<#3MnPjVqC6ze8ymN&nd%emrh-9%*Tz%N_=t7d~Y7dEcOq4a7SB zlM=7R;4Bd*@C^rk*_WcA({sZJPAKfCen}{`p!)osapt?NZ(W<6Z5wg5z6z8N0 zw~&jSG8(92vXIU>>1<}qQLU75qt(F@b<+cN2o;|cMrSIu!-_t$$E$Xy3mk8_ypi?K zPVUO`b;)#BLh#)GAd#WaWW4lt*nwYqMoJH&F`ko@8#hnv#o3jrzo_Lsw1PYeT?mG^yyv#6WT~5N80pmE`?cv*gSUl zRUs-wNx;Wkt|JCdQ~@kO!?$QX5P7)s9YH@=7Nd?A$ZwoZ?rd_j6c8U4uHydtDDzv% zOXq5-g`b#xR?7Z2TZP=Cs%t_~Z#j-Drb1xbHk*e6%GVy)1dO~?o6^*q#lalUry7kl zldc3yKq~jE?1NJ55v3X%l?i)QB%Y6^IIpL(`e!ev*83(M*73SPD#_p2o8NlPG(C0o zG#mPmt+CTnNxGSSK*?n>OC1CA&l(SF zLY2h+UGA?T1wlK`)ikTgfEGV4w;Wv`t12E!XnEWXDYe`{lvqxH{UE4{fm+IQe6Uq4 zxC&FLt?toWdp!1Y3&F-Jb-4v32>XW8THpeYEm1XmTGc(`#E+6aJa?YjU>=&6*eFZz z9uO#ki6XgSGySp)fM$D$P})D0U>=G79(ZK#J7sEK$M-3o!r;gyecm`)sH$x1qP7Db zeE7AsZ^L~Sba~_h(Pt?5RR?HN99sB-?xQP7UD=jCr;Q2K3*m8dHTDxj83gR0 z$9FMJ&4n{y>Ug0kvr7{=7cq2(cKUY3WyMvuOf$)XOm0;RI75#2t3k4&W@EPqA@KR) z+VZd`QrFdd+FKQJ>KV#7ZMQrN>@IrmX)1o$>7d_qcq43g%XKU_?~xcpz1$1MbgMSj zQ&KQt|87O=_7kl3+n+~kNPlV%hM}QG<}5o`4&<$bW%VF;(SGKwg@MYw=m8z)Ci7bx z3Zqr&s+m>&q&rr^5psrB0+(^NRMf)i`&OXDzZD;WEbS2=qeUb?`T9K9UeUA}a?<5cr z)mKtrHmaK-wB^P1Ok&Dl@c|RfU*O2fQ~xxSE)nJ~T6CNt?O3!%OCG7?^;Nx{44*<6 zU~g;_d6Lh;93O6+{b*tnTz}gG3{frTwFOTZ`a6v5{0PTMWq}ajuh8_aKG94ipR;9! zW@Gj&=ThJ2@Eq7?7X~w+fXA0Bs97lW6 z58TpL%-;z=w9meket*^0(k57LqrE)J*siq&2NvE_^ji)}3lg`>k_NEt3~cCSvWqR{ z^-Ma#X{)-d-7(}rSS~GbnOU`?&j%#RdWY-s$A4J#Z=9QDfWpxmftg>HK5quL+>{JK z77Lhl^_zP9=q>}nCcSeoI&xR=;|IlLS*Ohx9%enakYB2^H>OL3-pw8Vek>F?yDK;t z*hx`IpUk+O?P|7^wXinuxnKY<+(|0MJd*}il}^HM#FyPJX!!)5fTDU?IR!B#QM)Rn z`y=`8dlSjxrm~oLeT!3*0ZW%HrYW_F*?U9Lf(h00%o*k0sWp@=v6-2^lLv4jA+ww? zwl`PPtfumtwP~{%jl0+X5-yOmw-EHO)Vyp{LLaFx`oRIduHH8NT%HIwzV7ETnH#!T zMHaS)%{sLlyQVH?Btwaj#KNKT=+!o#BV&Jm|K)1^VHRuKtaMu5VaeT9kZ!8x&@ENw z7A$e@?-zHW$4?^PU|XV+B{vzPI)NvA9}n~q4YxL`^qnFgidAQO9|-{~)-7o(tobff zl-(u$h62moG}#EZ)=+99FkYimglTH4k4@mK07BS{m!UzLLgZIN9jEJ-ls_8%}>fcys1U%Nwi|0b4GwFcyl zI>$TE2tYr+5SYT9X;GtrA8*BcMSf#^#pd4H#_b4-G&t~X-l#ql3Sn0+9QZW+qJ^Et zY+tC~a8pOXlZG+L9!~aX-UNQ7hsU5XWirm2+H9QvHl-ow8@MA3cRL##yE8=?-gBE9 z4SxKy2pFCiw8PcU)S;)mhOCK(XwA)hxx#*ETVhd_S5q{<6DAGUdfTcvXjRBPL@yZr zL?B{c)1Daj+=hN=7DYiHpHDj)dZt|J7_z3#9Hv>XvaeO-Wy{W#al4eHH-Mxm)y~^D zf{s^|vY1qMBa@$>iOEEVcNdcpLY6iNc3u5 z)Ig7ce!iyz#^{@~{PdQ}}ue$jIJd{G_q=F1OM4ddq8t@k`9l{Qho ze`(tzgoC~O-`NH)WEE6i-h84ZS=v9+{UJcn&8u5DeC41g>!9~NE1oS;l^8T>rRH@g zG4?%)@!g{*@{}r?!kOpD>~XN3IP$IMVGq~l53?RBzp8h2K(SP|>YWLdvB@)66+RYP zc4k;+WpkY8RPcOk%%gCgp_LYV{5WUhZaz!!>?Pw_nag0s=`Ag4avJ!xVQ2FrZ0H6` zsU_3Rr69fddAC7*&l>-2cbqqusBN?78_J994OTrq!kHKmgPd#Tw&7ppa$q%|(nrrv z)p8Q_R?qfI1hE)xglDmfBKI6@ZP(cnHH15?4CkCW-o{nCldL>UjEr<< ze1LM(_q0o%G*#~sK%Z#_&;IrA1mviQFKrdw3YND6wAf&CPrQxC5Wf9tPhfqlcDpq+- zmIW}di_SZ3XNCzI@l_g3nZ=cvp6@NpCZ~KgM<^PSSx*S6wJ9afnO~28{|t~L1)cMp ztF>(9IjO55EE6zUZm_d(W|t65YQ^_*1oRG!uw|@ZceMNQ>*rj-b9eD`O-~8UL=%yn z#|al_hRPXBn!y|EeBs*-6VMH*LbxMDdHjqNUUs$YA^EulxT;2``!yKg1u21Mg2b;q zB{x=NXV1zDIVa@fBn&#2A!_ zzcP(;z6_Upsc+SQbb`2H$^gHw+$>L|{=MiXRKtMuBx{`LhA^g%q-C;dFFxi4;eZpU z7r%R+vlFdHM7+q-CpIQtQT00T_Ik9Eqqi?~@ofrdo$uB5wXupgL=I~CT+#{P^W4tE zgIQsNLq%vDLCa%0=2d#B>7ERLNzlnjax?E!ep?drFn+1d?e4%ZF06F$m;@;bS>4mv zf2R9EKs?(3RiEKkZ26fmezG~RQhCp+iOX_A4}3>WV!)gWj2l(-gqum;!x*B)U*K#Y zsjcK~3mCAmIU5Oeg6m^kDYfcQ&~zXI_#s0t|V+Oy^AHCb(*gaOT0r7dnG zwjK|c=+^vD2vD%PwV|8^Mz;NURchPQgssP}&_eg%@^jXOGrtc%MumLv@j5Zlt7k@6 z)VlCo=?Nh=KlFxUudY2-vE73Es7W8@7$OyZQ>gxL`}mMsuQ56yXGXVTuE%P8vv)bb zHXkdU^z3r_X7L60wy$b`d>QH`usV7>7VmGC`n{12CwonZ!TTVD}l2UbpzZOd_xu2Ii7AT*0sgv%*LW&$_tV7A0((rf|yB)su}(Gs?@j2R_kEjs~<`+j1C8 zcn^+#(|xNxRWRJuBV;_d{P9)vIWU|{VCr_&_xTh_>+CP}$b5hkmuYoPO~>gkvtD(K z?E}rl9#BE`hQNV~&bYN+yDD(x5v`?#y=j);;Vu8<4gXoGS@r#O>F&M=V1$yy11NU^ zto6=xjfZ;(Ff48=9AG}Fe7Wv~j7MXlW0f)JGe(#dzKN9@=YtfGGEJHBe=D9CbP!{@ z{qu`UmNRYYWvyS-g6#eElW){uT|FZAtQ{6q7i+s0&8RM(pmuCy@zJ73vvvGKL)M^! z3x`K`lpeuXHZnHgJj*2ye(==?p8CK4A=tagF@0f0d?SM)J6OuX?9NNeamN(GaUp_7 zrv6noog1bW*Ct2nMABaWdbYvNvJzrFFdL;jTld~fb3mWQk~TgMKZ{4qpTUOjyVchu zzPeid@IV5sV2K*+d5RkYi}F8qL=$yHpc~zJ~`a22;bk+3e9E?CdR1joHC1jEZWR zyv0FmUtYcbLmW=!^VTK=pal8;G%Ea)PE93HK(_bXX^&M`=s4;OokGnW*-9wIRmcQJ zS^wn|URyK%Y&AnplZ%vbGd;J~D zc2opj{FeApo)5^Z_J$PljeOwu0%T(jLsO-!y|RHzx*!O4N20H9<4U~;vLZ3!c*ZWQ zc(TIusXGDD;JSMIrUJumIowdrZ^*@!)*zwSvaT7^Op3_FQc_*D>dnd#LvMwJ4 z4j$9DY$p1oISgzk&MK|;BSkHH>{XM{2HL(;p(ldpFK7BifcSoDGe_z>{9@fU$S z{~%lM+~=S-CF1sUL(NbiES{A3dl>gv>U4&Jb%B3z>@U~2QM~)q9QyEmJ|TDX2k}V z--A~RKj16rQv3T5CF$9B@dr?l`$vlbq$qb!d)=*l7>5!vVld&XK+&U*76s7yG{yJQ z(;9VCx6oD%eyOggZUoCj-yeKjP2c^GzvlBb8Jd1=rbLo+rt&EkzUX1~(t*z#r8*>t zL+KpgjYW7Tg8AuAZOGphXU6*Bd(_nD8(wh+qZXFS~k~ zHgF7xy^{U!ANmbLp`K8~ba;&~$l7r2or1xaK;q`cMNwApB2I&irJwORsadULR#c%i zcB=jin*00}B0zF_UK0L;zL#vdUiURA`_$aOuibct27}ydrV0FU+JGV>$+6q(q z+0thbLQ*bAy?p{ceZx03TS4#78(+LsY?9wBKjR}(aD|Md?y zp~nGiM6KVOhV2rIW`+nFM%M0aU#P28q%PDWE20%fIKd{!g|?3qEJ=N?P?|{Of;EbNo5)& z-u!u|3oyIC{IYu<{hIjcbby&0NpQ#UbalHZ6li<={9coSRG=rzJx25-^b=5}jy2x+ zh?4Qe@A*<5Ah8#pW}HK-kS$w2`GR*2D^x&mJ4>-J#u$4N6R)$8Sw%yLmS+0Qv&%0OyaZ8a zJ}{>o82BPxUezYFG}(q{_`IrjdbVp3Y@l90#kfPfC`t7XRrN|R4!7Wx(XXYpN03Zo zZJ80mN!1)ek8SGb`B?sa!M_yW@Lc4^ zs}Huo6sW>I4xnmaL7&9@YFPC{i&7Bw_O69SfB|J+DLc{WJheJFws*IH9Enq8S|jnS z*L_JsXq{V_aY|^v_z(NTe?1ggy)S^Jb-MnoohZvLgbHQl2cd-X33DBq&_DBNYDy|M z`-1~N*PS~jsb_!M9*>^m)@V1l(d3$K)gaKJk&|Y(rQG8(P?BA3s8|IQmXu*q(DuTI zTg6fqEKuKP(;^-j3QZ9Hbf|=)b>}?~-5Ap+g?<)YpY5Pfl(*r3I@|;)}Md zF@>%Ucb<23cPra}`pVd4IptOJ$RKk)gZ78oo_`MzcWRfNXak>(MScaj{kf@o2?d%( zpuQCGkPX?k&^jR@kA~N$(MWq{=cyde#Y?vDF5zfr!iCNJ3rPLvTZ*jZT%YWeW=5^( zd&>y(fu#b>V*h|1`I(2yg#kxTl=YUgZwv6g#haSR@pvg%j_?R6N}C9@yQI8b&L9N*nNMz{KLxQj}}o~ zYkXYiFHxxB>D3m?dkXCai*|~BGD+p-eTsgN|F_J&*89IN<0}%6X>>D+8n!*9CO&CZ ztWcA}kV*qGC6OkZ^|41s8We%Z17%rNDVCck$+V5IMBuRHX~IlJq9TQl={ z+u_XkS5pqRng}O1q^RB|9-v)nc9zjs$!93l6btDS6b{*6V?aNj#g%Wb_1o06IZL8v z`kk%i%6F8q0PQc70NO{#66xDT#^E-W7jD#Bv8y#JxVb<6|BChhw_kBfXtXqExFW&m z-@O2gk)D?hf}D2kw(0?N8{uDoF9sK*3`h6h{5p^&nG^8?ge3X>P4B35)CrS;nV;{t z74P8Elov?^7!YS@hiYNmUKuf6by*P(IvI#_ZYmk**9B)sCKuRL!U{#LF$Nw&%2yauy#)St zP5IZjqAe`V{67;T>Y1MREzHM_c^9L+`nWI^mh`!5eqJdbvIRau+6Ge(^fB#LM5?|| z38T%>RyNc@u58N&QhBFQ<%*EvOlNa>8jXaCi_)aUZNIC$uLMb1wY|-#dy>r>9|^@6 zDS4#bbm6zwj`v7)Vt9x0CJijV(Kzsk^=6;bs2`{ooubW zxS<~=>zM9*QSa=fObphH-Nf;cJIyO^6o=Ik|6iUb?V}RC{l5uq<-& zeZOu2ZO6Rah_#ABm^DlYi%$>Vs0W4#C*GkYW%c8ePNmSSr9^~*Z-cgBo4ni4)D+6W zO66*J$&)@GhIiD$)|s*)i_U?t>Y&t;g)?#Z!KF$A2k{5E8qcY;&pfAXVkR3|Yft~* z#s;}qbiAkzs5{b^Fw0V>7?61O*vRI>N4Hq|@w^J-3>0UH`4B6Q$Ip74{&$7eN+au& z33G~wrqDs*z(#1c(u9g`l>(sBmcG+j7Mf;Jd8=KhhA~3nPK$G_@v+5PVvvVU{Tid( zJ5V+|Up=_`ZnxR@pj!d-%zJ(Q~syNU5)V$Ga?FQ@1DVy6U8|0i#^xDJ$qEcQ(PC-Wj1QY z9tBFLYPR)>D5iIXv$G7N&O6A(SN-`Ze^rrAMh{jqetl?IJv4Z%CKTZ#N9Qm>BFC@@ zhXlHa`r>vxw5IPTDm?Z?Z1!0`HT&3T7^dlXDRrDAjTF~Z>Ifk@f*RGa?t@G zi-l`3&fFuG<(lBisc=b$o^P=FYppzclA=79rP7!y#Qr*$IVJ4tS?))xYNmjCUircl zR#?wa41Mp&3l8Sg*SwFSynmkgGZ00kj+ff{BBYAGrJTywN$ntwo?52WF&2+!S^=sR z#&O%!4}vR6^IX=N!1!DnsP4`D_(i@z8Y;T{fw|)I{2D2Fi@l8mb(km|lKuUuXeBH5 zu&-}b%~*wcjhlSjlY{1j*gqW6%-Iq3Mo&vq&r~#RNnxphzgDOdu&u(bn_-&k4c1`(?Xx5Xmwdj(n(|>4bMkva7IX-0EGxAs`TXJNH>t5fdWfZgIDJ*s@Qd{Tviz&` zIJ>g5?B;{zKytlcsM>5!Qi2B6fR?fSp(0l$tqT@dApNCNh4;eDXPizm^-0`RKgk=u zp6*ppG0C-0gCme3{vB@(OBO5=I8%11UrfC*%|iz^b) zag~A)LZfy(VNzS;ipHa3%4!I*IRPV3vlNxYTX2k!HYGu3koXS9XvL@X!h#7Y(= zEKp$2%9^9+j&t>vi)I3#P93%(Mw`eIQ{ng$8q${2JAi4|S@a`ES@ zMcmWJMN-H*7Q+TKo`+Ju zkyI^6t32mu)su&wJg%;JHDOsn`2DoXBNUD(AX#PPMp{~~p6z|*fOcu3>2U}I0BSz8 zuQ9Rv{A>GIiX`htJwls0a+TJD@;}$f2lR9-FE>Z>NOT8X9{?@B03P{fp$ncM8U7F4 znYx);-oH;O*#BaLfLa_NRj+)1Ae|)~PIoe>xozM5(wHSZ`B3CV zGGb3Q!Msy+>q8N*BgBm3uOmuK>^H>O3oZv#|LLpc=bDZbA(1t>=J1ox>BQ=Ysg zeMbv3PW;c$4O5$+1?=m}eTz(~^O#s{yRxV(S^sy*kj!cD_rR#L3=;eL^xYdDojmV0 zBTtx5x0{XLx%|71zTL>~PIodAAs^D{W|)fu4oGMC89vnf*JeIM!_ws4EZS_98?MQ* z!T`5wRiTEHiGY^Cl9~#~==$LorG}S50wqY>*{D;eySOT&I;`#)b=f^`{JgRCU5X%7 zi136&NDhr?;SGOC1@lS%nos9SYfumTyWR!AIY&>qp>;K|E_xNcF>!R3j{ZzYue0wM zKPhF7ZNCWeZrwXxsi_sZ<7o?(aD-p{^}Z81?m2N582?_FV83lahDtmMV9TAXaDUY~ zMMe6DnA=V~rP-u;Y#Zk0)Mo0{)5@(t31Wpl2S0g(sfQ8EdqK?4iHN<=VS8?(1nCyX z?810riKDFdZO2_jk7n8z)j^;`>ZtM)ir~S-87mrDp3F$XKFWVf$_57!F09>E7r{#f zFBKX-DFREn%{6Mzt#JhXh5vyb-Uh+FW?Q;uUr^$+?cvh{d&N&?&QA+$#>;!=zv4y+ z1Ow`Q*y0jObHMz;k~nVz1R3R&X@H#ZX&X^O`56C;dhhuF7O*8%STtF2Qky253n_TC zg;=xfMMQl9#L^Et72?%b&9RLFV!!FeZ6d>}Q8@~E0^n!`VQ@tRVf~amkkWbzIcC!K-FoDS|hORvhlvSPdw;`{psH57bdOIH9LhAJs(P z4puBvjh$BaLKWaq74aD4;mqzD%~%j(tf^1K1!OisqlC{5X?vw97P%3sL;JMz5D4c( zB4)jQ2TQ&$gSXz-q&F|Oy*`Uvr6|P(Nn>w&8bi1y$g%)IVwfH8D*SEy=w1v}RT~R? zqP(A8NNSD^@0PT#dSV8%^?3XGtcBw>CGT&!_xcVJCAr&nFnGy0w>=k++7V8S@cf7e z1@gXrib`21!*3?K%+nvLqOn9##L2Nu%OFqbT}|H4q`i1^Ym2x4A+vyaB!3kTR>w9%woUr(+wZtfSkh$gyJk+$SpNSi38HgsC0<2LGD4K+5t}d zj|Ie~it(bAz@!c1d|1tWusQ)GFdXlEm;P>4Zjfbbj_lAm&L-3b*X-V0c+^>qZcRcp zBI4mDrqVuNZEu`cb_+|yZDyhVfSF3^lxOr;KdlUhF!dSNtXafr@*bzm?R>XO-|83{ zcBc2nxeVVn&XEbX3MSprKHP&3S_&%r(JYF008t71ZKzIG4J?>l*!WZG+g8B3%4Qh& zc%~@U`9zPCWz{xp34mPu!vHZg+8a*`;6_xXe*qX}?#_!3~O$|tC=oxLyOS;Rg4gp;&#;`fWCh8FV zjahz5yJQB}>`BShljN!;Ve5f~Fsd>(GX5@aI)h6VqyQ%$DT`oQ2t0TQGt869+$B9L z^QtdH*o#l((7c&$BPsi)#0jpqtv8njUGm}UE{inRW(uvt4BnPc5il2`2ECM!{#&am z+XP35S4pDtAXel8c|?&+Mmga}8bSD>y_Ka;!0EDbb3~C5G3fIVWt*8kVB%* zeWlv_?~u%c8R_uRSH#Tza$<$UBH{d1g`}P3PwgYoUe$AIjZ4Rv?Wy9EH|41BYkxeE zf??a~BwIh&WJu-Ot)7A8RDhiZ+bc&?II4$cBUm`t;w$=wF7BXJPOd&J9I7oXHE*at z7z`A{MP}dEgb79*-rS+Ml$tJJ=nwT#`KrN+PX&WjDS6>Y6{_lLgwNZGuNuBGw8)m3 zipb{`93ww&qNaF2M?yzpEdSL+?|=54(TX{@|Fjk$FaGvr$b~IfQK%0?AMW-A4BYS- zoM8jPc;%%Ve$t70$i^n{MDzg^?Ixo~AIx*ArY85=9?gL!M1XYg=KwR@M|U*onXI|1 zZ7JPWg)^n71%H^!_eXma{~xG4y8BdF*_q8ezMj!{c!I+)wXk>1_H2D+9{*%}#w|wloC8DGD~I;^5bdhiYhq@X zFFvkx*qa@QT_lNt5oXJIe<|KKsk|nnb3y0Z;cx_NP5SZ;W#zTkPkVii_-bZkRVkRE zx5hbP&X@FEBbFo~T@3H3&FXmii>*g{`2AD4xiA#0EuDkmJx3O6R3|%b&2>5svdnzR zL)GcWX8VVp6XHT}X7&2l_gLMzOAPOyR-`$!vpeR>r2-Vx7V5rCfuwpgROhq&2nCAy zvt*OcdFT24%F=(g_4(0S-Ikuh4ngL8=)bUk_AW8N(50S#c#xiXfMHwB8>j%Ov)wSe zV2l1$m)@8YabirD%h8q-L0ztKb1p^zZ!*5G3!ODzNShczbUJZUNU?{_C4LKg2OsN;vVs}ExX;zY`%v({?fDU2{lP|H@|EOB7NEOsw+*@YPX`I`8OTMRdd zEqkAQ6I(WZ-haEM?X9I93`HF=aXWWG8#2cG+cAe5-)}yz}bbl)r%9a+(7z= z(IyDk{UlO%LE(KD+N|VKNW2uM+OR~K4d|_x!JJ;{UhvPA5|^A2t{ce1X2#sGV-DU> zfkgM6+MSxM>|8LDrByK&Kg;X={#9o=j{g4MoW?b&2R2C*5K$cxXSFWq`QG^qd+_(~ zsbS>VA>(FZPU3K*%tjlec3khNhC)xae@@80#O%Ajx1je&YT;5x{YNb_(ttnTq|W=! zyp86s&E06mn@N3?@f+`<@x&l#o?t&Pa08U8yxNeX81|O>RZY4^cPHuWj8mBrExl6T z85$&h7%?(S?K-KlbQPf$K^Ei*pd)dg=DowK4SNFcX3q|3ub-=N4f6pQW{+PqGnt)0 z6EWp9oU+RyE)({Z#bQIxFWIObJ%<1j?Q0A(Ge=s}2=O}Fehge3`%mhv{T{_S8xBfd zhxwU)sOHw|-+!{TBVR6+$t-p~+SVIhXY1V^&+8*>n6qi7x$^pYzIXH#KcsrQ+nNZU zVZyyZ|naOwq8{w^)dfD;d+9m<1h|#w|Z!2h?jz6Hu+Lzh)r32Rbw0g7#yyS z)W8p%XoW`)K?zdLOOR z9m{gr2A4Mx<3HW1xda*#WA;zL(E1ej+v4Wj0#I?q)lIwi5Ss`6;DvHR3j07NnT{TK zNI;>Dd7Q>e<64W4v+-A^rW8xvHFCC_g0`lVFinBeInkU$8HI!QchZOZTQP3Y6@_tl zo~FfS?_Xh<65A;}P}At365!Id+G;gcb8RTaHtd z@8RU+Fvz>1%b4n?die%M42@yVt+ITOnW_+*)Q-)0KxFVq`cuwQ*>%!PrhODPxY+Kk z!V^2c?_aL20wD`YvL1Kb4N`vAFRIx75QiDSxz-r14QKku?P$hSJA-^Y;WLGPj@d3dJjO^cAiDKg4WO*- zb^DU8yAqxe%sE`L-}c+t^L3$uKP3 ze*GOfZ%9FkTA>b>iR1Bd5DvRD)j}dL5o*3gr&qL)b$5QM=jVeIentHVS6lbBEyNgk z@H$CiL+)E1M^!Z1))+(ugNA4u&(A{k5XQjWK9_u~#h!9lQK?>hOtM(oL&31c3Q(9X zNYJzKA^pUH^D7stYEpLY#&Jw@j)uMpBzRh<%cV0K5q+5yUQYaLDxb2PWUQ&Q8KW}e z(i5TQenMeXmzlE!E)=_w$=qongzrBvzoTYTLPdeqW@DE z` z;6=sXp{oa}O+UYc-$9UunFc}TE5v!_SvQ&6{<{DZv!d{6e#Pa_qG?TDsgi(~xJfyt z<}WF)K2T<)OcEIwqQoYQ+z0g-Bu#hoVQiNvL2gQ{oc)Gf)bH7QybZi;a*4rGY7MzF zZ6i4*UMd1qEsgz*i{3e>1=BfmP09j7P?8;a6&(%u@tJjFN9oeIbM@G0{E%wxQ1NCS zvW$saBkSlrI=Wf2)n~A`SlkQX!ZiD`y!sC5@vB#h#wQ^y%E#;X66+t*wcJ)>Go00_ zYtAzJVxbc``5g7S8DRR!a3mWJcXL9sH9MqbB2R`jsP_ZbwJEwpj^Yhx!S9W;;HXmB z(A_IsFCLxW-e+MYz!Q`kPWq0$x?$&Z5B-1Kj?Tzu(;RIBq28Vj>aRBc@}TFF#A7JS zZj(6~aL26jv>A0+;4%YtsyrkRK-)S~&sCOrj#Xthg-wFST#T^%;NRg=qoj?s34qY^ z-pb4&K*Rh`ByiybzL5gYKLw5kdktojGsuqBSlfW*uVG%OEqcHnWyUr3bm`uLvh2Cn zKIpDSLNm#(Ylj@ko9q#dap+D65uF7n!nTUX?efZ~bFIGzqrBqId< zm4bTKze;__yKLsbuAV1^{XVON)y1}CvJ?t4XB~LO>{wk!<>8FECxh?wajDDr8J#2^ zS?{R_jc@6tIT~xA00TVIfJnWt*3hvFbRv8qX<@zBk-z#)qWN7}l`D;3;156(2>0_Z z!onwof$|@;_Gfl>rWhHngmL>}R>*U+X7Oc}sDYr{b*;wpeFY49}#UF?)zUmC;cWP4v zrqZb{p)JF~t>z4K-Yv&5q7QK9#Wqc!xo^LyWbSp#sZN)@#Y2yBSrQWs#3ysDQ0#4I zKBwBg06zn$WV#l}s(tmu$vnVimI~%@SeR~U+EAQIY)SXvZ2P9A7Ps6`MfWNid$U8b zPYj|uyGr&}Q1dYvs_y%yCLgCqo_z_QtQ4nO%i#wmQIAe0;`MoMAiL2%4RIzik8~mz zJM*+Y5Ocy30Fd{m*ZZIBIm@y|b+)p`q44xI{h|~<0Js)N0D^`%Z$5zHxg>0s%ihm& z4pp)TKVDv069BCdbXBE-N@ag$h0{WVgUqaNlEtQrtCCH3 ze-HeyQ`_>iUHYX#tvzjY?gB-qY89KcAM}20z-J9OqPmmRC{&dTV+y|fn(UNkQ6kgss&eSsf%TWb-zC-H957{uRo&Hytir0Huzd+@5apGpo8QDBxBX-QW6`s z|HUB)%^WS=!w>Scp*=Lz{!i*FKLRjf2hdjh9pxFe#?_dAt>O#u3JLr(M8!mgisX&% zWO#21@*|>{&agdylitxf^Xubz9=6$3omfuF!w|{kh}I{l?nJgYzdaXB(b-FSHU@uJ z$4H|8Bb5ETVOsI*mXmm*lZcxWwGvMsx6#dQWLDL!1*r4pd$us;P)(YodQcBRiIV;p zi-wunWb49H*{{SIk~e@orxy=R)J&Cff8922z?@HAt2SMee4@AjF)t7JV({}-%xT^J z`(`Gq8yjoqWqHb;E){VWU9Pir8ZQZP<6t;U3d@0tCZd658Zv2~2!djVn&}wZ&ZZP? z1q8!_uCJ^a@Rf-Ruf|-*On!f_F6=+fB-`G=XZ>@nAptXXxI9Jufgqj^93jb5~gKHg6zS|DJzXGMQEris#dH3`gnvXkusH?Yp zTJp#+Q@O4)-lW+XpAlNN&SXSgFC_#g7t{o95v8td(90btK9J472Os)87&qjOss|;p zP|AnyN)%@fGSs7Wfs++f*1lYp)l>>fW^`pOxrlZwq%d2~<`s8TDF=W5?D>d~8q`V0 zAt!Sd2)d-qK+~+~bniJN&I7(lrdu!GJ}(^egy=)g*w#UrHt%e}R2hOucG}f1SAmuy3XHl)tG*@ide3Bn)mCL3XKDVmm$lslk`Ca_p+71tA%jUExW6qfgoK|kP|vCa z%76=iCKwH_y7gbt!{lG@+_1rHfn-f-Ug|AyllWoN{j-Zv|jQmrfe&;4jdZUDCVt_X@J{U=@H#cb#B@5+-;f<<`q2f-Uc1 zaS?-4h6{zq7}c65F`-~m1jcE6w$MZW%)-*Wi=qfum>lit_(^WsaxtXOZ@l#ob*ExI z;0v76?@Q-vO?O;X{H`4}{P=uRpPj2HEH{pLRI4hwcy}T8-9qTk1D8f7%n{|c0`DG{ zo{Sj)HB~TG{vZ^sY=jU;uaNA?jjxbAm$a7T6lA;ZzaFfB_Cq5Q_A5ra1|95%jp>0+KmY-CuBd7Dq?onMM&VpB+Egc1*NY5ND5A+oo-_TDj9ZK1aWD|Trk(B^HKDe=of`Q zTPcTVzC*IYvsvK+Q|I(kosr$evZ}Wuj(`02GAo>-f7!f>ywY=yaFF-CsXIwPWm_@e z=f$E7zxvF>X9Z6(a+scY?L@QmQ^vnNysnwPTj!S8J$AIfK9|Q}ne`Wu+mPASy2iPx zo}=#U(78Z65zixaOlH|mNfK@aKxiMm0#T$09oc|uc)Z?b`xQ9lSBu(lU&zUnYI z*S?d}!4xC{_VanQfz<+-%G<7aKQVA&7keXQ2nh+VZmv=H`K@q$aC7QF(0d}h+OG|1 zG1B@PW8AiLd|wkSnwDi@RPSgF^d-9$+D9FZU1P}9%p@m|-P4Wc3utcoRe1XrTEMIG z%-<53P6jVOEXsoKKFU*0wN)N;%`)L|>fGIIN_2CCKZlw(#1FhHaf0WXqGA|8T+|($ zvRfuEqkuf;bM3B^Qx&AzzvnP*Nb{mF&23a@j%`_z;2&ow&@^_UwAK?5>!5$$*7 z>=&QQtxQM{isTx_3Y>Y2oaA1!br-0f!EIEtu+uSH`=nhgZ?0&@z3LRp-#ZR<*Y$f9 zei(TY8J+8I*|!;PW!dF3jdK8!u{F`TFJN_1o&z1Mv#1xC#K6hTf|XLXlCVJ1V^d zLFo`$2qci8^bS%&2%$<15PE=+a2E6J{p~4^v;UpzI_LXGfR(k@`#$}>pXZifPQE`} zc?YFwdV&(OX7S1;4QAgun&=Y1x5-ooV`M?^RwuMbWww{^`p(f%4cdHA8rW?)9qE)J z_gZSP#cysD;jDKfYL25>YP9AhtM7_GKxYP5(hjyTfg1E43x4sOUC8h=R~im!Hck&K zyg4%@s=b|BLO#&n=J2QQ_!%GbT`Ugq&bJRubE_vfn9nx7ilrh}K?E0Kd(i%#50$uE zyFRYyAdJ&*Je8m3MO&Q8IfP0!Ym`&wJ$9KDvfdT%^j|I2a*-o{5lSXd2>?txmkUgk z&QP8AHh}V*LJah?n;z1QUasWqW}efa>^UOkQZd}KFXA3#dV?7)t65z-CpGd@V!^;W z&#RSY@gut_1=ZHUPGPENWW*vyC-dl%TICG6{u77E@^B{UAlW@uRy(8jrPqTW+jFIOR?iY# zN%X1%H7?oynr2>&{gD#-7S^)MG)wlf?U+A0wOBKiveLzyt#XE|357q4b~b$OHKV%T zGJ5}5md{Lv8v99BYuwIM{ZGc%QqykD}t1+hcq{Ty9;DPsdD+s)N?Hmzm)n zLv%_8OH7NI(rzep+V6D<)pr64K`WP&g9Fn_kTG;nRj&p5ecj33mW6`_^1GS1n^?lQ zaVBKSU~_k5e#{1ZQMBsAwC2+;TP!#H%gL?EF;u2Vv^`j%dT~<}6Xir6@@{>1XPS{Y zM1uKgIZw`+^L%W^4-?9zTjS!3R^tpLMV3{$*wgpW;FsNtmC@(@F4Y^*I-cvDdsm$J zY9UkRwjVgvDP4tDV!}G%n}?DZszK$YKY3i$o-|;8Z(xxx$`p>$b0Z46pSEKxw(Hsd zVb*P{5!GW~NV-1z{5$5YzPm|1W;L~(NwUj+`wcV*f3d5E48Z^?x-@;uz@*kI!5KE1 z-=JO{%}^}o9wI^&vK&e?Z7e#;@}i|nU->#da8J6+N_96SjlM0zcezq_%>2VkZKXr; zRD{8Ncg~{6*r0w2*0Pifa%)r|{JRWwm{dXaBIZr;!Pcgz?yzsg?6VZgD)sfN($d}r zdGevC>2NSHx&6IotXgcF$23A_LKBtyblSSQo+a3M)?*}8e_kIqS$CSDxe#7fI+`n> z!eYF;v2RAMJ^2`Liv}$1>49^~-d=~%F}**XNrJ;2&#b>J&UX&699-2`Ii_z}P2wc9 z^s4Wqc9Rbt1DZS@M?|9p*`Cxd)D7e;wTx|+JvfN#7^ay5E>wn{J>#&9ut;MK%;=K~ z>{FrGL-R=9tQWunu%M5>s+#m;elckY=J}hgHUQ#e@OD>mp5L=O8MT{p^MvT&bKG*% z!$F&`yLF^{nv+CP3A)@5z^44{Jzvja z|HwD8L@#GwwI8FnzdPW7`iYMZDmipHI4FeTzY{9-YIL#H&GJoo!h`Et^-Nk+@xHj| zl28vDU5j1=|gj=aQtRUp#WyY&R?2C8f zW4EKTjTlcnKaIvCv|WSTKSuOdxM-$>MNs{Qso zCuO$+Fk2m58?ogDt8P+m`J-7CLy2i>Gw+t5$&veA-l9&1=Kc12!(uE=-JM&pERW|A zuEv<^g4H`omM}+O#!{J}t>kyM+v7eYg=!&+iW_7P2RD)_ECO`jVg8avzR*rnXqq(S z^SXVPk%S$JlTFguxpjBvT6%Ef36aBnA^OxQDBo&T*zJrYjfcAC5(P4z+zJho5jXqx zI0H6`kwwWscqN7#_lSnnDBoGzaW$Ja<4=g(XcTI{h2lfZfcGIZw;|I6v zgAdy0=i2iQS-Y18AQT5V;yr2X=D-R+SLc?=(g4~>e+X<8|6=nCM-yWR0Y2BJR)pB)bvk`X<+sb1)@bV;PfxZ_qV{!AiD7t5N z{KkR!m!|~^Ff;flfI94_ejvPe$rwcpRR3&y0+w!9H(tfXFFcqW%gN%2w+u(w3lfYFIrBPVO zj^hKQrnKJ7dBFj7rF@|mZx=zX9D^D1zCQTIB`xD)3YKy5OXrkK_$HY*CbCd4nNm@^-MEE;kXKuX27GUpErMpTx~zUgaKzEVc`hk@ zZcv?lUu;fsYGbdcQD&`RqgjoFLYT3aKv=*|mz z=Q>Y*3|t;mU^*4CZCI$^KK6t*J69LI+43gHW;vv*O4L4?8%bX3!Cg+fQ(T$}eW?S> zs9&DHMwoqxfp!sDZqF6g+l@GQZ%v&{btAaAE)5mgVh|}K{zS@swl{=IBR`7L?zFcC zw(?|(igal1`W@}tG=@rQX#mU~q(&aq7`1dcfsIn^#hu3VX;Ke;t6&gIAMHyw7dXA! z!lf8t*~Q zwpfKagT=?O5DpJI#}pG7ORdTRdmZ}g3b^!kKoCXRT^5umKl5oj6jozcFO75LB=gqZ zqg#}ZLa{9m-N%ay@V3N8p%NKa$5wu&_Ly4A_sAYZqq`@lm!$YEJ0eb!uT(TI<4F}) z2obAINnIfy_)t4qDwum$?9sJ{ChcX_^1m6loovX$>B@g zY0t}5hvWVJ^WHO-zLcDdWr}Os@QrR!zvYeA%s}ZS6e@uxV<_A7fy$~B_55z1y<2qL z%8d99E2SCo#XR<*DFAtS5Y4|#&R_#&JTGz#!nrU;8IS|)Gi+32Fas+;he`5kY*GhV zG7?;nADA}Y390b}hB9BwPjg}%I3Haxs7mxOcr%qya~ls~%x)RGJ$T}UsGriDkn;AR zL@&SDkh$VQ-8TJd$S3Z+l-hx_pd70bLp}eNeZ5KeM0@Kye8q0`;F;3k$GicZpw(8g zV>Q~6c1G8WEz9wf3p#&^IJfessd_HMh(0 z+~ylh0?XwD5l`Y0PuYaU^&$hiZC|6al17$Fqa``Kn~hgeCv;$q1T_$e;X<%QwFSHE zBbf#!%yo~qp$PhUbv1$lX-r0LL(J1@TNpR2#yMvO7Z_=q>I`A@ zaj&~33X#D3;t=Zg65W2jtH*uz1hvKP+6qL31tD&%2WJ&801TpYt18zE+a3HUZXoj( z^eSX2PTC}?X+LOB*xu3+kksG_#(Ke*25>EIiS0_qY6$Vcs;DOyG3}Ltq*wi!7u3+A zr|UTY&d*IxL9#zP;C0QFz%eG$INOkv2`HsYug|y1H!1Co(PwZfNg_cwpIGN;>pk?I z7(Y|m4xqaEj<@<9aax-(lkfLBZ)q)pO)22E)N6b zkGysEZ@z0a+G2Nd8RddwKZ5Cd0d%S2t0^k4TVlIF$g}FE@TETbYQ62fH?gy%8w*ln zMk6P%pzKPRWwH@t3EP4pGI5IjQSKu&&s2>hq)bw8j$%J}Y+i2-GPRDkK#zEfkAiRV z50_9nvLI`#y3%FCLWQEQBc}(8mYZ}N;o?l+@>f5g8s@?n5F96F^!BB%nN5hQQ#kFO zU(Xuqwu{IWk|)<>IEVARx-aY$-Xaq`{Z@Vh{44^7^^hx%dpb@+)l#duG?da11y*-$ zbd_hCY*~=F;cZo0o9E4B9QJwVfzK0H(ho12vQNuG=ibmVi}O2;R?qM3JG z=2noi_x-eWLuW^ePN))|x%PLhaccQt*!y%R?}yblWv^@c*2|3jobeSrr26NbxrqTG* zMA)8m+sD%i`tM0<6qL_wj9$D1OVe7i zbqm}cYrjp06JlFT&vphw$kxD=YqFhde>o^@LS?F26z;OK*}=bv1ROhzaZA@G|@w1emszp4uA_h!f9v)P6=H$}a>l$;Mei6Xhw z4~U!FMdQF=nPhM$-76(V)Aih~)A!kOOJ7;|Z!byB!Dl;5+D!>(+;8M`HNFyIc6pvE zKEih2x#q!m{1D#Zt}ZE|{eY1fFQF5$5rS{Ro0+HqsR}rd#FAXK(@-{4M1p8pletr}`t0z)I|W}e*OKd*qxnct_PpDCUP8jb zj5c<})h9UZvD9(1nNAV&-8A&cDkpqUeQmKxWsC21YF<5AGT*Y7u|a}%koe3w2o@ga zZBo!7fC=e}T^(E~vY7cq+RQt7+{c17e20FS57TjX-e-GhC>Pvm_YB4J5fwGVIjonw zsI=^Rn~d9AMHi?<=jN(zcLhEY=u%{=EG;dN8rG%M$1dCdWYS;z34v*_APJ&a7UjCO zI5iCOU`Zw9AWKU}?1lMYXf(GxNWnZmKjPKRMtP#A~NeZ4AyyFMVIn)H5U+tm(1ANJS%#vp7V`%Oh)tWcZrU~lGa zNf94Ubx-qXlsn_P(@+=PUr(3t`azk_{KGJ78W{bUB(FqR5p;s8)LZtJH=6EVDHnh0 zPb$CLqS@Ez^sHX#vDdhS+c!dAWc+%m0wh8vP45K%*L#Om@MpOk;=Sim;r3mU+6vrU z$LWKt@g1^g%rZCojia{RzjUl*Jpt+=(etgt|FOzobF0xUcz1J>B`@w_EHu+;dA)`` z0D$HG=U@K)Q^N^C9na$mB4?R@`|7ZCHXS3B!OD|O-{p}7pgi6ts#EYtWx#uWr5kFi zI_fznG;0}l>)%)Z;Tu1{0A^6%^2)xYarBU3W9N1$dX)K=Ft0bK*nZCP&WoGtr@BjV z;XOx4ILkxZ432Ea;xCS!P%Z$7(nO>J{r)rl08s7(AtUK-7o&kM*X+-|_^xl;r|Z8V z-{c3HiRYUf=dOIQy!879tFR;1yX1cn&wlZp|J})?bws`yInZ_IiyQq1b0r3By4Zh1 zzL|bRzR8-G68@JSh`&gI4Il!x{|))(tt0YHel3ZU|4hEw1T0hQe?z|Mbws|Ym2P+T z>+~Q;vikzq*tq|OeDm`WJ8Zga_vQbB9Tu2kaJlloVTb(=SovF?skiR`Rf+zjSXYjC z@a+DF8u|}1{^xVQm#o0bdrwPxzxfM^t$KC!k-jLBC(&|JManKTcEpH!AEu zDD5A1S2;Kgp~sh;;BS!>a?JajfIY)dl8E;W3lqr9&q#Av_?d!p0^=`x_*#FX2mib8 zDP{=+Qgy^m{8Qxsj{MryZ-x^o$J>9VE(Xdmzt9i+{i6Kg?*IP78-75XXBS=i4gXo3 znE@%S4870(#V`83Oe+VRw~CqG@TqnX`r1a7Wikr-PWUR2!ZHGE3Nl5 zi*Svz+A1;~VDpY2~#~!Bs73Kb?ry=Qu)>vyKmFE4QCHsq$|D%0s z`#($ef3Ef)WaR(ft~Sfd<<<4y!cg<`$zt{;dv>d)61F{@{v(0#Pvs`#68E^%48*xp z4P(a9&kk7r98lc}s`UQ%EP&rxSLq{ud9EhM&sUBD+%878b397mgfr*n8K1Kvw>D`- ziM%x+_5mc9<)ui&x-U<*JGxa?9n)G-NPvQO6&LVP;GJ^eH|63l5){&nEq0$6NOo$o z3di05+qu)g_2$g_yRC^{&~GnE(&zD56vDLHA^pQBDI~$5$S`fF$Vg3o)FHn9l9Epn1T-t(>SFHBMi`sA9?O#m>zYlwW2x#4XRW}{hOmsmt;IGYO)hGtr`iE zC&^3TEYzbpM8!%>rDeEB-R(>z-8=TKmsjf4a9_WF+vy<6;VNSU)vxH074p+eUa=oY zF9P4pK`Aku^y`oN+S{J~?hT^G+VK;R6fn|+p2tYSF?7GG`82h&V(9HLu|UhdCT@Qv z2VfrJq60$xWTtK0?PB*W#dKrz4)4)i<7741MSINQRGx>fBr$)AZ|&qq8!26)-~Cn*U#T^qfQS7$nF<}H{eax*L(D+6J{enDe$1+P9H(jm`k z4^pS*My+M?PMng^j&pZM7X4KWf-8K8gMW zt*Y*?#^m?h>>TNx|00C)UkcL^0fgz|9+?2-(q*ozo5X|8nyZtseziw}&oY+Su|4MY z+1s9rJ;ZPz-H@(GX?|}1UvBNc2scj}nEI&36Eb@GqGXewbG>+ahEL$;*_D_>_2sDj zJXnYrtp_tAY+NI-Se??7zB7Azvok%wvnXVJ7N@$c_t0>1)@VXJ9L~do)B|Vp+Et=^ zm;ImKn(jLuBi;e+my)yUld@Z`;W!YsP=BDD(tu%;VKycjM0}WD*RuglQlWo0@9?)x znZ29kwdXMRe4=ArwBCB3B9Cb}&P^e6Fq^9h)}+$#f&156pI0VZ#4~ir@+vOH)hMI`r%-bvIcLQT3Y*-12$Z#+eisiQ!ieV- zx^B*8MrcgX0{OXda-~0y7juM_p&fi>N|fDE(-QFmv7ll({Um^^M75mGR47AKH2GTK z_5SL)=#4i$bUaBe3;^c#r~7U6=X%W!=H2^%Yu8g0>T?+?9J2eoAU`v#*B~u+znFd0 zZFuzd4xG?k&JSY+lgBCW*6wmO9v4A_LN+5clnlVGBZ&hkv>mLs!shCIL1%^1_PpeC zJyu2$pIlRYQcxp2hhD{m#n;-h;+#n)L|qEdu%P$$=V$K-&n1lD7Ot56z}!7wI>Y9C z1z3cHfTE>=I@HpX)Ac;|Z(!TBcoBJ&Aq7R`Ehxj{c&oz|C~tHYo#@c5;<-wv@kH-)=n`UUy06`OYIq ztguC0ldGQhOsr=3iS4niWy`r##{qj|Guxx`rGC)`(wwnSzS$OFtcJTP=e-s`DW%(c zLqvQO_1J5jK?#Arq<3`}LQ*7f#TFU3Ye56Kv6@4I2HAoknF_7NAoOrOaN-n6jRANP zKY}<+4Qn5PDU0r1q^Ck&gF=Oe ztd$O@oeM5Bcf`>De8Oft*x%;U?0klA;jMl-qvB*G`7Nxww#U4#R-{U6P@`{B zil!L?Xq(!oe>yZ1?)g^Uwf}?p?a8h;Bv=1r-|y= zTBo(pH?Xu6Y1tsWLF4N<4hy&iNaAx;v2o#FME zTLCj{uMN_*qQL)gD}Vm!jb_-vuelZYo-@@3XmYi;k4vv2#Y-|n_U&mi#)nY6ZxZ*x zV+;;JL0OK~y^d27Uxil4KzGK8>5#eWbLn(`G`7&7h|jE~A)wnu+RpeD0ImpfPOJGFShqIAQ;5x5wEx|ys7mxA@Pt-edIm7 z7ycUPq%*%`%%~)a+A`gohjmH{l?p`HOqg)+17m5M7cHSPEgps!oNe}|T4CL~{OJke z_lvj5HPk!az>2Gylw#rtC?Bu5CBA!iR$W;;A!O|+hET{#|A22!T1{NYVK;JO?}1U0xD~S?r}NGm z)_6e&1Va)Ki^;lW$RV6t>w7=PqWBDvLzrc}Glxv={Hj7sZ=qo#>6fz2XM@i}!gQ!t32Jx1~4CF$eA(WqL9b7$1V4bJ`-FD7;5w*~~q| zwsg5bBYF4NJ=|686rj_1-UtCQ7nQUJ-INW4L&?bA)!e((>QchvSa@+(*3)h#Ri*ug z6`)@!x&O8HA-{-2V7rQaT92RkVy$$e2`-c+D?~P+@XW53)Y0J@A!&!&HlwXZB?h#v z&ei0wNDsMk5IQiy8;6-&b|RFzn%o9_m$bN=Yk(YH3)oBC$^9hn-qk_sN904-hx4lxsW&k76Pt zcsnHw<#)!tjgoXrN$r-ba&Qhy#X&zTRL>lTM~-=hR2e}rQZ3$P~8cRE16jwdv;V8FUrFv959Dw&F#%H<~2ofIG(}7Nmi(8WO!qRQ)t!U|Im6W@BrmMD{0X4tnbS#GOqk9Igci+1C3OO=kq8v4Ysw^5oiI$OzN z2|Qq(o_VY07TwHDyn8xU;B&49CvBYV)6Dq!Giv>##BUmAomv zThxn)LZ$D7J+8-2#7z~KM;gevj(^C&dBm(Cm+h&O80gr>EBeGDMZ_OnhYk6_lH{P` z>&Wfr3Hr1G4*9cT{apb~#|Ov|Rgj&diBvM$o=qUQ<-{-A@_PB%X)taNqo)_ON^_4zmXa=TgNZYp7PttF9oh71EH8}?xaSx#M&|`>)b81WkOKqQ{b82@FhEMNbchG$9AABI!2|Sg*L9Cpi6@yq z@ss4K63=0G7(9L)AP$YGXhG0NrOHdgX*-2_A(7yz?1^lXLgxepUh8UvxNn|Dkd!I^ zK;+un@HlB96p(N3Nz=OR?jKV3k~X_p>@Nqe9rH{Xn5rPCv2+7n&0b)a2w>lQ@7O3= znoQp^iNNC{NrG^{rP%vY}!><%U8k(|ywR)2pDt=AMaGgNb^13AE>eyr;RvucITJ`mQB7H5S3Kp_RP{P8@Gq63bEPzIF{F& zsx=lA=88pWXc-MQ2aYsXTN<$_9tFcduSbToLLxq6)qHhWLb1pk*RapTMz(M5+#%XT z#3ty$fLwT9r0&9b_y<-h$SO{0Ux)HCyaL6%8E}@;F8Q@eu+C*b;G6q9r>i4)E3nLL zMIH7q$So$eqN1rh7&q^VBUTZjI#9sJ7PNV+*Q0J)pTyyZVW_pfDdP!YjEt5uTqgOQ zo+y(Xdl%L@xV<@A71zsVz%h1`yDZ+)<$|y|ng>1&_d_hV5!RvO?W{8{Me)sZX z3s>af5Bzpjd+yj9apF#Zj$M>!i}NIPviiAL(+V8AEMv9n?cqGRcftdM*(s=u6;o4? z2DRJ`OSj9d^&9DSOLPM@PQf;XbTN`gNJxnhacs>c+3x8*%QJ?2<2 zXrRH=A1Vb#FE<7P>ezkPqg*>rv^)B1?6&c;q7k=H9TqU#-;TmC%C}o)Q7&BH0qww z7_9Vq3DU9XLa^s3?k2%#yVw*ls$XzEgy2x`$h^4*MYarW&h%DxOWjpd*?a|_Qf1cD z_+`FAH)qTCs&kMnufVYE5UFTEBDL@*5k3FO?ldXiBFiV^mvV#a1LxKcss4jnH*{2M z1JC!tI=Zh$BsZOQL}pz)OaqddMZyhwI(v?s^w%8inR}rNcP`!fwPWM<#u~;B06^AW za9kdhpk`PBJ~kS9>vI;A_IMQ6Enn?>Or_-AWF~_f=Sf7CI0?-=CR=X`PdOt{DD?U9 z?7p*z7-x}`px>A&SX0_*qyem>y2*U&ZB{i(Q$$18uCfA1Pz?&)CDfrn#EK6V&%vR5 zomxxs0u%bv9SdHU(ekH^-j8jGa$PRfge z@1nqPapY^o9*(wR!;(~dp?5LKxg<~7(>PVi^)ja00cAmy>km}w%{&UscY%mH2k#rc zlAYrBv#k!lP{AC(ng5Qc_Xc;7B{}JqH|-t7_42n!_CBS(U#`0P&qv_bTa=`DHR*#I z)${z9JB_(OK8xpm3#S2SK}XbDu5IN*Roj#B-t}Zfz>r+xh17Jcs{&0&XI%XgUj1F7m#SvB=esqzss=dT@m%s87tD5RzS*&gJ$< z$&Kh%Ex+J#@VVyMrMg82wAnDM!?H6e4LAM-Hx@u{)Kx9y;X8G_8S{`ah6)TE`A<+@EY^0OWsEDL_J7vSE|d!PvPPkIxSrqH z_nCi`f|ld6{I~)jtVPO?Hssl2;Xc7cDOo?>xody5HGdx{%ic^7n#a~3L4Kj72lnbL zUI+8fjJoR&18BUi~{`x0teG3iox0f3j!o7oMUC}O!EiT(+c1gpv>jox36kOYR;+w`# z;G)mRK6zxgXce95tzUbn560@n`QpnvUTPkoSC{V0-IUL*6g`TTVUh%Q4(42;C%b^V zRO|d0NmvbPa6xurav|~1ZblBvl?y|KCIbQu^PMA}qBWj&G|1sT>EM|iZaf0zil&V8 z28pdWBORQ%lvz4~V&z3lY_8O56~pE}b*oY|XAgUb8D#WjA2}oO81b|M&2pP~ZKZ=sIe3T{F&M%aO+wMhK%XA%ugENVO&g&In^Gzio-P<``YP=zw zGTQEg@5zq#XC+}!F$GgUz$2w*$6V(!(D@~j#r1?!GCQi(ZLtzWG3U_1m423_k=(~D z5`00T3&H!Jgd$Q*CR)W65U)_S!CR4@ZcrK=0Ow^dhM#=_784SX+uxzaJRQLlThx!9 z<_=aLG-TYbUVg)2UJ61d;-Sb4TH)Lh$t}uz=qUivN##{Ik<>Pg$a_R z$6B;`p-p$(4qOg{2TLf)TyCgF?r2z^5qCsO%ubkpw;q?aMbb(M<=J&2lEAlWZV|B3 zR^wJ>kc9e(nWwAeIHCPXHk>kXvG&Un6^N%TLBV?nWVWx=+pL2UpgNP5v>JO_wB9D! z1hp0KfB^fskcw8GbXlK!QlwR0S_2E*SL(0r51?)182hVJ`=s7lRfuXv_h}+0q}LJd zNI|zm@uFm6y{O;IcDd-d*spBE9}IFYfx zTn`x=L8n}~epfd{F~{)ENM#SU%Vqp+Mt7>K`Uw^lDl1t0S@EJ<-K3f+R`l) zDT6V4yrNEYi8zLNOeI$XGkQ)fh65zQ^ti>UfRlRfAA-H-Z#0@)^|<>3X;CanX}r!> zp5rIOhCU8P@y4vpwdbH#Mp#KFXfsX`+%ItSdj6BGZjV<1%;?4 zAfQwKuz)5i4ys;yj2hi3EE!e2jRb(7B1_rEAUUD7mE)*+mb#AR3486z*zR+dz^O~l zX)u_y@744cN_%%}HUDnc_EmRpvQ+MRdmhV81wZ__%V@>3{u)%g!nR9QDWBvCuUv3W zY+;JJh@6l}BS>U_pu7OGgvWNpwltq5)z582GeKAOSBeY^Y*sHS48|4|2W-fH&L0@7 z9doJF>^|zoOb3cdR8oX_VS9d+JJLk5mMVyN5FIjZKa(xtjr z(_PuR0_P?~RX5{qZCK?j<6YPptwgbQ_w>Z9$GJ_@{kGVHIE z%o?vS#^@^Js59Y0cwZT?m{4a0m)7$zw3w1bFm;T3Q%?=NdytcFw?`RqgqV?$A_P-r zL%i6pvk4>ITjkQS5Y)<#@~7F1iVVBAMt%8n){lyQ?nY$zCP$4(E;e1UQJ6}CJ6N`I zLtiZVdkG7`1CM%}fs><0Z@xcR=_2QRQ_iAT#1K0X1PtR-xnsxZj6Gb>%a4^qcIX6C zi7e*d7`0|)+hVqJf5^>%Udg)oUXx_#=RzzMDdXe`kJB`%^`37Qw}IA&i`?$K{G?%= z?z&azS3q!OM?iSxGrbD4yXJwYUs{weU43Pfa;3=V*kJ@2J0y9g=lxQ#W2>{<@VoMW zdD@{0T4$$JypMg%us>jp7(uEZT_))o4N;V~-r1qO@XhlhR#1yvWE^xHzDfI$GFk}s z_6CAvwUqHx6SxJpUcsa0g=AmQg6Wg_`bbrxMFj41^YI=ZmXl2erd={;oD3vcvF)8v zIOkTogf2dWAX|YUW29PaFKN}B-Fysb`s+t%i%I_0EMa%f6N98u}9SobJ+QSt)N)4WqVjD|+Op^Oge8<`$j&WN1 zMxrUswrIJ(=@H%!%$<1iCTEZ6lFvmpBkqpz?sv-wwRi_9uv73lc-;6yYHj^W_3pU5 zYAQF%KX!PXwc|F?9j4GV(z@ygwEJN~5(sV9NrWq_Vj|9Xpdj0!H7whHTy(+xc<8#c zY2JewRiNyqns$7L(CL1jQYbujRL>ES!-k9n)i0KB?#$uyR%Pm4TMC}SB2{b!SkOD- z{uYhi?@K?boxzTtuC>Y1)sZO|P%d76%(u(J*^e<^1|2)fqsIApK3 zH3n6zW%zTYKF+NsV-!M8=IYjx_hGhI8|bJB`AjEh$ntnr^=WMpvB*onp>B@LUX$0V zNnWydxL7jcHgUUtDA6qW5TgGabiSwpluUMO8FBH8C8Zm0DK|7yfF=~H`{sP=+yqvZ zSh2uUR|zVYnyi(L!p5_G?v}WoI&V*1)&WKp|E?zV2LRZ6ew`0?T{OKwv}lUL;casa9K{iT{oDGu=^9h@-Vejnc0-tcMn=W%Go@x@e?MbbfEK3xU z9YdP2KW&OM{(A_9fDC|$v&|D6KI!GcqxaixapfydxCJ0-H;pfA@yoj~G_~s|NBkRX z_m}Nd-n(!_%FxSOas=~Wx{+m7P9H4UlJUlk;-pH?)*G!8ANso(w);4kiUpD{Y$*M` z7M~lr0zhru&*l9Z4xmc5uj%BdZs#v`BnYVtaDDuR;bQ>)V-UN`srlEozsDy3@f;!R zCd0jxOgYV{Q&(Q_g*g>M*Yf6x(=Ir}>2p8>|_)`0b@|Mtrb zy}w@^<+vjx^slVl)?Ybg{MRE{gesROO@0ku@IUsh?R%gy^1s0HKfd@MfpN+WK)}3n zzV4szed|X6;j5-48S&3R=qJG4EuI>EWdr|bU#_Sf0n6JW=Wc!_0{qwWiYjM~{4MF? z?~Dh4sSlgJe5?O5-`|^*fBpL;FmfI8-Ph8I|G9VAWdLrHew9D9@t?Q74q!zS{qR*N z=x^%z@+u%6-k}>9zBu*!kK*lh55P;ur8}Hu{pyY%z108r zgj461T3lrp?;K#HyFR^F@2&uM#^eI%2+$ls=a+puoS{+NF0Ek$ru#sN6tz71ArNrt7D9CN-)6$)$Yy2 zD|G==eaxS}d-YSAl(QYsk!)m`ma)0=qmKE*q|k~N+xMPDF*L<;O8qEn{&4;&@$2f% z22%d+<-Q(rAS&SciQLfg^Lz=8#mhDi@j#=1In(^TlqSXdV^xlNde~svBL_zGKd8YK zGy~H`8TawA?JVF_z?_41;9|?;3--8MTnK|M)oX*l)`qtHv_sTn{i$axR_g0}(-9wf z{u(>8p010AnD|5ddEkLQ&ff)T6#;+^@ksTHFyhx~;oc;N+E_t{sFid@4*kXE zsDGJT0Ay)!jF)S(yyB7TY591-POGtfafaI;m#_AyTSlfwLhYZO#67?OUS;C@Q-J>0 zk3d@#V12pfQ~nF5`y=3ZzGr0p_v`=d;wZZv`P|seF5Ryy#?j0D3iy*3Prp0>`1enH zMdrwvwFJxwd|je}Yr^gW#^v6JZyCUrR=}ai94v%>J>fZ>MwxhDVd?k94zK@xm6y=4 z4m|u7XJ=|ds+4mVaH%7}sXUijz23r=2K*-VVS-=@GMBzJpl!Faj4!1FtSKqLbfyv{ zboODhgnf@sE`WHI0=RgjxLm7rBQK*E2K4f@q(TnXBMS8L;_$#}p2jmK_J@xrmcRX~ zAF-kVi)f)#3eGlU+_CFQo_gBU&Z%Eyc%yP}E{%&Zt`Y>y{C71hHD4^eQF403g;&pU zIL`{pQrGD__3>FQFe>ocr*{(}heXCmgoGyP?QxL&-*l?|%gu#t2m#uAkcqCk0h4jh z+}kI?2TF64H$Jz&?oKpwa|EnyvLEq`<6CpKHGOqI6_PSYS$oWnmOFOS?&;Ml^)42#p;In=~ev*iB*%gudV_A2O1DQW7$E2Re zXE>Jbg)dMv-k(jfRF zj;+=DO^yr`Fx1YHQ)T8yQtI13YWyG8K^cF37Z5y9_@IO%uTeH|NW=sMMtcC6SzBL$LYvM(q$MOForW64L03!5t6Z`1In1i&eqN`1b)A0q<%7 z72gik)-36K8j#MZn_gYo!P5mRqteBCtrt=ko-pfup6^X~I#Ds^ms$1YBawVzUm6>U|HF z4;u2co1v2kBtjnamlKPn@`w+Xy>cQTb-6efcIcb|*1|V-Nz~whZ`|i1xks*+=4#OA zK0H1pY&&Sxxp9R$k@NBWT% zR|?AEg>N@H*!0KO116z<4^WZw9JQXY3FYalCuu=kj1m`an0{U|EW07>M9EEaGo7qS zskHBv?#LXDr&#y1>~eZboPI?*xyqurd&Rt4Q#<+Q52^gofL^GPnN$1XYk=_sSsE_k z3=vFq5i+(k9a_H{uL`M9XmNj=G*5_iYf+-_=A^;b?;6B+ASZ%J4Ew38_)2?>cMV7R1id85j>A@z-+VrW~!zZwrJ+L^jpoM##RVBqfAinAe_q0_+zI36) zZE27?r>#s~&!9peO1WB*V<1;E+BvhDGH2RXWF%VYG4*rK>aM}#cO*H36gl5Qh_P}f zWNTqc(J~UJKGPBlBA)@y5&NNfuoa(_eLMzeTU)c8+pk`bm?F!d+snQjAfM z2u13s(e%Y$SkT4;AJDFJ(0VgR@TUxa=<|<{#>kCjB1$FHdr!0n{G*r8W8psVezAA^ z8l===#Oo(oTJJipgI||gM*%mCni-6b#nN`nMs0^#Xh3&_XGo-w;6icegNiHV z!$Zbxfa9+1{6>AOI*a0z6fCQK8sK|4yy{p;bK<5slqefcvy7YO@*>#vhgdMyljfZj zCO$Lz9ug*8TQgN|Eu1(UM+O&TO^~jYxR^shH8m@i;X%syOZK%#H)Tv-8dK8%2g$r3 zDDMbmXFRD$U9PN|%$P{RWNqbOdt7bSaVZQJBUKc=HuxetC7|#X_T>9p!#jZ^%Ey{C zO4&1bONwy%Dmc4B($BaktRE}VNLldp`BDz82!a!)G-O4nyiIviFJ;L)PuZyAo9OC< z97RgOPDTxO4XW$A|jr9DX`}-deTXJ)Ebl4|)Ya z#?Ovgw7&Ihea(JYd*7E=xPvA$RQ?kIJ6CW9rJd&#u6S}kPRA_jkx%bS?TT;S^;FJj zXWqWGw91E9WILs~9Wb!p?5WSSbyK+z67h75$=Er-n2BT#NQcM7l+Van==gThGWFp^ zQu``( zj;v6eLv4?{&3ExRKXnJmcYO4(b!Yx-F-M2~?Z>7cX{(8j>j2pX%u(aV0h9@Rnfaxz z`A)CyTeJMrcX=Chl+{d}fm|#qxTd@6XWEnHkOl+Q_zJo0Lf%&P-9B;NWgS(WTq0hu zCd+RxJ4Tpxu6SkpYC-UmVV0UvY_Nyc5_n0h7YDZJyCM;>ZQYfC3>7*!DU7VItt zC*D1kRb({`QzPdH+rU)bFPCXVFYUOlnPhtzS>0FKFNzi1537ezR&`a=<5X6LylCIR zxypJk8d8U(Eu|-dHfw}$C@JBcFSfnRtElnm+J9?)ho{ydq+{~Yj&ZfZAYVw=C^Akj zjNw+qoy*+r7lmFWBh7Q8-OwIpjMzI8k6mkcK zHpA?p<3Xw}4gz^_%)TH~Tc;VJLsx`eSukT*CVQafARgTmfHGQwG@Lwc0LSF{XgwCO zT)9i}qU8+RyYLU^kRGbEzKzKA=_d6iP0KEAE``VLi~I`fc~;5}ilAePdh6WLJ4G(R z2s4Vv;Wsg>Xu90S9sLrsG4kD^Umx__bq<=;j93iaHy?j>5Zflvy?1-N*i6JCPI3oK*`GGqwl4u*|wkRP3H>r zC>~JJ#kxYHr#%_+YKJG*2)npf(gVRXsrQQ-m`(%_<6p!1Q;cj2LuAK&1yig$8{6)X z+`is*2N3f7Az>1T)Z9Jt25TDfgU@X%c?jv`DbZe(@|tg7EvJ9XYB;R)=l4j|pZ##i z{MCnhI>ud?%Bqey|9zuc>t;)x>f!gV zFD6~g$v$i-XWZypbI`~8ojsYuyF*(yTW$-WGCg>HMw zk~NabI`+vjmb+w6_T8N#`)=%%tqe2CGMKT1F~X1;jA8J7x!>>4_xrw=PyPP?z5nre zn0awsuj`!aoaZ^`d7aC+%<9sPro9l``_H@&eh*4cq7jRVx26NJcK>dw?ylTVElALK zI$7@jjiu0Wrn;+d>e$SCZad=6;$TT}i zEN3G&s8CLYE$lw_W27b~qBp)amWMdcg#l+?PpkDg~>CVu^(8F)dnCrC{P z$BLOt3GPy5bBo;m@W_F!;P8@kpYyij+`2iyeby0!|YBhL)+r}YCyH!N=?Rw~S?e>hpXfZrnihV6n;%Yqt0IovPFgQ8YJl6o(Dpd@)|!!f&A z;Unan;4{m};E#C2Nm+cGk^p;l;AVlq)MLFw*svJ>X7~%C_+#5(B#9fM{;42k14)p< zBbu|tN*-J2gota-1tAr@SW<%fE>}2bN6f3R^jWt%DUeE#UAD}ndzkHv8q-K+djusx zwmZbjrho4v`T35?S+`V@v(WMjh0dlVl9s%MVNP%PTV<5@T2!~6*K0oMX0^Z``8eFu zHcm73QnX+{FFvrMgK7~WyXayjiPq0L^Hc)mwB1H`N;E(8Pd_?CPw!S%?pxBNqo|{A z>4oJIe0Ss|)M!1_)tOhakwGf;r6<9BmD#-X--*_S;W?vBnA!XYbv7*|JzL*3mCbYV zXRpv@Z!~>N!8r)A@}a1A+4%-!zP!5dt2oMwwQ*J*#a*^0IQ;FU$j2Tvl~vO2fPex{ z_A`Y4eOR`X$O!$ZJ8tycI41vaKc81RozFUNgGCm&2|e3_pI;$9Y;)iGB*zdC?=mF| zrD)9z7u%x0A~)&ZnB>LYj^O}5oRt$+*5ySQC&(1=!T(H5SeN&(9y}EzG#| z?(rC@BbkZ}vOb?X265cJ&gPGmG zfR@pI+6R?ej&8kV(x9tcb#A!1jVVxlH5HaSv#CE#Jysc1zR(BS5ai!h&x2pGqr*|| zarS{or233Q&jk*!?pJG$KJ8&yYt_m4%4{iL7QFOWX$${`jd0%j{7C`jya4d3Q(r` zG!^K0$k?Yn9nE+Fngyo4kN()oCS(bw-%Zs2Io@x0j6ai3Ym{&Y-1kVS5RTI_2Genl zsuA%GYF-#uD@hc>?TUvp$GV`BU7C4rTAUZ3m+kEKvH@*#b5K@c+?VG9FqZuD&X;uA z3(S@MO8eLgVg+&ef%-A#lXu|Qu1jJr4ea8>P3f;y60x+)#(&S1fF?{C1(5gi8NS@^w>uk-;c-=MZ=&|U7c zU3RhI1=mwU9em<@Yn>dsSy1t@H~KavoB{f@z~*^A#KNHN(DKLm-;qHr?yJ%s~>u^ZjwIYwlOpkLBzQ$?%D>h%Sz$`BdoGu$+FpoE^Va zQt2}*7oKU=^lFEe2Iey|Y-xDmIdaF{*AzHOgo3n;Hg9D%@P#?_XC4D&G)b(G%mU-> zb=H1EA-V&@FDhHG4wGE!nMkcz$;ruwNWsb^()n$5+6G~7A~ILG@%raTM+t2j64vBk zTp&ENPSDmnv@ES=ai4cgcyFtQpp|z2jVtwa#nZiS+|NF?cQgO8+%QbAK)Sv122tWj zR4en|=EqH^m31xQuI_r2Y;(A)J_6w@7wwyAyAlHGseuZNPv@g$kb~6x2&utrqmOAW zHq()XHNIK72{Bp~kyAtzDJxzT+2(s?T>zcrj@Kr>AVC-U>QAa7Wg0gkLR(M2hXkFY z4A#G#)xf9nes^Qc@AI3JPg54Kk^y&kRWQQFN=e08+WhmfS;;O?KjOcQF8QU_-lmmpk;mBi}dtg7;?q_YCkWt+`kCwGJKg18DDpb8@MyG#9IMUn+!mw|ePW zsA!VP!@hpXsdTdzM{{O7OlcA0k=8)C4?3BttyGOwGEEM1+kBFQj;b*UajzcqP8WoT zA+^{9-^qN<=ws!5dn`B93{-FNTxCu1BiaRe;*1yqsGV7sK8vYxlDDlJIK;k54Xn+<>=iFmX^ZR-N9*-FeRL|bQiQ*4wVkSwk&r@5 z99k63pj*$3#Wi3h@>X?iNZAWi$7uHzvK;KCY;W}l%Fm)<#+z`#2?4Km&VqKU(K>`Q za`>Cr^Uo;Gfkmt9)Nn9eFt9YAhSC4+TEOdc@q8kWwh~ln-OgSRPkCjlA=2$41;68lmun^}}Reb)% z?`{0ows4`W>*<~$FTlk4@%g67RUK?@(_>Yc;8(7%_1jw6PUBl&a8=f!gY+S-^_S`5 z+^&RoiL)1Qk(6J3D^NRpWhZqd?RSjELc@p;!gRW#n-3!AD!T-naV+{JAD7Ub1}SngWFINVm;=%oamT3dwx0-`rVzm&+ley-_bIc$$`AwdoL z);kxhEi7|88uPr@t@xbHz$2nHGju8=#Ax3{gAK0Wc1lg#^Y$U%C)P8dOYsZ-=xQTJ#RFUVn5l8&82~gCHc?wpB+-IrOlSKd{*` zHtu4+h;0=K514UnYk8B81Pb>L&{02^hW5TThfZfYo9y5l8ak_f=?>7+i?-e6QUsVz zdFnPO%2G1i%xA4vLXbfTr>Q9qUS!`{z8uig+%(@n$cT(JzGQdsx;A&Kz`9lB-DL8( zzhRKtbtRRYl~ler1>p7Tu|ZGzRktr{bM<*Ovik_K#;R@&>ZwMC-vAqXkku=2^t)J5;S`97Hg2BO>T>ozrL{! zmyOTRg$W}o5%^$0?|R+wknM{Jx`Qjm<}$yumk+XytX-~8xrjn&kAU5-2zXaZguli< zyMSZ|IV`8f;7Bghs8&Mj>Gdqv5=7jx9KN#Q22Ip_Qa9!K(6X1T-Uv zX{lblCj^h1FxzV8K|pe%9P?U0H>_eQ5$#Ma&t;zbqQ8bz;v&`}R<8U7{^xg6Ir0Ur zo_r3?+bdke+qRguo#GZz6cGMaRMJ9A@UgZ=>8JxHPE2b!x8Z82ID~OFxT*X2OgaDpI|ymY)#r954GJuAvJL9BAy2r~hKvgvi25R^&84)6r;fxI1|Yyzn9! zeg8baQ215Wm*OJSRe^9(U&^A>oO--3)DwF{`gt>omM7e+?yW}%DXadyR!AqS*6A^X zn@EEdIg{*doYfq3G9&d^2(Xo^Ez|5kzs>6?RqIa|rJIraDRKi28u&(pR7YzqVL7b^ zk^^(xqj0`AAY4fc!@ooCbHDI50=1kou$)R5x7|u1M2g{6<8Mux8Q$-vXi^?^uTr?;S=}?;>VyqwI{N*mYWegVn1sH45ofB8ad zr1L0tKxmv(@X(>|SD*Yv!#la)bXAXUjc*0B?WMJt==2u!bYv()%DnFl`}L~iPG(MM zgNzwK+L0cy#Apnp82@pR0+DqmzWYt27mv+;~G)-%tM*?0SFXqfz zHL|HO3N-ZjO=yJ0^J+%nN-JY!g?4z~EA(qtMQNU_1R2to8B4OkzYNgE&^yQ_ux83k zvVpg=)TA%6q(w9@@DzkWLtK+nE|B6Br*(mwt3cM$f-Jq;m(s5n1v3rQ7)tSjxF8Nz z6u+*zsCRnXf(c8P)^(M1@*e={YcXQEhGoLOL!I!JHna*SrR{h-p|$9AT~vq)S*AO> zLHh$I^KzV@W^U{wM;~w31jdJTO}6C3;McB0k^yhe7z?;2sa&FGb4wC9(e16$#So$7 zzH{+7v_=E>A)|SKA(Hp=rOzysT3+Z4Om{kk(sxeHQT1f>1YQ)Z(I^uqYoBuLd0hc7 zs2^9-F7V7?)v9C5PJ%ISLdROSPH*5H-qi$|y&LJ6vyGA+5saEw78u5L=j|t97?X&U zq=sG@L)91Cx}H&Ze6R8=-{XCBD35UgKopXJVgwpxpN{4FFa#BiQESvPf`_i^orYLrTW zt5APGq{q8?o)_gaX;^Pi#e{ez9pjidA5EPQK*AtQ1oBghhgc%BFQ;3o_dS~rs_l%k zDKHu>zOMrzFPXP)20r9l4&8FFAr{7uS7twQS$N-G%O8c8!ILlK4Po%yxrI0Llqyr6^_&zouOp#H$=hlk#3W`Ia!p2=D`K- z+Ud^6CT-oGz7@r@eR~WdH7^(4C!xkj5qP@vyHltL9j(tD9cNz~2>l0E(F$QYOP`7_ zw+uzx*I1B<8u$7LH58BsmCLk}=o4KiB(gSM4t=Ex`4kiHN;k}UP4E#Ps%a2&DgPGB z@0p@5AjEhF_iOZj&8mMfF@7T${yfOxf2*A%m$#tILL^Mx!1yveEASKC!i{hC6fh+7gl z<1&nDXqJcx9%uVLm#{hBr_m5Jc*rv1NKtZ@P6+;X*(497&QQ~{VQG`#Z9OHU0VL8e zc#GK>3u`V>L0H|%e)cQBec z-?762{&?gHAE}Se-@3_&RWvGS59&0-;K0%uYhqQcL2DR(8kFoQBml>fQ<#bW>Ju%iA5mm!N^V-1P%AhTvTca-JcQn)&VItYzwD$%B%;A%fg{je{Ks4Q#<+|0eY5Cvd78hI3128pOFWdE5UGHwE_pM5J#>msZ#eo9#fZn^FYx3eopnFQFQO zOKx<$4x<@}b+t>e@bD&cq8b0W{#8OIkdo<1js>cSERQfuVNG5^N5-6zBq%>HqShxa` zZA9;-aCuwGUDd$q{o72Ul;y^t|L{9W3_%Y1)o`l&-KIsPoA<%%sa{`+#f^K;FY)Fx zXbEuagfyD?WP)1u=ltov@RX($ot6IaO8!PJdnrBCkdxW;cTN0a&3+)yafOPh1?AW` z)PWI*n;4!4-X2H8&eu#G^?kLtMGVYj&=Pak`t%gLT${J$Z97T^9cN&wIivCd6HYhk zOVQ7;$RmC(wnIa#;QYf?UlVQ^JAdYQuhU>4vLOvUbMLtYvyQeZdvbp6?iIv4wb$ZP zwpScM*RMD;oyPkzJjn4zD&vGyb{J+iUwsI7dMq$vV7iO+Z#Qia40jgE_WWhoA=n_n z@BOb4b#B{PD)*;^v#c-VO2j?}a!fIYJpJ?M3bj^~jqKYO!`{m0FRxwqT0Yiq5 z1Pt)kvo@VQpV!kjO*w_K0uHR$kddgnUYwDv*{-nwQ-+yV_{ zPHIbbDIvY<vSw5gn_^k2i-HC-|GBKV zFU^Btpd}V(p$o8xLi*En$0tt=W+5>R@g#Q&n=$vbcb_?Vmm8+kU@tG&+rpC9i^b>` zBi@)mH=EtU?O17gU(&eM#|>HX!_SG#gy=|lG2WScwHm7_jxS`Zj#uD%}FRChKrhsMhA8IB4*sKtYY*8I{aTpo!3k(}EpY1qm3! z6&zJ{P4@Gc_UQD==?sMd4f|`W@^xI!so$KM3WIs^n;Gv4OwwQt>X9-9ZSi~Q-1TtE z-|#!2jRWem$9lXgk>b^KB<)Z`&s@7t-a-uLNv*eYlMoU~-)5ab!thpH$_>v%Xq&Lg z)>|n-^H%FBTPs)Ywy5d-Y2-DjOrq&U1ZD{Q|N2c7{hGnf5MSm#cI88+Ic# zNc(Dk*?QkCk(@j6wjxW@Mm%4YyAz&{6g+IBa{l>^|074Q8RibWeG7xK9qsXlcS7^A zjXK+GRPPZ&tl3I7X1o!&_4HVE=ij%(za>X9v`0YUI%_ZgRP_045ctQ#nx(*7nejVc z{l{#Aa20oON;0{scRxSi1g4Wob=BD2Z7@?_uJNwo+n@aPD#IUtU2Dz;&pv*d!#Pf- zW9D&DPQL^e$d%nZT`fQo9{hXe@lWp*X32g*Slcz0qq@dYES4u%*giktd|v=qIy(fP z<1eY`g>i$&`8zIF{A4QPj5qXSxijFoxw!s!uG5;3;}5U`s|Ir2h-w8_V z(Sf3(o+ZABQ_c)Xz53dz)~bHMdn_H4Drs{M3p?CDex0DKIaXm~wFFnTkn8DtFXRk zn;vyC*0jv7qrFvrOPp(P@F!CFd^faZiH$UQKawvx$8Yiz4aB_qlLfE=ABSA`^X7Z6EmBBu6_SiRAaUOD(Zg~^{>0_F9^)o{=XLWzZUhs7WMy&>mPqY zjDB$B`NRCe#M~$ESVKvLbJ1A}sf@%7pYe8P>Nc(_nCmQpJW)ex%KiraG4S96ydK}F z8PWXCH;gPS8ZJH0i)l8Fg?)Vjb$r??dx#DRS(!^_R~x-s{?AFJe|um?tInlQA1YvE z+c4#epE)i(1j!UcmVRTzyDjNQlTXB`3bmX32mwS(gs7w}wsiILxJ&p$CJq7(x+~nr zJz)^w_%8WwYEKM&#f3tt`#xE}qS7*4+STm467y&h2%`+DajD)b5bP?+k^gWL83H?y z6OdxEXt2YRetCNR<&L^@`TDBt`>&>kZ&CLkFsRG2TJw&4kt@cx4f#B%;0GWA zh)loCJ+Eb0eKBYG8MgAxnbU5Uh~Aqda)HUk4tZ_lRhJw})pvQctsQl2y{)lU%x-5@ zBxZkms)*F|ZLk3L?QxE!bM8j!T(t^oez3nb#Hl-+v(l=tR&tj$8w z=~U7_%kHUQp=M_lv-hK*Zl8Xj{C?{G>I6ut1v1~;u4jIRcRNSx-q$=05@jweZ+C#W zz6B#9uQ?N^-AhgfR0gdUcV;OJ?rD+Kl?QsB5p^BtS|hL;nZ6c^Y|urMUYRNLA0b}* ziwsJy1L&Rvb_A2x?nS#bOf;{^wn=)T)XALwScr4e&hjX8>vnSm;)=8>T`Va6+N!9% zySV{BYrr4sWM%BZ#NX!^^ur*BGa7(YmKEPjbBVvrm3xG2iKl%S6qB~3qxcPB-7_^k ztfFZ=F|{$DB3c{I$O}eOR#H7M3OPkTklm>Pr=yfp+4fqSzZ+yP!QA)PHgs+v2gZ#& zo0#Zd-qa%ZBp^njLG(ylAS>l?u9dM$DR;a2=30${n97!e=R}g3mBg~zA-VkwPjr?? z`pY>M+&3)zbrRENjH2Vq$R2>Ti@>DZf_-Q2Hf?uZY=_XF95t z7G{q{A7dWxuxJfvd5nmgyRK%MuIUcP80dUXBjkGhvN!f=<;b#VIp~~)TPd6W07Ngs z0RbwTb%2EONFRn(-d6A=>eGC`ECmaOP*TiO5UfH6tKi_eovvY9yA!)AVn%KEGiCx7 z1$>0dmG<;-dwLojYO~(lApcwN5Ez_Rsc$#;Z9%Hx|+RLlNO`t50iD)1z=PUODrJoZw^14-z28B0$~ulfTG*e z;YidW2F58sU7GO^;dq(0hqK##HfK<&UOxV~=rYtxtCRc<6`HSdgz92U!gJD^`G)hYt^U2M(7Cm?-l&{;Bt@;;K# z@~$Yz;elssLwWbRc`^5=v&ud3SKX7ma(@tD3^<2v?=>Ou>Ujlw+f~Ihi{5$rv4-Cm zz6T$^7Vyy^EdsYDR6c%>8a`b0?RkB(?Krm3DOsa+K9+3A$W~;DvkH$z1#XBviRc#Z z)O|$U>5K$h+W4Aj$X(rLwk2jhHf*|(X6xs)aZ)PS05Wf&i2sBO)aE~2^R}Dh7J7J* zt`ik$M!rc9vnV zS}bz6*kr*isu`g**Sgx~g65P8?BU@7=CJox3f30TTYFH&Tb+yM+uTo4^jI#Nbh{?Q z$i}-!>7s7E@{(5eC{6RV(f)xNasYYQc+uPldFV1@-rb;mo|!d}(+s3{FB(Pdnn`|d z?|x=KZT;|(bLGJ-rNf*TxLA+&rdGPFwYTh$I;h6ZXWgBs-6CoYO7@5Q(E`?f zH#bHrVWsQ?)P*7qQ)=&U?(NkUlx+^qd3UfJzTX(|Nucfb+KIP%{#SBk<-`3c*O?_G zTd0x8_nrd9wNx!kTK4wyIa7hX6$c<)v@Cu&feNm6MBUIH;SbzQ1e`R+7)HY2eOx_S z_WOEfV7rHBJ)m7;sw?i3o5OQPGmwXn!FsfRVfB9B!3V*Z%X)sI2Y6VA>Ee`xaS*bJ zi^D^Gf54ERS$%UBxF$yM1$jIeJx67flswqivlGMH1Z=!DEohwqc8Z&aXalz9!2cq3 zJoJXLq1xARFA(uQLg2O`Vz28rXLXWdwB2rx4%fbDyE?c0BZO@LYnkUBBgFLq=9t{VL0&Lb<8 zwef46(1r+io-3>%3MUjDoI>o(b+9j$sa2NOB5IItTK#+i9I4 zlyO0^!DRIudtubPQ=9FV$Z2xL94j+;O)Dm3%VlO&UvPY9gE>$6Y!$XV+ppg=&$xBA z2_>FOt#kHIr5rp1b1NBBkzg3?oH5~nLT47dW2I6h%*B0+TAa{UX_{*nOqAm4pllWy zP8^tg3z#>6>reSn1Y*$b%WrY1A-ltQoO!!NgYc}7Z!vkQgXx-AwMg8*@8zslq?>(M z&+!l0m^mbsBkLlKjz0wNO#Az32OSO}4)S*}#TYf%j^p8H%jb%&ifr(-xVoY|;+r{>grIQQi2=&XS8$*8YC|ia(SS9V?IxLYy7c9Y7hVJ2zNGU+@t3AL(KOM$VqO!u`A6Q&b?>T3SU3 zzZDM-cB?i(h<^Q8nudnAe;WlX!&va{h<9Y3Uq3r6b&%6k&>WtgdKvc=E2Z(|3CHFH zZZk%*$>-i|;N%Azjco5VicRXHytXH!d_Dfi3Eo!qd4k6(D9Um0I+I3z071-d@?16I z1;+evsMtx`|H!?!Z{T2YH(hkTX%Niw#u+$FJK%=_cSSryC}++6$1cA+Ko%#qk7B~v zu^o?=p(tk<8H=o;4EiILD@l$tg?Z3kD;2Fdi?v&(g3Xp|F^3zYUSG@)_Fz26derdE zlDY%Y@ad<44=i0L8!4%uWj5fjd2_}C?Tm-Ap{hxk^KTQOKTN1~of?^m@f)|cfoX$m zulZViJC;g7PmHkXKT_I$Ilf&R-L}H+J@%0NsGsLnR8j!Ki#UkWwgJA*Wbf#-rrhQjSk)p znavmw_eR^eL{}Q92U1##@tZ__du=B2E6KLiex$<@ctY+iXI*DfsZuE>QTj{;lLa9z zt9|MlVb1z}bwE}qH z8K%CcpeGEg+H7;FZ93H&{DAK|Q&M`=&cZ*fw@9DW@| zC&T_Q+utK1{qqib9rk+K)CC*Uh<1?SULg8=SiwiWEu8kl0B$iR)FfTmHVK~{sFfJe zy^4&~*d_Ov=23Tw6X97po$=m^V2!h)16p%kY2Y^j-BEo;ah+2I7BvrURnIeFxL~9q zqqCF-#JsqX)wJq*>u}w~IuxC!zn2Zhs-ESY4%zU-y`f+9M&R&mJ7=NQ;m%b4pwG0& z1v}DK8y%Is6lYGTaX!2zg|P7^Y~kLdC{Afu*>MFuW=RSs7*Glx5Zb_JQ63B)ZikAWEdK-frPq%U2MxlJ&k4oc}7HMdIx8wPfm zpO4ZvK7e%WEIlUmZQZXdwh95yD;ixnR4_RHd(+^Kj>bX4O|0P__AD1?X`%a28>T`p z^6Cd-u)+*erv1h{59Z*p=si^&0<)~Y8TSJ`Dq2MYQqXz8*@;BLZGvV_g2$TnHxfW>2Bex6L7dWG_&qQeCde_y&heHigheq-j5Z3^0lp(+e> zQ&4@pzTI_%5CG-q@9_$(VXA&t^0@l_(+pXPs->TM-k^oTIRkpUk~Tl z{oy(oADvAb$~;&jn{njr!4FrH58_}{;@o}Hg1az0Ja;T(8xPz1-S9$_9AtTr$rK(Q z-|g)u1rP#u#>ryv)Q3XeZz`$e!(I$A1<%#-t$ckg)+blNn^R-8ZunI(Yh%K)D@DR% zFJ|sTj1(bg@r=^}Ne+A`8^}zgz$&5hn3%ny+;zRVjJ8DjFSh>sXEv&XY=G#d#}2l{ zK3s@nW>%f6qE-jsMhKZOz<)TO*P4@?CWR3vI&z{?6h#iGCP7)J-WHO=>+@hw!|#%|(p%0ElPp`|&fGS5i){)(50 zHV2-yv#nyLju+Y^Q4S;E1;(|9YYcx_GFq$+Jt_`Qs;_|CZwC=z?W6U;3gQ#a2ClYS zHSxl;Lr8i1H=X&yyL5ODJ`+g-?MvjC+Jkw7ll}H|AeytNW3MuX+=h3{ae{hx*6bGs zAMA5woADk&X04NwV$;N4bHgk&p$q6aWTLjVpmX-{;B93}jBGTeq_jeUANqx%$Hrw% zvRzfahSHmrnFq&696|-~Zi6+wud=W=QPzVjQ&hjxd(6T@_?MeW}$d(6Or z?dx4eN`ChebN9p3xJ1BnLJgukgcvE9X?yNb#k5wbqY+?Q;r$x zG*ZKnPig-qHSOLKuIUZ08&tq>2v9m(7?zmlc%SyaactZPEd8vei9ys#AR^$Z_`?XE^0bwrARR?5cI`Dp5J7f+TI4 z^_=;%C)w^$(Wt|~bKO|)<&0(!ugGPYk{uvcwi)gpI51sA1#L49hj*i%E>?1N?DG`) zM91?CUo!Ck-3ISQ!2pn#w^ue3KT%&P6};gcqm#4w{V^#3>A}R-LD{NISAYRvFMT^B z^!v$~z}>pXBl5U+C!!9in~GeK5@4Unnzj(7t!F4nCXJ;~)Ykn8cowRrdfo96Le2RUH|Z-{Fuc%pcd@>`y%{cVre8qq3zLf zM%9}bsivSwi6Vwm?7YZtpNxX6A=qjiUtx&YJiz(z)#!V1Cp=WS6Ju$=Jox%%46$#F ztX4!G&ZBB7zpih-Ks#p?v>+7|Or9u}b3xo}Q*0@}ikqEi+%Ieno(QZ|k1-0`8JIKA zBO>Z2WIpU(Uk^0j08$~rpMwh9)<&howlV(ek4aV3S}@dL;{$$wt?t{XtZD91CSaHj zSxu*9*|7IUL`?xObE9hM0bim-^pCfqL`w>?9dE9li#pipQx_Hv< zNngqcg~`ht-joU$OUBu0d@N{o?Z`W)Ma?vAAe3q6`@h5(*BUah;nh?#_I?TNZm!tIYTlbchxS-|2{6NOfaLdb_>^r~M)Te6SH<~z8iDJUEc)x0H+W~PJoX^f2uanm*pem{ zBHW5!jhT{a6aAKV9zM;*f-NT2KKaCzG>sZH~7@Kd|cX1NLFjE!O8+ns2%=6LCgMto?8)XopvHx z(Q3z#_PDS`XySXdN+pq8d20i6Y#rpDXF3HG*_2YRHCK|F_rLEx3kyKC6wzd}teIWl z|8S&RhXqu4GI8R`zLym>t&8Ta{U~pnFM65-bBe>*9!bw4zLYt za~9`+G{t{a1i~PU07n5{^)u=ZP}yfcq8@#?+CBGYKEgkRr}LaZRO)`8^h1a9hZKeN z0gk|LcVG1UcomLHmc0_d_}CANKcQ>^S!w-r0NFE?vc&wTyzsAi$65j7H$EE#$a6o? zs-v|^mj*K0>}D4^b$$epk*Ruzlg{G{84vc{CQ3V*v}?v zZ2#eiOk?rPg7@c-VuA7dBwP>vNs52^tn;@cGELQ~YNbDY{7*H2&c-7$jnz2y2P*n+ z?PEA%Y282T0Qt${0|mGL-z%zpDa50by2mx%2yv_idZf)w2d+yTvE*DEJ+4;+(c*!e z*O{M4TwA>A5#nag7{=8GV8Atw1Kl2h{A4Oe45!5T4YL9Ff2jjj=KyF!C}vi9&ju)& z@CBFQUjIbs1KBjkcoi!!7A%W82 z++Of0KQRDPV%AImR$_wj{JmJadsv_`F-}TyyccLai#YaTw(8c8w{F-i_F08t ztXC$Ot_u`r4nbp8&C-quGVrbg<)3)T+`8z}3q%fFxdTynnrD7at?+eXetwm|z zXyucfJgt7RPNxMsX+8pE$)a(mw44~AtkuMO0Xis;<~Ye8kY8PS-@dPPE!7b@lb(H9 z>VJ9xtnz@kZ}Do)0o=Y~rOnUR)s4@WW&ea>Nui6C7GAu$8WZ-a`@@T<3}I8i%!!XO z!m}1>zG_-FxK4H%*kyrX2t%Dt_xUlfUOnt!i)EY~{v0i$Vl_)|3264qcOUuiM{gf~)+Nx8&+i0$ z^`~THD6NA-cdsrha8vp1#vT=5O?|IAX8NtSkyDkWENdRpX zUnH1KbiYOqkql4?NuXA z+&*8)tErnmnJ|a(1ppl( z&$s^^$8d%MN^#uy)7tBwNZ^#{%OfWEnVFxHLK!uIO-uLgfxpTBqT>IL$n_CerP}A~ zKX|gJ8(7 z5qMay=KfcrYnTZD-ua)+{f65}%A>wnt>S-Ukr1>X2n5UH4lnQGw^Ybe{&w#LqBJ{y z(aTbBTV6f<1>;ZlUj)&ISO;&7;$#{ge0t>o@ceU|;!s@@`{gvuX@)$j;ZEik7v>qjE>fT63v55254=u(-B8?@tO?f4?p{ zj6K`9GZ*uR;ZyYlUivH`4l9(Z_mKlroC)%7q8{w6+wppK(yCXgeqx*ksQfwWc^m3M z3JAR_m5>2#nk@V3zxv>?CP_dgE70}^6ov;(lNrC*-{%c;2&WmU{w0xFWhvhQA<;&H z8f+=w3Og>QspC&&nZL*k{b+%_$;d#CS6y2IMNN2UYYqD3r(f8TfMajhcpTLy0c4{n zRhoZ3-?0uaRA{^6Sh0$F7Yn@hj4U_LOio2Bv7Z=ub!s^g*EFx24X0;GxT-rqmI+Zx zIyP>w7YI;NIKc-S2{A`)@C>of?gK56kAt5GZA9dU(7iS|LAtL(xH4&9S8Pj;5xCdW z3^F&EXPw1=5*k5voqU18@^?RYd@{<~TToqOR|5Vwh$f9XAK{Tdpz@}h6@dC^OLa2qDJgo!;Y=8?#K87x3CGQ+i+?d_(osiTgg?%o0S=sw z?ZUezz_K9u19|$GdOAG<+@HJwj&)s}PKAg0ZJI`QDW1}2pH-e#9uMp}YtG>d-?*&Q z{b)`X6t}~3(_sM&&yMeAJ~%Qx#O(%U-GLwtc9xKxE&+{lEGTBBpAWiQ5=z&_urS}Y zreoQKTbulo+3d8Z!#<7$^QXx@a+vz){8`P~W9^98wKh5to06V*m5lW^YEoAb6dY$6 zp64v|ageQCGs`-{;HqL!zk7z*hYGp-eIL1q@AN>|#+fLdV!93Qbw#wh>(%Ge4YA|G z2+|6S4L-Hd6yTHZ<$}4dGgh4=Dgfu50?H|>m=a9h^0od(AZGA7Y;+Ui8?kt4UdbS+ zk~*Q7J#M!g5mb`XZveerR4uT|4s`|k7XUYmmJciBcp3M#zVgb*9El)`2Oh&o5i1b|k7r&7n{tj7?HlJl zi6-3K|DqvHkUw)+U? zzc}?7s^FS1SX`zz-sPO{=%#uYPA5gF;@mkT?1iq9H{)`U)ssTJmEmwMH$k=JPTSOG z7yowE*a{=&?E$nx6<#WA0?RfhL2?t+Cx5kC_O+a}ZEs(23Er~)Sj~ZtP;xdtF^wmf zpdcaTsWPcfSqpvDeG1A_ci?GgC-q6TD4cS;Zk=4a-R!fAlsoW2wRnosn8k^XhV^`> zMfL-u3dQKml?wE0#|TC|SCGs?!qac^i6{G1eBTGg`x5ijB7}gpg73VSWhZqozGcP1 zRUQV1&L2^ioX>4{xbZ}+#;p7~$FqOxdwPjW5nY@AG}M{$v1Hk?v>DgeOig403^Z{R z#`zpTKhI21#|W2mbOCo|+a#GI(ea(J!#G!T?t#f*vVAvju}q=U{29aEh}6{eO6nJyJqbdc4^qfRgJMS4?BuJZe1N`X7k@1}55-wMTvTJAyPexiV^H6wz9^G$MG}AzC#N*h zhD=A}v8{0dM(5`IvO(TUOGRf!QsSuTd8(s=c>!>Oww?#P38%_Q9$)_}OHA2pBg>QQOkwi0mLOCs^r%iz_tH=YMhbK)V$I5+fID>Z-p zF|)O5Q~RsAfG8_t?=$utf)E1t-Ov|)c_=bfy2Q#@vHv~g1&x(8;&CM;n>`fKH~mQ@ z4;z3iJPoz1=wY>pbKCf}Ws!Fx+4B>;V=<5&k9gt$BWWCBQTy14_*>giB>i@?g&?JM zuLGcUVK8w4pbz-}(!Mtu99iH7VcdA-U75MV)K77&cxw_Q#l5&^9594z9P@N9 zBfXW}+j!e%iZZ}!S8y$uqf<)>5u``(L%Huckt83gKqR_(9ai1Z{^!_O?xLn0PjC}; zsB-J5hehR}_TgQtJR7@n&EK%i))^-(`h&mtEK4f{q-& zm2M=$$5-J7@cIf0vXB+rXZz~111iod3{_|2H*$k5s~vjkhBQW<=Dfu*BZ&yF0l^M> z3!wE}WnGcU6f!~K7puotd2*6{&iB<`DaGBdV+>0zydZ-}I*9?+UOs`E@$_+tl}cej z#48mf-;y%(pIuf*10_KWwvJ7aPQ`-L;41YWwiw9+mzjp zn0reAQ7H_}{p%ge*U8?Rxs@5bE8N1S_D^k%C|_=TlW0j+|Dky4?6FdG26jM-yN$^{)#YTSj;_k#!ETQIm7YYm60UUjrKdM@#+7L# z=rtyn5#p#@+#VY5`t&wuv_ZnVF|^0Lu6wQL#q2GBMNesaMl5tU!bMcR1SR^+Gja`K zYuOkKbH_gm;GBNMU_wmo!l`Fb9qNsqd_ap5)!eLicDxOhx|=#GvI5;dXxHa*2O#7O z`XJZBX9^n@q72J8^!ue_(^_u1IF_75yTCO;6~~z5Sp^FJ8jA_l+b?(BU%x&_{@J;^ zD8j+y`nn5!M~r7J%f`}}&z?KwXvCR?iQx{I~X#^+af?quON zo$o{q4h#&mFO*$ZdH4rm#AXF+-8=6N&oP}DqF*%Ra6F9`45nog|vd<2%A?jxgGp&$#vEoeA}{MK^?rpmiJ8s7i; zdq?_SUN4wG9WVBrC4vL|`ueiR%)MX*nADd zxP3EO)7oBK#T=4PM=wbmldI7=!*DtHEa~(?qLImVnHe=ClvP zlweH~;fQ4iTLG}pW6{A7?kBIedzpy?>9dB(4-M|8UfX-D^H!XM)}P?>?sr%4s|red zbIb^L-Aufy67E^-I$@tb)KBwME4iKS^5`{Mc}Da#K^3s0pzjakRU#z<9`l?N^%#G_0? z(~kuIT7l0F??B4Q14WBS@!iP&OAspn4)d8`+pkmsr1a_e%6)aPvYK~qUOsTV#YsRxh2Z2 zaqVSZ$xuYVzyp0IulB}g+NmT{3)+3PgVHy0UjLz0zvB8EWMDxK6Ys@1Gqh0Gp_wRM zX2l~&iM#xzU6xChdwZxnnW|1JZ5c!ei{*5bWcfV_kNb5DRIQR zR&h!lQxgo(GbMf4u!{553xB92w3(<`?CP?9@Ru(+ldef%Q?HoS*lBG%WsiXGVioXT z(d^+Li`sz#+=Tgi@4^DlK9xKhV|2< znTFqTdi~>6O?qpt54&(if!Xtpg|=B_>=HOq9oE8B?^*U*Eu!ouAKVsB_a{XTmGBMA z9^mfxe8qjI!gdU4TtA86iJ6uBva)q^T7j#z054#IK^~`nHHIS%kLpcU-cE;V>`RsN z)_m=#Po|ruhZ~qmloun;AVUiGzdxrdNbAm~N>Ebe2zB;g!a6o*8*}G!nZC^e15GS0 zLQJ}VIpRH6%5Cg{UriypTX*&jsxB#DCEhESq@XF$v)HvjQ>a{#q9}^q#!^=sOrsHP0P6PxF zvCxXeHqL@mM-A44BfRG&I)^iITrHWJjuZPAV;e9M$=_4hT^@{kio2h?wWb1KE*DB(%{3fx;(jO4RC4&JoGbwvZEOyS<=&a+?HK=dJ6uD$q%e|v z#GvcCjMidA46l4myXwaEru?A_rGbIOgPk#Nqc6=SK6?Jo3P&P;^~_&Qj&O3OC)8l^B2I6bn9TC^*_dPkqIdmxB;9fX6T!{-*f_#b zy+0C#0|g+OzSv=!j_KG%3+&rD!n`cSI$GJ6KJ}DCd{7`dR6-g_M*ZPASgS zu-Q_8q8_3q_yOEPoI~29n5H4~ws=M~_f>;QK))#S$)H|$PPtFdsC{qGaXeVp6YR0r z8xS1Qpd7!JZ&ZmM!h~AYD3)i`DOHMkX04{{vZk%(l|&KCNO7%qGtip4FJgFf5-G?f z-K-nL6`50{@-f->%0p(}+Jo|b$x?KF+}g;qA}sOr2;-?KT4HH)_lg~TqAJiYqRFAE z{1IkY2>Pf{6aMHZ=LEns@kqimjAYLym^9Z++t%*cI{zhKta`4E{hXTvCYfw=hj{K; z2kQJ$uT+AZy+2ClyEqZ`S>L(7y$Z{Jyga07UXm-!0TQxw;);WT@3ko2#_e>6#kL=- zk6tYEIxi%4&1VDFa|Ox4Utd|TbX}A>a4*HkQBacpO`X**A!Y|YE=Z+YuM|b~tId=9 zEonKbP4fI2?^$8G7qIv)b!2nLrS()8&%m2Bm?iGg`skNPA^Zk?RPYxmj8>_ViRffN z{Zb8bT80>e0_)(evzD}$wDa>;$m(4*aNWg%o$fJ7xr|zVr?)Ez^z++x7r}|1fqh}5 zyeA*u#bxAO>hKwL+iPOpm~Xkssz;6fb^lIl2Fbl+$-T?vG*ln9ZlZ?b3UQ(gnIpmB{o1_vmaY2I3x12q zGW5Xz+yQl8{ZRJWrKg=E6~}*b7MD*fR+^sJ)}j2vPKyQ)ntg&ry4+MgE=nMEDY=J7+avuScvhAf@L1^&fb)>}Yema%$_= z#B)R)g@1ISXLvXzMN{-s&SLWK4pud*R)ss3(Aoq=BG=|eMZFtyA2N<`ZyeH?g#Sn zn&rPv-Znnz`DCk$&!bGT>}HVg2=ndKnmkkUAh9TK?3Y?GJTvHOf2lQ}5%GMYw0~x3 zjHMkYbiU-g5iy`52BuQ(wsM(6DNcoJbjH&qEgc3cSZZ=+0IZ;D- z&RvA6-ve#={aeV^(L%^n2%DfQsX6Ml87N5JxcgJ-Ct2~pKBoQp!_`wa*@v;SY;E*B z#>WJIKgIh1JB0z%fG7bchYoad=ROb+V|a$W8LwAgs3{P$)e5gB4u3hT8C5~{o|Dwi z6kkiLv7Bh#P4Ba+U95()lMQ4QYbT?}64i?xGBWXl#W``PF^D13zG=;+mDW%nQ&nw> zQh)7b=W>72-YkRcwD2#d_!D17Ok)D-;cKvFuaOm}p$ALPUg?Z?slz)B+y2#t;KVF? z$S!nydrF5CkPAcl#R`ow)&j6rtypynzh!g$aV+xQ=+h(RS}RdIMMuBlE~zq8v2Sj+ zsrY5Xz}}L-T8`4Qr1zx=rmdt=-`!fV(t!uJQv#1P%p+~8Gst^xfWedy8Jq`Z1Kt(2 zcZ1_{;gL^n>~0R#G?K>%yNJ_%nwY9!uEqS_y$g6xFKV1u=Ci)gqSO+_7^~brDv z>_Iy79rh9?=w68J$IBN>o>g(BlTLkzIYt?6qh3TMVsKYl@*l@I5gwo3EEi>VDJTWJ zbvG;x@)Sww*D_e!O8rqoaP^DtraIl4WFUC}xXE0e&6*0Q@k!8`AX-UFg`pYnn&|D4 z6H}H20mH%0txIuJA!D$XJmV!5n_9NKtuam6bLmoXqP}a;p~eHOzXQ(SDm( zFni}@M_C$mb2CbzWFe+!&SHaeYV-uFvpg~M*yoyE4NBk1*#|sz{ay6TPtm58->?v^ z`DDk#khRMRinTC<22gC2r*)`Io&cJBuS~(R<4vt6lHGIr_Tx z6gC`NsZuf-ui8gvW-78K25`CWr9ub05dOBXLU+LJu&?(g2?gezhN~66O{@ITJ21iBh}C8GhuuVIY?gie1RH}wnu zE$|WHn%sA)07IYJ4I6t^mC{l9D8(GKa4}*3#pN)FK}w%*4@%P``16*(g+^$VCdVsI z<$|1`x)B+%fr(L{3?n5kGZ$a`7g`OT`HNrQkA-9=VpI(b$u+4-(G2FPR29z}`?1VV z-z;P}Iz=l3%+1GWuXQkxn6{IYVz_iWovBP@YcO8_Vlyl>_WL#0{a05fYQVJ}kY1N6 zwFA!mdZH2t@oXK|>ARM-#qgo|r-<6%1$C@M)LdigW0uX+%qm#}_#fLUhbcP!n30ZB zWYCdocUx=c+@-Eu>~Hj}ohyO+<>??>rg1!6x$H+YY+qe$Q2ev|5MGo)x#5tzjyN4L zT7^cXe1qiARIvA1pp(ea4g1x=i=NL9K7U^WXdC)I0GC>Gf*W)!Pugygp z5pdRHkcs)u%@^bfWbJ19&sF^1s$baQvU$f3IMxu{IE`5=+x?#9MBVi)oN?{W z`^1c3_rMqG0S_+9zds}D(U>bwc|k30Z87s1^&g^JG{$~i+T3c$h>d0PtUw{FWBoR! zF-?Z>XqC|6QcfY@!n7sh8m%+gfu0M!yWd3OYDIsWug~MmUMi7LMTn>R%oa>ZVnczu zqKq0*8~iX)(?C^7VnderjToBqH60=;y#PyZs$rlhGdm}6VuY)!vL=Spkf+r}M8UG& z1C?$8r0w^{r7S~{Ij*C-uei%*+pEvg>Q@`6FL(tRH0|7-_Ka`83L?c*B9S-jg4K*f z{&}?YI60WsG@(|%iglW%zT!0?bh`G+NFSrvr(Vx z&L=YKJ5;V#Bc5aXBHAP*z*A;^s4EZW8A$OnRXP=BTaK&|wV|$&6+T*u`-KC<>zwnm zu;1#;yugS|;o(b^_8=O8kn6M&fO_kJ%GmDZ4m8kIEl6)g+m*)I!Zk~RChR>xtu*bp z`+@3GavIk&nbroHc-HIM(`!f@4_4Ui3#~8hE6G_?Lk63{a#$#>zPLo^tMjgKF+8MS zffzbbF`L}Qr{vu;r?~8IWCoP}>Hp@cNtukkUyq$*5d)fU*B1Sj`!28H5 zTWhZATaJJBhtUtZ%2%mnPv`t*FXgBRJFcp-_kwYwlg%zz00nADd_=2mM+H0ZOtp#E?SA(~j zrkcOF-7SFHqYtAto+Y4r>kN9dM;wEhfpNadT(|L14ZPDFrN94Nh*~0B{aPt?w?3KC zN7&5yFh|w#@|uTSU@J!N(Oyo~$Iy_r43!Js#$-Zm6Z}Sql8u4Qg?>hrGc-JVx4(ri zp%h=231P*=tx0SWjqAnmPMwT7T7)JUT`r2Px<+*d|4xYqtd4Ox`__WUo{JJ zJUO|KOX@!P>mn1oH} zK`=a|v#)x`{waCMASgxzRMK!>Yu2AE5@u>VZn(NU?;&^`)YXNNa8OiO5(K4Ey*}!{ zl!N=jOceU%10&T0J{O!C=Q}!MzFK}l4;deueUtxMEK{uKKz_(Ox6PNuk1!f*b?f4u zCyAy_#3h0zukaZD!@l>Ea_curr+U^V%9iDlzeX{P&Ce|bwex@k>3%q5aAPMcTl4~N zIM9CywhLORUIdOL-LP~BMl}GPjZwa9??9nC0Rzptk!(dw?nULT;(bG(zQB3z(;HOjMC)P0B`C6~|lSISddtxh%8dS&bbQ($+ z>#z>C&~d(iW4;J5E_Yxy(RVDgVQchpRi)*|ROu@8xc3v~NeS{+KU3JNuU3>Q^~joi?+>~!16(pib1|_6t%LG%B%C^e z*0Jnem%i_F>!A8wod`%gJF#~rXeEf{)`gI?MKRkpQ*zV7Rsq)>fV>LG?=*$7xNj*gD+}iNE-g6q}ix?_Z?h6=bG<`%DA1L}6Ya;t~nPS+n$v5Ze!ZfsJjL69l1W z`8`jxn;VEB!3boXZGO}3khqG6s}1S0qsrorxHv^Px4~m+;NlJJ&df!6jc;HavYY6+ zF&GEico*${`AfK8V{T>*tcj3qX~PMX(tjc~8ID-#WL+#7va(!FI7M_AuUdisiu0e# z1S5Eoyrd{J>-EXV3aReI(josi-EDyOB8ksT3r8{KjK_oC&o;j-ebz1bAp$s|R&++K zM1Hzc2+^0sh!mOHJP`Bv0@vN8AOZFBwM|N+lC8HLgX_cZf*o?m+~+BdI8+8Sd^Vh8 zAQOD?GJ@1Wf?&2{86H)p^(!I?8AScDM6+ED86A=AqOt|!U_9BUN|XraWb|Aa+Wuf# zW!aP@ecjgUlv6@@&&s>yPCE&rDY#DLN?OSaMNF$Z(>h|8ycN-Qv&v?_D9-o1gG@(=X2sj%17%*G^j7 za;;~`@5|s^%sC5sJ``x!qm6M>Pj&=}p}-(IU8amc6Xs1^vs^aokhmmhIF->?I@7AodR=G3Hm^V^brv-Y%HOt)WH^8mo&}Zbgcu4;As+@GOTtb#f_v~#$(|`Qk}WBi ze=3PqV;fR+3;p6oHHY?|{qfvt63b+lCsLeAHU*PZAtj63jmNK$9?{bwto_qYos{5R z6RG@dQ+*|cl2>)7W=ECskL146>!CI_+U}3@h(!KxB>fLaJ!XDDJBrHV4;=R5L4 z3J%O3lCur-J3>H#;|RY=L`{!8hdi*}b!SCDxAF4uM1$k!wUD15&BlUOhtEAlAj}%~ z55wd6H@nFlJY(;WF7og$&+^VwBWaCbFU}rP)6%mJ#KDj(PR49t5udaEV<`M3C5`-` z?vqT~+|Vk6Vo?R#m^;rGD?)VQG|k4@jTb4mdd)(Lv88pI+~hYNc#u66zpxl2+f&#& zNp*l_igE{aD%hJ29aXOBWIZUZ{&ExOlbn#!-g+1W9!Z*52hLZc&6$V^+R^!yz`Zdq80vzTcADEHQ+Ce2n+@!TlFn0X(#ds7 zv#Ql0|9aw&F-XH<{7IUodqHxO?uZr8j%XzEJYR~~W)$PvFu14Zh235=<`Sv$e12Wa zY~|UIw&PQJJyQduAcs8`bSt_c9v^{1D4Altr$PPFEwc&GXelpregvV#d(uh*r4U-u z32}X+YqhlzWL?<8r*BfdZ`~Uz)N_Tjy*je6BVX%^UL6T#cRO{u{LWmnUKTpM*}Q@$ zqg?8>lq+-*esgo@{$t<%p>1Kl0t=I3i@t@;J8FUvk~mTxt%Xdc@h}XWlIZHA;#5-C z{qdKW`(9e!ZdG36!cm!r5PDVUA-T2J&n_bbs!hM=+Bye-*QO>Y-lxfA19E(AQWA2q zEJo6Bd@d%fkSP;7DNaBfyv%Wt{2&iM=4c~)?Q>oT`zr|E--YDd^fq~GYDYRe29Pw{ zjms(3-q5k&>>%-jCcNFujXk1wJgP@5I>DU*35-uHgCPA0s@G1}mUq3KomYKRe?L7LKC-j2TI_;xB)a#p6f0ouoZM(^GoMtAt#C;YQ;E; z@qjZ3StkEfc_8mqHf(Y}&C8zityI}Pc1YZlD^+IW5o_o5TQrQ(UGbr6cUoHT_|2|O zGUzqv!fd6YPPDgjk2e^h*oiqy8hFPQG^Z$$-(;&KtDFRgn7);%gJXgBLkw=u`D2D1 z{MJtkYQMx2^!}uDPI0 zq$q}N`6QM$6*t*B*!wwvf_4qaiBwai{B+QTAey$~XiYwliU}rJ$FyEq-^<<~jcZL> zPdPQEOoAN)ClFLhqbb~j@E={dtlvsr?FeqhQKiLW1Jjc5!2OO%W+7ss0{E$DNW&LA z%Sz_Y+I@gy<^09B-UrcF8ZAohZU*|0KF*+swmCtd0UOk=t#L2?v^>1x!lFEfC^OGv z!+T-5%Wy7QJ}gdqfhBaL2&DbKTge{c~@w*HOCl_ zV_<$dLN-9i*%L{HED2)!#P$QQC|v!SRJ1=ppYiBc%FTa zoV=!OWNKXaVm3MrsongII7DA8-U}MFZ@ydF-{YwtG$&FLE%0)o2wtw(6PKx1FKe># zIePMuU2bNTKI}t8<62AOsDf&snV!h6i| z0tPbO1!mlg3R$m}DH)rr^9?HOw@Vl+pu64;+Q?tm4U|BQlw^2fBnWEsgQ@*Ja$eEu z?wx+&a{2jFVvwDf3q(+&&3bK7OYgZ&lmejZpt|MB0`KhJI&qsldwJqzWti7a`r#h) z^Yty6NB%3K0tBPm`~)0D)o=Qt&tW=3$tI`aI$Xyc-+kiV-+)EEi9kl34RM?t6h5UI zD61b8*m?0i(23m|{lF4ADLh!wC?9A-7Swa;iVceVC2aMGzXXbK3(mnAD05tUf&O_# zWxSCY88q+e^&@`!&t+H$1>6o+E6?zc!0uL)?uJ|x80~1kBFuaFX{T79#CV#O3gQ0v zb{|s4p!Cf+Qtk!0w_PZSdVKQ&S2BJKViT9v2S)jYkeaYA9Jd;tFb4hzIiwNF5v=gN z?I52ypD;b})}P-ese3=`=z^)%XB(3}VD=ZUbN&eA#6E7cuvQi3&#ht1#(aj1%r+jE z+>U+P0i?-7MjyBHq>0~bwRHx!Re$_dAs%#3MmOumfwdf&H@{2kX5lypT6DvxMHQ*e z>r&cczRkFkcYb!Qjz0=}BHpQ}YtH#2cHOX*d|@A#<-Q>kCwL%hKe-k(uMwk*=rtA9 zj?M{39s`Mmk&;;#mjsbPUC7(>w=n@=$L^j5BkkmQ!U{7Xi<6`8sK3NzuXC~oIJ5i9bL6|3u>dxss1kp6DX*awo6- z^^5=f#g7kRVP_>xv@VX`Q2bf<{vt8S&zBB_krGYMNBs4RpRx9!o^F=^{i@g2Cw_X% zzjFkC`b(A*kS0*fF+cv-FMb~D;TOOZBx;7r-n2*M^reAH`7lc+|1|5J+mg60@1i2M z8>!m~7;uGwvNA>(oUQS(=w4+Y^Lmu*&%^!cS)au@!0L&PPAjGg)yL&l+uiREzccL| zaPCbW2#2BMeln9k&E?0C_her>I)W{}gkDC>RRCkYmy?u~bk8}3#66T}L?uhh1J{`T z?f>franF6Vic6@56%Sv{s^cpSBPZIFXpC+w4wTE?~FRSr7UEs zF+b#9+nHSM7$t3-0lC}^T)!EN_6e8%C-H$e_>ZPRP!F;+aNRre-Eo64z*AIjD)v@x z4BpBwcf_4Kr?cK6pm^b)=}&V1%QHt|lDLmj>ZepLy>ABWh4n_;!VWb;>br=X@7(}e zHMjJ5MF03FajqS?cI?8#2mF$8Uz3MJ07csymMJW42|NSwc@zjVH*-d_-t&ki{{17e zHaYlzixal$_$VbKQ=JdY0$P1*H;_bD4h^_$1+ZB#(kkn)qaPpqgUJLc9Y>yk^(p{; zG^d=q7c(2TERM7X72YTNDwg$L!oLO1L}_XR-CX)a7MBG2VsRZQxkt3iaq!M9#Szp2 z@Baeizi1jDxdoT7dzH?)@@9E&-<##1x*R$Ec;mNIzpYYUERw!+blUy0xi#IRyz-~A6Rfd3=e|53fatk3_b-p|JLKS=i%ILP`Rr27ew|38Uz|Gz-D zw(nadnLqZ``|F&m0qBqhMkI3)s1lw$)Iq&266dhZO6I}w-4ZEMui9z&x)A!aVgA`M zICl+*w~n|z%D-{{27knp?@qP7?>Qrc4!@uhG)oDD@;xIP%ypIrdA4k_$New*{GZRc z|NYm9m}8SQD|FSgOHOV(_z_~hk$wGx(MRo@JGu8auY#Q2VHlLstz8e z7Qg=D;I*RW+<9`eq9ykH=Es=Ru9)Dh+sfuk5{qdrXq{%?MoUo7C=CU;|3*kkzhaM= z4%+{WN;Y*0+M5Pa?#pbxXrkfq^|PAyTz4X|u=eOK%cwR_TF3h;BKf2Ou_vKbyR??? zTMZfZa=S38ui%UF({`4}9%Wozg?bLXq;a#gDb-e5 zf%{+xm)PWkml|#$)RO9A2LUY~yprZ(8PuETF4p|WZ!D;FX||s|ZZ|rw ztx=;LhSMqcGB$OtPeRzQrTop{fFzF<=Z4&Bc;YU8s^k_9>*Lne2=}V?LLaQkbOpj5w=)MB#!Eko<7jFpnFHRqvs=xkHqbAnD({`sf5|`(<%>E$dzOm3u{xUXH zSQ~gZ>XM96$oalZZCN2REr4L)ZPnn{$WGc#N8H1DX+zYH_A7r+c9~k zm3s}TMRZ!#>ZtPV0qHs&ZPqe!F{NjAT~>C)BXIjwr0e$g=Slr%3QH3KUK0zC(B*a6 z^LoE4r-S^eZ|=e3giR|QLiqKZr%;UGPOugQ?M>7wMoX=yA2YUs2 z56-DMcJ)^E^eE(Re#tP=0+~$#foop4cz~KVWPSZ6E9M5ZMD=Vj1Ws1_z5At-ALc>% zS_@-;qeI~6*fiO_*mT{xa%sxxfU>hc>C>mhFfIw>*VxoQ3@k&^%SH=k2NfL3L#kJn z@B#2q`iT7;K2AKNE6%RG`_g*3>|m>cUOH^VZxYd1HGHRPTZfiRU zWQ%5Ng5aZ7snR9YT}zis@yb$90=trojfWhPO^f2D5pSvRI{9Eyhq54^%+By$Iz5tv z)!2&IDDi80zXRd$cVU00EV@wat?VrABgFgigZKZX=G*)6$$bw^RyZY(uH00WUTihV zXydy}+#|`loYXVIMS+@UgGV-@+x?b9r_#Lc1ZN0^Y*O(Gj0y^vd3TeK`Ha^eh7asWqVP>V-u)bY(9bLEVb zmc0oHP|?+1v%MKiW*uPAmtfP(2u%7*fE~Ez1#3Qm0EIT+>B>KII8ypHXD}2+OBx&@ zPA2=gZtohBcfhN!vBgJDB&q6{FJ|nQFhqY~IxU2%3B3suN8U+=hK2+XJMCK-kx4y^ z1t5gzZ%^A**pIbRZ;4iv9n$Fe$UoJS|!P{2E7rw%@sr@g_L!-(LmHy4- zLiOU3*0K{>iVU|2`ex(XK2WezIr}nkPkUnHVVep^I8>Wqxi}{fC%-kZg3>g9!DC-6 zT(M0|$gBnjZ*=pShwH?hjZeE_xlc}&x}Huiu}UFqqcmG)4y6Go^V$I(}EH zYnQ7i*xUJ`E9|?}rk(A(LjXN{dNyM_!*Aks$u7QnWSWi%^$iIeOPew|b~)28VzM4* zajZu^kRRZ*_jMhuTGW@+ej$aaS(58~{0*zIKD;rlhcT~gTIXrZty7XQ3Ov8}-g}Si zi9?g(H-^@*DeB++bKE9rSJNGh5fdI&GlB9?Kj#=I+7<109!eVKe3b)Y$&`*?)HXd` zz*=&L7g_J^<%Q;nu-6Gyd+s&;or+t?V?1K=^8%~If*(lF^(+Wf??=?hf@j`;pId4^ z2P7A4Dl+1Oy05TKw!>hhgVzUMs|WfTM9ri+R5QEub-mJx9q#gUpAvD{b?801yBa)o zrx#?)E4WrZ##X#Ewf^+b(LqAU`rzXs82bD4#y+E^(R%y=bpff{^LTx>?5V zAYAaMT)5Z7qWa!c5k$RJV55!JMcL6$w)t9v49>M_)mYUMCL8nhcqf%tHs=o(2JEB8 zGA@Uauxwg(YG#MTPFv96n{ZK+zR-_2j3G>nwI~=yj2o<{>@_&EKrK7yU5?$Vb&3D zL55zmXW2evk&C@nxOfnTE)srPXZsu=M)r$>ZK%bbEpX%=^K0?M-Xu(}lRb=-VSMwg zdyCm_`)~BZEt2RT)28ZRPD&&ITo>-{4zu&j52k@{-_9&;oS9FZXx$mAvma)WrYrQ^=GyX~DBS9mEQlTcK>(<_84gUHUkR5n6~JYYD`swuN{I zt9w{MHY;)Ye&Ua6>fV@(;H&wa{%t5E^)Sg3qL+HnN!HImP|*Z#{E#>&u;g1U99&MmW2 zkwf8*xCI_avv&-)OwtfgKEf*pHK8e?tM3!}uMEcZ&uX(;r8{$4hl&%#Tfm)V1A6B_ zY*O;Hv6gqSvP~MjenoYAag7Dk_R2{$uXGhh=g!^dJxyImT0u4!a#P>wjn*)IGHAtG zY!GM)d_j(T)2Q6AT}vM6qybEzmSPqViw-`4Gb3y#Wt$e~DfT2$!$kD3*q+H*j(!o( zGW%*5+6}C>_5_L$3V>c_$3VX{Y-95C=gSFtW%d2}#?>ahGn&4gZkd@%bIfN!*2bzV zTscrd{ePbOleYT;K%V#YDKQY7EQm>_Y$fOL8mWY@CbjIWq)z67_GhutC*@jZr~LA7 zr*A(WPWmu&iApEjE^V=`_Ks^#u{fhC<5{t8U?(p1v`#G7pxH~a$HpG&vO1p4wX)pv zxYs3w6`nBDq^#%#cMG7+jsCKKQz3ToaqC?RxW{SaRb}JUdy6Yg8uEeap?e!|U*TNb+%vG8SR6qkArE_l+K3!I3=W{`DT#IDNaRieRDB)gzPm_h{e89Z{y1 zGt3w8ux{E58{ihm3JJG3*i^eujq5>odM#UcN)e~~Y)f}R;$ z42LqEj9zNIaGvx{Q&d3imZSr{&-GQy3r>T6EsV;g$9?3FM5|_Br6n9Td3XjrARrOc z)Z~t0I^mTq?c$z0V)pN!X&tRgMKL>%CQN_7l#{4rUsAb+r!T>wYgO&;ceGaP8T&pf zn{2uHZ!I%xwZrlBXHj^dG?u^%_HRwyjHW+qUvIzDD>j}+=VH&#KW~5b-!A;>?5WrX zxnBDe&rVZmxw^G6#$wrF6*d*{oYVLH$x~5Uud#uy+!S8Fg0aVVs8{JdyU6R0WMBw@UGdS!?=v%iEMH#TEAEq?>UD_%U0F z^GQb9Y-T9{g)7V*yTSMQq7Hz6zHjb21Y`@8TZ?pH-$OcY-Va_He%=m)exH_hr5ici zo3OW!06!PWRdI9+_YYxOyT28ynaVT$l2{sDUpvdc`24i$phW4m4TQRkH7Cf<&%*Vy z{GW~3pW%f(?JO1-->I;fxA5%26_ix{mvi@jg}Bok%KA2kwf}6j|l)h8KvI}nw^`=x7 zl+r)8V+;-XeLUk69pK*tJu`;x=C+~ildz6fG&Ha3DaVCHjj08FQB1gah?33OP}=yy5MxGZA9B+US0=Z{Kw2zqt#LK8sR}mK01AORQ2!o1;BG-?=Ox2}ZZDyBn_Z9ED)d%FR;&<^KJ8Vw42Sy_HeHE8X5v zlW)QLpH(?x$bxiS*LX!n?m)tuKk$oQ#Br05Z)BQ9Sbxa{kZ*n3#iB z$i@&o%1EWkCtymUqye&aM+>*xDVs?nr}`FuG6~>s?ndr-o%?04bEx&6z=i#(`1~OR zcGtFl@H?}QwQOn^x=0Q+r0mj&BgBB2Kk)k|$8Kd3W`I-x*~kdWuDm3=v%j|Yc-^1X z2pm$LcswM<(t)h!ic=p5wBN3kTB#g~$WC4`wOu_hK2(uYr1f?)t;XIzCNt4B^lzb9 zxABN@U<1bH_r1+8#zAZK?mH_I`?jJ?sNY1|mgd6un~xvAu3Ci%3|8QkUW@xr&-ftE zJ+a|EQCdB}8ZIH$MN-Pa>i`7h5lpm$lXzrW+9_@Zk+{Ux~UaAVxIm$h+a3ij^rTiG0Q|g;%&&3=XJ^M_){Rp{9 zJ0W+^ebo-#f9lhR3RRxJ(BIff_Y~u+&ZMic(t}RIOhEGjac8qNse7!wcKcuV{d{6< zEj%!i>pC{fh~@5{z`t9^`}cv?t#Q5;KU#j=4RBd9Untd+Z7gJu4%qobP`++y`P7{( ziRe{k4N0J-_1mg-#I{#!M~Zs%G@-lQAMMlm@9W%et+k+cY;lW+tidRNW6CA*y?<%T z;HXOcy3JITWnZ4~)$J%KaH9PEG3{}~r~vm?dY|&h@3Fd0oaqwig;z^E$yf%BCy%K)-rEHt-J0e)8+gG-KIj=oy5A3Elt3T+EIRwc(7k zeBT=;iWKsT&(3J2b>1e%^9STG{CH2mF7AEJfqpgRm*@C`1+RatS}Ck8nvYc1WPiLh zbemA0C39UADT(zaAx6*>_k@9jw9}NMd!Q-Igf*-js+zHt;5EV-nUh54;sa-CcKW4& z80X_+9GX2Ws%0N1X{vVv2Lt4!Wv`Gf%=~SdR5<3}naBcuTg{yay}^HnC|m!kh7^5R znzc0gcRWM94Lzq{R%1Hqjv?lj1~cQcYuy8aHq`qUe`IiQ5{wx^a)KGB6O^@x$z`-M zCdeZRlC#-Kod^l=1+jo_)+zZM(@D9afgRyN=CW6Cy^MbfpXi4nka<6O7aN+t;FL}T z(QU<_1}&87QzzfWc<&_^Y)Hc6vthoRwbURdN)!L`-ev&wdgJPyF&20}OoxG+vVUvz zu=Ohi<`Sy-BV(x~bGd;2wOql1V?ICMZr_XL*3wjFmMgRdhVEEcdref+3tqH?F`_Ej zCm!p_H?L*+jc|I?jS<@Doqu8S*6zlsx(yEopw_iKzLqUfxgmaur0`=xdo-3lhCwSE z3&2{oJbdP}*a*zQ)hj6_m(RMa>+BOHhSFJQ(9K0|Sx!A&zI6p6J?_N9-{oo03sjE+I zftugyxLVLwdi;4rvy;JPac9uHP9~5NbN%66Da4As?m0kq=2H*d-+kCWKGPnaEBwFO z`_8bYwry=iP!yy{RYDO|KspvW2?_*7stBkMnp>qCkdn{@1vEf{B1MoQ9g(7d^d?n$ z@4ffl>z#3*v&FryIX}KX_j$gfe|SPzbFI1N7~>u79kD2+I(dmCYU#GIW=uh1%2yM$ z@G|bWl(kn|%E2odWp=h_E2u*eg@CyQ0?uVPV7XhIE{6#jQII^4Xm8;H>eIlntYwX zF14;7MTqjQrt*7YvAg`ae<&-djF6tPyZXWgZj<`k0=363Kk^_v6~m4 z{DK`HLAh5K4=q_m`BAf3L>vCLWJ0h5l&lzP=WfA^U)tdFSwppVk7!S=0LU`bib*qZ}Ycx7`kNS zv*KQm1E=MeEa<;nE5RV(WFEV1D$ejbW%(xzy9~-Ldx+og622VcEg<{P6Rw1k{T7z| zxLhz5@Mwr`&=X#h`~rCB;+Eh#_ka6=e}{yBzA_~MbVOXw{f9M=`aOd!>YFHyU;E#$ z@~iou`{MT+ayZ}{3Gzwn`uchXphr7sqMtY3D=|dgCjhMd`-PO&mB$b*X`k#o=bam7?v`OF=e!Mr4MP36$(#&$p_H z3qyZAa}HtPPS-S*n9`?m1*uKwUZohc@%5lbnaq6I*XX##Z^_@fWugK^!})RG#0_Ai zMusXAlfK@gKF1VJ^T9@1Tks4GMgcyQAXHM|26vsdMTfnH5$e~=R0S1t^`Zi zSs>pm9tG+DngOW6BLd}ksj|mM7(4J)rz@8Ox>SHDuy#Ed_197cY2I`ZIcSH(x!-F< zSU>5AKC&NDtCqu~KQTFn$>1u~Q%)yU7a}NfPw}OR-4AU4cf0kkQOn&B>A|Fm+T{3!&8sAy)HAoU=^~;~Y z_D|p^O>wiFaxJ3^l5(&OzHHnyG-m<*Rel@h!IHau#1{`dE==!)>@$17xX;5f%g7rB z`cKe%o8!&a^-_WKyxI-I!G6l(+~)WmN+V>ox!h^wwdL`86-wyNldWOa{h&JO?U|v? z*Kii&$<43i$uctQ{VpzDhF*M4fZijdPITD&s2JL^l}sSdcSs1}-B`*~?HV(efH1_a z8^?KrJ}L)fuk#28t8tu3!Y|O_XkO96VV9%IznqrmvTQ{0U_P`pdgv7==2Fgp8(9;W zmD2ns&0*}_qo%jqw=rMXRVBfpvvbs5LS08O7Vn5Pj1$0>WVpdX7*#YZ)P z0`?r;!h)_rki$-C+rQtxr6~@0p_&>#Q12_{$k%&jx$ zup9`h@8_MY2c$)B5+`tEZI-a^rJi|^1f2nm8_n+?g}no$wCgdTM8C~ad3mf>5%9t* zf`fq{wspAWoUJjTd58v#d(q&_&< zwF|R6q2st%cBO28h@ns+d~YSL8PxTgv{9xrP&@hz7VK~L@6+GGg(X$6IGA;T7OG~v z)c})b0@xMy{8JobI-FNqwc>CtJHwO5pc^u^3Gz=aqs!&*8Tni>`I<2$cd6p#*|?Of zAq=gQE*M?5t;u{(?4a~8wh%2^%KX`)kt(Hm_YZ$*$>8e;khh>OF=(B6?Yvp7$FDvL z-QTRA^s#gtW}iORgfUIdjs=FwG${k~A&vtEf~HO!8#(O=hZv`|&bSGrBT)IV-~Bk_ zYs2fRCd3;7qiB&*#t@MER=AMxmvu54AY63i zPJ_8&_tkLnCE@LXgI5k}zN23NcsY{2+7kK`^}EV&s8W!EVPSof^Dg@i)aUl5_4b1= zCgP2CbcmjF7mFs$g%OXc-iru%C0{E0g{6e9r+rvjHpMxRrN zbD87E992u^t=!Sl^~cjCzmKuUe)uTT`M_@~2S{mr4unjo#)iDx3D66)eZn!MIP^|8 zf%C6v)D>zZE4Z4G;$*Hi2ljW+Flc7$xg?NIIftE%3TSe_#(TY*&w0JJ-F#n?3N=eN z!=1M#x!pQP)4^;Sy5j946rvG%IS<9qF5zIqS`CVq%kk|+Ql%=Ai=~$hOJC;YjENCt z=xsFxHi?bD2T3K5XsExncjxSy!I;3iPUgRqJ3d!};f9LaBI`U4Z0})CD%Sy0QaHN1 zyDtEg&b?b16DiELXWp+a{xpTjaVK4H9wgJ#KBoZbWVc@f&6-fT`R{e=JXI6%J6icu zcUqMSK}b?O3A9M+?M`Vi;7HZ^m2*bkC^pUI&i4r_Kt|N+H+V*>JjJl-2PDKt>*AC-K|wI^nF&V%ClY|9HrjKBSMr9-R`VE z)3$gieDl^;(J|@NAB89lYn;%{8)~-1FQ#v=3`qk$N{>&uPJ}{SyD!yCra5QIA>d(I zd(+QLUkjm+dD=*}$60mDMdU)xCD%_MJafad?;U@C3beg{J_2p{U|m9o@2Eghs>)hx42n<(zdL<+6c%l_rCbqz|^ z!^8MFKp+^~R7TY@NY_39`mWwbdfKg&*KvDb_>biRtiO?7Mt?urU-Mo9qs4Zj-mVEX z3_Cp?>BWDuk$!X~$#EvHQ)7csbY`X zeya8<02T~o@Q90)#*o7;z|YAf4JZ?=5~{b{g~oDlotDzF79V^;(*$Oj1?bUVhZOG4 zTB}#c3H@sZard~N_99d$WDwX9&jO;gE)@kdY5TRdaLGrI^1fz9`xjC;RmnP z19WXW*&l0#W!lyR37tHGYP=+|e@)6PQp-dc49v8ke(Td$`lHi1+l*ra;yN}5;RtM> z*mNV~R|rI)jX!hONp#r)ILVEuK$TM-j;we}jNsVa8Heh9f*~1g>16pRR5$hZG9NpK zh7b4P2fB>~auelJx6~PfS!M#sAHdkNRBf9UH8`C$)p(KPN)MKHmN6e4#bBolhI%0| ztfV7@8NczIqgW`I5hphwRp`7st*1vW z5a#({zh_)pYu5l)=NI07&on9Wk+tM6&CuHFAZ8NyHo!LQr=>lb4I_41@3t)DuNQVJ z#hZ1Equ$<9U^Bi`a8$Z6Dl^lqjQS83_9U9oHb*jWnyG)Id}^VL`fVa5EfnzHwMIN9 zZmW_hbX0F7yNuQ;UugCL3g@wg#cocJ=~cI)Mz5>KtcJzL9@Z^ig6m)D5<~~sUg5NM z+T64u4qHxXSx5WvrvW7vS{-?wlDfw$%!DiI)_1tcoAc=q|1Y~n`P{+#rg5|B99k313 z0`xL-6QR1{Sm~_^!}vu;wD?q`8m6zot(19ETjeL}EYhcA6xUX%K^@dWpyCaWpIS}k zK)FThsP;<($7RWUp%)g}aC0*p;++^!%<}n2qito0*|%wYOFbB2dhYb+bkr}s^?UC= z3cW*)(+%ev3-U1+%0rUB>S5wWHwN4$ztj7XcGk~x6N~uTXq}~oZ#kvz@MdL`#SP6J zeZ}OTEjn$bYt-MfoT(;;WFC78CKipB@rRdic)Z5Gb%FTWZPmN%`7xM!q$MXiBF}=B z`>tf)l;zmcL+*>VL`3!pQ#QH1VOJ5nd5oPlvI-- z>0Te4C2lq<0vmGmM8VfcZ!EU^@h-rbO<+>4t|TB>(YJ3_5{bdvNI8N24$d|>Wjkr4 zWoq$JAIzkc?n;)ASg>W*pogKw0O=`dVBo!od2@N=nNANeikTN|jxisdvknWBtk7uz zezeJLxl-Q-N-&*^9aSyUjB}bXngo;I?w?cn#1*c!LX2IjHZFD}k=u5+-XUcha|M_n z&UEcNs=8jN5UDJLvxX<6O^c%+jMKI&!V`4%v-+4;HE}5a^9S)euQBkT6z_RS=>7ck zxpqY7GYb|A@gI^u*aS%LDTI0y@sy-Tk%_st{}b3e~dBibJ(+F?5?%=BKYauRp{f zz6*KVsx-AE+%^Raej27%-y$S|`_Zbk80EoubqnvuB%9g}f&qvEU&?LnvsmOi@6&1N ze0ILGYJ-{^eK^G;fp4I0-Q_zFYHY{^I2;CWIC?FmYE3-wYl>1-c0yM{)5QZ}7Ff#h zx8rH%V{FvEUWjM`ht6uu%xG8;PHIgXqyGe+5QTC85D#84Gq*LH*%q2ixNCD{D z6dPjEW#Ip)J}guo3weic;Wi1A+_(Jnl1aMJ0twO4B~FN)LA~L zl4YF8NB6?4hP#JE_k2PfZo+fn3wdgo!Ez(gsgAzKEW`5K+6S6tVbdfRZUr@LUGwkY z0D&u(MH@?yfk$LJNK`5v&;Sv1_P}tOpbeb?2Aq5nKQzG{a>Njr|9P)xIz@RybSd#$ zrK;7;t5_^d$IS9AKpIh2%Q=jQ?F#4~5)%e8H3ew&FjIe!$Q8%BdqWt-SZ7Ve|Nx0?6PQlAV ztJOg5%Pnck{CqBeJefhashye8FOD}FWHvxa+DL!70FclJb>a6RG?JMnMN0Y%r zt7o4s6dw~IKUE9npuh~_5)gtV>(v8b@ zT3!o`an4Qp_NrkI#$sHH*&6J1VvC>?{Rv;6fY|&FNQ%)sN9-i6#L+P?TJ41Pm+t@oq@YT3Yz78Q$(C zrc{#&jhmq?1_1+Pa$fcYQWmM`Y0vWr^kmHKu*A*m@vy{q`(g~VQkiLb2rXy+^+Og< zy{-D>t1Vr&yAuHy0^b&8Y^Ac{?l-fu7nkVAO_Q(MAAAoek2*+ zu5&Zqy?a?8RbqS9k&H6#AR|SW1{;xoW*^#AEtfRNfBg7dd=>Sxnr+!bvtZ+q0bGGMlo56h8do@Ood{Y51Ck~6@ZG*FA#{KaW{j91yGjEg+ukv}mlu1?# ztuGvtUT{q>KFUOUxB?$21rS7c5&&b{)mWJtltY8eu~u4U8=tT^B=&VA=d=WH=+ev2 zn9`0zriXesvKS8++vQK|%J~Yq&EUgbIZIVMrXJhaoz;YQMlJJzGfHM36t0~wd!LP@ zIoCA?Ft}+Keik<(o0dBw!sJfI;`_}5pjt$gx9kM+&yZG2;BD%@L5iA2eu;P!TqXAGl9V`7U2XeHJ7aswBkFx^ z?8wQ6l-hxTzh1Lq7Hnmx$FgwuggZXIR)TVJ@B^Gdl~gjy`-Ug1NFal<%hK}qj=hY^n5l4WiSL?+_rLhMZBPG^6TuV0-1P(lgK6DPKb zl0@}_*p1$bh56;T3j)rIUXh-rbs^!)Cp|?oCyIAUX;em(uGBXkkr=yy*Q577;Ucyl zhL~&m#U&cudN2B!G`3tBKgD^glv7$Q&4rwgIX^4SuC~0%YeI^-c;?}4Vhg(-WCiilRN-P0b-6 zjgLW9zUk%~$8}TVhe*a99`sX@77Im|pO$Nos*~~%N>N!n@(LdXSWPrVzz=PKP=&#n zD<&E?yZE35?e(v4469`+j`(27DO!bX-yyni7|&xhBQ-gucy;CJvZmDmM+^p7yGB`J zvPB+gkyKR5qQZtq-Nsz>E1ap#yuon{7NgHD(&coCQKGJ zRS6TEvYS-<>P>2IZ$x+_S&o{o=-Ml^ysr~*_>47VF^Jf~TmY2$zDRxI=s1nCx=48? zwB5$Wdr8`7EZ1ii7^!xa(mIhthiJQQ$jWFm2~|MSDznS8;YG_sUIKHPpkGl_+G6iD zEZO=|eKuC;vRxEyerP?g^B&OPms|4ls}ols`R>)+V|v(SE=rEj%nBS)0gw`g<eJ-H~^U*i_nn3V2 zG(gXJ^Rn~KM(}{)r3w(_u7z-VgcOX2IjGkt=1qM}{DE?lzEoYTow-*_b6uhywG$)vs~agyO#Cv?+t&71hNuQluLI;Zb|&A zK`R9FGlF$6fcRrYfE;fH34Um=(p(q<32cF|Zo}COQEOhlHrJVhX7(R{c*s?m*u(Zq z?S+Hy^#E@D_cJoSz~@|2)j|oT&}>irzvNs5;ebV|Y~;J2SJb=2EXqb^LQj0iRSwuY z#Ea4sgzuc(;zy>mpM;Elr9_RHASF>h*-M|*eAFd4EtiFSf*cCpRQVQPJw)#x&q!#Q zS-wB>b$uDSRwkUPJ)Y+IE$WJN@DL#*c~+Lz4}Uuqzr2N&@S>f~)fMVjOFE`IfLmcLB@*Ic||8fQJx&XN{J518x+Y5Ycux-K{v<9&( z@AiVQwlnBmezo1uP3RW7;6*xRv(5BsG}l$u4IE0#Saqj=@y7ms9u(ieN-F`sPGT7e z5PvBT^3&=kX;s3drIbRhCQ;(%2$O#)4*;d2OewXZipkO@#&G-~vicCWoyzAy!F!9-T=_~GtTc~(&Hs5RH4Sja z5SlF)E2_4#F%T@=D!@)qVc<+bQ8~Mx2O#gCQVk=eC?iBtPUcu zf1H9Onk)co6Wx6PSvts6F}{N#{bTv)0h?CVX)v~if~hc!5g4*y;XiH)=`Gg0yJSjK zhN@ZHx~Lo^c-9BxWpnNoxYO#;V;A-E1ra&}-_G-sw7Y^D3SW#dt0Di}t&!cqAOBxN z8Ss@Be-=^{Upu^hnqGV(cVONPUv+{1;ab5_{1+q(n}CBr&IVt6(XGML$0erSbJbs+nvmG>0UlhW z#Oei$iYx=N?A$wVrTakFjP98I^-`wmg!1!(j3ixr3X+UI4pzz@VRNq#lr2;B(GD0iuvo_hlwer&Nq{$KL70$|AbPTRy0`;rk6)lA#Bzb!HSv*Mwb7U)Nj;s=DNz+lgcV1Utz zWr8n$kHCF^Uroe<@`(Rl?Jtx1e|+DxBZMDDgr6!P z+m^@4XxpzAY7sgO0pzX_o|?}k=Z8N@NLQZ+d40!?)NA^6J=x5P9Ry4HiX!MLKYxQr zK+-w8MqcqzyAHkPkf((JP6UfLa#sn7F7Z)67?y6VV^Ci_vrH@yESlG%&OsIeDD&b_ z%i{uAKPN(RNW~aIM9tx?Z4eC>kBDpfVj=(*2fP$_p5f^gYTsRo`@{YOAr%0N*d<|? zXZ-w4LCm&eFJrFv3_rL6=*xe$w*1?f@8 zF`gwdA+;0+i#u}#Ng~}U9GmQaM+v_e(N*Ff7r|DZ;JEY@LX5q@g{{utxMEt;7-8Pl zZEGztzsA#-MM>~2Y`7A=CXv6u*s1^E#cMio1PS&cxJtp!ZsKZ=O+_Fs(YCjjF>&tb zXY$qQlY`!WN!vevfc@cGxyOFQn^#b{xbi3m$J0E92^&>-=G}R#Y!jChi>~Imgp(e- z7JN%-S3|w;7rH#n-rUYD<|K5++)2K6_d2T#XOQzRS}|G z!*`Ywr@Ox*m;=JYlq+U;9J`VfrOkF}To>DWEIuCe1%KH%@WLt?I_7M3_cAT)?7Y5e z^R7dws+B52O4rWsb!$igv2`mZp>w7stW@9(YdFM76~en{3tsex-hKgT%_S{D zyFgr!U)0%|-fg?qa=$b@=xxNtNp(q^RC^tkFwqHwQoq^PZ71)3$1-k$r==gpw#Ldr zzGdEG&d20HrNTM9Yf{`Fk#UNT6r<>U!;g@D0sBQG`kg0+kn4b7T!V^=Rx9WY3n55G z#pcV!#-&`ULC`liOhiXaTt*ig4JWuWq~}hMVrYc?3F?VwFp|SENO$hZ*Z(}V2S0g5 z^=gm3=8+x4&gf&F*8{*>MSp00mO-+zVi gPlfz{S9fh3(~Rr&S%fLyIt2dRkW-dTzNYW?KL+iGfdBvi literal 0 HcmV?d00001 diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 3793aec037f..11effc82fe7 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -541,7 +541,14 @@ const sidebars = { }, "realtime", "rerank", - "response_api", + { + type: "category", + label: "/responses", + items: [ + "response_api", + "response_api_compact", + ] + }, { type: "category", label: "/search", diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 7ffbe95be13..a1fa4ef8440 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -10,6 +10,7 @@ from pathlib import Path from typing import Optional from litellm_proxy_extras._logging import logger +from litellm.caching.redis_cache import RedisCache def str_to_bool(value: Optional[str]) -> bool: @@ -18,6 +19,103 @@ def str_to_bool(value: Optional[str]) -> bool: return value.lower() in ("true", "1", "t", "y", "yes") +class MigrationLockManager: + """Redis-based lock manager for database migrations""" + + MIGRATION_LOCK_KEY = "migration_lock" + LOCK_TTL_SECONDS = 300 # 5 minutes TTL + + def __init__(self, redis_cache: Optional[RedisCache] = None): + self.redis_cache = redis_cache + self.lock_acquired = False + self.pod_id = f"pod_{os.getpid()}_{int(time.time())}" + + def _get_redis_lock_key(self) -> str: + """Get Redis lock key for migration""" + return f"migration_lock:{self.MIGRATION_LOCK_KEY}" + + def acquire_lock(self) -> bool: + """Acquire migration lock""" + if self.redis_cache is None: + logger.warning( + "Redis cache is not available, running migration without lock protection" + ) + self.lock_acquired = True + return True + + try: + lock_key = self._get_redis_lock_key() + + # Redis SET with NX (only if not exists) and EX (expiration) + acquired = self.redis_cache.set_cache( + key=lock_key, value=self.pod_id, nx=True, ttl=self.LOCK_TTL_SECONDS + ) + + if acquired: + self.lock_acquired = True + logger.info(f"Migration lock acquired by pod {self.pod_id}") + return True + else: + logger.info("Migration lock is already held by another pod") + return False + + except Exception as e: + logger.warning(f"Failed to acquire migration lock: {e}") + return False + + def wait_for_lock_release( + self, check_interval: int = 5, max_wait: int = 300 + ) -> bool: + """Wait for another process to release the lock""" + if self.redis_cache is None: + logger.warning("Redis cache is not available, cannot wait for lock") + return False + + logger.info(f"Waiting for migration lock to be released (max {max_wait}s)...") + start_time = time.time() + + while time.time() - start_time < max_wait: + # Try to acquire lock using the public acquire_lock method + if self.acquire_lock(): + logger.info( + f"Migration lock acquired after waiting by pod {self.pod_id}" + ) + return True + + time.sleep(check_interval) + + logger.warning(f"Failed to acquire migration lock within {max_wait} seconds") + return False + + def release_lock(self): + """Release migration lock""" + if not self.lock_acquired or self.redis_cache is None: + return + + try: + lock_key = self._get_redis_lock_key() + + # Verify current pod owns the lock + current_value = self.redis_cache.get_cache(lock_key) + if current_value and str(current_value) == self.pod_id: + self.redis_cache.delete_cache(lock_key) + logger.info(f"Migration lock released by pod {self.pod_id}") + else: + logger.warning(f"Pod {self.pod_id} cannot release lock (not owner)") + + except Exception as e: + logger.warning(f"Failed to release migration lock: {e}") + finally: + self.lock_acquired = False + + def __enter__(self): + """Context manager entry - acquire lock when entering with statement""" + self.acquire_lock() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Context manager exit - release lock when exiting with statement""" + self.release_lock() def _get_prisma_env() -> dict: """Get environment variables for Prisma, handling offline mode if configured.""" @@ -346,19 +444,50 @@ class ProxyExtrasDBManager: ) @staticmethod - def setup_database(use_migrate: bool = False) -> bool: + def setup_database( + use_migrate: bool = False, redis_cache: Optional[RedisCache] = None + ) -> bool: """ Set up the database using either prisma migrate or prisma db push - Uses migrations from litellm-proxy-extras package + Uses migrations from litellm-proxy-extras package. + In multi-instance environment, use redis lock to prevent concurrent execution. Args: schema_path (str): Path to the Prisma schema file use_migrate (bool): Whether to use prisma migrate instead of db push + redis_cache: Redis cache instance for distributed locking Returns: bool: True if setup was successful, False otherwise """ schema_path = ProxyExtrasDBManager._get_prisma_dir() + "/schema.prisma" + + database_url = os.getenv("DATABASE_URL") + if not database_url: + logger.error("DATABASE_URL environment variable is not set") + return False + + # Use MigrationLockManager to prevent concurrent migration execution + with MigrationLockManager(redis_cache) as lock_manager: + # Lock is already acquired in __enter__, check if it was successful + if not lock_manager.lock_acquired: + # Cannot acquire lock, another process is running migration + logger.info( + "Another pod is running migration, waiting for completion..." + ) + + # Wait for other process to complete migration + if not lock_manager.wait_for_lock_release(): + logger.error("Failed to acquire migration lock after waiting") + return False + + # Successfully acquired lock, proceed with migration + logger.info("Acquired migration lock, proceeding with migration") + return ProxyExtrasDBManager._execute_migration(use_migrate, schema_path) + + @staticmethod + def _execute_migration(use_migrate: bool, schema_path: str) -> bool: + """Execute the actual migration""" for attempt in range(4): original_dir = os.getcwd() migrations_dir = ProxyExtrasDBManager._get_prisma_dir() diff --git a/litellm/__init__.py b/litellm/__init__.py index dfe959bf747..7f7ee21f692 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -197,6 +197,7 @@ retry = True api_key: Optional[str] = None openai_key: Optional[str] = None groq_key: Optional[str] = None +gigachat_key: Optional[str] = None databricks_key: Optional[str] = None openai_like_key: Optional[str] = None azure_key: Optional[str] = None @@ -275,6 +276,7 @@ banned_keywords_list: Optional[Union[str, List]] = None llm_guard_mode: Literal["all", "key-specific", "request-specific"] = "all" guardrail_name_config_map: Dict[str, GuardrailItem] = {} include_cost_in_streaming_usage: bool = False +reasoning_auto_summary: bool = False ### PROMPTS #### from litellm.types.prompts.init_prompts import PromptSpec @@ -1440,6 +1442,8 @@ if TYPE_CHECKING: from .llms.github_copilot.chat.transformation import GithubCopilotConfig as GithubCopilotConfig from .llms.github_copilot.responses.transformation import GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig + from .llms.gigachat.chat.transformation import GigaChatConfig as GigaChatConfig + from .llms.gigachat.embedding.transformation import GigaChatEmbeddingConfig as GigaChatEmbeddingConfig from .llms.nebius.chat.transformation import NebiusConfig as NebiusConfig from .llms.wandb.chat.transformation import WandbConfig as WandbConfig from .llms.dashscope.chat.transformation import DashScopeChatConfig as DashScopeChatConfig diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 8c8266d5ca4..26133ebc222 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -255,6 +255,8 @@ LLM_CONFIG_NAMES = ( "GithubCopilotEmbeddingConfig", "NebiusConfig", "WandbConfig", + "GigaChatConfig", + "GigaChatEmbeddingConfig", "DashScopeChatConfig", "MoonshotChatConfig", "DockerModelRunnerChatConfig", @@ -644,6 +646,8 @@ _LLM_CONFIGS_IMPORT_MAP = { "GithubCopilotEmbeddingConfig": (".llms.github_copilot.embedding.transformation", "GithubCopilotEmbeddingConfig"), "NebiusConfig": (".llms.nebius.chat.transformation", "NebiusConfig"), "WandbConfig": (".llms.wandb.chat.transformation", "WandbConfig"), + "GigaChatConfig": (".llms.gigachat.chat.transformation", "GigaChatConfig"), + "GigaChatEmbeddingConfig": (".llms.gigachat.embedding.transformation", "GigaChatEmbeddingConfig"), "DashScopeChatConfig": (".llms.dashscope.chat.transformation", "DashScopeChatConfig"), "MoonshotChatConfig": (".llms.moonshot.chat.transformation", "MoonshotChatConfig"), "DockerModelRunnerChatConfig": (".llms.docker_model_runner.chat.transformation", "DockerModelRunnerChatConfig"), diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 5b206317b29..a89efc4e82b 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -3,6 +3,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req """ import json +import os from typing import ( TYPE_CHECKING, Any, @@ -22,6 +23,7 @@ from typing import ( from openai.types.responses.tool_param import FunctionToolParam from pydantic import BaseModel +import litellm from litellm import ModelResponse from litellm._logging import verbose_logger from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator @@ -691,19 +693,26 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if isinstance(reasoning_effort, dict): return Reasoning(**reasoning_effort) # type: ignore[typeddict-item] - # If string is passed, map with summary="detailed" + # Check if auto-summary is enabled via flag or environment variable + # Priority: litellm.reasoning_auto_summary flag > LITELLM_REASONING_AUTO_SUMMARY env var + auto_summary_enabled = ( + litellm.reasoning_auto_summary + or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true" + ) + + # If string is passed, map with optional summary based on flag/env var if reasoning_effort == "none": - return Reasoning(effort="none", summary="detailed") # type: ignore + return Reasoning(effort="none", summary="detailed") if auto_summary_enabled else Reasoning(effort="none") # type: ignore elif reasoning_effort == "high": - return Reasoning(effort="high", summary="detailed") + return Reasoning(effort="high", summary="detailed") if auto_summary_enabled else Reasoning(effort="high") elif reasoning_effort == "xhigh": - return Reasoning(effort="xhigh", summary="detailed") # type: ignore[typeddict-item] + return Reasoning(effort="xhigh", summary="detailed") if auto_summary_enabled else Reasoning(effort="xhigh") # type: ignore[typeddict-item] elif reasoning_effort == "medium": - return Reasoning(effort="medium", summary="detailed") + return Reasoning(effort="medium", summary="detailed") if auto_summary_enabled else Reasoning(effort="medium") elif reasoning_effort == "low": - return Reasoning(effort="low", summary="detailed") + return Reasoning(effort="low", summary="detailed") if auto_summary_enabled else Reasoning(effort="low") elif reasoning_effort == "minimal": - return Reasoning(effort="minimal", summary="detailed") + return Reasoning(effort="minimal", summary="detailed") if auto_summary_enabled else Reasoning(effort="minimal") return None def _transform_response_format_to_text_format( diff --git a/litellm/constants.py b/litellm/constants.py index e8524a87c41..1cd2da549ca 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -375,6 +375,7 @@ LITELLM_CHAT_PROVIDERS = [ "perplexity", "mistral", "groq", + "gigachat", "nvidia_nim", "cerebras", "baseten", diff --git a/litellm/integrations/callback_configs.json b/litellm/integrations/callback_configs.json index 88f7908e9a2..6b30b6b736e 100644 --- a/litellm/integrations/callback_configs.json +++ b/litellm/integrations/callback_configs.json @@ -187,6 +187,12 @@ "ui_name": "Sampling Rate", "description": "Sampling rate for logging (0.0 to 1.0, default: 1.0)", "required": false + }, + "langsmith_tenant_id": { + "type": "text", + "ui_name": "Tenant ID", + "description": "LangSmith tenant ID for organization-scoped API keys (required when using org-scoped keys)", + "required": false } }, "description": "Langsmith Logging Integration" diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index f0f355b4895..7e62613a7e4 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -50,6 +50,42 @@ else: Langfuse = Any +def _extract_cache_read_input_tokens(usage_obj) -> int: + """ + Extract cache_read_input_tokens from usage object. + + Checks both: + 1. Top-level cache_read_input_tokens (Anthropic format) + 2. prompt_tokens_details.cached_tokens (Gemini, OpenAI format) + + See: https://github.com/BerriAI/litellm/issues/18520 + + Args: + usage_obj: Usage object from LLM response + + Returns: + int: Number of cached tokens read, defaults to 0 + """ + cache_read_input_tokens = usage_obj.get("cache_read_input_tokens") or 0 + + # Check prompt_tokens_details.cached_tokens (used by Gemini and other providers) + if hasattr(usage_obj, "prompt_tokens_details"): + prompt_tokens_details = getattr(usage_obj, "prompt_tokens_details", None) + if ( + prompt_tokens_details is not None + and hasattr(prompt_tokens_details, "cached_tokens") + ): + cached_tokens = getattr(prompt_tokens_details, "cached_tokens", None) + if ( + cached_tokens is not None + and isinstance(cached_tokens, (int, float)) + and cached_tokens > 0 + ): + cache_read_input_tokens = cached_tokens + + return cache_read_input_tokens + + class LangFuseLogger: # Class variables or attributes def __init__( @@ -757,8 +793,8 @@ class LangFuseLogger: cache_creation_input_tokens = ( _usage_obj.get("cache_creation_input_tokens") or 0 ) - cache_read_input_tokens = ( - _usage_obj.get("cache_read_input_tokens") or 0 + cache_read_input_tokens = _extract_cache_read_input_tokens( + _usage_obj ) usage = { diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index cc9b361b69d..570b78f2927 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -40,6 +40,7 @@ class LangsmithLogger(CustomBatchLogger): langsmith_project: Optional[str] = None, langsmith_base_url: Optional[str] = None, langsmith_sampling_rate: Optional[float] = None, + langsmith_tenant_id: Optional[str] = None, **kwargs, ): self.flush_lock = asyncio.Lock() @@ -48,6 +49,7 @@ class LangsmithLogger(CustomBatchLogger): langsmith_api_key=langsmith_api_key, langsmith_project=langsmith_project, langsmith_base_url=langsmith_base_url, + langsmith_tenant_id=langsmith_tenant_id, ) self.sampling_rate: float = ( langsmith_sampling_rate @@ -76,6 +78,7 @@ class LangsmithLogger(CustomBatchLogger): langsmith_api_key: Optional[str] = None, langsmith_project: Optional[str] = None, langsmith_base_url: Optional[str] = None, + langsmith_tenant_id: Optional[str] = None, ) -> LangsmithCredentialsObject: _credentials_api_key = langsmith_api_key or os.getenv("LANGSMITH_API_KEY") _credentials_project = ( @@ -86,11 +89,13 @@ class LangsmithLogger(CustomBatchLogger): or os.getenv("LANGSMITH_BASE_URL") or "https://api.smith.langchain.com" ) + _credentials_tenant_id = langsmith_tenant_id or os.getenv("LANGSMITH_TENANT_ID") return LangsmithCredentialsObject( LANGSMITH_API_KEY=_credentials_api_key, LANGSMITH_BASE_URL=_credentials_base_url, LANGSMITH_PROJECT=_credentials_project, + LANGSMITH_TENANT_ID=_credentials_tenant_id, ) def _prepare_log_data( @@ -365,8 +370,11 @@ class LangsmithLogger(CustomBatchLogger): """ langsmith_api_base = credentials["LANGSMITH_BASE_URL"] langsmith_api_key = credentials["LANGSMITH_API_KEY"] + langsmith_tenant_id = credentials.get("LANGSMITH_TENANT_ID") url = self._add_endpoint_to_url(langsmith_api_base, "runs/batch") headers = {"x-api-key": langsmith_api_key} + if langsmith_tenant_id: + headers["x-tenant-id"] = langsmith_tenant_id elements_to_log = [queue_object["data"] for queue_object in queue_objects] try: @@ -418,6 +426,7 @@ class LangsmithLogger(CustomBatchLogger): api_key=credentials["LANGSMITH_API_KEY"], project=credentials["LANGSMITH_PROJECT"], base_url=credentials["LANGSMITH_BASE_URL"], + tenant_id=credentials.get("LANGSMITH_TENANT_ID"), ) if key not in log_queue_by_credentials: @@ -466,6 +475,9 @@ class LangsmithLogger(CustomBatchLogger): langsmith_base_url=standard_callback_dynamic_params.get( "langsmith_base_url", None ), + langsmith_tenant_id=standard_callback_dynamic_params.get( + "langsmith_tenant_id", None + ), ) else: credentials = self.default_credentials @@ -491,13 +503,16 @@ class LangsmithLogger(CustomBatchLogger): def get_run_by_id(self, run_id): langsmith_api_key = self.default_credentials["LANGSMITH_API_KEY"] - langsmith_api_base = self.default_credentials["LANGSMITH_BASE_URL"] + langsmith_tenant_id = self.default_credentials.get("LANGSMITH_TENANT_ID") url = f"{langsmith_api_base}/runs/{run_id}" + headers = {"x-api-key": langsmith_api_key} + if langsmith_tenant_id: + headers["x-tenant-id"] = langsmith_tenant_id response = litellm.module_level_client.get( url=url, - headers={"x-api-key": langsmith_api_key}, + headers=headers, ) return response.json() diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 12e60bc25bb..a7d2326d938 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -196,50 +196,88 @@ class OpenTelemetry(CustomLogger): litellm.service_callback.append(self) setattr(proxy_server, "open_telemetry_logger", self) + def _get_or_create_provider( + self, + provider, + provider_name: str, + get_existing_provider_fn, + sdk_provider_class, + create_new_provider_fn, + set_provider_fn, + ): + """ + Generic helper to get or create an OpenTelemetry provider (Tracer, Meter, or Logger). + + Args: + provider: The provider instance passed to the init function (can be None) + provider_name: Name for logging (e.g., "TracerProvider") + get_existing_provider_fn: Function to get the existing global provider + sdk_provider_class: The SDK provider class to check for (e.g., TracerProvider from SDK) + create_new_provider_fn: Function to create a new provider instance + set_provider_fn: Function to set the provider globally + + Returns: + The provider to use (either existing, new, or explicitly provided) + """ + if provider is not None: + # Provider explicitly provided (e.g., for testing) + # Do NOT call set_provider_fn - the caller is responsible for managing global state + # If they want it to be global, they've already set it before passing it to us + verbose_logger.debug( + "OpenTelemetry: Using provided TracerProvider: %s", + type(provider).__name__, + ) + return provider + + # Check if a provider is already set globally + try: + existing_provider = get_existing_provider_fn() + + # If a real SDK provider exists (set by another SDK like Langfuse), use it + # This uses a positive check for SDK providers instead of a negative check for proxy providers + if isinstance(existing_provider, sdk_provider_class): + verbose_logger.debug( + "OpenTelemetry: Using existing %s: %s", + provider_name, + type(existing_provider).__name__, + ) + provider = existing_provider + # Don't call set_provider to preserve existing context + else: + # Default proxy provider or unknown type, create our own + verbose_logger.debug("OpenTelemetry: Creating new %s", provider_name) + provider = create_new_provider_fn() + set_provider_fn(provider) + except Exception as e: + # Fallback: create a new provider if something goes wrong + verbose_logger.debug( + "OpenTelemetry: Exception checking existing %s, creating new one: %s", + provider_name, + str(e), + ) + provider = create_new_provider_fn() + set_provider_fn(provider) + + return provider + def _init_tracing(self, tracer_provider): from opentelemetry import trace from opentelemetry.sdk.trace import TracerProvider from opentelemetry.trace import SpanKind - # use provided tracer or create a new one - if tracer_provider is None: - # Check if a TracerProvider is already set globally (e.g., by Langfuse SDK) - try: - from opentelemetry.trace import ProxyTracerProvider + def create_tracer_provider(): + provider = TracerProvider(resource=_get_litellm_resource()) + provider.add_span_processor(self._get_span_processor()) + return provider - existing_provider = trace.get_tracer_provider() - - # If an actual provider exists (not the default proxy), use it - if not isinstance(existing_provider, ProxyTracerProvider): - verbose_logger.debug( - "OpenTelemetry: Using existing TracerProvider: %s", - type(existing_provider).__name__, - ) - tracer_provider = existing_provider - # Don't call set_tracer_provider to preserve existing context - else: - # No real provider exists yet, create our own - verbose_logger.debug("OpenTelemetry: Creating new TracerProvider") - tracer_provider = TracerProvider(resource=_get_litellm_resource()) - tracer_provider.add_span_processor(self._get_span_processor()) - trace.set_tracer_provider(tracer_provider) - except Exception as e: - # Fallback: create a new provider if something goes wrong - verbose_logger.debug( - "OpenTelemetry: Exception checking existing provider, creating new one: %s", - str(e), - ) - tracer_provider = TracerProvider(resource=_get_litellm_resource()) - tracer_provider.add_span_processor(self._get_span_processor()) - trace.set_tracer_provider(tracer_provider) - else: - # Tracer provider explicitly provided (e.g., for testing) - # Do NOT call set_tracer_provider - the caller is responsible for managing global state - # If they want it to be global, they've already set it before passing it to us - verbose_logger.debug( - "OpenTelemetry: Using provided TracerProvider: %s", - type(tracer_provider).__name__, - ) + tracer_provider = self._get_or_create_provider( + provider=tracer_provider, + provider_name="TracerProvider", + get_existing_provider_fn=trace.get_tracer_provider, + sdk_provider_class=TracerProvider, + create_new_provider_fn=create_tracer_provider, + set_provider_fn=trace.set_tracer_provider, + ) # Grab our tracer from the TracerProvider (not from global context) # This ensures we use the provided TracerProvider (e.g., for testing) @@ -257,39 +295,24 @@ class OpenTelemetry(CustomLogger): return from opentelemetry import metrics - from opentelemetry.sdk.metrics import Histogram, MeterProvider + from opentelemetry.sdk.metrics import MeterProvider - # Only create OTLP infrastructure if no custom meter provider is provided - if meter_provider is None: - from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( - OTLPMetricExporter, - ) - from opentelemetry.sdk.metrics.export import ( - AggregationTemporality, - PeriodicExportingMetricReader, + def create_meter_provider(): + metric_reader = self._get_metric_reader() + return MeterProvider( + metric_readers=[metric_reader], resource=_get_litellm_resource() ) - normalized_endpoint = self._normalize_otel_endpoint( - self.config.endpoint, "metrics" - ) - _metric_exporter = OTLPMetricExporter( - endpoint=normalized_endpoint, - headers=OpenTelemetry._get_headers_dictionary(self.config.headers), - preferred_temporality={Histogram: AggregationTemporality.DELTA}, - ) - _metric_reader = PeriodicExportingMetricReader( - _metric_exporter, export_interval_millis=10000 - ) + meter_provider = self._get_or_create_provider( + provider=meter_provider, + provider_name="MeterProvider", + get_existing_provider_fn=metrics.get_meter_provider, + sdk_provider_class=MeterProvider, + create_new_provider_fn=create_meter_provider, + set_provider_fn=metrics.set_meter_provider, + ) - meter_provider = MeterProvider( - metric_readers=[_metric_reader], resource=_get_litellm_resource() - ) - meter = meter_provider.get_meter(__name__) - else: - # Use the provided meter provider as-is, without creating additional OTLP infrastructure - meter = meter_provider.get_meter(__name__) - - metrics.set_meter_provider(meter_provider) + meter = meter_provider.get_meter(__name__) self._operation_duration_histogram = meter.create_histogram( name="gen_ai.client.operation.duration", # Replace with semconv constant in otel 1.38 @@ -327,22 +350,26 @@ class OpenTelemetry(CustomLogger): if not self.config.enable_events: return - from opentelemetry._logs import set_logger_provider + from opentelemetry._logs import get_logger_provider, set_logger_provider from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider from opentelemetry.sdk._logs.export import BatchLogRecordProcessor - # set up log pipeline - if logger_provider is None: - litellm_resource = _get_litellm_resource() - logger_provider = OTLoggerProvider(resource=litellm_resource) - # Only add OTLP exporter if we created the logger provider ourselves + def create_logger_provider(): + provider = OTLoggerProvider(resource=_get_litellm_resource()) log_exporter = self._get_log_exporter() - if log_exporter: - logger_provider.add_log_record_processor( - BatchLogRecordProcessor(log_exporter) # type: ignore[arg-type] - ) + provider.add_log_record_processor( + BatchLogRecordProcessor(log_exporter) # type: ignore[arg-type] + ) + return provider - set_logger_provider(logger_provider) + self._get_or_create_provider( + provider=logger_provider, + provider_name="LoggerProvider", + get_existing_provider_fn=get_logger_provider, + sdk_provider_class=OTLoggerProvider, + create_new_provider_fn=create_logger_provider, + set_provider_fn=set_logger_provider, + ) def log_success_event(self, kwargs, response_obj, start_time, end_time): self._handle_success(kwargs, response_obj, start_time, end_time) @@ -944,6 +971,15 @@ class OpenTelemetry(CustomLogger): if not self.config.enable_events: return + # NOTE: Semantic logs (gen_ai.content.prompt/completion events) have compatibility issues + # with OTEL SDK >= 1.39.0 due to breaking changes in PR #4676: + # - LogRecord moved from opentelemetry.sdk._logs to opentelemetry.sdk._logs._internal + # - LogRecord constructor no longer accepts 'resource' parameter (now inherited from LoggerProvider) + # - LogData class was removed entirely + # These logs work correctly in OTEL SDK < 1.39.0 but may fail in >= 1.39.0. + # See: https://github.com/open-telemetry/opentelemetry-python/pull/4676 + # TODO: Refactor to use the proper OTEL Logs API instead of directly creating SDK LogRecords + from opentelemetry._logs import SeverityNumber, get_logger, get_logger_provider from opentelemetry.sdk._logs import LogRecord as SdkLogRecord @@ -1807,7 +1843,8 @@ class OpenTelemetry(CustomLogger): ) return self.OTEL_EXPORTER - if self.OTEL_EXPORTER == "console": + otel_logs_exporter = os.getenv("OTEL_LOGS_EXPORTER") + if self.OTEL_EXPORTER == "console" or otel_logs_exporter == "console": from opentelemetry.sdk._logs.export import ConsoleLogExporter verbose_logger.debug( @@ -1854,6 +1891,67 @@ class OpenTelemetry(CustomLogger): return ConsoleLogExporter() + def _get_metric_reader(self): + """ + Get the appropriate metric reader based on the configuration. + """ + from opentelemetry.sdk.metrics import Histogram + from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + ConsoleMetricExporter, + PeriodicExportingMetricReader, + ) + + verbose_logger.debug( + "OpenTelemetry Logger, initializing metric reader\nself.OTEL_EXPORTER: %s\nself.OTEL_ENDPOINT: %s\nself.OTEL_HEADERS: %s", + self.OTEL_EXPORTER, + self.OTEL_ENDPOINT, + self.OTEL_HEADERS, + ) + + _split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS) + normalized_endpoint = self._normalize_otel_endpoint(self.OTEL_ENDPOINT, "metrics") + + if self.OTEL_EXPORTER == "console": + exporter = ConsoleMetricExporter() + return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) + + elif ( + self.OTEL_EXPORTER == "otlp_http" + or self.OTEL_EXPORTER == "http/protobuf" + or self.OTEL_EXPORTER == "http/json" + ): + from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter, + ) + + exporter = OTLPMetricExporter( + endpoint=normalized_endpoint, + headers=_split_otel_headers, + preferred_temporality={Histogram: AggregationTemporality.DELTA}, + ) + return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) + + elif self.OTEL_EXPORTER == "otlp_grpc" or self.OTEL_EXPORTER == "grpc": + from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, + ) + + exporter = OTLPMetricExporter( + endpoint=normalized_endpoint, + headers=_split_otel_headers, + preferred_temporality={Histogram: AggregationTemporality.DELTA}, + ) + return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) + + else: + verbose_logger.warning( + "OpenTelemetry: Unknown metric exporter '%s', defaulting to console. Supported: console, otlp_http, otlp_grpc", + self.OTEL_EXPORTER, + ) + exporter = ConsoleMetricExporter() + return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) + def _normalize_otel_endpoint( self, endpoint: Optional[str], signal_type: str ) -> Optional[str]: diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index d92af417175..6baaae7ae3f 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2000,24 +2000,56 @@ class CustomStreamWrapper: ) ## Map to OpenAI Exception try: - raise exception_type( + mapped_exception = exception_type( model=self.model, custom_llm_provider=self.custom_llm_provider, original_exception=e, completion_kwargs={}, extra_kwargs={}, ) - except Exception as e: - from litellm.exceptions import MidStreamFallbackError + except Exception as mapping_error: + mapped_exception = mapping_error - raise MidStreamFallbackError( - message=str(e), - model=self.model, - llm_provider=self.custom_llm_provider or "anthropic", - original_exception=e, - generated_content=self.response_uptil_now, - is_pre_first_chunk=not self.sent_first_chunk, - ) + def _normalize_status_code(exc: Exception) -> Optional[int]: + """ + Best-effort status_code extraction. + Uses status_code on the exception, then falls back to the response. + """ + try: + code = getattr(exc, "status_code", None) + if code is not None: + return int(code) + except Exception: + pass + + response = getattr(exc, "response", None) + if response is not None: + try: + status_code = getattr(response, "status_code", None) + if status_code is not None: + return int(status_code) + except Exception: + pass + return None + + mapped_status_code = _normalize_status_code(mapped_exception) + original_status_code = _normalize_status_code(e) + + if mapped_status_code is not None and 400 <= mapped_status_code < 500: + raise mapped_exception + if original_status_code is not None and 400 <= original_status_code < 500: + raise mapped_exception + + from litellm.exceptions import MidStreamFallbackError + + raise MidStreamFallbackError( + message=str(mapped_exception), + model=self.model, + llm_provider=self.custom_llm_provider or "anthropic", + original_exception=mapped_exception, + generated_content=self.response_uptil_now, + is_pre_first_chunk=not self.sent_first_chunk, + ) @staticmethod def _strip_sse_data_from_chunk(chunk: Optional[str]) -> Optional[str]: diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index facabbda72a..7a4da985528 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -242,3 +242,30 @@ class BaseResponsesAPIConfig(ABC): ######################################################### ########## END CANCEL RESPONSE API TRANSFORMATION ####### ######################################################### + + ######################################################### + ########## COMPACT RESPONSE API TRANSFORMATION ########## + ######################################################### + @abstractmethod + def transform_compact_response_api_request( + self, + model: str, + input: Union[str, ResponseInputParam], + response_api_optional_request_params: Dict, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + pass + + @abstractmethod + def transform_compact_response_api_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + pass + + ######################################################### + ########## END COMPACT RESPONSE API TRANSFORMATION ###### + ######################################################### diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 34ea598a655..ea740400664 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -91,6 +91,7 @@ from litellm.types.rerank import RerankResponse from litellm.types.responses.main import DeleteResponseResult from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( + CallTypes, EmbeddingResponse, FileTypes, LiteLLMBatch, @@ -850,7 +851,9 @@ class BaseLLMHTTPHandler: ) if client is None or not isinstance(client, HTTPHandler): - sync_httpx_client = _get_httpx_client() + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) else: sync_httpx_client = client @@ -896,7 +899,8 @@ class BaseLLMHTTPHandler: ) -> EmbeddingResponse: if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider) + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: async_httpx_client = client @@ -2004,6 +2008,10 @@ class BaseLLMHTTPHandler: """ Handles responses API requests. When _is_async=True, returns a coroutine instead of making the call directly. + + Keeps the pre-transform request context for streaming so post-call hooks/metadata + (added for Responses API parity with chat) receive the original params instead of + the provider-shaped body that caused them to be skipped before. """ if _is_async: @@ -2060,6 +2068,18 @@ class BaseLLMHTTPHandler: if extra_body: data.update(extra_body) + # Preserve the OpenAI-style request context (not sent to the provider) for streaming + # hooks/metadata; the streaming iterator now consumes this to run deployment hooks + # with the same info as chat, including litellm_params. + request_context: Dict[str, Any] = {"input": input} + try: + request_context.update(response_api_optional_request_params) + except Exception: + pass + # Needed by streaming callbacks/metadata helpers to reconstruct api_base/model_id + # but never included in the outbound provider payload. + request_context["litellm_params"] = dict(litellm_params) + ## LOGGING logging_obj.pre_call( input=input, @@ -2097,6 +2117,8 @@ class BaseLLMHTTPHandler: responses_api_provider_config=responses_api_provider_config, litellm_metadata=litellm_metadata, custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, ) return SyncResponsesAPIStreamingIterator( @@ -2106,6 +2128,8 @@ class BaseLLMHTTPHandler: responses_api_provider_config=responses_api_provider_config, litellm_metadata=litellm_metadata, custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, ) else: # For non-streaming requests @@ -2189,6 +2213,18 @@ class BaseLLMHTTPHandler: if extra_body: data.update(extra_body) + # Preserve the OpenAI-style request context (not sent to the provider) for streaming + # hooks/metadata; the streaming iterator now consumes this to run deployment hooks + # with the same info as chat, including litellm_params. + request_context: Dict[str, Any] = {"input": input} + try: + request_context.update(response_api_optional_request_params) + except Exception: + pass + # Needed by streaming callbacks/metadata helpers to reconstruct api_base/model_id + # but never included in the outbound provider payload. + request_context["litellm_params"] = dict(litellm_params) + ## LOGGING logging_obj.pre_call( input=input, @@ -2227,6 +2263,8 @@ class BaseLLMHTTPHandler: responses_api_provider_config=responses_api_provider_config, litellm_metadata=litellm_metadata, custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, ) # Return the streaming iterator @@ -2237,6 +2275,8 @@ class BaseLLMHTTPHandler: responses_api_provider_config=responses_api_provider_config, litellm_metadata=litellm_metadata, custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, ) else: # For non-streaming, proceed as before @@ -3526,6 +3566,174 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, ) + def compact_response_api_handler( + self, + model: str, + input: Union[str, "ResponseInputParam"], + responses_api_provider_config: BaseResponsesAPIConfig, + response_api_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str], + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + _is_async: bool = False, + shared_session: Optional["ClientSession"] = None, + ) -> Union[ResponsesAPIResponse, Coroutine[Any, Any, ResponsesAPIResponse]]: + """ + Handler for the compact responses API. + """ + if _is_async: + return self.async_compact_response_api_handler( + model=model, + input=input, + responses_api_provider_config=responses_api_provider_config, + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, + client=client, + shared_session=shared_session, + ) + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = responses_api_provider_config.validate_environment( + headers=extra_headers or {}, model=model, litellm_params=litellm_params + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = responses_api_provider_config.get_complete_url( + api_base=litellm_params.api_base, + litellm_params=dict(litellm_params), + ) + + url, data = responses_api_provider_config.transform_compact_response_api_request( + model=model, + input=input, + response_api_optional_request_params=response_api_optional_request_params, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input=input, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + }, + ) + + try: + response = sync_httpx_client.post( + url=url, headers=headers, json=data, timeout=timeout + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=responses_api_provider_config, + ) + + return responses_api_provider_config.transform_compact_response_api_response( + raw_response=response, + logging_obj=logging_obj, + ) + + async def async_compact_response_api_handler( + self, + model: str, + input: Union[str, "ResponseInputParam"], + responses_api_provider_config: BaseResponsesAPIConfig, + response_api_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str], + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + _is_async: bool = False, + shared_session: Optional["ClientSession"] = None, + ) -> ResponsesAPIResponse: + """ + Async version of the compact response API handler. + """ + if client is None or not isinstance(client, AsyncHTTPHandler): + verbose_logger.debug( + f"Creating HTTP client for compact_response with shared_session: {id(shared_session) if shared_session else None}" + ) + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + shared_session=shared_session, + ) + else: + async_httpx_client = client + + headers = responses_api_provider_config.validate_environment( + headers=extra_headers or {}, model=model, litellm_params=litellm_params + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = responses_api_provider_config.get_complete_url( + api_base=litellm_params.api_base, + litellm_params=dict(litellm_params), + ) + + url, data = responses_api_provider_config.transform_compact_response_api_request( + model=model, + input=input, + response_api_optional_request_params=response_api_optional_request_params, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input=input, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + }, + ) + + try: + response = await async_httpx_client.post( + url=url, headers=headers, json=data, timeout=timeout + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=responses_api_provider_config, + ) + + return responses_api_provider_config.transform_compact_response_api_response( + raw_response=response, + logging_obj=logging_obj, + ) + def list_files(self): """ Lists all files @@ -8288,4 +8496,4 @@ class BaseLLMHTTPHandler: return skills_api_provider_config.transform_delete_skill_response( raw_response=response, logging_obj=logging_obj, - ) \ No newline at end of file + ) diff --git a/litellm/llms/gigachat/__init__.py b/litellm/llms/gigachat/__init__.py new file mode 100644 index 00000000000..3ddbd7864d9 --- /dev/null +++ b/litellm/llms/gigachat/__init__.py @@ -0,0 +1,23 @@ +""" +GigaChat Provider for LiteLLM + +GigaChat is Sber AI's large language model (Russia's leading LLM). +Supports: +- Chat completions (sync/async) +- Streaming (sync/async) +- Function calling / Tools +- Structured output via JSON schema (emulated through function calls) +- Image input (base64 and URL) +- Embeddings + +API Documentation: https://developers.sber.ru/docs/ru/gigachat/api/overview +""" + +from .chat.transformation import GigaChatConfig, GigaChatError +from .embedding.transformation import GigaChatEmbeddingConfig + +__all__ = [ + "GigaChatConfig", + "GigaChatEmbeddingConfig", + "GigaChatError", +] diff --git a/litellm/llms/gigachat/authenticator.py b/litellm/llms/gigachat/authenticator.py new file mode 100644 index 00000000000..e61015a4a21 --- /dev/null +++ b/litellm/llms/gigachat/authenticator.py @@ -0,0 +1,241 @@ +""" +GigaChat OAuth Authenticator + +Handles OAuth 2.0 token management for GigaChat API. +Based on official GigaChat SDK authentication flow. +""" + +import time +import uuid +from typing import Optional, Tuple + +import httpx + +from litellm._logging import verbose_logger +from litellm.caching.caching import InMemoryCache +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.custom_httpx.http_handler import ( + HTTPHandler, + _get_httpx_client, + get_async_httpx_client, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.utils import LlmProviders + +# GigaChat OAuth endpoint +GIGACHAT_AUTH_URL = "https://ngw.devices.sberbank.ru:9443/api/v2/oauth" + +# Default scope for personal API access +GIGACHAT_SCOPE = "GIGACHAT_API_PERS" + +# Token expiry buffer in milliseconds (refresh token 60s before expiry) +TOKEN_EXPIRY_BUFFER_MS = 60000 + +# Cache for access tokens +_token_cache = InMemoryCache() + + +class GigaChatAuthError(BaseLLMException): + """GigaChat authentication error.""" + + pass + + +def _get_credentials() -> Optional[str]: + """Get GigaChat credentials from environment.""" + return get_secret_str("GIGACHAT_CREDENTIALS") or get_secret_str("GIGACHAT_API_KEY") + + +def _get_auth_url() -> str: + """Get GigaChat auth URL from environment or use default.""" + return get_secret_str("GIGACHAT_AUTH_URL") or GIGACHAT_AUTH_URL + + +def _get_scope() -> str: + """Get GigaChat scope from environment or use default.""" + return get_secret_str("GIGACHAT_SCOPE") or GIGACHAT_SCOPE + + +def _get_http_client() -> HTTPHandler: + """Get cached httpx client with SSL verification disabled.""" + return _get_httpx_client(params={"ssl_verify": False}) + + +def get_access_token( + credentials: Optional[str] = None, + scope: Optional[str] = None, + auth_url: Optional[str] = None, +) -> str: + """ + Get valid access token, using cache if available. + + Args: + credentials: Base64-encoded credentials (client_id:client_secret) + scope: API scope (GIGACHAT_API_PERS, GIGACHAT_API_CORP, etc.) + auth_url: OAuth endpoint URL + + Returns: + Access token string + + Raises: + GigaChatAuthError: If authentication fails + """ + credentials = credentials or _get_credentials() + if not credentials: + raise GigaChatAuthError( + status_code=401, + message="GigaChat credentials not provided. Set GIGACHAT_CREDENTIALS or GIGACHAT_API_KEY environment variable.", + ) + + scope = scope or _get_scope() + auth_url = auth_url or _get_auth_url() + + # Check cache + cache_key = f"gigachat_token:{credentials[:16]}" + cached = _token_cache.get_cache(cache_key) + if cached: + token, expires_at = cached + # Check if token is still valid (with buffer) + if time.time() * 1000 < expires_at - TOKEN_EXPIRY_BUFFER_MS: + verbose_logger.debug("Using cached GigaChat access token") + return token + + # Request new token + token, expires_at = _request_token_sync(credentials, scope, auth_url) + + # Cache token + ttl_seconds = max(0, (expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000) + if ttl_seconds > 0: + _token_cache.set_cache(cache_key, (token, expires_at), ttl=ttl_seconds) + + return token + + +async def get_access_token_async( + credentials: Optional[str] = None, + scope: Optional[str] = None, + auth_url: Optional[str] = None, +) -> str: + """Async version of get_access_token.""" + credentials = credentials or _get_credentials() + if not credentials: + raise GigaChatAuthError( + status_code=401, + message="GigaChat credentials not provided. Set GIGACHAT_CREDENTIALS or GIGACHAT_API_KEY environment variable.", + ) + + scope = scope or _get_scope() + auth_url = auth_url or _get_auth_url() + + # Check cache + cache_key = f"gigachat_token:{credentials[:16]}" + cached = _token_cache.get_cache(cache_key) + if cached: + token, expires_at = cached + if time.time() * 1000 < expires_at - TOKEN_EXPIRY_BUFFER_MS: + verbose_logger.debug("Using cached GigaChat access token") + return token + + # Request new token + token, expires_at = await _request_token_async(credentials, scope, auth_url) + + # Cache token + ttl_seconds = max(0, (expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000) + if ttl_seconds > 0: + _token_cache.set_cache(cache_key, (token, expires_at), ttl=ttl_seconds) + + return token + + +def _request_token_sync( + credentials: str, + scope: str, + auth_url: str, +) -> Tuple[str, int]: + """ + Request new access token from GigaChat OAuth endpoint (sync). + + Returns: + Tuple of (access_token, expires_at_ms) + """ + headers = { + "Authorization": f"Basic {credentials}", + "RqUID": str(uuid.uuid4()), + "Content-Type": "application/x-www-form-urlencoded", + } + data = {"scope": scope} + + verbose_logger.debug(f"Requesting GigaChat access token from {auth_url}") + + try: + client = _get_http_client() + response = client.post(auth_url, headers=headers, data=data, timeout=30) + response.raise_for_status() + return _parse_token_response(response) + except httpx.HTTPStatusError as e: + raise GigaChatAuthError( + status_code=e.response.status_code, + message=f"GigaChat authentication failed: {e.response.text}", + ) + except httpx.RequestError as e: + raise GigaChatAuthError( + status_code=500, + message=f"GigaChat authentication request failed: {str(e)}", + ) + + +async def _request_token_async( + credentials: str, + scope: str, + auth_url: str, +) -> Tuple[str, int]: + """Async version of _request_token_sync.""" + headers = { + "Authorization": f"Basic {credentials}", + "RqUID": str(uuid.uuid4()), + "Content-Type": "application/x-www-form-urlencoded", + } + data = {"scope": scope} + + verbose_logger.debug(f"Requesting GigaChat access token from {auth_url}") + + try: + client = get_async_httpx_client( + llm_provider=LlmProviders.GIGACHAT, + params={"ssl_verify": False}, + ) + response = await client.post(auth_url, headers=headers, data=data, timeout=30) + response.raise_for_status() + return _parse_token_response(response) + except httpx.HTTPStatusError as e: + raise GigaChatAuthError( + status_code=e.response.status_code, + message=f"GigaChat authentication failed: {e.response.text}", + ) + except httpx.RequestError as e: + raise GigaChatAuthError( + status_code=500, + message=f"GigaChat authentication request failed: {str(e)}", + ) + + +def _parse_token_response(response: httpx.Response) -> Tuple[str, int]: + """Parse OAuth token response.""" + data = response.json() + + # GigaChat returns either 'tok'/'exp' or 'access_token'/'expires_at' + access_token = data.get("tok") or data.get("access_token") + expires_at = data.get("exp") or data.get("expires_at") + + if not access_token: + raise GigaChatAuthError( + status_code=500, + message=f"Invalid token response: {data}", + ) + + # expires_at is in milliseconds + if isinstance(expires_at, str): + expires_at = int(expires_at) + + verbose_logger.debug("GigaChat access token obtained successfully") + return access_token, expires_at diff --git a/litellm/llms/gigachat/chat/__init__.py b/litellm/llms/gigachat/chat/__init__.py new file mode 100644 index 00000000000..3e030497a1a --- /dev/null +++ b/litellm/llms/gigachat/chat/__init__.py @@ -0,0 +1,12 @@ +""" +GigaChat Chat Module +""" + +from .transformation import GigaChatConfig, GigaChatError +from .streaming import GigaChatModelResponseIterator + +__all__ = [ + "GigaChatConfig", + "GigaChatError", + "GigaChatModelResponseIterator", +] diff --git a/litellm/llms/gigachat/chat/streaming.py b/litellm/llms/gigachat/chat/streaming.py new file mode 100644 index 00000000000..3565559e43c --- /dev/null +++ b/litellm/llms/gigachat/chat/streaming.py @@ -0,0 +1,134 @@ +""" +GigaChat Streaming Response Handler +""" + +import json +import uuid +from typing import Any, Optional + +from litellm.types.llms.openai import ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk +from litellm.types.utils import GenericStreamingChunk + + +class GigaChatModelResponseIterator: + """Iterator for GigaChat streaming responses.""" + + def __init__( + self, + streaming_response: Any, + sync_stream: bool, + json_mode: Optional[bool] = False, + ): + self.streaming_response = streaming_response + self.response_iterator = self.streaming_response + self.json_mode = json_mode + + def chunk_parser(self, chunk: dict) -> GenericStreamingChunk: + """Parse a single streaming chunk from GigaChat.""" + text = "" + tool_use: Optional[ChatCompletionToolCallChunk] = None + is_finished = False + finish_reason: Optional[str] = None + + choices = chunk.get("choices", []) + if not choices: + return GenericStreamingChunk( + text="", + tool_use=None, + is_finished=False, + finish_reason="", + usage=None, + index=0, + ) + + choice = choices[0] + delta = choice.get("delta", {}) + finish_reason = choice.get("finish_reason") + + # Extract text content + text = delta.get("content", "") or "" + + # Handle function_call in stream + if finish_reason == "function_call" and delta.get("function_call"): + func_call = delta["function_call"] + args = func_call.get("arguments", {}) + + if isinstance(args, dict): + args = json.dumps(args, ensure_ascii=False) + + tool_use = ChatCompletionToolCallChunk( + id=f"call_{uuid.uuid4().hex[:24]}", + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=func_call.get("name", ""), + arguments=args, + ), + index=0, + ) + finish_reason = "tool_calls" + + if finish_reason is not None: + is_finished = True + + return GenericStreamingChunk( + text=text, + tool_use=tool_use, + is_finished=is_finished, + finish_reason=finish_reason or "", + usage=None, + index=choice.get("index", 0), + ) + + def __iter__(self): + return self + + def __next__(self) -> GenericStreamingChunk: + try: + chunk = self.response_iterator.__next__() + if isinstance(chunk, str): + # Parse SSE format: data: {...} + if chunk.startswith("data: "): + chunk = chunk[6:] + if chunk.strip() == "[DONE]": + raise StopIteration + try: + chunk = json.loads(chunk) + except json.JSONDecodeError: + return GenericStreamingChunk( + text="", + tool_use=None, + is_finished=False, + finish_reason="", + usage=None, + index=0, + ) + return self.chunk_parser(chunk) + except StopIteration: + raise + + def __aiter__(self): + return self + + async def __anext__(self) -> GenericStreamingChunk: + try: + chunk = await self.response_iterator.__anext__() + if isinstance(chunk, str): + # Parse SSE format + if chunk.startswith("data: "): + chunk = chunk[6:] + if chunk.strip() == "[DONE]": + raise StopAsyncIteration + try: + chunk = json.loads(chunk) + except json.JSONDecodeError: + return GenericStreamingChunk( + text="", + tool_use=None, + is_finished=False, + finish_reason="", + usage=None, + index=0, + ) + return self.chunk_parser(chunk) + except StopAsyncIteration: + raise diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py new file mode 100644 index 00000000000..4ce333a1309 --- /dev/null +++ b/litellm/llms/gigachat/chat/transformation.py @@ -0,0 +1,473 @@ +""" +GigaChat Chat Transformation + +Transforms OpenAI-format requests to GigaChat format and back. +""" + +import json +import time +import uuid +from typing import TYPE_CHECKING, Any, AsyncIterator, Iterator, List, Optional, Union + +import httpx + +from litellm._logging import verbose_logger +from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import Choices, Message, ModelResponse, Usage + +from ..authenticator import get_access_token +from ..file_handler import upload_file_sync + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + +# GigaChat API endpoint +GIGACHAT_BASE_URL = "https://gigachat.devices.sberbank.ru/api/v1" + + +class GigaChatError(BaseLLMException): + """GigaChat API error.""" + + pass + + +class GigaChatConfig(BaseConfig): + """ + Configuration class for GigaChat API. + + GigaChat is Sber's (Russia's largest bank) LLM API. + + Supported parameters: + temperature: Sampling temperature (0-2, default 0.87) + top_p: Nucleus sampling parameter + max_tokens: Maximum tokens to generate + repetition_penalty: Repetition penalty factor + profanity_check: Enable content filtering + stream: Enable streaming + """ + + temperature: Optional[float] = None + top_p: Optional[float] = None + max_tokens: Optional[int] = None + repetition_penalty: Optional[float] = None + profanity_check: Optional[bool] = None + + def __init__( + self, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + max_tokens: Optional[int] = None, + repetition_penalty: Optional[float] = None, + profanity_check: Optional[bool] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + # Instance variables for current request context + self._current_credentials: Optional[str] = None + self._current_api_base: Optional[str] = None + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """Get complete API URL for chat completions.""" + base = api_base or get_secret_str("GIGACHAT_API_BASE") or GIGACHAT_BASE_URL + return f"{base}/chat/completions" + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Set up headers with OAuth token. + """ + # Get access token + credentials = api_key or get_secret_str("GIGACHAT_CREDENTIALS") or get_secret_str("GIGACHAT_API_KEY") + access_token = get_access_token(credentials=credentials) + + # Store credentials for image uploads + self._current_credentials = credentials + self._current_api_base = api_base + + headers["Authorization"] = f"Bearer {access_token}" + headers["Content-Type"] = "application/json" + headers["Accept"] = "application/json" + + return headers + + def get_supported_openai_params(self, model: str) -> List[str]: + """Return list of supported OpenAI parameters.""" + return [ + "stream", + "temperature", + "top_p", + "max_tokens", + "max_completion_tokens", + "stop", + "tools", + "tool_choice", + "functions", + "function_call", + "response_format", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """Map OpenAI parameters to GigaChat parameters.""" + for param, value in non_default_params.items(): + if param == "stream": + optional_params["stream"] = value + elif param == "temperature": + # GigaChat: temperature 0 means use top_p=0 instead + if value == 0: + optional_params["top_p"] = 0 + else: + optional_params["temperature"] = value + elif param == "top_p": + optional_params["top_p"] = value + elif param in ("max_tokens", "max_completion_tokens"): + optional_params["max_tokens"] = value + elif param == "stop": + # GigaChat doesn't support stop sequences + pass + elif param == "tools": + # Convert tools to functions format + optional_params["functions"] = self._convert_tools_to_functions(value) + elif param == "tool_choice": + if isinstance(value, dict) and value.get("function"): + optional_params["function_call"] = {"name": value["function"]["name"]} + elif value == "auto": + pass # Default behavior + elif value == "required": + # GigaChat doesn't have 'required', handled differently + pass + elif param == "functions": + optional_params["functions"] = value + elif param == "function_call": + optional_params["function_call"] = value + elif param == "response_format": + # Handle structured output via function calling + if value.get("type") == "json_schema": + json_schema = value.get("json_schema", {}) + schema_name = json_schema.get("name", "structured_output") + schema = json_schema.get("schema", {}) + + function_def = { + "name": schema_name, + "description": f"Output structured response: {schema_name}", + "parameters": schema, + } + + if "functions" not in optional_params: + optional_params["functions"] = [] + optional_params["functions"].append(function_def) + optional_params["function_call"] = {"name": schema_name} + optional_params["_structured_output"] = True + + return optional_params + + def _convert_tools_to_functions(self, tools: List[dict]) -> List[dict]: + """Convert OpenAI tools format to GigaChat functions format.""" + functions = [] + for tool in tools: + if tool.get("type") == "function": + func = tool.get("function", {}) + functions.append({ + "name": func.get("name", ""), + "description": func.get("description", ""), + "parameters": func.get("parameters", {}), + }) + return functions + + def _upload_image(self, image_url: str) -> Optional[str]: + """ + Upload image to GigaChat and return file_id. + + Args: + image_url: URL or base64 data URL of the image + + Returns: + file_id string or None if upload failed + """ + try: + return upload_file_sync( + image_url=image_url, + credentials=self._current_credentials, + api_base=self._current_api_base, + ) + except Exception as e: + verbose_logger.error(f"Failed to upload image: {e}") + return None + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """Transform OpenAI request to GigaChat format.""" + # Transform messages + giga_messages = self._transform_messages(messages) + + # Build request + request_data = { + "model": model.replace("gigachat/", ""), + "messages": giga_messages, + } + + # Add optional params + for key in ["temperature", "top_p", "max_tokens", "stream", + "repetition_penalty", "profanity_check"]: + if key in optional_params: + request_data[key] = optional_params[key] + + # Add functions if present + if "functions" in optional_params: + request_data["functions"] = optional_params["functions"] + if "function_call" in optional_params: + request_data["function_call"] = optional_params["function_call"] + + return request_data + + def _transform_messages(self, messages: List[AllMessageValues]) -> List[dict]: + """Transform OpenAI messages to GigaChat format.""" + transformed = [] + + for i, msg in enumerate(messages): + message = dict(msg) + + # Remove unsupported fields + message.pop("name", None) + + # Transform roles + role = message.get("role", "user") + if role == "developer": + message["role"] = "system" + elif role == "system" and i > 0: + # GigaChat only allows system message as first message + message["role"] = "user" + elif role == "tool": + message["role"] = "function" + content = message.get("content", "") + if not isinstance(content, str): + message["content"] = json.dumps(content, ensure_ascii=False) + + # Handle None content + if message.get("content") is None: + message["content"] = "" + + # Handle list content (multimodal) - extract text and images + content = message.get("content") + if isinstance(content, list): + texts = [] + attachments = [] + for part in content: + if isinstance(part, dict): + if part.get("type") == "text": + texts.append(part.get("text", "")) + elif part.get("type") == "image_url": + # Extract image URL and upload to GigaChat + image_url = part.get("image_url", {}) + if isinstance(image_url, str): + url = image_url + else: + url = image_url.get("url", "") + if url: + file_id = self._upload_image(url) + if file_id: + attachments.append(file_id) + message["content"] = "\n".join(texts) if texts else "" + if attachments: + message["attachments"] = attachments + + # Transform tool_calls to function_call + tool_calls = message.get("tool_calls") + if tool_calls and isinstance(tool_calls, list) and len(tool_calls) > 0: + tool_call = tool_calls[0] + func = tool_call.get("function", {}) + args = func.get("arguments", "{}") + if isinstance(args, str): + try: + args = json.loads(args) + except json.JSONDecodeError: + args = {} + message["function_call"] = { + "name": func.get("name", ""), + "arguments": args, + } + message.pop("tool_calls", None) + + transformed.append(message) + + # Collapse consecutive user messages + return self._collapse_user_messages(transformed) + + def _collapse_user_messages(self, messages: List[dict]) -> List[dict]: + """Collapse consecutive user messages into one.""" + collapsed: List[dict] = [] + prev_user_msg: Optional[dict] = None + content_parts: List[str] = [] + + for msg in messages: + if msg.get("role") == "user" and prev_user_msg is not None: + content_parts.append(msg.get("content", "")) + else: + if content_parts and prev_user_msg: + prev_user_msg["content"] = "\n".join( + [prev_user_msg.get("content", "")] + content_parts + ) + content_parts = [] + collapsed.append(msg) + prev_user_msg = msg if msg.get("role") == "user" else None + + if content_parts and prev_user_msg: + prev_user_msg["content"] = "\n".join( + [prev_user_msg.get("content", "")] + content_parts + ) + + return collapsed + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + """Transform GigaChat response to OpenAI format.""" + try: + response_json = raw_response.json() + except Exception: + raise GigaChatError( + status_code=raw_response.status_code, + message=f"Invalid JSON response: {raw_response.text}", + ) + + is_structured_output = optional_params.get("_structured_output", False) + + choices = [] + for choice in response_json.get("choices", []): + message_data = choice.get("message", {}) + finish_reason = choice.get("finish_reason", "stop") + + # Transform function_call to tool_calls or content + if finish_reason == "function_call" and message_data.get("function_call"): + func_call = message_data["function_call"] + args = func_call.get("arguments", {}) + + if is_structured_output: + # Convert to content for structured output + if isinstance(args, dict): + content = json.dumps(args, ensure_ascii=False) + else: + content = str(args) + message_data["content"] = content + message_data.pop("function_call", None) + message_data.pop("functions_state_id", None) + finish_reason = "stop" + else: + # Convert to tool_calls format + if isinstance(args, dict): + args = json.dumps(args, ensure_ascii=False) + message_data["tool_calls"] = [{ + "id": f"call_{uuid.uuid4().hex[:24]}", + "type": "function", + "function": { + "name": func_call.get("name", ""), + "arguments": args, + } + }] + message_data.pop("function_call", None) + finish_reason = "tool_calls" + + # Clean up GigaChat-specific fields + message_data.pop("functions_state_id", None) + + choices.append( + Choices( + index=choice.get("index", 0), + message=Message( + role=message_data.get("role", "assistant"), + content=message_data.get("content"), + tool_calls=message_data.get("tool_calls"), + ), + finish_reason=finish_reason, + ) + ) + + # Build usage + usage_data = response_json.get("usage", {}) + usage = Usage( + prompt_tokens=usage_data.get("prompt_tokens", 0), + completion_tokens=usage_data.get("completion_tokens", 0), + total_tokens=usage_data.get("total_tokens", 0), + ) + + model_response.id = response_json.get("id", f"chatcmpl-{uuid.uuid4().hex[:12]}") + model_response.created = response_json.get("created", int(time.time())) + model_response.model = model + model_response.choices = choices # type: ignore + setattr(model_response, "usage", usage) + + return model_response + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: Union[dict, httpx.Headers], + ) -> BaseLLMException: + """Return GigaChat error class.""" + return GigaChatError( + status_code=status_code, + message=error_message, + headers=headers, + ) + + def get_model_response_iterator( + self, + streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse], + sync_stream: bool, + json_mode: Optional[bool] = False, + ): + """Return streaming response iterator.""" + from .streaming import GigaChatModelResponseIterator + + return GigaChatModelResponseIterator( + streaming_response=streaming_response, + sync_stream=sync_stream, + json_mode=json_mode, + ) diff --git a/litellm/llms/gigachat/embedding/__init__.py b/litellm/llms/gigachat/embedding/__init__.py new file mode 100644 index 00000000000..af237e49aab --- /dev/null +++ b/litellm/llms/gigachat/embedding/__init__.py @@ -0,0 +1,7 @@ +""" +GigaChat Embedding Module +""" + +from .transformation import GigaChatEmbeddingConfig + +__all__ = ["GigaChatEmbeddingConfig"] diff --git a/litellm/llms/gigachat/embedding/transformation.py b/litellm/llms/gigachat/embedding/transformation.py new file mode 100644 index 00000000000..0da6565050e --- /dev/null +++ b/litellm/llms/gigachat/embedding/transformation.py @@ -0,0 +1,212 @@ +""" +GigaChat Embedding Transformation + +Transforms OpenAI /v1/embeddings format to GigaChat format. +API Documentation: https://developers.sber.ru/docs/ru/gigachat/api/reference/rest/post-embeddings +""" + +import types +from typing import List, Optional, Tuple, Union + +import httpx + +from litellm import LlmProviders +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues +from litellm.types.utils import EmbeddingResponse + +from ..authenticator import get_access_token + +# GigaChat API endpoint +GIGACHAT_BASE_URL = "https://gigachat.devices.sberbank.ru/api/v1" + + +class GigaChatEmbeddingError(BaseLLMException): + """GigaChat Embedding API error.""" + + pass + + +class GigaChatEmbeddingConfig(BaseEmbeddingConfig): + """ + Configuration class for GigaChat Embeddings API. + + GigaChat embeddings endpoint: POST /api/v1/embeddings + """ + + def __init__(self) -> None: + pass + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self, model: str) -> List[str]: + """GigaChat embeddings don't support additional parameters.""" + return [] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """Map OpenAI params to GigaChat format (no special mapping needed).""" + return optional_params + + def _get_openai_compatible_provider_info( + self, + api_base: Optional[str], + api_key: Optional[str], + ) -> Tuple[str, Optional[str], Optional[str]]: + """ + Returns provider info for GigaChat. + + Returns: + Tuple of (custom_llm_provider, api_base, dynamic_api_key) + """ + api_base = api_base or GIGACHAT_BASE_URL + return LlmProviders.GIGACHAT.value, api_base, api_key + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """Get the complete URL for embeddings endpoint.""" + base = api_base or GIGACHAT_BASE_URL + return f"{base}/embeddings" + + def transform_embedding_request( + self, + model: str, + input: AllEmbeddingInputValues, + optional_params: dict, + headers: dict, + ) -> dict: + """ + Transform OpenAI embedding request to GigaChat format. + + GigaChat format: + { + "model": "Embeddings", + "input": ["text1", "text2", ...] + } + """ + # Normalize input to list + if isinstance(input, str): + input_list: list = [input] + elif isinstance(input, list): + input_list = input + else: + input_list = [input] + + # Remove gigachat/ prefix from model if present + if model.startswith("gigachat/"): + model = model[9:] + + return { + "model": model, + "input": input_list, + } + + def transform_embedding_response( + self, + model: str, + raw_response: httpx.Response, + model_response: EmbeddingResponse, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + request_data: dict, + optional_params: dict, + litellm_params: dict, + ) -> EmbeddingResponse: + """ + Transform GigaChat embedding response to OpenAI format. + + GigaChat returns: + { + "object": "list", + "data": [{"object": "embedding", "embedding": [...], "index": 0, "usage": {...}}], + "model": "Embeddings" + } + """ + response_json = raw_response.json() + + # Log response + logging_obj.post_call( + input=request_data.get("input"), + api_key=api_key, + additional_args={"complete_input_dict": request_data}, + original_response=response_json, + ) + + # Calculate total tokens from individual embeddings + total_tokens = 0 + if "data" in response_json: + for emb in response_json["data"]: + if "usage" in emb and "prompt_tokens" in emb["usage"]: + total_tokens += emb["usage"]["prompt_tokens"] + # Remove usage from individual embeddings (not part of OpenAI format) + if "usage" in emb: + del emb["usage"] + + # Set overall usage + response_json["usage"] = { + "prompt_tokens": total_tokens, + "total_tokens": total_tokens, + } + + return EmbeddingResponse(**response_json) + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Set up headers with OAuth token for GigaChat. + """ + # Get access token via OAuth + access_token = get_access_token(api_key) + + default_headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {access_token}", + } + return {**default_headers, **headers} + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + """Return GigaChat-specific error class.""" + return GigaChatEmbeddingError( + status_code=status_code, + message=error_message, + ) diff --git a/litellm/llms/gigachat/file_handler.py b/litellm/llms/gigachat/file_handler.py new file mode 100644 index 00000000000..200428a747a --- /dev/null +++ b/litellm/llms/gigachat/file_handler.py @@ -0,0 +1,211 @@ +""" +GigaChat File Handler + +Handles file uploads to GigaChat API for image processing. +GigaChat requires files to be uploaded first, then referenced by file_id. +""" + +import base64 +import hashlib +import re +import uuid +from typing import Dict, Optional, Tuple + +from litellm._logging import verbose_logger +from litellm.llms.custom_httpx.http_handler import ( + _get_httpx_client, + get_async_httpx_client, +) +from litellm.types.utils import LlmProviders + +from .authenticator import get_access_token, get_access_token_async + +# GigaChat API endpoint +GIGACHAT_BASE_URL = "https://gigachat.devices.sberbank.ru/api/v1" + +# Simple in-memory cache for file IDs +_file_cache: Dict[str, str] = {} + + +def _get_url_hash(url: str) -> str: + """Generate hash for URL to use as cache key.""" + return hashlib.sha256(url.encode()).hexdigest() + + +def _parse_data_url(data_url: str) -> Optional[Tuple[bytes, str, str]]: + """ + Parse data URL (base64 image). + + Returns: + Tuple of (content_bytes, content_type, extension) or None + """ + match = re.match(r"data:([^;]+);base64,(.+)", data_url) + if not match: + return None + + content_type = match.group(1) + base64_data = match.group(2) + content_bytes = base64.b64decode(base64_data) + ext = content_type.split("/")[-1].split(";")[0] or "jpg" + + return content_bytes, content_type, ext + + +def _download_image_sync(url: str) -> Tuple[bytes, str, str]: + """Download image from URL synchronously.""" + client = _get_httpx_client(params={"ssl_verify": False}) + response = client.get(url) + response.raise_for_status() + + content_type = response.headers.get("content-type", "image/jpeg") + ext = content_type.split("/")[-1].split(";")[0] or "jpg" + + return response.content, content_type, ext + + +async def _download_image_async(url: str) -> Tuple[bytes, str, str]: + """Download image from URL asynchronously.""" + client = get_async_httpx_client( + llm_provider=LlmProviders.GIGACHAT, + params={"ssl_verify": False}, + ) + response = await client.get(url) + response.raise_for_status() + + content_type = response.headers.get("content-type", "image/jpeg") + ext = content_type.split("/")[-1].split(";")[0] or "jpg" + + return response.content, content_type, ext + + +def upload_file_sync( + image_url: str, + credentials: Optional[str] = None, + api_base: Optional[str] = None, +) -> Optional[str]: + """ + Upload file to GigaChat and return file_id (sync). + + Args: + image_url: URL or base64 data URL of the image + credentials: GigaChat credentials for auth + api_base: Optional custom API base URL + + Returns: + file_id string or None if upload failed + """ + url_hash = _get_url_hash(image_url) + + # Check cache + if url_hash in _file_cache: + verbose_logger.debug(f"Image found in cache: {url_hash[:16]}...") + return _file_cache[url_hash] + + try: + # Get image data + parsed = _parse_data_url(image_url) + if parsed: + content_bytes, content_type, ext = parsed + verbose_logger.debug("Decoded base64 image") + else: + verbose_logger.debug(f"Downloading image from URL: {image_url[:80]}...") + content_bytes, content_type, ext = _download_image_sync(image_url) + + filename = f"{uuid.uuid4()}.{ext}" + + # Get access token + access_token = get_access_token(credentials) + + # Upload to GigaChat + base_url = api_base or GIGACHAT_BASE_URL + upload_url = f"{base_url}/files" + + client = _get_httpx_client(params={"ssl_verify": False}) + response = client.post( + upload_url, + headers={"Authorization": f"Bearer {access_token}"}, + files={"file": (filename, content_bytes, content_type)}, + data={"purpose": "general"}, + timeout=60, + ) + response.raise_for_status() + result = response.json() + + file_id = result.get("id") + if file_id: + _file_cache[url_hash] = file_id + verbose_logger.debug(f"File uploaded successfully, file_id: {file_id}") + + return file_id + + except Exception as e: + verbose_logger.error(f"Error uploading file to GigaChat: {e}") + return None + + +async def upload_file_async( + image_url: str, + credentials: Optional[str] = None, + api_base: Optional[str] = None, +) -> Optional[str]: + """ + Upload file to GigaChat and return file_id (async). + + Args: + image_url: URL or base64 data URL of the image + credentials: GigaChat credentials for auth + api_base: Optional custom API base URL + + Returns: + file_id string or None if upload failed + """ + url_hash = _get_url_hash(image_url) + + # Check cache + if url_hash in _file_cache: + verbose_logger.debug(f"Image found in cache: {url_hash[:16]}...") + return _file_cache[url_hash] + + try: + # Get image data + parsed = _parse_data_url(image_url) + if parsed: + content_bytes, content_type, ext = parsed + verbose_logger.debug("Decoded base64 image") + else: + verbose_logger.debug(f"Downloading image from URL: {image_url[:80]}...") + content_bytes, content_type, ext = await _download_image_async(image_url) + + filename = f"{uuid.uuid4()}.{ext}" + + # Get access token + access_token = await get_access_token_async(credentials) + + # Upload to GigaChat + base_url = api_base or GIGACHAT_BASE_URL + upload_url = f"{base_url}/files" + + client = get_async_httpx_client( + llm_provider=LlmProviders.GIGACHAT, + params={"ssl_verify": False}, + ) + response = await client.post( + upload_url, + headers={"Authorization": f"Bearer {access_token}"}, + files={"file": (filename, content_bytes, content_type)}, + data={"purpose": "general"}, + timeout=60, + ) + response.raise_for_status() + result = response.json() + + file_id = result.get("id") + if file_id: + _file_cache[url_hash] = file_id + verbose_logger.debug(f"File uploaded successfully, file_id: {file_id}") + + return file_id + + except Exception as e: + verbose_logger.error(f"Error uploading file to GigaChat: {e}") + return None diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 96598c1dfe6..cc2439b431a 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -500,3 +500,69 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): response._hidden_params["headers"] = raw_response_headers return response + + ######################################################### + ########## COMPACT RESPONSE API TRANSFORMATION ########## + ######################################################### + def transform_compact_response_api_request( + self, + model: str, + input: Union[str, ResponseInputParam], + response_api_optional_request_params: Dict, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """ + Transform the compact response API request into a URL and data + + OpenAI API expects the following request + - POST /v1/responses/compact + """ + url = f"{api_base}/compact" + + input = self._validate_input_param(input) + data = dict( + ResponsesAPIRequestParams( + model=model, input=input, **response_api_optional_request_params + ) + ) + + return url, data + + def transform_compact_response_api_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + """ + Transform the compact response API response into a ResponsesAPIResponse + """ + try: + logging_obj.post_call( + original_response=raw_response.text, + additional_args={"complete_input_dict": {}}, + ) + raw_response_json = raw_response.json() + raw_response_json["created_at"] = _safe_convert_created_field( + raw_response_json["created_at"] + ) + except Exception: + raise OpenAIError( + message=raw_response.text, status_code=raw_response.status_code + ) + raw_response_headers = dict(raw_response.headers) + processed_headers = process_response_headers(raw_response_headers) + + try: + response = ResponsesAPIResponse(**raw_response_json) + except Exception: + verbose_logger.debug( + f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct" + ) + response = ResponsesAPIResponse.model_construct(**raw_response_json) + + response._hidden_params["additional_headers"] = processed_headers + response._hidden_params["headers"] = raw_response_headers + + return response diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 826f151df35..d23c698cd7a 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -6,7 +6,7 @@ Handles Authentication and generating request urls for Vertex AI and Google AI S import json import os -from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple, cast import litellm from litellm._logging import verbose_logger @@ -168,7 +168,6 @@ class VertexBase: ) def _credentials_from_default_auth(self, scopes): - import google.auth as google_auth return google_auth.default(scopes=scopes) @@ -392,7 +391,7 @@ class VertexBase: Returns token, url """ - version: Optional[Literal["v1beta1", "v1"]] = None + version: Optional[Literal["v1", "v1beta1"]] = None if custom_llm_provider == "gemini": url, endpoint = _get_gemini_url( mode=mode, @@ -415,7 +414,7 @@ class VertexBase: stream=stream, vertex_project=vertex_project, vertex_location=vertex_location, - vertex_api_version=version, + vertex_api_version=cast(Literal["v1", "v1beta1"], version), ) return self._check_custom_proxy( diff --git a/litellm/main.py b/litellm/main.py index f4f27eb5841..e8a8b504d96 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2141,6 +2141,49 @@ def completion( # type: ignore # noqa: PLR0915 logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, ) + elif custom_llm_provider == "gigachat": + # GigaChat - Sber AI's LLM (Russia) + api_key = ( + api_key + or litellm.api_key + or litellm.gigachat_key + or get_secret("GIGACHAT_API_KEY") + or get_secret("GIGACHAT_CREDENTIALS") + ) + + headers = headers or litellm.headers or {} + + ## COMPLETION CALL + try: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + elif custom_llm_provider == "sap": headers = headers or litellm.headers ## LOAD CONFIG - if set @@ -5224,6 +5267,28 @@ def embedding( # noqa: PLR0915 aembedding=aembedding, litellm_params={}, ) + elif custom_llm_provider == "gigachat": + api_key = ( + api_key + or litellm.api_key + or litellm.gigachat_key + or get_secret_str("GIGACHAT_CREDENTIALS") + or get_secret_str("GIGACHAT_API_KEY") + ) + response = base_llm_http_handler.embedding( + model=model, + input=input, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + logging_obj=logging, + timeout=timeout, + model_response=EmbeddingResponse(), + optional_params=optional_params, + client=client, + aembedding=aembedding, + litellm_params={"ssl_verify": kwargs.get("ssl_verify", None)}, + ) else: raise LiteLLMUnknownProvider( model=model, custom_llm_provider=custom_llm_provider diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e823dd5dc6b..c7a2f60856d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -15831,6 +15831,68 @@ "max_tokens": 8191, "mode": "embedding" }, + "gigachat/GigaChat-2-Lite": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0.0, + "supports_function_calling": true, + "supports_system_messages": true + }, + "gigachat/GigaChat-2-Max": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0.0, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_vision": true + }, + "gigachat/GigaChat-2-Pro": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0.0, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_vision": true + }, + "gigachat/Embeddings": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024 + }, + "gigachat/Embeddings-2": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024 + }, + "gigachat/EmbeddingsGigaR": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 4096, + "max_tokens": 4096, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 2560 + }, "google.gemma-3-12b-it": { "input_cost_per_token": 9e-08, "litellm_provider": "bedrock_converse", @@ -32092,3 +32154,4 @@ "mode": "chat" } } + diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index ffa17a5b7c4..ded591a8f53 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -15,6 +15,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( ) from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.types.mcp_server.mcp_server_manager import MCPServer +from litellm.proxy.utils import get_server_root_path router = APIRouter( tags=["mcp"], @@ -381,13 +382,30 @@ async def callback(code: str, state: str): # ------------------------------ # Optional .well-known endpoints for MCP + OAuth discovery # ------------------------------ -@router.get("/.well-known/oauth-protected-resource/{mcp_server_name}/mcp") +""" + Per SEP-985, the client MUST: + 1. Try resource_metadata from WWW-Authenticate header (if present) + 2. Fall back to path-based well-known URI: /.well-known/oauth-protected-resource/{path} + ( + If the resource identifier value contains a path or query component, any terminating slash (/) + following the host component MUST be removed before inserting /.well-known/ and the well-known + URI path suffix between the host component and the path(include root path) and/or query components. + https://datatracker.ietf.org/doc/html/rfc9728#section-3.1) + 3. Fall back to root-based well-known URI: /.well-known/oauth-protected-resource +""" +@router.get(f"/.well-known/oauth-protected-resource{'' if get_server_root_path() == '/' else get_server_root_path()}/{{mcp_server_name}}/mcp") @router.get("/.well-known/oauth-protected-resource") async def oauth_protected_resource_mcp( request: Request, mcp_server_name: Optional[str] = None ): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) # Get the correct base URL considering X-Forwarded-* headers request_base_url = get_request_base_url(request) + mcp_server: Optional[MCPServer] = None + if mcp_server_name: + mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name) return { "authorization_servers": [ ( @@ -401,14 +419,25 @@ async def oauth_protected_resource_mcp( if mcp_server_name else f"{request_base_url}/mcp" ), # this is what Claude will call + "scopes_supported": mcp_server.scopes if mcp_server else [], } - -@router.get("/.well-known/oauth-authorization-server/{mcp_server_name}") +""" + https://datatracker.ietf.org/doc/html/rfc8414#section-3.1 + RFC 8414: Path-aware OAuth discovery + If the issuer identifier value contains a path component, any + terminating "/" MUST be removed before inserting "/.well-known/" and + the well-known URI suffix between the host component and the path(include root path) + component. +""" +@router.get(f"/.well-known/oauth-authorization-server{'' if get_server_root_path() == '/' else get_server_root_path()}/{{mcp_server_name}}") @router.get("/.well-known/oauth-authorization-server") async def oauth_authorization_server_mcp( request: Request, mcp_server_name: Optional[str] = None ): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) # Get the correct base URL considering X-Forwarded-* headers request_base_url = get_request_base_url(request) @@ -423,16 +452,21 @@ async def oauth_authorization_server_mcp( else f"{request_base_url}/token" ) + mcp_server: Optional[MCPServer] = None + if mcp_server_name: + mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name) + return { "issuer": request_base_url, # point to your proxy "authorization_endpoint": authorization_endpoint, "token_endpoint": token_endpoint, "response_types_supported": ["code"], - "grant_types_supported": ["authorization_code"], + "scopes_supported": mcp_server.scopes if mcp_server else [], + "grant_types_supported": ["authorization_code", "refresh_token"], "code_challenge_methods_supported": ["S256"], "token_endpoint_auth_methods_supported": ["client_secret_post"], # Claude expects a registration endpoint, even if we just fake it - "registration_endpoint": f"{request_base_url}/{mcp_server_name}/register", + "registration_endpoint": f"{request_base_url}/{mcp_server_name}/register" if mcp_server_name else f"{request_base_url}/register", } diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index a5ac966062e..3a548e203c5 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -660,14 +660,14 @@ class MCPServerManager: """ allowed_mcp_servers = await self.get_allowed_mcp_servers(user_api_key_auth) - list_tools_result: List[MCPTool] = [] verbose_logger.debug("SERVER MANAGER LISTING TOOLS") - for server_id in allowed_mcp_servers: + async def _fetch_server_tools(server_id: str) -> List[MCPTool]: + """Fetch tools from a single server with error handling.""" server = self.get_mcp_server_by_id(server_id) if server is None: verbose_logger.warning(f"MCP Server {server_id} not found") - continue + return [] # Get server-specific auth header if available server_auth_header = None @@ -685,15 +685,21 @@ class MCPServerManager: server=server, mcp_auth_header=server_auth_header, ) - list_tools_result.extend(tools) - verbose_logger.info( - f"Successfully fetched {len(tools)} tools from server {server.name}" - ) + return tools except Exception as e: verbose_logger.warning( f"Failed to list tools from server {server.name}: {str(e)}. Continuing with other servers." ) - # Continue with other servers instead of failing completely + return [] + + # Fetch tools from all servers in parallel + tasks = [_fetch_server_tools(server_id) for server_id in allowed_mcp_servers] + results = await asyncio.gather(*tasks) + + # Flatten results into single list + list_tools_result: List[MCPTool] = [ + tool for tools in results for tool in tools + ] verbose_logger.info( f"Successfully fetched {len(list_tools_result)} tools total from all servers" @@ -2003,6 +2009,9 @@ class MCPServerManager: Note: This now handles prefixed tool names """ for server in self.get_registry().values(): + if server.auth_type == MCPAuth.oauth2: + # Skip OAuth2 servers for now as they may require user-specific tokens + continue tools = await self._get_tools_from_server(server) for tool in tools: # The tool.name here is already prefixed from _get_tools_from_server @@ -2284,14 +2293,7 @@ class MCPServerManager: # Check all accessible servers target_server_ids = allowed_server_ids - # Run health checks concurrently - tasks = [self.health_check_server(server_id) for server_id in target_server_ids] - results = await asyncio.gather(*tasks) - - # Filter out None results (servers that were not found) - list_mcp_servers = [server for server in results if server is not None] - - return list_mcp_servers + return await self._run_health_checks(target_server_ids) async def get_all_allowed_mcp_servers( self, @@ -2306,8 +2308,6 @@ class MCPServerManager: Returns: List of MCP server objects without health status """ - from datetime import datetime - # Get allowed server IDs allowed_server_ids = await self.get_allowed_mcp_servers(user_api_key_auth) @@ -2319,40 +2319,56 @@ class MCPServerManager: verbose_logger.warning(f"MCP Server {server_id} not found in registry") continue - # Build LiteLLM_MCPServerTable without health check - mcp_server_table = LiteLLM_MCPServerTable( - server_id=server.server_id, - server_name=server.server_name, - alias=server.alias, - description=( - server.mcp_info.get("description") if server.mcp_info else None - ), - url=server.url, - transport=server.transport, - auth_type=server.auth_type, - created_at=datetime.now(), - updated_at=datetime.now(), - teams=[], - mcp_access_groups=server.access_groups or [], - allowed_tools=server.allowed_tools or [], - extra_headers=server.extra_headers or [], - mcp_info=server.mcp_info, - static_headers=server.static_headers, - status=None, # No health check performed - last_health_check=None, # No health check performed - health_check_error=None, - command=getattr(server, "command", None), - args=getattr(server, "args", None) or [], - env=getattr(server, "env", None) or {}, - authorization_url=server.authorization_url, - token_url=server.token_url, - registration_url=server.registration_url, - allow_all_keys=server.allow_all_keys, - ) + mcp_server_table = self._build_mcp_server_table(server) list_mcp_servers.append(mcp_server_table) return list_mcp_servers + def _build_mcp_server_table(self, server: MCPServer) -> LiteLLM_MCPServerTable: + from datetime import datetime + + return LiteLLM_MCPServerTable( + server_id=server.server_id, + server_name=server.server_name, + alias=server.alias, + description=( + server.mcp_info.get("description") if server.mcp_info else None + ), + url=server.url, + transport=server.transport, + auth_type=server.auth_type, + created_at=datetime.now(), + updated_at=datetime.now(), + teams=[], + mcp_access_groups=server.access_groups or [], + allowed_tools=server.allowed_tools or [], + extra_headers=server.extra_headers or [], + mcp_info=server.mcp_info, + static_headers=server.static_headers, + status=None, # No health check performed + last_health_check=None, # No health check performed + health_check_error=None, + command=getattr(server, "command", None), + args=getattr(server, "args", None) or [], + env=getattr(server, "env", None) or {}, + authorization_url=server.authorization_url, + token_url=server.token_url, + registration_url=server.registration_url, + allow_all_keys=server.allow_all_keys, + ) + + async def get_all_mcp_servers_unfiltered(self) -> List[LiteLLM_MCPServerTable]: + """Return all MCP servers from registry without applying access controls.""" + + registry = self.get_registry() + if not registry: + return [] + + servers: List[LiteLLM_MCPServerTable] = [] + for server in registry.values(): + servers.append(self._build_mcp_server_table(server)) + return servers + async def reload_servers_from_database(self): """ Public method to reload all MCP servers from database into registry. @@ -2360,5 +2376,34 @@ class MCPServerManager: """ await self._add_mcp_servers_from_db_to_in_memory_registry() + async def get_all_mcp_servers_with_health_unfiltered( + self, server_ids: Optional[List[str]] = None + ) -> List[LiteLLM_MCPServerTable]: + """Return health info for all servers in registry regardless of user access.""" + + registry = self.get_registry() + if not registry: + return [] + + if server_ids: + target_server_ids = [sid for sid in server_ids if sid in registry] + else: + target_server_ids = list(registry.keys()) + + if not target_server_ids: + return [] + + return await self._run_health_checks(target_server_ids) + + async def _run_health_checks( + self, target_server_ids: List[str] + ) -> List[LiteLLM_MCPServerTable]: + if not target_server_ids: + return [] + + tasks = [self.health_check_server(server_id) for server_id in target_server_ids] + results = await asyncio.gather(*tasks) + return [server for server in results if server is not None] + global_mcp_server_manager: MCPServerManager = MCPServerManager() diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index e00fdbfb930..9c7001266f0 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -709,7 +709,8 @@ if MCP_AVAILABLE: extra_headers: Optional[Dict[str, str]] = None if server.auth_type == MCPAuth.oauth2: - extra_headers = oauth2_headers + # Copy to avoid mutating the original dict (important for parallel fetching) + extra_headers = oauth2_headers.copy() if oauth2_headers else None if server.extra_headers and raw_headers: if extra_headers is None: @@ -755,11 +756,10 @@ if MCP_AVAILABLE: # Decide whether to add prefix based on number of allowed servers add_prefix = not (len(allowed_mcp_servers) == 1) - # Get tools from each allowed server - all_tools = [] - for server in allowed_mcp_servers: + async def _fetch_and_filter_server_tools(server: MCPServer) -> List[MCPTool]: + """Fetch and filter tools from a single server with error handling.""" if server is None: - continue + return [] server_auth_header, extra_headers = _prepare_mcp_server_headers( server=server, @@ -786,16 +786,24 @@ if MCP_AVAILABLE: user_api_key_auth=user_api_key_auth, ) - all_tools.extend(filtered_tools) - verbose_logger.debug( f"Successfully fetched {len(tools)} tools from server {server.name}, {len(filtered_tools)} after filtering" ) + return filtered_tools except Exception as e: verbose_logger.exception( f"Error getting tools from server {server.name}: {str(e)}" ) - # Continue with other servers instead of failing completely + return [] + + # Fetch tools from all servers in parallel + tasks = [ + _fetch_and_filter_server_tools(server) for server in allowed_mcp_servers + ] + results = await asyncio.gather(*tasks) + + # Flatten results into single list + all_tools: List[MCPTool] = [tool for tools in results for tool in tools] verbose_logger.info( f"Successfully fetched {len(all_tools)} tools total from all MCP servers" diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b77cc40d6dc..954c26e2cb2 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1908,6 +1908,9 @@ class UserHeaderMapping(LiteLLMPydanticObjectBase): } +UserMCPManagementMode = Literal["restricted", "view_all"] + + class ConfigGeneralSettings(LiteLLMPydanticObjectBase): """ Documents all the fields supported by `general_settings` in config.yaml @@ -2025,6 +2028,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="Fine-grained control over which object types to load from the database when store_model_in_db is True. Available types: 'models', 'mcp', 'guardrails', 'vector_stores', 'pass_through_endpoints', 'prompts', 'model_cost_map'. If not set, all objects are loaded (default behavior).", ) + user_mcp_management_mode: Optional[UserMCPManagementMode] = Field( + None, + description="Controls how non-admin users interact with MCP servers in the dashboard. 'restricted' shows only accessible servers, 'view_all' lists every server in read-only mode.", + ) class ConfigYAML(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 34049a44c8c..537b48f06ed 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -319,6 +319,7 @@ class ProxyBaseLLMRequestProcessing: "aget_responses", "adelete_responses", "acancel_responses", + "acompact_responses", "acreate_batch", "aretrieve_batch", "alist_batches", @@ -457,6 +458,7 @@ class ProxyBaseLLMRequestProcessing: "aget_responses", "adelete_responses", "acancel_responses", + "acompact_responses", "atext_completion", "aimage_edit", "alist_input_items", diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index 406ddceabf5..fe27af78d58 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -154,7 +154,11 @@ class PrismaManager: prisma_dir = PrismaManager._get_prisma_dir() - return ProxyExtrasDBManager.setup_database(use_migrate=use_migrate) + from litellm.proxy.proxy_server import redis_usage_cache + + return ProxyExtrasDBManager.setup_database( + use_migrate=use_migrate, redis_cache=redis_usage_cache + ) else: # Use prisma db push with increased timeout subprocess.run( diff --git a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py index ea8f1b0a97f..5850103132c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py @@ -118,7 +118,7 @@ class LassoGuardrail(CustomGuardrail): Falls back to UUID if ULID library is not available. """ if ULID_AVAILABLE and ulid is not None: - return str(ulid.new()) # type: ignore + return str(ulid.ULID()) # type: ignore else: verbose_proxy_logger.debug("ULID library not available, using UUID") return str(uuid.uuid4()) diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index a871a6637a2..47793c8fc8e 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -32,8 +32,8 @@ from fastapi import ( from fastapi.responses import JSONResponse import litellm -from litellm._uuid import uuid from litellm._logging import verbose_logger, verbose_proxy_logger +from litellm._uuid import uuid from litellm.constants import LITELLM_PROXY_ADMIN_NAME from litellm.proxy._experimental.mcp_server.utils import ( validate_and_normalize_mcp_server_payload, @@ -67,7 +67,6 @@ if MCP_AVAILABLE: from litellm.proxy._experimental.mcp_server.ui_session_utils import ( build_effective_auth_contexts, ) - from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy._types import ( LiteLLM_MCPServerTable, LitellmUserRoles, @@ -76,8 +75,10 @@ if MCP_AVAILABLE: SpecialMCPServerName, UpdateMCPServerRequest, UserAPIKeyAuth, + UserMCPManagementMode, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view from litellm.proxy.management_helpers.utils import management_endpoint_wrapper from litellm.types.mcp import MCPCredentials @@ -302,6 +303,20 @@ if MCP_AVAILABLE: return {"access_groups": access_groups_list} ## FastAPI Routes + def _get_user_mcp_management_mode() -> UserMCPManagementMode: + proxy_general_settings: dict = {} + try: + from litellm.proxy.proxy_server import ( + general_settings as proxy_general_settings, + ) + except Exception: + pass + + mode = proxy_general_settings.get("user_mcp_management_mode") + if mode == "view_all": + return "view_all" + return "restricted" + @router.get( "/server", description="Returns the mcp server list with associated teams", @@ -319,18 +334,26 @@ if MCP_AVAILABLE: ``` """ - auth_contexts = await build_effective_auth_contexts(user_api_key_dict) + user_mcp_management_mode = _get_user_mcp_management_mode() - aggregated_servers: Dict[str, LiteLLM_MCPServerTable] = {} - for auth_context in auth_contexts: - servers = await global_mcp_server_manager.get_all_allowed_mcp_servers( - user_api_key_auth=auth_context + if user_mcp_management_mode == "view_all": + servers = await global_mcp_server_manager.get_all_mcp_servers_unfiltered() + redacted_mcp_servers = _redact_mcp_credentials_list(servers) + else: + auth_contexts = await build_effective_auth_contexts(user_api_key_dict) + + aggregated_servers: Dict[str, LiteLLM_MCPServerTable] = {} + for auth_context in auth_contexts: + servers = await global_mcp_server_manager.get_all_allowed_mcp_servers( + user_api_key_auth=auth_context + ) + for server in servers: + if server.server_id not in aggregated_servers: + aggregated_servers[server.server_id] = server + + redacted_mcp_servers = _redact_mcp_credentials_list( + aggregated_servers.values() ) - for server in servers: - if server.server_id not in aggregated_servers: - aggregated_servers[server.server_id] = server - - redacted_mcp_servers = _redact_mcp_credentials_list(aggregated_servers.values()) # augment the mcp servers with public status if litellm.public_mcp_servers is not None: @@ -372,6 +395,17 @@ if MCP_AVAILABLE: --header 'Authorization: Bearer your_api_key_here' ``` """ + user_mcp_management_mode = _get_user_mcp_management_mode() + + if user_mcp_management_mode == "view_all": + servers = await global_mcp_server_manager.get_all_mcp_servers_with_health_unfiltered( + server_ids=server_ids + ) + return [ + {"server_id": server.server_id, "status": server.status} + for server in servers + ] + auth_contexts = await build_effective_auth_contexts(user_api_key_dict) server_status_map: Dict[ diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 623e8408862..ec1bc5497bd 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -698,6 +698,88 @@ async def get_response_input_items( ) +@router.post( + "/v1/responses/compact", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], +) +@router.post( + "/responses/compact", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], +) +@router.post( + "/openai/v1/responses/compact", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], +) +async def compact_response( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Compact a response by running a compaction pass over a conversation. + + Returns encrypted, opaque items that can be used to reduce context size. + + Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/compact + + ```bash + curl -X POST http://localhost:4000/v1/responses/compact \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-4o", + "input": [{"role": "user", "content": "Hello"}] + }' + ``` + """ + from litellm.proxy.proxy_server import ( + _read_request_body, + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + data = await _read_request_body(request=request) + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="acompact_responses", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=None, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + @router.post( "/v1/responses/{response_id}/cancel", dependencies=[Depends(user_api_key_auth)], diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index fd00cfc1c0a..a321e25a9a5 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -25,6 +25,7 @@ ROUTE_ENDPOINT_MAPPING = { "alist_input_items": "/responses/{response_id}/input_items", "aimage_edit": "/images/edits", "acancel_responses": "/responses/{response_id}/cancel", + "acompact_responses": "/responses/compact", "aocr": "/ocr", "asearch": "/search", "avideo_generation": "/videos", @@ -116,6 +117,7 @@ async def route_request( "aget_responses", "adelete_responses", "acancel_responses", + "acompact_responses", "acreate_response_reply", "alist_input_items", "_arealtime", # private function for realtime API diff --git a/litellm/responses/main.py b/litellm/responses/main.py index e837346df23..8177b177fe6 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -1361,3 +1361,205 @@ def cancel_responses( completion_kwargs=local_vars, extra_kwargs=kwargs, ) + + +@client +async def acompact_responses( + input: Union[str, ResponseInputParam], + model: str, + instructions: Optional[str] = None, + previous_response_id: Optional[str] = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + # LiteLLM specific params, + custom_llm_provider: Optional[str] = None, + **kwargs, +) -> ResponsesAPIResponse: + """ + Async version of the POST Compact Responses API + + POST /v1/responses/compact endpoint in the responses API + + Runs a compaction pass over a conversation, returning encrypted, opaque items. + """ + local_vars = locals() + try: + loop = asyncio.get_event_loop() + kwargs["acompact_responses"] = True + + # get custom llm provider so we can use this for mapping exceptions + if custom_llm_provider is None: + _, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=model, api_base=local_vars.get("base_url", None) + ) + + func = partial( + compact_responses, + input=input, + model=model, + instructions=instructions, + previous_response_id=previous_response_id, + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + + # Update the responses_api_response_id with the model_id + if isinstance(response, ResponsesAPIResponse): + response = ResponsesAPIRequestUtils._update_responses_api_response_id_with_model_id( + responses_api_response=response, + litellm_metadata=kwargs.get("litellm_metadata", {}), + custom_llm_provider=custom_llm_provider, + ) + + return response + except Exception as e: + raise litellm.exception_type( + model=model, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +@client +def compact_responses( + input: Union[str, ResponseInputParam], + model: str, + instructions: Optional[str] = None, + previous_response_id: Optional[str] = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + # LiteLLM specific params, + custom_llm_provider: Optional[str] = None, + **kwargs, +) -> Union[ResponsesAPIResponse, Coroutine[Any, Any, ResponsesAPIResponse]]: + """ + Synchronous version of the POST Compact Responses API + + POST /v1/responses/compact endpoint in the responses API + + Runs a compaction pass over a conversation, returning encrypted, opaque items. + """ + local_vars = locals() + try: + litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore + litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) + _is_async = kwargs.pop("acompact_responses", False) is True + + # get llm provider logic + litellm_params = GenericLiteLLMParams(**kwargs) + + ( + model, + custom_llm_provider, + dynamic_api_key, + dynamic_api_base, + ) = litellm.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + + if custom_llm_provider is None: + raise ValueError("custom_llm_provider is required but passed as None") + + # get provider config + responses_api_provider_config: Optional[BaseResponsesAPIConfig] = ( + ProviderConfigManager.get_provider_responses_api_config( + model=model, + provider=litellm.LlmProviders(custom_llm_provider), + ) + ) + + if responses_api_provider_config is None: + raise ValueError( + f"COMPACT responses is not supported for {custom_llm_provider}" + ) + + local_vars.update(kwargs) + + # Build optional params for compact endpoint + response_api_optional_params: ResponsesAPIOptionalRequestParams = ( + ResponsesAPIRequestUtils.get_requested_response_api_optional_param( + local_vars + ) + ) + + # Get optional parameters for the responses API + responses_api_request_params: Dict = ( + ResponsesAPIRequestUtils.get_optional_params_responses_api( + model=model, + responses_api_provider_config=responses_api_provider_config, + response_api_optional_params=response_api_optional_params, + allowed_openai_params=None, + ) + ) + + # Pre Call logging + litellm_logging_obj.update_environment_variables( + model=model, + optional_params=dict(responses_api_request_params), + litellm_params={ + **responses_api_request_params, + "litellm_call_id": litellm_call_id, + }, + custom_llm_provider=custom_llm_provider, + ) + + # Call the handler with _is_async flag instead of directly calling the async handler + response = base_llm_http_handler.compact_response_api_handler( + model=model, + input=input, + responses_api_provider_config=responses_api_provider_config, + response_api_optional_request_params=responses_api_request_params, + litellm_params=litellm_params, + logging_obj=litellm_logging_obj, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout or request_timeout, + _is_async=_is_async, + client=kwargs.get("client"), + shared_session=kwargs.get("shared_session"), + ) + + # Update the responses_api_response_id with the model_id + if isinstance(response, ResponsesAPIResponse): + response = ResponsesAPIRequestUtils._update_responses_api_response_id_with_model_id( + responses_api_response=response, + litellm_metadata=kwargs.get("litellm_metadata", {}), + custom_llm_provider=custom_llm_provider, + ) + + return response + except Exception as e: + raise litellm.exception_type( + model=model, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 0407776029d..0b838f916e2 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -1,5 +1,6 @@ import asyncio import json +import traceback from datetime import datetime from typing import Any, Dict, Optional @@ -11,6 +12,9 @@ from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base +from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + update_response_metadata, +) from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.responses.utils import ResponsesAPIRequestUtils @@ -22,7 +26,8 @@ from litellm.types.llms.openai import ( ResponsesAPIStreamEvents, ResponsesAPIStreamingResponse, ) -from litellm.utils import CustomStreamWrapper +from litellm.types.utils import CallTypes +from litellm.utils import CustomStreamWrapper, async_post_call_success_deployment_hook class BaseResponsesAPIStreamingIterator: @@ -40,6 +45,8 @@ class BaseResponsesAPIStreamingIterator: logging_obj: LiteLLMLoggingObj, litellm_metadata: Optional[Dict[str, Any]] = None, custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict[str, Any]] = None, + call_type: Optional[str] = None, ): self.response = response self.model = model @@ -47,21 +54,25 @@ class BaseResponsesAPIStreamingIterator: self.finished = False self.responses_api_provider_config = responses_api_provider_config self.completed_response: Optional[ResponsesAPIStreamingResponse] = None - self.start_time = datetime.now() + self.start_time = getattr(logging_obj, "start_time", datetime.now()) - # set request kwargs + # track request context for hooks self.litellm_metadata = litellm_metadata self.custom_llm_provider = custom_llm_provider + self.request_data: Dict[str, Any] = request_data or {} + self.call_type: Optional[str] = call_type # set hidden params for response headers (e.g., x-litellm-model-id) - # This matches ths stream wrapper in litellm/litellm_core_utils/streaming_handler.py + # This matches the stream wrapper in litellm/litellm_core_utils/streaming_handler.py _api_base = get_api_base( model=model or "", optional_params=self.logging_obj.model_call_details.get( "litellm_params", {} ), ) - _model_info: Dict = litellm_metadata.get("model_info", {}) if litellm_metadata else {} + _model_info: Dict = ( + litellm_metadata.get("model_info", {}) if litellm_metadata else {} + ) self._hidden_params = { "model_id": _model_info.get("id", None), "api_base": _api_base, @@ -102,13 +113,21 @@ class BaseResponsesAPIStreamingIterator: # if "response" in parsed_chunk, then encode litellm specific information like custom_llm_provider response_object = getattr(openai_responses_api_chunk, "response", None) if response_object: - response = ResponsesAPIRequestUtils._update_responses_api_response_id_with_model_id( - responses_api_response=response_object, - litellm_metadata=self.litellm_metadata, - custom_llm_provider=self.custom_llm_provider, + response = ( + ResponsesAPIRequestUtils._update_responses_api_response_id_with_model_id( + responses_api_response=response_object, + litellm_metadata=self.litellm_metadata, + custom_llm_provider=self.custom_llm_provider, + ) ) setattr(openai_responses_api_chunk, "response", response) + # Allow callbacks to modify chunk before returning + openai_responses_api_chunk = run_async_function( + async_function=self._call_post_streaming_deployment_hook, + chunk=openai_responses_api_chunk, + ) + # Store the completed response if ( openai_responses_api_chunk @@ -149,11 +168,159 @@ class BaseResponsesAPIStreamingIterator: except json.JSONDecodeError: # If we can't parse the chunk, continue return None + except Exception as e: + # Ensure failures trigger failure hooks + self._handle_failure(e) + raise def _handle_logging_completed_response(self): """Base implementation - should be overridden by subclasses""" pass + async def _call_post_streaming_deployment_hook(self, chunk): + """ + Allow callbacks to modify streaming chunks before returning (parity with chat). + """ + try: + # Align with chat pipeline: use logging_obj model_call_details + call_type + typed_call_type: Optional[CallTypes] = None + if self.call_type is not None: + try: + typed_call_type = CallTypes(self.call_type) + except ValueError: + typed_call_type = None + if typed_call_type is None: + try: + typed_call_type = CallTypes(getattr(self.logging_obj, "call_type", None)) + except Exception: + typed_call_type = None + + request_data = self.request_data or getattr( + self.logging_obj, "model_call_details", {} + ) + callbacks = getattr(litellm, "callbacks", None) or [] + hooks_ran = False + for callback in callbacks: + if hasattr(callback, "async_post_call_streaming_deployment_hook"): + hooks_ran = True + result = await callback.async_post_call_streaming_deployment_hook( + request_data=request_data, + response_chunk=chunk, + call_type=typed_call_type, + ) + if result is not None: + chunk = result + if hooks_ran: + setattr(chunk, "_post_streaming_hooks_ran", True) + return chunk + except Exception: + return chunk + + async def call_post_streaming_hooks_for_testing(self, chunk): + """ + Helper to invoke streaming deployment hooks explicitly (used in tests). + """ + return await self._call_post_streaming_deployment_hook(chunk) + + def _run_post_success_hooks(self, end_time: datetime): + """ + Run post-call deployment hooks and update metadata similar to chat pipeline. + """ + if self.completed_response is None: + return + + request_payload: Dict[str, Any] = {} + if isinstance(self.request_data, dict): + request_payload.update(self.request_data) + try: + if hasattr(self.logging_obj, "model_call_details"): + request_payload.update(self.logging_obj.model_call_details) + except Exception: + pass + if "litellm_params" not in request_payload: + try: + request_payload["litellm_params"] = getattr( + self.logging_obj, "model_call_details", {} + ).get("litellm_params", {}) + except Exception: + request_payload["litellm_params"] = {} + + try: + update_response_metadata( + result=self.completed_response, + logging_obj=self.logging_obj, + model=self.model, + kwargs=request_payload, + start_time=self.start_time, + end_time=end_time, + ) + except Exception: + # Non-blocking + pass + + try: + typed_call_type: Optional[CallTypes] = None + if self.call_type is not None: + try: + typed_call_type = CallTypes(self.call_type) + except ValueError: + typed_call_type = None + except Exception: + typed_call_type = None + if typed_call_type is None: + try: + typed_call_type = CallTypes.responses + except Exception: + typed_call_type = None + + try: + # Call synchronously; async hook will be executed via asyncio.run in a new loop + run_async_function( + async_function=async_post_call_success_deployment_hook, + request_data=request_payload, + response=self.completed_response, + call_type=typed_call_type, + ) + except Exception: + pass + + def _handle_failure(self, exception: Exception): + """ + Trigger failure handlers before bubbling the exception. + """ + traceback_exception = traceback.format_exc() + try: + run_async_function( + async_function=self.logging_obj.async_failure_handler, + exception=exception, + traceback_exception=traceback_exception, + start_time=self.start_time, + end_time=datetime.now(), + ) + except Exception: + pass + + try: + executor.submit( + self.logging_obj.failure_handler, + exception, + traceback_exception, + self.start_time, + datetime.now(), + ) + except Exception: + pass + + +async def call_post_streaming_hooks_for_testing(iterator, chunk): + """ + Module-level helper for tests to ensure hooks can be invoked even if the iterator is wrapped. + """ + hook_fn = getattr(iterator, "_call_post_streaming_deployment_hook", None) + if hook_fn is None: + return chunk + return await hook_fn(chunk) + class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): """ @@ -168,6 +335,8 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): logging_obj: LiteLLMLoggingObj, litellm_metadata: Optional[Dict[str, Any]] = None, custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict[str, Any]] = None, + call_type: Optional[str] = None, ): super().__init__( response, @@ -176,6 +345,8 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): logging_obj, litellm_metadata, custom_llm_provider, + request_data, + call_type, ) self.stream_iterator = response.aiter_lines() @@ -203,16 +374,21 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): except httpx.HTTPError as e: # Handle HTTP errors self.finished = True + self._handle_failure(e) + raise e + except Exception as e: + self.finished = True + self._handle_failure(e) raise e def _handle_logging_completed_response(self): """Handle logging for completed responses in async context""" # Create a deep copy for logging to avoid modifying the response object that will be returned to the user - # The logging handlers may transform usage from Responses API format (input_tokens/output_tokens) + # The logging handlers may transform usage from Responses API format (input_tokens/output_tokens) # to chat completion format (prompt_tokens/completion_tokens) for internal logging import copy logging_response = copy.deepcopy(self.completed_response) - + asyncio.create_task( self.logging_obj.async_success_handler( result=logging_response, @@ -229,6 +405,7 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): start_time=self.start_time, end_time=datetime.now(), ) + self._run_post_success_hooks(end_time=datetime.now()) class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): @@ -244,6 +421,8 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): logging_obj: LiteLLMLoggingObj, litellm_metadata: Optional[Dict[str, Any]] = None, custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict[str, Any]] = None, + call_type: Optional[str] = None, ): super().__init__( response, @@ -252,6 +431,8 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): logging_obj, litellm_metadata, custom_llm_provider, + request_data, + call_type, ) self.stream_iterator = response.iter_lines() @@ -279,16 +460,21 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): except httpx.HTTPError as e: # Handle HTTP errors self.finished = True + self._handle_failure(e) + raise e + except Exception as e: + self.finished = True + self._handle_failure(e) raise e def _handle_logging_completed_response(self): """Handle logging for completed responses in sync context""" # Create a deep copy for logging to avoid modifying the response object that will be returned to the user - # The logging handlers may transform usage from Responses API format (input_tokens/output_tokens) + # The logging handlers may transform usage from Responses API format (input_tokens/output_tokens) # to chat completion format (prompt_tokens/completion_tokens) for internal logging import copy logging_response = copy.deepcopy(self.completed_response) - + run_async_function( async_function=self.logging_obj.async_success_handler, result=logging_response, @@ -304,6 +490,7 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): start_time=self.start_time, end_time=datetime.now(), ) + self._run_post_success_hooks(end_time=datetime.now()) class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): @@ -324,6 +511,8 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): logging_obj: LiteLLMLoggingObj, litellm_metadata: Optional[Dict[str, Any]] = None, custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict[str, Any]] = None, + call_type: Optional[str] = None, ): super().__init__( response=response, @@ -332,6 +521,8 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): logging_obj=logging_obj, litellm_metadata=litellm_metadata, custom_llm_provider=custom_llm_provider, + request_data=request_data, + call_type=call_type, ) # one-time transform diff --git a/litellm/router.py b/litellm/router.py index 6821ab9e6c6..d980b5f74d8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -713,6 +713,23 @@ class Router: self, routing_strategy: Union[RoutingStrategy, str], routing_strategy_args: dict ): verbose_router_logger.info(f"Routing strategy: {routing_strategy}") + + # Validate routing_strategy value to fail fast with helpful error + # See: https://github.com/BerriAI/litellm/issues/11330 + # Derive valid strategies from RoutingStrategy enum + "simple-shuffle" (default, not in enum) + valid_strategy_strings = ["simple-shuffle"] + [s.value for s in RoutingStrategy] + + if routing_strategy is not None: + is_valid_string = isinstance(routing_strategy, str) and routing_strategy in valid_strategy_strings + is_valid_enum = isinstance(routing_strategy, RoutingStrategy) + if not is_valid_string and not is_valid_enum: + raise ValueError( + f"Invalid routing_strategy: '{routing_strategy}'. " + f"Valid options: {valid_strategy_strings}. " + f"Check 'router_settings.routing_strategy' in your config.yaml " + f"or the 'routing_strategy' parameter if using the Router SDK directly." + ) + if ( routing_strategy == RoutingStrategy.LEAST_BUSY.value or routing_strategy == RoutingStrategy.LEAST_BUSY @@ -812,6 +829,9 @@ class Router: self.acancel_responses = self.factory_function( litellm.acancel_responses, call_type="acancel_responses" ) + self.acompact_responses = self.factory_function( + litellm.acompact_responses, call_type="acompact_responses" + ) self.adelete_responses = self.factory_function( litellm.adelete_responses, call_type="adelete_responses" ) @@ -3924,6 +3944,7 @@ class Router: "anthropic_messages", "aresponses", "acancel_responses", + "acompact_responses", "responses", "aget_responses", "adelete_responses", @@ -4152,6 +4173,7 @@ class Router: elif call_type in ( "aget_responses", "acancel_responses", + "acompact_responses", "adelete_responses", "alist_input_items", ): diff --git a/litellm/types/integrations/langsmith.py b/litellm/types/integrations/langsmith.py index 23f760ecf32..9c026a117fd 100644 --- a/litellm/types/integrations/langsmith.py +++ b/litellm/types/integrations/langsmith.py @@ -31,6 +31,7 @@ class LangsmithCredentialsObject(TypedDict): LANGSMITH_API_KEY: Optional[str] LANGSMITH_PROJECT: Optional[str] LANGSMITH_BASE_URL: str + LANGSMITH_TENANT_ID: Optional[str] class LangsmithQueueObject(TypedDict): @@ -52,6 +53,7 @@ class CredentialsKey(NamedTuple): api_key: str project: str base_url: str + tenant_id: Optional[str] @dataclass diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3eec67d9d26..784c8403c3f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2677,6 +2677,7 @@ class StandardCallbackDynamicParams(TypedDict, total=False): langsmith_project: Optional[str] langsmith_base_url: Optional[str] langsmith_sampling_rate: Optional[float] + langsmith_tenant_id: Optional[str] # Humanloop dynamic params humanloop_api_key: Optional[str] @@ -2946,6 +2947,7 @@ class LlmProviders(str, Enum): MISTRAL = "mistral" MILVUS = "milvus" GROQ = "groq" + GIGACHAT = "gigachat" NVIDIA_NIM = "nvidia_nim" CEREBRAS = "cerebras" AI21_CHAT = "ai21_chat" diff --git a/litellm/utils.py b/litellm/utils.py index 6b9aeca0934..fbbaa94f7a1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7521,6 +7521,8 @@ class ProviderConfigManager: return litellm.CompactifAIChatConfig() elif litellm.LlmProviders.GITHUB_COPILOT == provider: return litellm.GithubCopilotConfig() + elif litellm.LlmProviders.GIGACHAT == provider: + return litellm.GigaChatConfig() elif litellm.LlmProviders.RAGFLOW == provider: return litellm.RAGFlowConfig() elif ( @@ -7716,6 +7718,8 @@ class ProviderConfigManager: return litellm.CometAPIEmbeddingConfig() elif litellm.LlmProviders.GITHUB_COPILOT == provider: return litellm.GithubCopilotEmbeddingConfig() + elif litellm.LlmProviders.GIGACHAT == provider: + return litellm.GigaChatEmbeddingConfig() elif litellm.LlmProviders.SAGEMAKER == provider: from litellm.llms.sagemaker.embedding.transformation import ( SagemakerEmbeddingConfig, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e823dd5dc6b..c7a2f60856d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -15831,6 +15831,68 @@ "max_tokens": 8191, "mode": "embedding" }, + "gigachat/GigaChat-2-Lite": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0.0, + "supports_function_calling": true, + "supports_system_messages": true + }, + "gigachat/GigaChat-2-Max": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0.0, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_vision": true + }, + "gigachat/GigaChat-2-Pro": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0.0, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_vision": true + }, + "gigachat/Embeddings": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024 + }, + "gigachat/Embeddings-2": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024 + }, + "gigachat/EmbeddingsGigaR": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 4096, + "max_tokens": 4096, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 2560 + }, "google.gemma-3-12b-it": { "input_cost_per_token": 9e-08, "litellm_provider": "bedrock_converse", @@ -32092,3 +32154,4 @@ "mode": "chat" } } + diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 45ee47c01bc..bc5dea7b97c 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -28,7 +28,8 @@ "list_container_files": "Supports GET /containers/{id}/files endpoint", "retrieve_container_file": "Supports GET /containers/{id}/files/{file_id} endpoint", "retrieve_container_file_content": "Supports GET /containers/{id}/files/{file_id}/content endpoint", - "delete_container_file": "Supports DELETE /containers/{id}/files/{file_id} endpoint" + "delete_container_file": "Supports DELETE /containers/{id}/files/{file_id} endpoint", + "compact": "Supports /responses/compact endpoint" } } }, @@ -1519,6 +1520,7 @@ "retrieve_container_file": true, "retrieve_container_file_content": true, "delete_container_file": true, + "compact": true, "a2a": true, "interactions": true } diff --git a/requirements.txt b/requirements.txt index 06a7c17336c..249b899b86b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ google-cloud-aiplatform==1.47.0 # for vertex ai calls google-cloud-iam==2.19.1 # for GCP IAM Redis authentication google-genai==1.22.0 anthropic[vertex]==0.54.0 -mcp==1.23.0 ; python_version >= "3.10" # for MCP server +mcp==1.25.0 ; python_version >= "3.10" # for MCP server google-generativeai==0.5.0 # for vertex ai calls async_generator==1.10.0 # for async ollama calls langfuse==2.59.7 # for langfuse self-hosted logging diff --git a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py index 9a96919da87..60d4f479733 100644 --- a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py +++ b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py @@ -61,13 +61,19 @@ async def test_bedrock_apply_guardrail_blocked(): guardrailVersion="DRAFT", ) - # Mock the make_bedrock_api_request method + # Mock the make_bedrock_api_request method to raise an exception for blocked content with patch.object( - guardrail, "make_bedrock_api_request", new_callable=AsyncMock + guardrail, "make_bedrock_api_request", new_callable=AsyncMock ) as mock_api_request: - # Mock a blocked response from Bedrock - mock_response = {"action": "BLOCKED", "reason": "Content violates policy"} - mock_api_request.return_value = mock_response + # Mock the method to raise an HTTPException as it would for blocked content + from fastapi import HTTPException + mock_api_request.side_effect = HTTPException( + status_code=400, + detail={ + "error": "Violated guardrail policy", + "bedrock_guardrail_response": "", + }, + ) # Test the apply_guardrail method should raise an exception with pytest.raises(Exception) as exc_info: @@ -77,8 +83,9 @@ async def test_bedrock_apply_guardrail_blocked(): input_type="request", ) - assert "Content blocked by Bedrock guardrail" in str(exc_info.value) - assert "Content violates policy" in str(exc_info.value) + # The apply_guardrail method wraps the original exception in a generic Exception + assert "Bedrock guardrail failed:" in str(exc_info.value) + assert "Violated guardrail policy" in str(exc_info.value) @pytest.mark.asyncio @@ -253,7 +260,15 @@ async def test_bedrock_apply_guardrail_filters_request_messages_when_flag_enable with patch.object( guardrail, "make_bedrock_api_request", new_callable=AsyncMock ) as mock_api: - mock_api.return_value = {"action": "BLOCKED", "reason": "policy"} + # Mock the method to raise an HTTPException as it would for blocked content + from fastapi import HTTPException + mock_api.side_effect = HTTPException( + status_code=400, + detail={ + "error": "Violated guardrail policy", + "bedrock_guardrail_response": "policy", + }, + ) with pytest.raises(Exception, match="policy") as exc_info: await guardrail.apply_guardrail( @@ -265,7 +280,8 @@ async def test_bedrock_apply_guardrail_filters_request_messages_when_flag_enable assert mock_api.called _, kwargs = mock_api.call_args assert kwargs["messages"] == [request_messages[-1]] - assert "Content blocked by Bedrock guardrail" in str(exc_info.value) + # The apply_guardrail method wraps the original exception in a generic Exception + assert "Bedrock guardrail failed:" in str(exc_info.value) def test_bedrock_guardrail_filters_latest_user_message_when_enabled(): diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py index 5714cd5c487..c981ccd8dac 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py @@ -1,11 +1,14 @@ import os import sys +import time +from unittest.mock import Mock, patch sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path -from litellm_proxy_extras.utils import ProxyExtrasDBManager +from litellm_proxy_extras.utils import ProxyExtrasDBManager, MigrationLockManager + def test_custom_prisma_dir(monkeypatch): @@ -27,101 +30,279 @@ def test_custom_prisma_dir(monkeypatch): assert os.path.exists(migrations_dir) -class TestPermissionErrorDetection: - """Test cases for permission error detection in Prisma migrations""" +class TestMigrationLockManager: + """Test cases for MigrationLockManager""" - def test_is_permission_error_postgres_42501(self): - """Test detection of PostgreSQL 42501 error code (insufficient privilege)""" - error_message = "Database error code: 42501 - permission denied for table users" - assert ProxyExtrasDBManager._is_permission_error(error_message) is True + def test_acquire_lock_without_redis(self): + """Test lock acquisition when Redis is not available""" + lock_manager = MigrationLockManager() + result = lock_manager.acquire_lock() + assert result is True # Should return True when Redis is not available + assert lock_manager.lock_acquired is True # Redis 없을 때도 lock_acquired는 True - def test_is_permission_error_must_be_owner(self): - """Test detection of 'must be owner of table' error""" - error_message = "ERROR: must be owner of table my_table" - assert ProxyExtrasDBManager._is_permission_error(error_message) is True + def test_acquire_lock_with_redis_success(self): + """Test successful lock acquisition with Redis""" + mock_redis = Mock() + mock_redis.set_cache.return_value = True + lock_manager = MigrationLockManager(mock_redis) - def test_is_permission_error_permission_denied_schema(self): - """Test detection of 'permission denied for schema' error""" - error_message = "permission denied for schema public" - assert ProxyExtrasDBManager._is_permission_error(error_message) is True + result = lock_manager.acquire_lock() - def test_is_permission_error_permission_denied_table(self): - """Test detection of 'permission denied for table' error""" - error_message = "permission denied for table my_table" - assert ProxyExtrasDBManager._is_permission_error(error_message) is True + assert result is True + assert lock_manager.lock_acquired is True + mock_redis.set_cache.assert_called_once() - def test_is_permission_error_must_be_owner_schema(self): - """Test detection of 'must be owner of schema' error""" - error_message = "must be owner of schema public" - assert ProxyExtrasDBManager._is_permission_error(error_message) is True + def test_acquire_lock_with_redis_failure(self): + """Test failed lock acquisition with Redis""" + mock_redis = Mock() + mock_redis.set_cache.return_value = False + lock_manager = MigrationLockManager(mock_redis) - def test_is_permission_error_case_insensitive(self): - """Test that permission error detection is case insensitive""" - error_message = "PERMISSION DENIED FOR TABLE my_table" - assert ProxyExtrasDBManager._is_permission_error(error_message) is True + result = lock_manager.acquire_lock() - def test_is_permission_error_negative(self): - """Test that non-permission errors are not detected as permission errors""" - error_message = "column 'id' already exists" - assert ProxyExtrasDBManager._is_permission_error(error_message) is False + assert result is False + assert lock_manager.lock_acquired is False + mock_redis.set_cache.assert_called_once() + + def test_acquire_lock_with_redis_exception(self): + """Test lock acquisition with Redis exception""" + mock_redis = Mock() + mock_redis.set_cache.side_effect = Exception("Redis error") + lock_manager = MigrationLockManager(mock_redis) + + result = lock_manager.acquire_lock() + + assert result is False + assert lock_manager.lock_acquired is False + + def test_wait_for_lock_release_success(self): + """Test successful waiting for lock release""" + mock_redis = Mock() + # First call returns False (lock held), second call returns True (lock acquired) + mock_redis.set_cache.side_effect = [False, True] + lock_manager = MigrationLockManager(mock_redis) + + result = lock_manager.wait_for_lock_release(check_interval=0.1, max_wait=1) + + assert result is True + assert lock_manager.lock_acquired is True + assert mock_redis.set_cache.call_count == 2 + + def test_wait_for_lock_release_timeout(self): + """Test timeout while waiting for lock release""" + mock_redis = Mock() + mock_redis.set_cache.return_value = False # Lock always held + lock_manager = MigrationLockManager(mock_redis) + + result = lock_manager.wait_for_lock_release(check_interval=0.1, max_wait=0.2) + + assert result is False + assert lock_manager.lock_acquired is False + + def test_release_lock_not_acquired(self): + """Test releasing lock when not acquired""" + mock_redis = Mock() + lock_manager = MigrationLockManager(mock_redis) + + lock_manager.release_lock() + + mock_redis.get_cache.assert_not_called() + mock_redis.delete_cache.assert_not_called() + + def test_release_lock_success(self): + """Test successful lock release""" + mock_redis = Mock() + mock_redis.get_cache.return_value = "pod_123_456" + lock_manager = MigrationLockManager(mock_redis) + lock_manager.pod_id = "pod_123_456" + lock_manager.lock_acquired = True + + lock_manager.release_lock() + + mock_redis.get_cache.assert_called_once() + mock_redis.delete_cache.assert_called_once() + assert lock_manager.lock_acquired is False + + def test_release_lock_wrong_owner(self): + """Test releasing lock when not the owner""" + mock_redis = Mock() + mock_redis.get_cache.return_value = "pod_999_999" # Different pod + lock_manager = MigrationLockManager(mock_redis) + lock_manager.pod_id = "pod_123_456" + lock_manager.lock_acquired = True + + lock_manager.release_lock() + + mock_redis.get_cache.assert_called_once() + mock_redis.delete_cache.assert_not_called() + assert lock_manager.lock_acquired is False + + def test_context_manager(self): + """Test MigrationLockManager as context manager""" + mock_redis = Mock() + mock_redis.set_cache.return_value = True + # Mock get_cache to return the same pod_id for successful release + mock_redis.get_cache.return_value = "pod_123_456" + + lock_manager = MigrationLockManager(mock_redis) + lock_manager.pod_id = "pod_123_456" # Set consistent pod_id + + with lock_manager: + assert lock_manager.lock_acquired is True + + # Should call release_lock when exiting context + mock_redis.get_cache.assert_called_once() + mock_redis.delete_cache.assert_called_once() -class TestIdempotentErrorDetection: - """Test cases for idempotent error detection in Prisma migrations""" +class TestProxyExtrasDBManagerMigrationLock: + """Test cases for ProxyExtrasDBManager with migration locking""" - def test_is_idempotent_error_already_exists(self): - """Test detection of generic 'already exists' error""" - error_message = "object already exists" - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + @patch("litellm_proxy_extras.utils.ProxyExtrasDBManager._resolve_all_migrations") + @patch("litellm_proxy_extras.utils.subprocess.run") + @patch("litellm_proxy_extras.utils.ProxyExtrasDBManager._get_prisma_dir") + @patch("os.chdir") + def test_setup_database_with_redis_lock_success( + self, mock_chdir, mock_get_prisma_dir, mock_subprocess, mock_resolve_migrations + ): + """Test successful database setup with Redis lock""" + # Setup mocks + mock_get_prisma_dir.return_value = "/test/prisma" + mock_subprocess.return_value = Mock(stdout="Migration completed", stderr="") + mock_resolve_migrations.return_value = None - def test_is_idempotent_error_column_already_exists(self): - """Test detection of 'column already exists' error""" - error_message = "column 'email' already exists" - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + # Mock Redis cache + mock_redis = Mock() + mock_redis.set_cache.return_value = True # Lock acquired successfully + mock_redis.get_cache.return_value = "pod_123_456" - def test_is_idempotent_error_duplicate_key(self): - """Test detection of duplicate key violation error""" - error_message = "duplicate key value violates unique constraint" - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + # Set DATABASE_URL + with patch.dict( + os.environ, {"DATABASE_URL": "postgresql://test:test@localhost/test"} + ): + result = ProxyExtrasDBManager.setup_database( + use_migrate=True, redis_cache=mock_redis + ) - def test_is_idempotent_error_relation_already_exists(self): - """Test detection of 'relation already exists' error""" - error_message = "relation 'users_pkey' already exists" - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + assert result is True + # set_cache is called once in acquire_lock (__enter__ calls acquire_lock) + assert mock_redis.set_cache.call_count == 1 + mock_subprocess.assert_called_once() - def test_is_idempotent_error_constraint_already_exists(self): - """Test detection of 'constraint already exists' error""" - error_message = "constraint 'fk_user_id' already exists" - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + @patch("litellm_proxy_extras.utils.ProxyExtrasDBManager._resolve_all_migrations") + @patch("litellm_proxy_extras.utils.subprocess.run") + @patch("litellm_proxy_extras.utils.ProxyExtrasDBManager._get_prisma_dir") + @patch("os.chdir") + def test_setup_database_with_redis_lock_wait_and_skip( + self, mock_chdir, mock_get_prisma_dir, mock_subprocess, mock_resolve_migrations + ): + """Test database setup when lock is held by another pod, then acquired after waiting""" + # Setup mocks + mock_get_prisma_dir.return_value = "/test/prisma" + mock_resolve_migrations.return_value = None - def test_is_idempotent_error_case_insensitive(self): - """Test that idempotent error detection is case insensitive""" - error_message = "COLUMN 'ID' ALREADY EXISTS" - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + # Mock Redis cache - first call fails, second call succeeds + mock_redis = Mock() + mock_redis.set_cache.side_effect = [False, True] # First fails, then succeeds + mock_redis.get_cache.return_value = "pod_123_456" - def test_is_idempotent_error_negative(self): - """Test that non-idempotent errors are not detected as idempotent errors""" - error_message = "Database error code: 42501 - permission denied" - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is False + # Set DATABASE_URL + with patch.dict( + os.environ, {"DATABASE_URL": "postgresql://test:test@localhost/test"} + ): + result = ProxyExtrasDBManager.setup_database( + use_migrate=True, redis_cache=mock_redis + ) + assert result is True # Should return True after waiting and acquiring lock + # set_cache is called 2 times: once in __enter__, once in wait_for_lock_release + assert mock_redis.set_cache.call_count == 2 + # Proceed for case handling in case of migration failure + mock_subprocess.assert_called_once() -class TestErrorClassificationPriority: - """Test cases to ensure errors are correctly classified""" + @patch("litellm_proxy_extras.utils.ProxyExtrasDBManager._resolve_all_migrations") + @patch("litellm_proxy_extras.utils.subprocess.run") + @patch("litellm_proxy_extras.utils.ProxyExtrasDBManager._get_prisma_dir") + @patch("os.chdir") + def test_setup_database_without_redis( + self, mock_chdir, mock_get_prisma_dir, mock_subprocess, mock_resolve_migrations + ): + """Test database setup without Redis cache""" + # Setup mocks + mock_get_prisma_dir.return_value = "/test/prisma" + mock_subprocess.return_value = Mock(stdout="Migration completed", stderr="") + mock_resolve_migrations.return_value = None - def test_permission_error_not_classified_as_idempotent(self): - """Ensure permission errors are not mistakenly classified as idempotent""" - error_message = "Database error code: 42501 - must be owner of table users" - assert ProxyExtrasDBManager._is_permission_error(error_message) is True - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is False + # Set DATABASE_URL + with patch.dict( + os.environ, {"DATABASE_URL": "postgresql://test:test@localhost/test"} + ): + result = ProxyExtrasDBManager.setup_database( + use_migrate=True, redis_cache=None + ) - def test_idempotent_error_not_classified_as_permission(self): - """Ensure idempotent errors are not mistakenly classified as permission errors""" - error_message = "column 'created_at' already exists" - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True - assert ProxyExtrasDBManager._is_permission_error(error_message) is False + assert result is True + # Redis가 없을 때는 락 보호 없이 마이그레이션을 실행해야 함 + mock_subprocess.assert_called_once() - def test_unknown_error_classified_as_neither(self): - """Ensure unknown errors are classified as neither permission nor idempotent""" - error_message = "connection timeout" - assert ProxyExtrasDBManager._is_permission_error(error_message) is False - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is False + def test_setup_database_no_database_url(self): + """Test database setup without DATABASE_URL""" + with patch.dict(os.environ, {}, clear=True): + result = ProxyExtrasDBManager.setup_database( + use_migrate=True, redis_cache=None + ) + + assert result is False + + @patch("litellm_proxy_extras.utils.subprocess.run") + @patch("litellm_proxy_extras.utils.ProxyExtrasDBManager._get_prisma_dir") + @patch("os.chdir") + @patch.object( + MigrationLockManager, "LOCK_TTL_SECONDS", 1 + ) # Set short TTL for testing + def test_setup_database_lock_timeout( + self, mock_chdir, mock_get_prisma_dir, mock_subprocess + ): + """Test database setup when lock acquisition times out""" + # Setup mocks + mock_get_prisma_dir.return_value = "/test/prisma" + + # Mock Redis cache - always fails to acquire lock + mock_redis = Mock() + mock_redis.set_cache.return_value = False # Always fails + + # Set DATABASE_URL + with patch.dict( + os.environ, {"DATABASE_URL": "postgresql://test:test@localhost/test"} + ): + # Patch the wait_for_lock_release method to use shorter timeout + with patch.object( + MigrationLockManager, "wait_for_lock_release" + ) as mock_wait: + mock_wait.return_value = False # Simulate timeout + + result = ProxyExtrasDBManager.setup_database( + use_migrate=True, redis_cache=mock_redis + ) + + assert result is False # Should return False after timeout + mock_subprocess.assert_not_called() # Should not run migration + # Verify that wait_for_lock_release was called with default parameters + mock_wait.assert_called_once_with() + + def test_wait_for_lock_release_actual_timeout(self): + """Test actual timeout behavior of wait_for_lock_release with real timing""" + mock_redis = Mock() + mock_redis.set_cache.return_value = False # Always fails to acquire lock + lock_manager = MigrationLockManager(mock_redis) + + # Test with very short timeout to verify actual timeout behavior + start_time = time.time() + result = lock_manager.wait_for_lock_release(check_interval=0.1, max_wait=0.5) + end_time = time.time() + + assert result is False # Should timeout + assert end_time - start_time >= 0.5 # Should wait at least the max_wait time + assert end_time - start_time < 1.0 # But not too much longer + # Should have called set_cache multiple times during the wait + assert mock_redis.set_cache.call_count > 1 diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 7553c670774..5f35d6837c0 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -1814,3 +1814,49 @@ async def test_extra_body_merges_with_request_data(extra_body_mock_response_data assert "temperature" in request_body assert "custom_field" in request_body assert request_body["custom_field"] == "custom_value" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_openai_compact_responses_api(sync_mode): + """ + Test the compact_responses API for OpenAI. + + This test verifies that the compact_responses endpoint works correctly + for compressing conversation history. + """ + litellm._turn_on_debug() + litellm.set_verbose = True + + input_messages = [ + {"role": "user", "content": "Hello, how are you?"}, + {"role": "assistant", "content": "I'm doing well, thank you for asking!"}, + {"role": "user", "content": "What is the weather like today?"}, + ] + + try: + if sync_mode: + response = litellm.compact_responses( + model="openai/gpt-4o", + input=input_messages, + instructions="Be helpful and concise", + ) + else: + response = await litellm.acompact_responses( + model="openai/gpt-4o", + input=input_messages, + instructions="Be helpful and concise", + ) + except litellm.InternalServerError: + pytest.skip("Skipping test due to InternalServerError") + except litellm.BadRequestError as e: + # compact_responses may not be available for all models/accounts + pytest.skip(f"Skipping test due to BadRequestError: {e}") + + print("compact_responses response=", json.dumps(response, indent=4, default=str)) + + # Validate response structure + assert response is not None + assert "id" in response, "Response should have an 'id' field" + assert "output" in response, "Response should have an 'output' field" + assert isinstance(response["output"], list), "Output should be a list" diff --git a/tests/llm_responses_api_testing/test_responses_hooks.py b/tests/llm_responses_api_testing/test_responses_hooks.py new file mode 100644 index 00000000000..8c0f7dab2af --- /dev/null +++ b/tests/llm_responses_api_testing/test_responses_hooks.py @@ -0,0 +1,165 @@ +import asyncio +from datetime import datetime +from types import SimpleNamespace + +import httpx +import pytest + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.responses import streaming_iterator as streaming_module +from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator +from litellm.types.llms.openai import ResponsesAPIStreamEvents +from litellm.types.utils import CallTypes + + +class _FakeLoggingObj: + def __init__(self): + self.success_calls = 0 + self.async_success_calls = 0 + self.failure_calls = 0 + self.async_failure_calls = 0 + self.start_time = datetime.now() + self.model_call_details = {"litellm_params": {}} + + # Signature alignment with Logging handlers + def success_handler(self, *args, **kwargs): + self.success_calls += 1 + + async def async_success_handler(self, *args, **kwargs): + self.async_success_calls += 1 + + def failure_handler(self, *args, **kwargs): + self.failure_calls += 1 + + async def async_failure_handler(self, *args, **kwargs): + self.async_failure_calls += 1 + + +@pytest.mark.asyncio +async def test_responses_streaming_triggers_hooks(monkeypatch): + """ + Ensure streaming iterator fires success + post-call hooks for responses API. + """ + hook_calls = {"post_call": 0, "metadata": 0} + seen = {} + + async def fake_post_call(request_data, response, call_type): + hook_calls["post_call"] += 1 + seen["request_data"] = request_data + seen["call_type"] = call_type + + def fake_update_metadata(**kwargs): + hook_calls["metadata"] += 1 + + monkeypatch.setattr( + streaming_module, + "async_post_call_success_deployment_hook", + fake_post_call, + ) + monkeypatch.setattr( + streaming_module, + "update_response_metadata", + fake_update_metadata, + ) + + logging_obj = _FakeLoggingObj() + + iterator = ResponsesAPIStreamingIterator( + response=httpx.Response(200), + model="test-model", + responses_api_provider_config=SimpleNamespace(), # not used in this test + logging_obj=logging_obj, + request_data={"foo": "bar", "litellm_params": {}}, + call_type=CallTypes.responses.value, + ) + + # Simulate completed streaming event + iterator.completed_response = SimpleNamespace( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=SimpleNamespace() + ) + + iterator._handle_logging_completed_response() + await asyncio.sleep(0.2) # allow async tasks to run + + assert logging_obj.success_calls == 1 + assert logging_obj.async_success_calls == 1 + assert hook_calls["post_call"] == 1 + assert hook_calls["metadata"] == 1 + assert seen["request_data"]["foo"] == "bar" + assert seen["request_data"].get("litellm_params") is not None + assert seen["call_type"] == CallTypes.responses + + +@pytest.mark.asyncio +async def test_responses_streaming_calls_post_streaming_deployment_hook(monkeypatch): + """ + Ensure per-chunk streaming deployment hook can modify chunks. + """ + + class _HookLogger(CustomLogger): + async def async_post_call_streaming_deployment_hook( + self, request_data, response_chunk, call_type + ): + response_chunk.tagged = True + return response_chunk + + # Set callbacks to our fake hook + original_callbacks = litellm.callbacks + litellm.callbacks = [_HookLogger()] + + logging_obj = _FakeLoggingObj() + + class _StubConfig: + def transform_streaming_response(self, **kwargs): + return SimpleNamespace( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, response=None + ) + + iterator = ResponsesAPIStreamingIterator( + response=httpx.Response(200), + model="test-model", + responses_api_provider_config=_StubConfig(), + logging_obj=logging_obj, + request_data={"foo": "bar"}, + call_type=CallTypes.responses.value, + ) + + # Call hook helper directly to verify chunk is modified/flagged + chunk = SimpleNamespace(type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, response=None) + chunk = await streaming_module.call_post_streaming_hooks_for_testing(iterator, chunk) + assert getattr(chunk, "_post_streaming_hooks_ran", False) is True + assert getattr(chunk, "tagged", False) is True + + # reset callbacks + litellm.callbacks = original_callbacks + + +@pytest.mark.asyncio +async def test_responses_streaming_failure_triggers_failure_handlers(): + """ + If transform raises, failure handlers should be called. + """ + + class _FailConfig: + def transform_streaming_response(self, **kwargs): + raise ValueError("boom") + + logging_obj = _FakeLoggingObj() + + iterator = ResponsesAPIStreamingIterator( + response=httpx.Response(200), + model="test-model", + responses_api_provider_config=_FailConfig(), + logging_obj=logging_obj, + request_data={"foo": "bar"}, + call_type=CallTypes.responses.value, + ) + + with pytest.raises(ValueError): + iterator._process_chunk('{"delta": "chunk"}') + + # allow failure callbacks to run + await asyncio.sleep(0.2) + assert logging_obj.failure_calls >= 1 + assert logging_obj.async_failure_calls >= 1 diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 7c849650bf6..ab5709cd72d 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -385,7 +385,7 @@ def test_anthropic_tool_use(tool_type, tool_config, message_content): "computer_tool_used, prompt_caching_set, expected_beta_header", [ (True, False, True), - (False, True, True), + (False, True, False), (True, True, True), (False, False, False), ], diff --git a/tests/llm_translation/test_databricks.py b/tests/llm_translation/test_databricks.py index 40fc712f2b7..3013d00288f 100644 --- a/tests/llm_translation/test_databricks.py +++ b/tests/llm_translation/test_databricks.py @@ -15,6 +15,7 @@ import litellm from litellm.exceptions import BadRequestError from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.utils import CustomStreamWrapper +from litellm._version import version from base_llm_unit_tests import BaseLLMChatTest, BaseAnthropicChatTest try: @@ -725,6 +726,7 @@ def test_embeddings_with_sync_http_handler(monkeypatch): headers={ "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", + "User-Agent": f"litellm/{version}", }, data=json.dumps( { @@ -767,6 +769,7 @@ def test_embeddings_with_async_http_handler(monkeypatch): headers={ "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", + "User-Agent": f"litellm/{version}", }, data=json.dumps( { @@ -823,6 +826,7 @@ def test_embeddings_uses_databricks_sdk_if_api_key_and_base_not_specified(monkey headers={ "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", + "User-Agent": f"litellm/{version}", }, data=json.dumps( { @@ -895,6 +899,7 @@ async def test_databricks_embeddings(sync_mode, monkeypatch): headers={ "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", + "User-Agent": f"litellm/{version}", }, data=json.dumps( { @@ -923,6 +928,7 @@ async def test_databricks_embeddings(sync_mode, monkeypatch): headers={ "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", + "User-Agent": f"litellm/{version}", }, data=json.dumps( { diff --git a/tests/llm_translation/test_gigachat.py b/tests/llm_translation/test_gigachat.py new file mode 100644 index 00000000000..80bf51b4646 --- /dev/null +++ b/tests/llm_translation/test_gigachat.py @@ -0,0 +1,349 @@ +""" +Tests for GigaChat LiteLLM Provider + +Tests message transformation, parameter handling, and response transformation. +Run with: pytest tests/llm_translation/test_gigachat.py -v +""" + +import json +import pytest +from unittest.mock import Mock, MagicMock + + +class TestGigaChatMessageTransformation: + """Tests for message transformation (OpenAI -> GigaChat format)""" + + @pytest.fixture + def config(self): + from litellm.llms.gigachat.chat.transformation import GigaChatConfig + return GigaChatConfig() + + def test_simple_user_message(self, config): + """Basic user message should pass through""" + messages = [{"role": "user", "content": "Hello"}] + result = config._transform_messages(messages) + + assert len(result) == 1 + assert result[0]["role"] == "user" + assert result[0]["content"] == "Hello" + + def test_developer_role_to_system(self, config): + """Developer role should be converted to system""" + messages = [{"role": "developer", "content": "You are helpful"}] + result = config._transform_messages(messages) + + assert result[0]["role"] == "system" + + def test_system_after_first_becomes_user(self, config): + """System message after first position should become user""" + messages = [ + {"role": "assistant", "content": "Response"}, + {"role": "system", "content": "Additional instruction"}, + ] + result = config._transform_messages(messages) + + assert result[0]["role"] == "assistant" + assert result[1]["role"] == "user" # system after first becomes user + + def test_tool_role_to_function(self, config): + """Tool role should be converted to function""" + messages = [{"role": "tool", "content": "result data"}] + result = config._transform_messages(messages) + + assert result[0]["role"] == "function" + + def test_tool_calls_to_function_call(self, config): + """tool_calls should be converted to function_call""" + messages = [{ + "role": "assistant", + "content": "", + "tool_calls": [{ + "id": "call_123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Moscow"}' + } + }] + }] + result = config._transform_messages(messages) + + assert "function_call" in result[0] + assert result[0]["function_call"]["name"] == "get_weather" + assert result[0]["function_call"]["arguments"] == {"city": "Moscow"} + assert "tool_calls" not in result[0] + + def test_none_content_becomes_empty_string(self, config): + """None content should become empty string""" + messages = [{"role": "assistant", "content": None}] + result = config._transform_messages(messages) + + assert result[0]["content"] == "" + + def test_name_field_removed(self, config): + """name field should be removed (not supported by GigaChat)""" + messages = [{"role": "user", "content": "Hi", "name": "John"}] + result = config._transform_messages(messages) + + assert "name" not in result[0] + + +class TestGigaChatCollapseUserMessages: + """Tests for collapsing consecutive user messages""" + + @pytest.fixture + def config(self): + from litellm.llms.gigachat.chat.transformation import GigaChatConfig + return GigaChatConfig() + + def test_no_collapse_single_message(self, config): + """Single message should not be changed""" + messages = [{"role": "user", "content": "Hello"}] + result = config._collapse_user_messages(messages) + + assert len(result) == 1 + assert result[0]["content"] == "Hello" + + def test_collapse_consecutive_user_messages(self, config): + """Consecutive user messages should be collapsed""" + messages = [ + {"role": "user", "content": "First"}, + {"role": "user", "content": "Second"}, + {"role": "user", "content": "Third"}, + ] + result = config._collapse_user_messages(messages) + + assert len(result) == 1 + assert "First" in result[0]["content"] + assert "Second" in result[0]["content"] + assert "Third" in result[0]["content"] + + def test_no_collapse_with_assistant_between(self, config): + """Messages with assistant between should not be collapsed""" + messages = [ + {"role": "user", "content": "First"}, + {"role": "assistant", "content": "Response"}, + {"role": "user", "content": "Second"}, + ] + result = config._collapse_user_messages(messages) + + assert len(result) == 3 + + +class TestGigaChatToolsTransformation: + """Tests for tools -> functions conversion""" + + @pytest.fixture + def config(self): + from litellm.llms.gigachat.chat.transformation import GigaChatConfig + return GigaChatConfig() + + def test_single_tool_conversion(self, config): + """Single tool should be converted correctly""" + tools = [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a city", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"} + } + } + } + }] + result = config._convert_tools_to_functions(tools) + + assert len(result) == 1 + assert result[0]["name"] == "get_weather" + assert result[0]["description"] == "Get weather for a city" + + def test_multiple_tools_conversion(self, config): + """Multiple tools should all be converted""" + tools = [ + {"type": "function", "function": {"name": "func1", "description": "First", "parameters": {"type": "object", "properties": {}}}}, + {"type": "function", "function": {"name": "func2", "description": "Second", "parameters": {"type": "object", "properties": {}}}}, + ] + result = config._convert_tools_to_functions(tools) + + assert len(result) == 2 + assert result[0]["name"] == "func1" + assert result[1]["name"] == "func2" + + +class TestGigaChatParamsTransformation: + """Tests for parameter transformation""" + + @pytest.fixture + def config(self): + from litellm.llms.gigachat.chat.transformation import GigaChatConfig + return GigaChatConfig() + + def test_temperature_zero_becomes_top_p_zero(self, config): + """temperature=0 should become top_p=0""" + params = {"temperature": 0} + result = config.map_openai_params( + non_default_params=params, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + + assert "top_p" in result + assert result["top_p"] == 0 + assert "temperature" not in result + + def test_temperature_nonzero_preserved(self, config): + """Non-zero temperature should be preserved""" + params = {"temperature": 0.7} + result = config.map_openai_params( + non_default_params=params, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + + assert result["temperature"] == 0.7 + + def test_max_completion_tokens_to_max_tokens(self, config): + """max_completion_tokens should become max_tokens""" + params = {"max_completion_tokens": 100} + result = config.map_openai_params( + non_default_params=params, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + + assert result["max_tokens"] == 100 + + def test_structured_output_via_json_schema(self, config): + """json_schema response_format should trigger structured output mode""" + params = { + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "person", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"} + } + } + } + } + } + result = config.map_openai_params( + non_default_params=params, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + + assert "_structured_output" in result + assert result["_structured_output"] is True + assert "function_call" in result + assert result["function_call"]["name"] == "person" + + +class TestGigaChatProviderRegistration: + """Tests for provider registration in LiteLLM""" + + def test_gigachat_in_provider_list(self): + """GigaChat should be in provider list""" + from litellm.types.utils import LlmProviders + + assert hasattr(LlmProviders, "GIGACHAT") + assert LlmProviders.GIGACHAT.value == "gigachat" + + def test_gigachat_in_chat_providers(self): + """GigaChat should be in LITELLM_CHAT_PROVIDERS""" + from litellm.constants import LITELLM_CHAT_PROVIDERS + + assert "gigachat" in LITELLM_CHAT_PROVIDERS + + def test_gigachat_key_exists(self): + """gigachat_key should be available""" + import litellm + + assert hasattr(litellm, "gigachat_key") + + def test_gigachat_config_exists(self): + """GigaChatConfig should be available""" + import litellm + + assert hasattr(litellm, "GigaChatConfig") + + +class TestGigaChatTransformRequest: + """Tests for request transformation""" + + @pytest.fixture + def config(self): + from litellm.llms.gigachat.chat.transformation import GigaChatConfig + return GigaChatConfig() + + def test_basic_request(self, config): + """Basic request should be transformed correctly""" + messages = [{"role": "user", "content": "Hello"}] + result = config.transform_request( + model="gigachat/GigaChat", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert result["model"] == "GigaChat" + assert len(result["messages"]) == 1 + assert result["messages"][0]["role"] == "user" + + def test_request_with_temperature(self, config): + """Request with temperature should include it""" + messages = [{"role": "user", "content": "Hello"}] + result = config.transform_request( + model="gigachat/GigaChat", + messages=messages, + optional_params={"temperature": 0.7}, + litellm_params={}, + headers={}, + ) + + assert result["temperature"] == 0.7 + + def test_request_with_functions(self, config): + """Request with functions should include them""" + messages = [{"role": "user", "content": "Hello"}] + functions = [{"name": "test", "description": "Test", "parameters": {}}] + result = config.transform_request( + model="gigachat/GigaChat", + messages=messages, + optional_params={"functions": functions}, + litellm_params={}, + headers={}, + ) + + assert "functions" in result + assert len(result["functions"]) == 1 + + +class TestGigaChatSupportedParams: + """Tests for supported parameters""" + + @pytest.fixture + def config(self): + from litellm.llms.gigachat.chat.transformation import GigaChatConfig + return GigaChatConfig() + + def test_supported_params(self, config): + """Check supported parameters list""" + supported = config.get_supported_openai_params("GigaChat") + + assert "temperature" in supported + assert "max_tokens" in supported + assert "max_completion_tokens" in supported + assert "tools" in supported + assert "response_format" in supported + assert "stream" in supported diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index 2b01b4c2a12..5e92c10fbdc 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -286,7 +286,7 @@ def test_completion_claude_3_empty_response(): }, ] try: - response = litellm.completion(model="claude-3-opus-20240229", messages=messages) + response = litellm.completion(model="claude-3-7-sonnet-20250219", messages=messages) print(response) except litellm.InternalServerError as e: pytest.skip(f"InternalServerError - {str(e)}") @@ -313,7 +313,7 @@ def test_completion_claude_3(): try: # test without max tokens response = completion( - model="anthropic/claude-3-opus-20240229", + model="anthropic/claude-3-7-sonnet-20250219", messages=messages, ) # Add any assertions, here to check response args @@ -326,7 +326,7 @@ def test_completion_claude_3(): @pytest.mark.parametrize( "model", - ["anthropic/claude-3-opus-20240229", "anthropic.claude-3-sonnet-20240229-v1:0"], + ["anthropic/claude-3-7-sonnet-20250219", "anthropic.claude-3-sonnet-20240229-v1:0"], ) def test_completion_claude_3_function_call(model): litellm.set_verbose = True @@ -411,7 +411,7 @@ def test_completion_claude_3_function_call(model): "model, api_key, api_base", [ ("gpt-3.5-turbo", None, None), - ("claude-3-opus-20240229", None, None), + ("claude-3-7-sonnet-20250219", None, None), ("anthropic.claude-3-sonnet-20240229-v1:0", None, None), # ( # "azure_ai/command-r-plus", @@ -512,7 +512,7 @@ async def test_anthropic_no_content_error(): try: litellm.drop_params = True response = await litellm.acompletion( - model="anthropic/claude-3-opus-20240229", + model="anthropic/claude-3-7-sonnet-20250219", api_key=os.getenv("ANTHROPIC_API_KEY"), messages=[ { @@ -630,7 +630,7 @@ def test_completion_claude_3_multi_turn_conversations(): ] try: response = completion( - model="anthropic/claude-3-opus-20240229", + model="anthropic/claude-3-7-sonnet-20250219", messages=messages, ) print(response) @@ -644,7 +644,7 @@ def test_completion_claude_3_stream(): try: # test without max tokens response = completion( - model="anthropic/claude-3-opus-20240229", + model="anthropic/claude-3-7-sonnet-20250219", messages=messages, max_tokens=10, stream=True, @@ -669,7 +669,7 @@ def encode_image(image_path): [ "gpt-4o", "azure/gpt-4.1-mini", - "anthropic/claude-3-opus-20240229", + "anthropic/claude-3-7-sonnet-20250219", ], ) # def test_completion_base64(model): diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index 0d9f84a301c..b9b5d0fdb07 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -1418,7 +1418,7 @@ def test_bedrock_claude_3_streaming(): @pytest.mark.parametrize( "model", [ - "claude-3-opus-20240229", + "claude-3-7-sonnet-20250219", "cohere.command-r-plus-v1:0", # bedrock "gpt-3.5-turbo", ], @@ -2914,7 +2914,7 @@ def test_completion_claude_3_function_call_with_streaming(): try: # test without max tokens response = completion( - model="claude-3-opus-20240229", + model="claude-3-7-sonnet-20250219", messages=messages, tools=tools, tool_choice="required", @@ -2946,7 +2946,7 @@ def test_completion_claude_3_function_call_with_streaming(): "model", [ "gemini/gemini-2.5-flash-lite", - ], # "claude-3-opus-20240229" + ], ) # @pytest.mark.asyncio async def test_acompletion_function_call_with_streaming(model): diff --git a/tests/logging_callback_tests/test_langsmith_unit_test.py b/tests/logging_callback_tests/test_langsmith_unit_test.py index e63ce9f8b38..bde2b944579 100644 --- a/tests/logging_callback_tests/test_langsmith_unit_test.py +++ b/tests/logging_callback_tests/test_langsmith_unit_test.py @@ -47,6 +47,19 @@ async def test_get_credentials_from_env(): credentials = logger.get_credentials_from_env() assert credentials["LANGSMITH_BASE_URL"] == "https://api.smith.langchain.com" + # Test with tenant_id + credentials = logger.get_credentials_from_env( + langsmith_tenant_id="test-tenant-id" + ) + assert credentials["LANGSMITH_TENANT_ID"] == "test-tenant-id" + + # Test tenant_id from environment variable + import os + os.environ["LANGSMITH_TENANT_ID"] = "env-tenant-id" + credentials = logger.get_credentials_from_env() + assert credentials["LANGSMITH_TENANT_ID"] == "env-tenant-id" + del os.environ["LANGSMITH_TENANT_ID"] + @pytest.mark.asyncio async def test_group_batches_by_credentials(): @@ -60,6 +73,7 @@ async def test_group_batches_by_credentials(): "LANGSMITH_API_KEY": "key1", "LANGSMITH_PROJECT": "proj1", "LANGSMITH_BASE_URL": "url1", + "LANGSMITH_TENANT_ID": None, }, ) @@ -69,6 +83,7 @@ async def test_group_batches_by_credentials(): "LANGSMITH_API_KEY": "key1", "LANGSMITH_PROJECT": "proj1", "LANGSMITH_BASE_URL": "url1", + "LANGSMITH_TENANT_ID": None, }, ) @@ -95,6 +110,7 @@ async def test_group_batches_by_credentials_multiple_credentials(): "LANGSMITH_API_KEY": "key1", "LANGSMITH_PROJECT": "proj1", "LANGSMITH_BASE_URL": "url1", + "LANGSMITH_TENANT_ID": None, }, ) @@ -104,6 +120,7 @@ async def test_group_batches_by_credentials_multiple_credentials(): "LANGSMITH_API_KEY": "key2", # Different API key "LANGSMITH_PROJECT": "proj1", "LANGSMITH_BASE_URL": "url1", + "LANGSMITH_TENANT_ID": None, }, ) @@ -113,6 +130,7 @@ async def test_group_batches_by_credentials_multiple_credentials(): "LANGSMITH_API_KEY": "key1", "LANGSMITH_PROJECT": "proj2", # Different project "LANGSMITH_BASE_URL": "url1", + "LANGSMITH_TENANT_ID": None, }, ) @@ -127,6 +145,57 @@ async def test_group_batches_by_credentials_multiple_credentials(): assert len(batch_group.queue_objects) == 1 # Each group should have one object +@pytest.mark.asyncio +async def test_group_batches_by_credentials_with_tenant_id(): + + # Test that different tenant_ids create separate groups + logger = LangsmithLogger(langsmith_api_key="test-key") + + queue_obj1 = LangsmithQueueObject( + data={"test": "data1"}, + credentials={ + "LANGSMITH_API_KEY": "key1", + "LANGSMITH_PROJECT": "proj1", + "LANGSMITH_BASE_URL": "url1", + "LANGSMITH_TENANT_ID": "tenant1", + }, + ) + + queue_obj2 = LangsmithQueueObject( + data={"test": "data2"}, + credentials={ + "LANGSMITH_API_KEY": "key1", + "LANGSMITH_PROJECT": "proj1", + "LANGSMITH_BASE_URL": "url1", + "LANGSMITH_TENANT_ID": "tenant2", # Different tenant_id + }, + ) + + queue_obj3 = LangsmithQueueObject( + data={"test": "data3"}, + credentials={ + "LANGSMITH_API_KEY": "key1", + "LANGSMITH_PROJECT": "proj1", + "LANGSMITH_BASE_URL": "url1", + "LANGSMITH_TENANT_ID": "tenant1", # Same as queue_obj1 + }, + ) + + logger.log_queue = [queue_obj1, queue_obj2, queue_obj3] + + grouped = logger._group_batches_by_credentials() + + # Should have two groups: one for tenant1 (queue_obj1 and queue_obj3), one for tenant2 (queue_obj2) + assert len(grouped) == 2 + for key, batch_group in grouped.items(): + assert isinstance(key, CredentialsKey) + assert key.tenant_id in ["tenant1", "tenant2"] + if key.tenant_id == "tenant1": + assert len(batch_group.queue_objects) == 2 + else: + assert len(batch_group.queue_objects) == 1 + + # Test make_dot_order @pytest.mark.asyncio async def test_make_dot_order(): @@ -201,10 +270,43 @@ async def test_async_send_batch(): call_args = logger.async_httpx_client.post.call_args assert "runs/batch" in call_args[1]["url"] assert "x-api-key" in call_args[1]["headers"] + # tenant_id should not be in headers if not provided + assert "x-tenant-id" not in call_args[1]["headers"] @pytest.mark.asyncio -async def test_langsmith_key_based_logging(mocker): +async def test_async_send_batch_with_tenant_id(): + logger = LangsmithLogger( + langsmith_api_key="test-key", + langsmith_tenant_id="test-tenant-id" + ) + + # Mock the httpx client + mock_response = AsyncMock() + mock_response.status_code = 200 + logger.async_httpx_client = AsyncMock() + logger.async_httpx_client.post.return_value = mock_response + + # Add test data to queue + logger.log_queue = [ + LangsmithQueueObject( + data={"test": "data"}, credentials=logger.default_credentials + ) + ] + + await logger.async_send_batch() + + # Verify the API call includes tenant_id header + logger.async_httpx_client.post.assert_called_once() + call_args = logger.async_httpx_client.post.call_args + assert "runs/batch" in call_args[1]["url"] + assert "x-api-key" in call_args[1]["headers"] + assert "x-tenant-id" in call_args[1]["headers"] + assert call_args[1]["headers"]["x-tenant-id"] == "test-tenant-id" + + +@pytest.mark.asyncio +async def test_langsmith_key_based_logging(): """ In key based logging langsmith_api_key and langsmith_project are passed directly to litellm.acompletion """ @@ -219,10 +321,11 @@ async def test_langsmith_key_based_logging(mocker): mock_response.text = "" mock_async_httpx_handler.post = AsyncMock(return_value=mock_response) - mock_get_client = mocker.patch( + mock_get_client = patch( "litellm.integrations.langsmith.get_async_httpx_client", return_value=mock_async_httpx_handler ) + mock_get_client.start() litellm.set_verbose = True litellm.DEFAULT_FLUSH_INTERVAL_SECONDS = 1 @@ -253,6 +356,8 @@ async def test_langsmith_key_based_logging(mocker): # Check headers contain the correct API key assert call_args[1]["headers"]["x-api-key"] == "fake_key_project2" + # tenant_id should not be in headers if not provided + assert "x-tenant-id" not in call_args[1]["headers"] # Verify the request body contains the expected data request_body = call_args[1]["json"] @@ -344,6 +449,8 @@ async def test_langsmith_key_based_logging(mocker): actual_body["post"][0]["session_name"] == expected_body["post"][0]["session_name"] ) + + mock_get_client.stop() except Exception as e: pytest.fail(f"Error occurred: {e}") diff --git a/tests/logging_callback_tests/test_opentelemetry_unit_tests.py b/tests/logging_callback_tests/test_opentelemetry_unit_tests.py index 3d0682d9033..04f8abe64de 100644 --- a/tests/logging_callback_tests/test_opentelemetry_unit_tests.py +++ b/tests/logging_callback_tests/test_opentelemetry_unit_tests.py @@ -65,31 +65,6 @@ class TestOpentelemetryUnitTests(BaseLoggingCallbackTest): # External spans should only be closed by their creators parent_otel_span.end.assert_not_called() - def test_init_tracing_respects_existing_tracer_provider(self): - """ - Unit test: _init_tracing() should respect existing TracerProvider. - - When a TracerProvider already exists (e.g., set by Langfuse SDK), - LiteLLM should use it instead of creating a new one. - """ - from opentelemetry import trace - from opentelemetry.sdk.trace import TracerProvider - from litellm.integrations.opentelemetry import OpenTelemetry - - # Setup: Create and set an existing TracerProvider - tracer_provider = TracerProvider() - trace.set_tracer_provider(tracer_provider) - existing_provider = trace.get_tracer_provider() - - # Act: Initialize OpenTelemetry integration (should detect existing provider) - otel_integration = OpenTelemetry() - - # Assert: The existing provider should still be active - current_provider = trace.get_tracer_provider() - assert current_provider is existing_provider, ( - "Existing TracerProvider should be respected and not overridden" - ) - def test_get_span_context_detects_active_span(self): """ Unit test: _get_span_context() should auto-detect active spans from global context. diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 40e223ffe07..45aae3b9aee 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -79,6 +79,73 @@ def test_routing_strategy_init(model_list): ) +def test_routing_strategy_init_invalid_strategy(model_list): + """Test that invalid routing_strategy raises ValueError with helpful message. + + See: https://github.com/BerriAI/litellm/issues/11330 + Invalid strategies like 'simple' (without '-shuffle') should fail fast + with a clear error, not silently cause 'No deployments available' errors. + """ + router = Router(model_list=model_list) + + # Test common mistake: "simple" instead of "simple-shuffle" + with pytest.raises(ValueError) as exc_info: + router.routing_strategy_init( + routing_strategy="simple", + routing_strategy_args={} + ) + + # Verify error message is helpful + error_msg = str(exc_info.value) + assert "Invalid routing_strategy" in error_msg + assert "simple" in error_msg + assert "simple-shuffle" in error_msg # Suggests the correct option + # Verify error message tells user WHERE to fix it + assert "config.yaml" in error_msg + assert "router_settings.routing_strategy" in error_msg + assert "Router SDK" in error_msg + + # Test completely invalid strategy + with pytest.raises(ValueError) as exc_info: + router.routing_strategy_init( + routing_strategy="not-a-real-strategy", + routing_strategy_args={} + ) + assert "Invalid routing_strategy" in str(exc_info.value) + + +def test_routing_strategy_init_valid_string_strategies(model_list): + """Test that all valid string routing strategies work without error. + + Valid strategies are derived from RoutingStrategy enum values plus 'simple-shuffle'. + """ + from litellm.types.router import RoutingStrategy + + router = Router(model_list=model_list) + + # All strategies from enum + simple-shuffle (default, not in enum) + valid_strategies = ["simple-shuffle"] + [s.value for s in RoutingStrategy] + + for strategy in valid_strategies: + # Should not raise + router.routing_strategy_init( + routing_strategy=strategy, routing_strategy_args={} + ) + + +def test_routing_strategy_init_valid_enum_strategies(model_list): + """Test that RoutingStrategy enum values work without error.""" + from litellm.types.router import RoutingStrategy + + router = Router(model_list=model_list) + + for strategy in RoutingStrategy: + # Should not raise when passing enum directly + router.routing_strategy_init( + routing_strategy=strategy, routing_strategy_args={} + ) + + def test_print_deployment(model_list): """Test if the api key is masked correctly""" diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 6490352c39b..596398e639f 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -1011,39 +1011,87 @@ def test_multiple_tool_calls_in_single_choice(): def test_map_reasoning_effort_adds_summary_detailed(): """ - Test that _map_reasoning_effort adds summary="detailed" when user provides reasoning_effort as a string. + Test that _map_reasoning_effort behavior with reasoning_auto_summary flag. - This ensures that when users pass reasoning_effort in the completions API for OpenAI responses/models, - the transformation automatically includes summary="detailed" in the reasoning parameter. + By default (flag=False), summary should NOT be added to avoid: + 1. Breaking for users without verified OpenAI orgs (400 errors) + 2. Making requests more expensive by including summary reasoning tokens + + When flag is enabled (flag=True or env var), summary="detailed" is added. """ + import os + + import litellm from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, ) handler = LiteLLMResponsesTransformationHandler() - # Test all string effort levels + # Test all string effort levels - DEFAULT BEHAVIOR (no summary) effort_levels = ["none", "low", "medium", "high", "xhigh", "minimal"] - for effort in effort_levels: - result = handler._map_reasoning_effort(effort) + # Save original flag value + original_flag = litellm.reasoning_auto_summary + original_env = os.environ.get("LITELLM_REASONING_AUTO_SUMMARY") + + try: + # Test 1: Default behavior (flag=False, no env var) - NO summary + litellm.reasoning_auto_summary = False + if "LITELLM_REASONING_AUTO_SUMMARY" in os.environ: + del os.environ["LITELLM_REASONING_AUTO_SUMMARY"] - assert result is not None, f"Result should not be None for effort={effort}" - assert result["effort"] == effort, f"Effort should be {effort}" - assert result["summary"] == "detailed", f"Summary should be 'detailed' for effort={effort}" + for effort in effort_levels: + result = handler._map_reasoning_effort(effort) + + assert result is not None, f"Result should not be None for effort={effort}" + assert result["effort"] == effort, f"Effort should be {effort}" + assert "summary" not in result, f"Summary should NOT be present by default for effort={effort}" + + print(f"✓ reasoning_effort='{effort}' correctly maps to effort='{effort}' (no summary by default)") - print(f"✓ reasoning_effort='{effort}' correctly maps to effort='{effort}', summary='detailed'") + # Test 2: With flag enabled - summary IS added + litellm.reasoning_auto_summary = True + + for effort in effort_levels: + result = handler._map_reasoning_effort(effort) + + assert result is not None, f"Result should not be None for effort={effort}" + assert result["effort"] == effort, f"Effort should be {effort}" + assert result["summary"] == "detailed", f"Summary should be 'detailed' when flag is enabled for effort={effort}" + + print(f"✓ reasoning_effort='{effort}' correctly maps to effort='{effort}', summary='detailed' (flag enabled)") + + # Test 3: With env var enabled (flag disabled) - summary IS added + litellm.reasoning_auto_summary = False + os.environ["LITELLM_REASONING_AUTO_SUMMARY"] = "true" + + result = handler._map_reasoning_effort("high") + assert result["summary"] == "detailed", "Summary should be 'detailed' when env var is enabled" + print("✓ LITELLM_REASONING_AUTO_SUMMARY env var works correctly") + + # Test 4: Dict input is passed through as-is (no modification) + litellm.reasoning_auto_summary = False + if "LITELLM_REASONING_AUTO_SUMMARY" in os.environ: + del os.environ["LITELLM_REASONING_AUTO_SUMMARY"] + + dict_input = {"effort": "high", "summary": "custom_summary"} + result_dict = handler._map_reasoning_effort(dict_input) + assert result_dict["effort"] == "high" + assert result_dict["summary"] == "custom_summary" + print("✓ Dict input is passed through without modification") + + # Test 5: None/unknown values return None + result_unknown = handler._map_reasoning_effort("unknown_value") + assert result_unknown is None + print("✓ Unknown reasoning_effort values return None") + + print("✓ All reasoning_effort behaviors work correctly with flag/env var control") - # Test that dict input is passed through as-is (no modification) - dict_input = {"effort": "high", "summary": "custom_summary"} - result_dict = handler._map_reasoning_effort(dict_input) - assert result_dict["effort"] == "high" - assert result_dict["summary"] == "custom_summary" - print("✓ Dict input is passed through without modification") - - # Test that None/unknown values return None - result_unknown = handler._map_reasoning_effort("unknown_value") - assert result_unknown is None - print("✓ Unknown reasoning_effort values return None") - - print("✓ All reasoning_effort string values correctly map to summary='detailed'") + finally: + # Restore original values + litellm.reasoning_auto_summary = original_flag + if original_env is not None: + os.environ["LITELLM_REASONING_AUTO_SUMMARY"] = original_env + elif "LITELLM_REASONING_AUTO_SUMMARY" in os.environ: + del os.environ["LITELLM_REASONING_AUTO_SUMMARY"] diff --git a/tests/test_litellm/integrations/langfuse/test_gemini_cached_tokens.py b/tests/test_litellm/integrations/langfuse/test_gemini_cached_tokens.py new file mode 100644 index 00000000000..e717840ec95 --- /dev/null +++ b/tests/test_litellm/integrations/langfuse/test_gemini_cached_tokens.py @@ -0,0 +1,90 @@ +""" +Test for Langfuse integration with Gemini cached_tokens bug +https://github.com/BerriAI/litellm/issues/18520 +""" +import pytest +from litellm.types.utils import PromptTokensDetailsWrapper, Usage + + +def test_cached_tokens_extraction(): + """ + Test that we can extract cached_tokens from prompt_tokens_details. + This is the core logic fix for https://github.com/BerriAI/litellm/issues/18520 + """ + # Create usage object like Gemini returns + usage = Usage( + prompt_tokens=20209, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=20203, + text_tokens=6, + ), + completion_tokens=541, + ) + + # Simulate the logic from langfuse.py lines 745-757 (after the fix) + cache_read_input_tokens = 0 # Default value + + # Check prompt_tokens_details.cached_tokens (the fix) + if hasattr(usage, "prompt_tokens_details"): + prompt_tokens_details = getattr(usage, "prompt_tokens_details", None) + if ( + prompt_tokens_details is not None + and hasattr(prompt_tokens_details, "cached_tokens") + ): + cached_tokens = getattr(prompt_tokens_details, "cached_tokens", None) + if cached_tokens is not None and cached_tokens > 0: + cache_read_input_tokens = cached_tokens + + # Verify the fix works + assert cache_read_input_tokens == 20203, f"Expected 20203, got {cache_read_input_tokens}" + + +def test_cached_tokens_not_present(): + """Test backward compatibility when cached_tokens is not present""" + # Usage without prompt_tokens_details + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + ) + + cache_read_input_tokens = 0 + + if hasattr(usage, "prompt_tokens_details"): + prompt_tokens_details = getattr(usage, "prompt_tokens_details", None) + if ( + prompt_tokens_details is not None + and hasattr(prompt_tokens_details, "cached_tokens") + ): + cached_tokens = getattr(prompt_tokens_details, "cached_tokens", None) + if cached_tokens is not None and cached_tokens > 0: + cache_read_input_tokens = cached_tokens + + # Should remain 0 + assert cache_read_input_tokens == 0 + + +def test_cached_tokens_is_zero(): + """Test when cached_tokens is explicitly 0""" + usage = Usage( + prompt_tokens=100, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=0, + text_tokens=100, + ), + completion_tokens=50, + ) + + cache_read_input_tokens = 0 + + if hasattr(usage, "prompt_tokens_details"): + prompt_tokens_details = getattr(usage, "prompt_tokens_details", None) + if ( + prompt_tokens_details is not None + and hasattr(prompt_tokens_details, "cached_tokens") + ): + cached_tokens = getattr(prompt_tokens_details, "cached_tokens", None) + if cached_tokens is not None and cached_tokens > 0: + cache_read_input_tokens = cached_tokens + + # Should remain 0 when cached_tokens is 0 + assert cache_read_input_tokens == 0 diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 5d648f601f6..6c17570e135 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -172,6 +172,86 @@ class TestOpenTelemetryCostBreakdown(unittest.TestCase): assert ("gen_ai.cost.original_cost", 0.004) not in call_args_list +class TestOpenTelemetryProviderInitialization(unittest.TestCase): + """Test suite for verifying provider initialization respects existing providers""" + + def test_init_tracing_respects_existing_tracer_provider(self): + """ + Unit test: _init_tracing() should respect existing TracerProvider. + + When a TracerProvider already exists (e.g., set by Langfuse SDK), + LiteLLM should use it instead of creating a new one. + """ + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + + # Setup: Create and set an existing TracerProvider + tracer_provider = TracerProvider() + trace.set_tracer_provider(tracer_provider) + existing_provider = trace.get_tracer_provider() + + # Act: Initialize OpenTelemetry integration (should detect existing provider) + otel_integration = OpenTelemetry() + + # Assert: The existing provider should still be active + current_provider = trace.get_tracer_provider() + assert current_provider is existing_provider, ( + "Existing TracerProvider should be respected and not overridden" + ) + + @patch.dict(os.environ, {"LITELLM_OTEL_INTEGRATION_ENABLE_METRICS": "true"}, clear=True) + def test_init_metrics_respects_existing_meter_provider(self): + """ + Unit test: _init_metrics() should respect existing MeterProvider. + + When a MeterProvider already exists (e.g., set by Langfuse SDK), + LiteLLM should use it instead of creating a new one. + """ + from opentelemetry import metrics + from opentelemetry.sdk.metrics import MeterProvider + + # Create and set an existing MeterProvider + meter_provider = MeterProvider() + metrics.set_meter_provider(meter_provider) + existing_provider = metrics.get_meter_provider() + + # Act: Initialize OpenTelemetry integration (should detect existing provider) + config = OpenTelemetryConfig.from_env() + otel_integration = OpenTelemetry(config=config) + + # Assert: The existing provider should still be active + current_provider = metrics.get_meter_provider() + assert current_provider is existing_provider, ( + "Existing MeterProvider should be respected and not overridden" + ) + + @patch.dict(os.environ, {"LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS": "true"}, clear=True) + def test_init_logs_respects_existing_logger_provider(self): + """ + Unit test: _init_logs() should respect existing LoggerProvider. + + When a LoggerProvider already exists (e.g., set by Langfuse SDK), + LiteLLM should use it instead of creating a new one. + """ + from opentelemetry._logs import get_logger_provider, set_logger_provider + from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider + + # Create and set an existing LoggerProvider + logger_provider = OTLoggerProvider() + set_logger_provider(logger_provider) + existing_provider = get_logger_provider() + + # Act: Initialize OpenTelemetry integration (should detect existing provider) + config = OpenTelemetryConfig.from_env() + otel_integration = OpenTelemetry(config=config) + + # Assert: The existing provider should still be active + current_provider = get_logger_provider() + assert current_provider is existing_provider, ( + "Existing LoggerProvider should be respected and not overridden" + ) + + class TestOpenTelemetry(unittest.TestCase): POLL_INTERVAL = 0.05 POLL_TIMEOUT = 2.0 @@ -620,7 +700,6 @@ class TestOpenTelemetry(unittest.TestCase): self.assertEqual(attributes.get("extra.attr"), "extra-value") - def test_handle_success_spans_only(self): # make sure neither events nor metrics is on os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", None) @@ -687,11 +766,8 @@ class TestOpenTelemetry(unittest.TestCase): logs = log_exporter.get_finished_logs() self.assertFalse(logs, "Did not expect any logs") + @patch.dict(os.environ, {"LITELLM_OTEL_INTEGRATION_ENABLE_METRICS": "true"}, clear=True) def test_handle_success_spans_and_metrics(self): - # only metrics on - os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", None) - os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_METRICS"] = "true" - # ─── build in‐memory OTEL providers/exporters ───────────────────────────── span_exporter = InMemorySpanExporter() tracer_provider = TracerProvider() @@ -1320,6 +1396,23 @@ class TestOpenTelemetryProtocolSelection(unittest.TestCase): ) self.assertEqual(normalized, "http://collector:4317/v1/logs") + def test_get_metric_reader_uses_http_exporter_for_http_protobuf(self): + """Test that http/protobuf protocol uses OTLPMetricExporterHTTP""" + from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter, + ) + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + + config = OpenTelemetryConfig( + exporter="http/protobuf", endpoint="http://collector:4318" + ) + otel = OpenTelemetry(config=config) + + reader = otel._get_metric_reader() + + self.assertIsInstance(reader, PeriodicExportingMetricReader) + self.assertIsInstance(reader._exporter, OTLPMetricExporter) + class TestOpenTelemetryExternalSpan(unittest.TestCase): """ diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 6a528fef8f0..ec2f528a35d 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -691,6 +691,29 @@ async def test_streaming_completion_start_time(logging_obj: Logging): ) +@pytest.mark.asyncio +async def test_vertex_streaming_bad_request_not_midstream(logging_obj: Logging): + """Ensure Vertex bad request errors surface as 400, not mid-stream fallbacks.""" + from litellm.llms.vertex_ai.common_utils import VertexAIError + + async def _raise_bad_request(**kwargs): + raise VertexAIError(status_code=400, message="invalid maxOutputTokens", headers=None) + + response = CustomStreamWrapper( + completion_stream=None, + model="gemini-3-pro-preview", + logging_obj=logging_obj, + custom_llm_provider="vertex_ai_beta", + make_call=_raise_bad_request, + ) + + with pytest.raises(litellm.BadRequestError) as excinfo: + await response.__anext__() + + assert getattr(excinfo.value, "status_code", None) == 400 + assert "invalid maxOutputTokens" in str(excinfo.value) + + def test_streaming_handler_with_created_time_propagation( initialized_custom_stream_wrapper: CustomStreamWrapper, logging_obj: Logging ): diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_function_call_args_serialization.py b/tests/test_litellm/llms/vertex_ai/gemini/test_function_call_args_serialization.py new file mode 100644 index 00000000000..0f369fbb8b9 --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_function_call_args_serialization.py @@ -0,0 +1,355 @@ +""" +Test cases for functionCall args serialization in Vertex AI Gemini. + +This test file specifically tests the edge cases where Vertex AI might return +functionCall args in unexpected formats that could lead to invalid JSON strings +like: {"x":"x"}{"a":"a"} +""" +import json +from typing import List, Optional + +import pytest + +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, +) +from litellm.types.llms.vertex_ai import HttpxPartType + + +class TestFunctionCallArgsSerialization: + """Test cases for functionCall args serialization edge cases.""" + + def test_normal_dict_args(self): + """Test normal case: args is a dict.""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": {"location": "Boston", "unit": "celsius"}, + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + assert tools[0]["function"]["name"] == "get_weather" + + # Verify arguments is a valid JSON string + arguments = tools[0]["function"]["arguments"] + assert isinstance(arguments, str) + # Should be valid JSON + parsed = json.loads(arguments) + assert parsed == {"location": "Boston", "unit": "celsius"} + + def test_none_args(self): + """Test case: args is None.""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": None, + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + # Should serialize None to "null" or empty dict + assert isinstance(arguments, str) + parsed = json.loads(arguments) + # json.dumps(None) returns "null" + assert parsed is None or parsed == {} + + def test_args_as_string_valid_json(self): + """Test case: args is already a valid JSON string.""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": '{"location": "Boston"}', # String, not dict + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + # If args is a string, json.dumps will double-encode it + # This would result in: "{\"location\": \"Boston\"}" + assert isinstance(arguments, str) + # This is the problematic case - string gets double-encoded + # The result would be a JSON string containing a JSON string + parsed = json.loads(arguments) + # If it's double-encoded, parsed would be a string, not a dict + if isinstance(parsed, str): + # Double-encoded case + inner_parsed = json.loads(parsed) + assert inner_parsed == {"location": "Boston"} + else: + # Normal case (shouldn't happen if args is string) + assert parsed == {"location": "Boston"} + + def test_args_as_string_invalid_json_concatenated(self): + """Test case: args is a string with concatenated JSON objects (the bug case). + + When args is a string like '{"x":"x"}{"a":"a"}', json.dumps() will serialize it + as a JSON string, resulting in: "{\"x\":\"x\"}{\"a\":\"a\"}" + This is a valid JSON string (the outer quotes), but the content inside is invalid JSON. + When you try to parse the inner content, it fails. + """ + # This simulates the case where Vertex might return something like: + # args = '{"x":"x"}{"a":"a"}' # Two JSON objects concatenated + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": '{"x":"x"}{"a":"a"}', # Invalid concatenated JSON + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + assert isinstance(arguments, str) + + # json.dumps() on a string will escape it, so we get: + # arguments = '"{\\"x\\":\\"x\\"}{\\"a\\":\\"a\\"}"' + # This is a valid JSON string (the outer quotes), but the inner content is invalid + parsed_outer = json.loads(arguments) + assert isinstance(parsed_outer, str) + + # The inner string is invalid JSON (two objects concatenated) + # This is the bug: the inner content cannot be parsed as valid JSON + with pytest.raises(json.JSONDecodeError): + json.loads(parsed_outer) + + # The arguments string would be: "{\"x\":\"x\"}{\"a\":\"a\"}" + # Which when parsed gives: '{"x":"x"}{"a":"a"}' (invalid JSON) + + def test_args_as_array(self): + """Test case: args is an array (unexpected but possible).""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": [{"x": "x"}, {"a": "a"}], # Array of objects + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + assert isinstance(arguments, str) + # Should serialize array correctly + parsed = json.loads(arguments) + assert parsed == [{"x": "x"}, {"a": "a"}] + + def test_args_missing_key(self): + """Test case: args key is missing from functionCall. + + This will raise a KeyError because the code directly accesses part["functionCall"]["args"] + without checking if the key exists. This is a bug that should be fixed. + """ + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + # args key missing + } + } + ] + + # This should raise KeyError because args key is missing + with pytest.raises(KeyError): + VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + def test_multiple_function_calls(self): + """Test case: multiple function calls in parts.""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": {"location": "Boston"}, + } + }, + { + "functionCall": { + "name": "get_time", + "args": {"timezone": "EST"}, + } + }, + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 2 + assert tools[0]["function"]["name"] == "get_weather" + assert tools[1]["function"]["name"] == "get_time" + + # Both should have valid JSON arguments + args1 = json.loads(tools[0]["function"]["arguments"]) + args2 = json.loads(tools[1]["function"]["arguments"]) + assert args1 == {"location": "Boston"} + assert args2 == {"timezone": "EST"} + + def test_args_with_vertex_protobuf_format(self): + """Test case: args in Vertex protobuf format with string_value, etc.""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": { + "location": {"string_value": "Boston, MA"}, + "unit": {"string_value": "celsius"}, + }, + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + assert isinstance(arguments, str) + # Should serialize the nested structure correctly + parsed = json.loads(arguments) + assert "location" in parsed + assert "unit" in parsed + + def test_args_as_empty_dict(self): + """Test case: args is an empty dict.""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": {}, + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + assert isinstance(arguments, str) + parsed = json.loads(arguments) + assert parsed == {} + + def test_args_with_special_characters(self): + """Test case: args contains special characters that need escaping.""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": { + "location": 'Boston, MA "downtown"', + "note": "Line 1\nLine 2", + }, + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + assert isinstance(arguments, str) + # Should handle special characters correctly + parsed = json.loads(arguments) + assert parsed["location"] == 'Boston, MA "downtown"' + assert parsed["note"] == "Line 1\nLine 2" + + def test_args_as_list_of_strings_that_look_like_json(self): + """Test case: args is a list containing strings that look like JSON objects.""" + # This could potentially cause issues if not handled correctly + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": ['{"x":"x"}', '{"a":"a"}'], # List of JSON strings + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + assert isinstance(arguments, str) + # Should serialize list correctly + parsed = json.loads(arguments) + assert isinstance(parsed, list) + assert parsed == ['{"x":"x"}', '{"a":"a"}'] + + def test_args_as_dict_with_nested_structures(self): + """Test case: args contains nested dicts and lists.""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "complex_function", + "args": { + "nested": {"key": "value"}, + "list": [1, 2, 3], + "mixed": [{"a": 1}, {"b": 2}], + }, + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + assert isinstance(arguments, str) + parsed = json.loads(arguments) + assert parsed["nested"] == {"key": "value"} + assert parsed["list"] == [1, 2, 3] + assert parsed["mixed"] == [{"a": 1}, {"b": 2}] + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) + diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 91a28ee6ec9..d09de3a0f26 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -10,11 +10,13 @@ from pydantic import BaseModel import litellm from litellm import ModelResponse, completion +from litellm.llms.vertex_ai.common_utils import VertexAIError from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, ) from litellm.types.llms.vertex_ai import UsageMetadata from litellm.types.utils import ChoiceLogprobs, Usage +from litellm.utils import CustomStreamWrapper def test_top_logprobs(): @@ -1605,6 +1607,39 @@ def test_vertex_ai_annotation_streaming_events(): assert "Weather information" in annotation["url_citation"]["title"] +@pytest.mark.asyncio +async def test_vertex_ai_streaming_bad_request_is_not_wrapped(): + class DummyLogging: + def __init__(self): + self.model_call_details = {"litellm_params": {}} + self.optional_params = {} + self.messages = [] + self.completion_start_time = None + self.stream_options = None + + def failure_handler(self, *args, **kwargs): + return None + + async def async_failure_handler(self, *args, **kwargs): + return None + + async def failing_make_call(client=None, **kwargs): + raise VertexAIError(status_code=400, message="bad input", headers={}) + + stream = CustomStreamWrapper( + completion_stream=None, + make_call=failing_make_call, + model="gemini-3-pro-preview", + logging_obj=DummyLogging(), + custom_llm_provider="vertex_ai_beta", + ) + + with pytest.raises(litellm.BadRequestError) as exc_info: + await stream.__anext__() + + assert getattr(exc_info.value, "status_code", None) == 400 + + def test_vertex_ai_annotation_conversion(): """ Test the conversion of Vertex AI grounding metadata to OpenAI annotations. diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 6df9abd3fee..4c5723b8284 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -354,7 +354,7 @@ async def test_register_client_remote_registration_success(): request_payload = { "client_name": "Litellm Proxy", - "grant_types": ["authorization_code"], + "grant_types": ["authorization_code", "refresh_token"], "response_types": ["code"], "token_endpoint_auth_method": "client_secret_post", } @@ -556,9 +556,33 @@ async def test_oauth_protected_resource_respects_x_forwarded_proto(): from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( oauth_protected_resource_mcp, ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + from litellm.proxy._types import MCPTransport from fastapi import Request except ImportError: pytest.skip("MCP discoverable endpoints not available") + # Clear registry + global_mcp_server_manager.registry.clear() + + # Create mock OAuth2 server + oauth2_server = MCPServer( + server_id="test_oauth_server", + name="test_oauth", + server_name="test_oauth", + alias="test_oauth", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id="test_client_id", + client_secret="test_client_secret", + authorization_url="https://provider.com/oauth/authorize", + token_url="https://provider.com/oauth/token", + scopes=["read", "write"], + ) + global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server # Mock request with http base_url but X-Forwarded-Proto: https mock_request = MagicMock(spec=Request) @@ -568,13 +592,14 @@ async def test_oauth_protected_resource_respects_x_forwarded_proto(): # Call the endpoint response = await oauth_protected_resource_mcp( request=mock_request, - mcp_server_name="test_server", + mcp_server_name="test_oauth", ) # Verify response uses HTTPS URLs assert response["authorization_servers"][0].startswith( "https://litellm.example.com/" ) + assert response["scopes_supported"] == oauth2_server.scopes @pytest.mark.asyncio @@ -584,9 +609,33 @@ async def test_oauth_authorization_server_respects_x_forwarded_proto(): from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( oauth_authorization_server_mcp, ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + from litellm.proxy._types import MCPTransport from fastapi import Request except ImportError: pytest.skip("MCP discoverable endpoints not available") + # Clear registry + global_mcp_server_manager.registry.clear() + + # Create mock OAuth2 server + oauth2_server = MCPServer( + server_id="test_oauth_server", + name="test_oauth", + server_name="test_oauth", + alias="test_oauth", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id="test_client_id", + client_secret="test_client_secret", + authorization_url="https://provider.com/oauth/authorize", + token_url="https://provider.com/oauth/token", + scopes=["read", "write"], + ) + global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server # Mock request with http base_url but X-Forwarded-Proto: https mock_request = MagicMock(spec=Request) @@ -596,13 +645,15 @@ async def test_oauth_authorization_server_respects_x_forwarded_proto(): # Call the endpoint response = await oauth_authorization_server_mcp( request=mock_request, - mcp_server_name="test_server", + mcp_server_name="test_oauth", ) # Verify response uses HTTPS URLs assert response["authorization_endpoint"].startswith("https://litellm.example.com/") assert response["token_endpoint"].startswith("https://litellm.example.com/") assert response["registration_endpoint"].startswith("https://litellm.example.com/") + assert response["grant_types_supported"] == ["authorization_code", "refresh_token"] + assert response["scopes_supported"] == oauth2_server.scopes @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 6491e11024a..d59b3f04ef5 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -594,7 +594,26 @@ class TestMCPServerManager: assert ( server.registration_url == "https://discovered.example.com/register" ) + @pytest.mark.asyncio + async def test_config_oauth_initialize_tool_name_to_mcp_server_name_mapping(self): + manager = MCPServerManager() + config = { + "example": { + "url": "https://example.com/mcp", + "transport": MCPTransport.http, + "auth_type": MCPAuth.oauth2, + "scopes": ["config"], + "authorization_url": "https://config.example.com/auth", + } + } + + await manager.load_servers_from_config(config) + + # Initialize the tool mapping + await manager._initialize_tool_name_to_mcp_server_name_mapping() + assert manager.tool_name_to_mcp_server_name_mapping == {} + @pytest.mark.asyncio async def test_list_tools_handles_missing_server_alias(self): """Test that list_tools handles servers without alias gracefully""" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_qualifire.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_qualifire.py index 6d6129b17bb..35ed49a84ed 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_qualifire.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_qualifire.py @@ -2,6 +2,7 @@ Unit tests for Qualifire guardrail integration. """ +import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -139,76 +140,98 @@ class TestQualifireGuardrailEvaluateKwargs: @pytest.mark.asyncio async def test_evaluate_called_with_prompt_injections(self): """Test that evaluate is called with prompt_injections enabled.""" - from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( - QualifireGuardrail, - ) + # Mock the qualifire module and its types + mock_qualifire_types = MagicMock() + mock_llm_message = MagicMock() + mock_llm_tool_call = MagicMock() + mock_message_instance = MagicMock() + mock_llm_message.return_value = mock_message_instance + + mock_qualifire_types.LLMMessage = mock_llm_message + mock_qualifire_types.LLMToolCall = mock_llm_tool_call + + with patch.dict('sys.modules', {'qualifire': MagicMock(), 'qualifire.types': mock_qualifire_types}): + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) - guardrail = QualifireGuardrail( - api_key="test_key", - prompt_injections=True, - guardrail_name="test_guardrail", - ) + guardrail = QualifireGuardrail( + api_key="test_key", + prompt_injections=True, + guardrail_name="test_guardrail", + ) - # Mock the client - mock_client = MagicMock() - mock_result = MagicMock() - mock_result.score = 100 - mock_result.status = "completed" - mock_result.evaluationResults = [] - mock_client.evaluate.return_value = mock_result - guardrail._client = mock_client + # Mock the client + mock_client = MagicMock() + mock_result = MagicMock() + mock_result.score = 100 + mock_result.status = "completed" + mock_result.evaluationResults = [] + mock_client.evaluate.return_value = mock_result + guardrail._client = mock_client - messages = [{"role": "user", "content": "Hello, world!"}] + messages = [{"role": "user", "content": "Hello, world!"}] - await guardrail._run_qualifire_check( - messages=messages, output=None, dynamic_params={} - ) + await guardrail._run_qualifire_check( + messages=messages, output=None, dynamic_params={} + ) - # Verify evaluate was called with correct kwargs - mock_client.evaluate.assert_called_once() - call_kwargs = mock_client.evaluate.call_args[1] - assert call_kwargs["prompt_injections"] is True - assert "messages" in call_kwargs + # Verify evaluate was called with correct kwargs + mock_client.evaluate.assert_called_once() + call_kwargs = mock_client.evaluate.call_args[1] + assert call_kwargs["prompt_injections"] is True + assert "messages" in call_kwargs @pytest.mark.asyncio async def test_evaluate_called_with_multiple_checks(self): """Test that evaluate is called with multiple checks enabled.""" - from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( - QualifireGuardrail, - ) + # Mock the qualifire module and its types + mock_qualifire_types = MagicMock() + mock_llm_message = MagicMock() + mock_llm_tool_call = MagicMock() + mock_message_instance = MagicMock() + mock_llm_message.return_value = mock_message_instance + + mock_qualifire_types.LLMMessage = mock_llm_message + mock_qualifire_types.LLMToolCall = mock_llm_tool_call + + with patch.dict('sys.modules', {'qualifire': MagicMock(), 'qualifire.types': mock_qualifire_types}): + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) - guardrail = QualifireGuardrail( - api_key="test_key", - prompt_injections=True, - pii_check=True, - hallucinations_check=True, - assertions=["Output must be valid JSON"], - guardrail_name="test_guardrail", - ) + guardrail = QualifireGuardrail( + api_key="test_key", + prompt_injections=True, + pii_check=True, + hallucinations_check=True, + assertions=["Output must be valid JSON"], + guardrail_name="test_guardrail", + ) - # Mock the client - mock_client = MagicMock() - mock_result = MagicMock() - mock_result.score = 100 - mock_result.status = "completed" - mock_result.evaluationResults = [] - mock_client.evaluate.return_value = mock_result - guardrail._client = mock_client + # Mock the client + mock_client = MagicMock() + mock_result = MagicMock() + mock_result.score = 100 + mock_result.status = "completed" + mock_result.evaluationResults = [] + mock_client.evaluate.return_value = mock_result + guardrail._client = mock_client - messages = [{"role": "user", "content": "Hello, world!"}] + messages = [{"role": "user", "content": "Hello, world!"}] - await guardrail._run_qualifire_check( - messages=messages, output="Test output", dynamic_params={} - ) + await guardrail._run_qualifire_check( + messages=messages, output="Test output", dynamic_params={} + ) - # Verify evaluate was called with correct kwargs - mock_client.evaluate.assert_called_once() - call_kwargs = mock_client.evaluate.call_args[1] - assert call_kwargs["prompt_injections"] is True - assert call_kwargs["pii_check"] is True - assert call_kwargs["hallucinations_check"] is True - assert call_kwargs["assertions"] == ["Output must be valid JSON"] - assert call_kwargs["output"] == "Test output" + # Verify evaluate was called with correct kwargs + mock_client.evaluate.assert_called_once() + call_kwargs = mock_client.evaluate.call_args[1] + assert call_kwargs["prompt_injections"] is True + assert call_kwargs["pii_check"] is True + assert call_kwargs["hallucinations_check"] is True + assert call_kwargs["assertions"] == ["Output must be valid JSON"] + assert call_kwargs["output"] == "Test output" class TestQualifireGuardrailCheckIfFlagged: diff --git a/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py b/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py index 011031c1e4f..97c1733a935 100644 --- a/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py +++ b/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py @@ -40,6 +40,7 @@ class TestKeyManagementEventHooksIndependentOperations: mock_data = MagicMock() mock_data.key_alias = "test-key-alias" mock_data.team_id = None + mock_data.send_invite_email = True mock_response = MagicMock() mock_response.model_dump.return_value = {"key": "sk-test", "token": "test-token"} @@ -59,6 +60,10 @@ class TestKeyManagementEventHooksIndependentOperations: KeyManagementEventHooks, "_store_virtual_key_in_secret_manager", side_effect=mock_store_secret, + ), patch.object( + KeyManagementEventHooks, + "_is_email_sending_enabled", + return_value=True, ), patch( "litellm.store_audit_logs", False ), patch( @@ -96,6 +101,7 @@ class TestKeyManagementEventHooksIndependentOperations: mock_data = MagicMock() mock_data.key_alias = "test-key-alias" mock_data.team_id = None + mock_data.send_invite_email = True mock_response = MagicMock() mock_response.model_dump.return_value = {"key": "sk-test", "token": "test-token"} @@ -115,6 +121,10 @@ class TestKeyManagementEventHooksIndependentOperations: KeyManagementEventHooks, "_store_virtual_key_in_secret_manager", side_effect=mock_store_secret_raises, + ), patch.object( + KeyManagementEventHooks, + "_is_email_sending_enabled", + return_value=True, ), patch( "litellm.store_audit_logs", False ), patch( diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index cc268ab9925..f2bae2cb14a 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -231,6 +231,40 @@ class TestListMCPServers: assert server.url == "https://mcp.deepwiki.com/mcp" assert server.transport == "http" + @pytest.mark.asyncio + async def test_list_mcp_servers_view_all_mode(self): + """Users should see all MCP servers when view_all mode is enabled.""" + + mock_user_auth = generate_mock_user_api_key_auth( + user_role=LitellmUserRoles.INTERNAL_USER + ) + + mock_servers = [ + generate_mock_mcp_server_db_record(server_id="server-1", alias="One"), + generate_mock_mcp_server_db_record(server_id="server-2", alias="Two"), + ] + + mock_manager = MagicMock() + mock_manager.get_all_mcp_servers_unfiltered = AsyncMock( + return_value=mock_servers + ) + + with patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._get_user_mcp_management_mode", + return_value="view_all", + ), patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ): + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + fetch_all_mcp_servers, + ) + + result = await fetch_all_mcp_servers(user_api_key_dict=mock_user_auth) + + assert len(result) == 2 + assert {server.server_id for server in result} == {"server-1", "server-2"} + @pytest.mark.asyncio async def test_list_mcp_servers_combined_config_and_db(self): """ @@ -1096,6 +1130,51 @@ class TestHealthCheckServers: assert result[0]["server_id"] == "server-1" assert result[0]["status"] == "healthy" + @pytest.mark.asyncio + async def test_health_check_view_all_mode(self): + """view_all mode should return health info for all MCP servers.""" + + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + health_check_servers, + ) + + mock_user_auth = generate_mock_user_api_key_auth( + user_role=LitellmUserRoles.INTERNAL_USER + ) + + health_result_one = generate_mock_mcp_server_db_record( + server_id="server-1", alias="One" + ) + health_result_one.status = "healthy" + + health_result_two = generate_mock_mcp_server_db_record( + server_id="server-2", alias="Two" + ) + health_result_two.status = "unhealthy" + + mock_manager = MagicMock() + mock_manager.get_all_mcp_servers_with_health_unfiltered = AsyncMock( + return_value=[health_result_one, health_result_two] + ) + + with patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._get_user_mcp_management_mode", + return_value="view_all", + ), patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ): + result = await health_check_servers( + server_ids=None, + user_api_key_dict=mock_user_auth, + ) + + assert len(result) == 2 + assert result[0]["server_id"] == "server-1" + assert result[0]["status"] == "healthy" + assert result[1]["server_id"] == "server-2" + assert result[1]["status"] == "unhealthy" + @pytest.mark.asyncio async def test_health_check_unauthorized_servers(self): """ diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index 8fdfd6897a8..ad4f53dac4b 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -742,18 +742,16 @@ class TestProxySettingEndpoints: ): """Test updating UI settings with an allowlisted field""" from unittest.mock import AsyncMock, MagicMock + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy._types import UserAPIKeyAuth - class MockUser: - def __init__(self, user_role): - self.user_role = user_role - - async def mock_admin_auth(): - return MockUser(LitellmUserRoles.PROXY_ADMIN) - - monkeypatch.setattr( - "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.user_api_key_auth", - mock_admin_auth, + # Override the FastAPI dependency with a proper mock + mock_user_auth = UserAPIKeyAuth( + user_id="test-user-123", + user_role=LitellmUserRoles.PROXY_ADMIN, ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) mock_prisma = MagicMock() mock_prisma.db.litellm_uisettings.upsert = AsyncMock() @@ -761,7 +759,11 @@ class TestProxySettingEndpoints: payload = {"disable_model_add_for_internal_users": True} - response = client.patch("/update/ui_settings", json=payload) + try: + response = client.patch("/update/ui_settings", json=payload) + finally: + # Clean up the dependency override + app.dependency_overrides.clear() assert response.status_code == 200 data = response.json() @@ -780,18 +782,16 @@ class TestProxySettingEndpoints: ): """Test non-allowlisted UI settings are ignored on update""" from unittest.mock import AsyncMock, MagicMock + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy._types import UserAPIKeyAuth - class MockUser: - def __init__(self, user_role): - self.user_role = user_role - - async def mock_admin_auth(): - return MockUser(LitellmUserRoles.PROXY_ADMIN) - - monkeypatch.setattr( - "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.user_api_key_auth", - mock_admin_auth, + # Override the FastAPI dependency with a proper mock + mock_user_auth = UserAPIKeyAuth( + user_id="test-user-123", + user_role=LitellmUserRoles.PROXY_ADMIN, ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) mock_prisma = MagicMock() mock_prisma.db.litellm_uisettings.upsert = AsyncMock() @@ -802,7 +802,11 @@ class TestProxySettingEndpoints: "unsupported_flag": True, } - response = client.patch("/update/ui_settings", json=payload) + try: + response = client.patch("/update/ui_settings", json=payload) + finally: + # Clean up the dependency override + app.dependency_overrides.clear() assert response.status_code == 200 data = response.json() diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useSSOSettings.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useSSOSettings.ts index 3e09c3c2ca8..f03f3977115 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useSSOSettings.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useSSOSettings.ts @@ -1,7 +1,7 @@ +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { getSSOSettings } from "@/components/networking"; import { useQuery, UseQueryResult } from "@tanstack/react-query"; import { createQueryKeys } from "../common/queryKeysFactory"; -import { getSSOSettings } from "@/components/networking"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; export interface SSOFieldSchema { description: string; @@ -27,13 +27,15 @@ export interface SSOSettingsValues { proxy_base_url: string | null; user_email: string | null; ui_access_mode: string | null; - role_mappings: { - provider: string; - group_claim: string; - default_role: "internal_user" | "internal_user_viewer" | "proxy_admin" | "proxy_admin_viewer"; - roles: { - [key: string]: string[]; - }; + role_mappings: RoleMappings; +} + +export interface RoleMappings { + provider: string; + group_claim: string; + default_role: "internal_user" | "internal_user_viewer" | "proxy_admin" | "proxy_admin_viewer"; + roles: { + [key: string]: string[]; }; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx index 65418179595..a8b1d2cddc9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx @@ -18,6 +18,7 @@ import { useQueryClient } from "@tanstack/react-query"; import { Col, Grid, Icon, Tab, TabGroup, TabList, TabPanel, TabPanels, Text } from "@tremor/react"; import type { UploadProps } from "antd"; import { Form, Typography } from "antd"; +import { PlusCircleOutlined } from "@ant-design/icons"; import React, { useEffect, useMemo, useState } from "react"; import AddModelTab from "../../../components/add_model/add_model_tab"; import HealthCheckComponent from "../../../components/model_dashboard/HealthCheckComponent"; @@ -274,6 +275,30 @@ const ModelsAndEndpointsView: React.FC = ({ premiumUser, te )} + + {/* Missing Provider Banner */} + {selectedModelId && !isLoading ? ( = ({ form, onFormS prevValues.use_role_mappings !== currentValues.use_role_mappings} + shouldUpdate={(prevValues, currentValues) => + prevValues.use_role_mappings !== currentValues.use_role_mappings || + prevValues.sso_provider !== currentValues.sso_provider + } > {({ getFieldValue }) => { const useRoleMappings = getFieldValue("use_role_mappings"); - return useRoleMappings ? ( + const provider = getFieldValue("sso_provider"); + const supportsRoleMappings = provider === "okta" || provider === "generic"; + return useRoleMappings && supportsRoleMappings ? ( = ({ form, onFormS prevValues.use_role_mappings !== currentValues.use_role_mappings} + shouldUpdate={(prevValues, currentValues) => + prevValues.use_role_mappings !== currentValues.use_role_mappings || + prevValues.sso_provider !== currentValues.sso_provider + } > {({ getFieldValue }) => { const useRoleMappings = getFieldValue("use_role_mappings"); - return useRoleMappings ? ( + const provider = getFieldValue("sso_provider"); + const supportsRoleMappings = provider === "okta" || provider === "generic"; + return useRoleMappings && supportsRoleMappings ? ( <> diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.test.tsx index 7d8a35b7f44..ef6ec6c7055 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.test.tsx @@ -1,60 +1,20 @@ -import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { render, screen } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; import DeleteSSOSettingsModal from "./DeleteSSOSettingsModal"; -vi.mock("@/app/(dashboard)/hooks/sso/useSSOSettings", () => ({ - useSSOSettings: vi.fn(() => ({ - data: { - values: { - google_client_id: "test-client-id", - }, - }, - })), -})); - -vi.mock("@/app/(dashboard)/hooks/sso/useEditSSOSettings", () => ({ - useEditSSOSettings: vi.fn(() => ({ - mutateAsync: vi.fn(), - isPending: false, - })), -})); - -vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ - default: vi.fn(() => ({ - accessToken: "test-token", - userId: "test-user-id", - userRole: "proxy_admin", - })), -})); - -const createQueryClient = () => - new QueryClient({ - defaultOptions: { - queries: { - retry: false, - gcTime: 0, - }, - }, - }); - describe("DeleteSSOSettingsModal", () => { it("should render", () => { const onCancel = vi.fn(); const onSuccess = vi.fn(); - const queryClient = createQueryClient(); render( - - - , + , ); expect(screen.getByText("Confirm Clear SSO Settings")).toBeInTheDocument(); expect( - screen.getByText( - "Are you sure you want to clear all SSO settings? Users will no longer be able to login using SSO after this change.", - ), + screen.getByText("Are you sure you want to clear all SSO settings? This action cannot be undone."), ).toBeInTheDocument(); + expect(screen.getByText("Users will no longer be able to login using SSO after this change.")).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.tsx index 44cbf0020eb..6a28b6490e0 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.tsx @@ -1,66 +1,79 @@ -import { useEditSSOSettings } from "@/app/(dashboard)/hooks/sso/useEditSSOSettings"; -import { useSSOSettings } from "@/app/(dashboard)/hooks/sso/useSSOSettings"; +import { Modal } from "antd"; import React from "react"; -import DeleteResourceModal from "../../../../common_components/DeleteResourceModal"; import NotificationsManager from "../../../../molecules/notifications_manager"; +import { updateSSOSettings } from "../../../../networking"; import { parseErrorMessage } from "../../../../shared/errorUtils"; -import { detectSSOProvider } from "../utils"; interface DeleteSSOSettingsModalProps { isVisible: boolean; onCancel: () => void; onSuccess: () => void; + accessToken: string | null; } -const DeleteSSOSettingsModal: React.FC = ({ isVisible, onCancel, onSuccess }) => { - const { data: ssoSettings } = useSSOSettings(); - const { mutateAsync: editSSOSettings, isPending: isEditingSSOSettings } = useEditSSOSettings(); - +const DeleteSSOSettingsModal: React.FC = ({ + isVisible, + onCancel, + onSuccess, + accessToken, +}) => { // Handle clearing SSO settings const handleClearSSO = async () => { - const clearSettings = { - google_client_id: null, - google_client_secret: null, - microsoft_client_id: null, - microsoft_client_secret: null, - microsoft_tenant: null, - generic_client_id: null, - generic_client_secret: null, - generic_authorization_endpoint: null, - generic_token_endpoint: null, - generic_userinfo_endpoint: null, - proxy_base_url: null, - user_email: null, - sso_provider: null, - role_mappings: null, - }; + if (!accessToken) { + NotificationsManager.fromBackend("No access token available"); + return; + } - await editSSOSettings(clearSettings, { - onSuccess: () => { - NotificationsManager.success("SSO settings cleared successfully"); - onCancel(); - onSuccess(); - }, - onError: (error) => { - NotificationsManager.fromBackend("Failed to clear SSO settings: " + parseErrorMessage(error)); - }, - }); + try { + // Clear all SSO settings + const clearSettings = { + google_client_id: null, + google_client_secret: null, + microsoft_client_id: null, + microsoft_client_secret: null, + microsoft_tenant: null, + generic_client_id: null, + generic_client_secret: null, + generic_authorization_endpoint: null, + generic_token_endpoint: null, + generic_userinfo_endpoint: null, + proxy_base_url: null, + user_email: null, + sso_provider: null, + }; + + await updateSSOSettings(accessToken, clearSettings); + + NotificationsManager.success("SSO settings cleared successfully"); + + // Close modal and trigger success callback + onCancel(); + onSuccess(); + } catch (error) { + console.error("Failed to clear SSO settings:", error); + NotificationsManager.fromBackend("Failed to clear SSO settings: " + parseErrorMessage(error)); + } }; return ( - + onCancel={onCancel} + okText="Yes, Clear" + cancelText="Cancel" + okButtonProps={{ + danger: true, + style: { + backgroundColor: "#dc2626", + borderColor: "#dc2626", + }, + }} + > +

Are you sure you want to clear all SSO settings? This action cannot be undone.

+

Users will no longer be able to login using SSO after this change.

+ ); }; diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RoleMappings.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RoleMappings.test.tsx deleted file mode 100644 index f4b7b9aadd4..00000000000 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RoleMappings.test.tsx +++ /dev/null @@ -1,92 +0,0 @@ -import type { RoleMappings as RoleMappingsType } from "@/app/(dashboard)/hooks/sso/useSSOSettings"; -import { screen } from "@testing-library/react"; -import { describe, expect, it } from "vitest"; -import { renderWithProviders } from "../../../../../tests/test-utils"; -import RoleMappings from "./RoleMappings"; - -describe("RoleMappings", () => { - it("should render successfully", () => { - const roleMappings: RoleMappingsType = { - provider: "generic", - group_claim: "groups", - default_role: "internal_user", - roles: { - proxy_admin: ["admin-group"], - proxy_admin_viewer: [], - internal_user: ["user-group"], - internal_user_viewer: [], - }, - }; - - renderWithProviders(); - - expect(screen.getByText("Role Mappings")).toBeInTheDocument(); - }); - - it("should return null when roleMappings is undefined", () => { - const { container } = renderWithProviders(); - - expect(container.firstChild).toBeNull(); - }); - - it("should display Group Claim and Default Role with correct values and display names", () => { - const testCases: Array<{ role: RoleMappingsType["default_role"]; displayName: string; groupClaim: string }> = [ - { role: "internal_user_viewer", displayName: "Internal Viewer", groupClaim: "custom-groups-1" }, - { role: "internal_user", displayName: "Internal User", groupClaim: "custom-groups-2" }, - { role: "proxy_admin_viewer", displayName: "Proxy Admin Viewer", groupClaim: "custom-groups-3" }, - { role: "proxy_admin", displayName: "Proxy Admin", groupClaim: "custom-groups-4" }, - ]; - - testCases.forEach(({ role, displayName, groupClaim }) => { - const roleMappings: RoleMappingsType = { - provider: "generic", - group_claim: groupClaim, - default_role: role, - roles: { - proxy_admin: [], - proxy_admin_viewer: [], - internal_user: [], - internal_user_viewer: [], - }, - }; - - const { unmount } = renderWithProviders(); - - expect(screen.getByText("Group Claim")).toBeInTheDocument(); - expect(screen.getByText(groupClaim)).toBeInTheDocument(); - expect(screen.getByText("Default Role")).toBeInTheDocument(); - const displayNameElements = screen.getAllByText(displayName); - expect(displayNameElements.length).toBeGreaterThan(0); - unmount(); - }); - }); - - it("should display table with roles, groups as Tags when mapped, and 'No groups mapped' when empty", () => { - const roleMappings: RoleMappingsType = { - provider: "generic", - group_claim: "groups", - default_role: "internal_user", - roles: { - proxy_admin: ["admin-group-1", "admin-group-2", "admin-group-3"], - proxy_admin_viewer: ["viewer-group"], - internal_user: ["user-group"], - internal_user_viewer: [], - }, - }; - - renderWithProviders(); - - expect(screen.getByText("Role")).toBeInTheDocument(); - expect(screen.getByText("Mapped Groups")).toBeInTheDocument(); - expect(screen.getAllByText("Proxy Admin").length).toBeGreaterThan(0); - expect(screen.getAllByText("Proxy Admin Viewer").length).toBeGreaterThan(0); - expect(screen.getAllByText("Internal User").length).toBeGreaterThan(0); - expect(screen.getAllByText("Internal Viewer").length).toBeGreaterThan(0); - expect(screen.getByText("admin-group-1")).toBeInTheDocument(); - expect(screen.getByText("admin-group-2")).toBeInTheDocument(); - expect(screen.getByText("admin-group-3")).toBeInTheDocument(); - expect(screen.getByText("viewer-group")).toBeInTheDocument(); - expect(screen.getByText("user-group")).toBeInTheDocument(); - expect(screen.getByText("No groups mapped")).toBeInTheDocument(); - }); -}); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RoleMappings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RoleMappings.tsx deleted file mode 100644 index 3750ee88183..00000000000 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RoleMappings.tsx +++ /dev/null @@ -1,74 +0,0 @@ -import type { RoleMappings as RoleMappingsType } from "@/app/(dashboard)/hooks/sso/useSSOSettings"; -import { Card, Divider, Table, Tag, Typography } from "antd"; -import { Users } from "lucide-react"; -import { defaultRoleDisplayNames } from "./constants"; -const { Title, Text } = Typography; - -export default function RoleMappings({ roleMappings }: { roleMappings: RoleMappingsType | undefined }) { - if (!roleMappings) { - return null; - } - - const roleMappingsColumns = [ - { - title: "Role", - dataIndex: "role", - key: "role", - render: (text: string) => {defaultRoleDisplayNames[text]}, - }, - { - title: "Mapped Groups", - dataIndex: "groups", - key: "groups", - render: (groups: string[]) => ( - <> - {groups.length > 0 ? ( - groups.map((group, index) => ( - - {group} - - )) - ) : ( - No groups mapped - )} - - ), - }, - ]; - return ( - -
- - Role Mappings -
-
-
-
- Group Claim -
- {roleMappings.group_claim} -
-
-
- Default Role -
- {defaultRoleDisplayNames[roleMappings.default_role]} -
-
-
- - ({ - role, - groups, - }))} - pagination={false} - bordered - size="small" - className="w-full" - /> - - - ); -} diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx index adc1251cde2..27ff96af05f 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx @@ -1,23 +1,23 @@ "use client"; import { useSSOSettings, type SSOSettingsValues } from "@/app/(dashboard)/hooks/sso/useSSOSettings"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { Button, Card, Descriptions, Space, Typography } from "antd"; import { Edit, Shield, Trash2 } from "lucide-react"; import { useState } from "react"; -import { ssoProviderDisplayNames, ssoProviderLogoMap } from "./constants"; import AddSSOSettingsModal from "./Modals/AddSSOSettingsModal"; import DeleteSSOSettingsModal from "./Modals/DeleteSSOSettingsModal"; import EditSSOSettingsModal from "./Modals/EditSSOSettingsModal"; import RedactableField from "./RedactableField"; -import RoleMappings from "./RoleMappings"; import SSOSettingsEmptyPlaceholder from "./SSOSettingsEmptyPlaceholder"; import SSOSettingsLoadingSkeleton from "./SSOSettingsLoadingSkeleton"; -import { detectSSOProvider } from "./utils"; +import { ssoProviderDisplayNames, ssoProviderLogoMap } from "./constants"; const { Title, Text } = Typography; export default function SSOSettings() { const { data: ssoSettings, refetch, isLoading } = useSSOSettings(); + const { accessToken } = useAuthorized(); const [isDeleteModalVisible, setIsDeleteModalVisible] = useState(false); const [isAddModalVisible, setIsAddModalVisible] = useState(false); const [isEditModalVisible, setIsEditModalVisible] = useState(false); @@ -26,8 +26,24 @@ export default function SSOSettings() { Boolean(ssoSettings?.values.microsoft_client_id) || Boolean(ssoSettings?.values.generic_client_id); + // Determine the SSO provider based on the configuration + const detectSSOProvider = (values: SSOSettingsValues): string | null => { + if (values.google_client_id) return "google"; + if (values.microsoft_client_id) return "microsoft"; + if (values.generic_client_id) { + // Check if it looks like Okta/Auth0 based on endpoints + if ( + values.generic_authorization_endpoint?.includes("okta") || + values.generic_authorization_endpoint?.includes("auth0") + ) { + return "okta"; + } + return "generic"; + } + return null; + }; + const selectedProvider = ssoSettings?.values ? detectSSOProvider(ssoSettings.values) : null; - const isRoleMappingsEnabled = Boolean(ssoSettings?.values.role_mappings); const renderEndpointValue = (value?: string | null) => ( @@ -169,52 +185,46 @@ export default function SSOSettings() { {isLoading ? ( ) : ( - - - - {/* Header Section */} -
-
- -
- SSO Configuration - Manage Single Sign-On authentication settings -
-
- -
- {isSSOConfigured && ( - <> - - - - )} + + + {/* Header Section */} +
+
+ +
+ SSO Configuration + Manage Single Sign-On authentication settings
- {isSSOConfigured ? ( - renderSSOSettings() - ) : ( - setIsAddModalVisible(true)} /> - )} - - - {isRoleMappingsEnabled && } - +
+ {isSSOConfigured && ( + <> + + + + )} +
+
+ + {isSSOConfigured ? ( + renderSSOSettings() + ) : ( + setIsAddModalVisible(true)} /> + )} + + )} setIsDeleteModalVisible(false)} onSuccess={() => refetch()} + accessToken={accessToken} /> = { okta: "Okta / Auth0 SSO", generic: "Generic SSO", }; - -export const defaultRoleDisplayNames: Record = { - internal_user_viewer: "Internal Viewer", - internal_user: "Internal User", - proxy_admin_viewer: "Proxy Admin Viewer", - proxy_admin: "Proxy Admin", -}; diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.test.ts b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.test.ts index 718302d35fe..1c878d7b7b3 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.test.ts +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.test.ts @@ -55,7 +55,6 @@ describe("processSSOSettingsPayload", () => { default_role: "proxy_admin", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", other_field: "value", }; @@ -84,7 +83,6 @@ describe("processSSOSettingsPayload", () => { default_role: "internal_user", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -102,7 +100,6 @@ describe("processSSOSettingsPayload", () => { default_role: "internal_user_viewer", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -124,7 +121,6 @@ describe("processSSOSettingsPayload", () => { default_role: "proxy_admin_viewer", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -146,7 +142,6 @@ describe("processSSOSettingsPayload", () => { default_role: "internal_user", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -165,7 +160,6 @@ describe("processSSOSettingsPayload", () => { default_role: "internal_user", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -180,7 +174,6 @@ describe("processSSOSettingsPayload", () => { default_role: "internal_user_viewer", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -193,7 +186,6 @@ describe("processSSOSettingsPayload", () => { default_role: "internal_user", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -206,7 +198,6 @@ describe("processSSOSettingsPayload", () => { default_role: "proxy_admin_viewer", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -219,7 +210,6 @@ describe("processSSOSettingsPayload", () => { default_role: "proxy_admin", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -232,7 +222,6 @@ describe("processSSOSettingsPayload", () => { default_role: "unknown_role", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -244,7 +233,6 @@ describe("processSSOSettingsPayload", () => { const formValues = { group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.ts b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.ts index c199048df3e..3533e1226c2 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.ts +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.ts @@ -1,5 +1,3 @@ -import { SSOSettingsValues } from "@/app/(dashboard)/hooks/sso/useSSOSettings"; - /** * Processes SSO settings form values and transforms them into the payload format expected by the API * Handles role mappings transformation and field extraction @@ -20,7 +18,7 @@ export const processSSOSettingsPayload = (formValues: Record): Reco ...rest, }; - // Add role mappings only if use_role_mappings is checked AND provider supports role mappings + // Add role mappings if use_role_mappings is checked if (use_role_mappings) { // Helper function to split comma-separated string into array const splitTeams = (teams: string | undefined): string[] => { @@ -54,20 +52,3 @@ export const processSSOSettingsPayload = (formValues: Record): Reco return payload; }; - -// Determine the SSO provider based on the configuration -export const detectSSOProvider = (values: SSOSettingsValues): string | null => { - if (values.google_client_id) return "google"; - if (values.microsoft_client_id) return "microsoft"; - if (values.generic_client_id) { - // Check if it looks like Okta/Auth0 based on endpoints - if ( - values.generic_authorization_endpoint?.includes("okta") || - values.generic_authorization_endpoint?.includes("auth0") - ) { - return "okta"; - } - return "generic"; - } - return null; -}; diff --git a/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx b/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx index a62d8baa75b..9600c962564 100644 --- a/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx +++ b/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx @@ -136,7 +136,7 @@ export const useMcpOAuthFlow = ({ if (!hasPreconfiguredCredentials) { const registration = await registerMcpOAuthClient(accessToken, serverId, { client_name: temporaryPayload.alias || temporaryPayload.server_name || serverId, - grant_types: ["authorization_code", "refresh_token"], + grant_types: ["authorization_code"], response_types: ["code"], token_endpoint_auth_method: temporaryPayload.credentials && temporaryPayload.credentials.client_secret ? "client_secret_post" : "none", From 0756b48963c0065199cbb61f81596531ba3e698e Mon Sep 17 00:00:00 2001 From: Felipe Peter Date: Tue, 6 Jan 2026 10:33:25 -0800 Subject: [PATCH 07/56] Add Anthropic cache control option to image tool call results (#18674) --- .../prompt_templates/factory.py | 7 +- ...llm_core_utils_prompt_templates_factory.py | 246 +++++++++++++++--- 2 files changed, 211 insertions(+), 42 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 12570a02de7..cbb31be146c 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1645,9 +1645,12 @@ def convert_to_anthropic_tool_result( ) elif content["type"] == "image_url": format = content["image_url"].get("format") if isinstance(content["image_url"], dict) else None - anthropic_content_list.append( - create_anthropic_image_param(content["image_url"], format=format) + _anthropic_image_param = create_anthropic_image_param(content["image_url"], format=format) + _anthropic_image_param = add_cache_control_to_content( + anthropic_content_element=_anthropic_image_param, + original_content_element=content, ) + anthropic_content_list.append(_anthropic_image_param) anthropic_content = anthropic_content_list anthropic_tool_result: Optional[AnthropicMessagesToolResultParam] = None diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index c8fe6efeaa1..a4188f03cde 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -620,26 +620,26 @@ def test_bedrock_tools_unpack_defs(): def test_bedrock_image_processor_content_type_fallback_url_extension(): """ - Test that _post_call_image_processing falls back to URL extension + Test that _post_call_image_processing falls back to URL extension when content-type is binary/octet-stream or application/octet-stream """ import base64 - + # Create mock response with binary/octet-stream content-type mock_response = MagicMock() mock_response.headers.get.return_value = "binary/octet-stream" - + # Create a simple PNG header (magic bytes) png_header = b"\x89\x50\x4e\x47\x0d\x0a\x1a\x0a" png_content = png_header + b"\x00" * 100 # Add some padding mock_response.content = png_content - + # Test with .png URL image_url = "https://example.com/test-image.png" base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( mock_response, image_url ) - + assert content_type == "image/png" assert base64_bytes == base64.b64encode(png_content).decode("utf-8") @@ -650,22 +650,22 @@ def test_bedrock_image_processor_content_type_fallback_binary_detection(): when content-type is missing and URL extension is not recognized """ import base64 - + # Create mock response with no content-type mock_response = MagicMock() mock_response.headers.get.return_value = None - + # Create a JPEG header (magic bytes) jpeg_header = b"\xff\xd8\xff" jpeg_content = jpeg_header + b"\x00" * 100 # Add some padding mock_response.content = jpeg_content - + # Test with URL without extension image_url = "https://example.com/test-image-without-extension" base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( mock_response, image_url ) - + assert content_type == "image/jpeg" assert base64_bytes == base64.b64encode(jpeg_content).decode("utf-8") @@ -675,22 +675,22 @@ def test_bedrock_image_processor_content_type_fallback_application_octet_stream( Test that _post_call_image_processing handles application/octet-stream correctly """ import base64 - + # Create mock response with application/octet-stream content-type mock_response = MagicMock() mock_response.headers.get.return_value = "application/octet-stream" - + # Create a GIF header (magic bytes) gif_header = b"GIF8" + b"\x00" + b"a" gif_content = gif_header + b"\x00" * 100 # Add some padding mock_response.content = gif_content - + # Test with .gif URL image_url = "https://s3.amazonaws.com/bucket/image.gif" base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( mock_response, image_url ) - + assert content_type == "image/gif" assert base64_bytes == base64.b64encode(gif_content).decode("utf-8") @@ -700,22 +700,22 @@ def test_bedrock_image_processor_content_type_with_query_params(): Test that _post_call_image_processing correctly extracts extension from URL with query parameters """ import base64 - + # Create mock response with binary/octet-stream content-type mock_response = MagicMock() mock_response.headers.get.return_value = "binary/octet-stream" - + # Create a WebP header (magic bytes) webp_header = b"RIFF" + b"\x00\x00\x00\x00" + b"WEBP" webp_content = webp_header + b"\x00" * 100 # Add some padding mock_response.content = webp_content - + # Test with URL containing query parameters (common in S3 signed URLs) image_url = "https://s3.amazonaws.com/bucket/image.webp?AWSAccessKeyId=123&Expires=456&Signature=789" base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( mock_response, image_url ) - + assert content_type == "image/webp" assert base64_bytes == base64.b64encode(webp_content).decode("utf-8") @@ -725,21 +725,21 @@ def test_bedrock_image_processor_content_type_normal_header(): Test that _post_call_image_processing works normally when content-type is correctly set """ import base64 - + # Create mock response with correct content-type mock_response = MagicMock() mock_response.headers.get.return_value = "image/png" - + # Create a PNG header png_header = b"\x89\x50\x4e\x47\x0d\x0a\x1a\x0a" png_content = png_header + b"\x00" * 100 mock_response.content = png_content - + image_url = "https://example.com/test-image.png" base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( mock_response, image_url ) - + assert content_type == "image/png" assert base64_bytes == base64.b64encode(png_content).decode("utf-8") @@ -751,16 +751,16 @@ def test_bedrock_image_processor_content_type_fallback_failure(): # Create mock response with binary/octet-stream content-type mock_response = MagicMock() mock_response.headers.get.return_value = "binary/octet-stream" - + # Create content with unrecognizable image format mock_response.content = b"\x00" * 100 - + # Test with URL without recognizable extension image_url = "https://example.com/unknown-file" - + with pytest.raises(ValueError) as excinfo: BedrockImageProcessor._post_call_image_processing(mock_response, image_url) - + assert "Unable to determine content type" in str(excinfo.value) @@ -771,18 +771,18 @@ def test_bedrock_image_processor_content_type_jpeg_variants(): # Create mock response with binary/octet-stream mock_response = MagicMock() mock_response.headers.get.return_value = "binary/octet-stream" - + jpeg_header = b"\xff\xd8\xff" jpeg_content = jpeg_header + b"\x00" * 100 mock_response.content = jpeg_content - + # Test with .jpg extension image_url_jpg = "https://example.com/photo.jpg" _, content_type_jpg = BedrockImageProcessor._post_call_image_processing( mock_response, image_url_jpg ) assert content_type_jpg == "image/jpeg" - + # Test with .jpeg extension image_url_jpeg = "https://example.com/photo.jpeg" _, content_type_jpeg = BedrockImageProcessor._post_call_image_processing( @@ -797,22 +797,22 @@ def test_bedrock_image_processor_content_type_pdf_document(): when content-type is binary/octet-stream """ import base64 - + # Create mock response with binary/octet-stream content-type mock_response = MagicMock() mock_response.headers.get.return_value = "binary/octet-stream" - + # Create a PDF header (magic bytes: %PDF) pdf_header = b"%PDF-1.4" pdf_content = pdf_header + b"\x00" * 100 mock_response.content = pdf_content - + # Test with .pdf URL pdf_url = "https://s3.amazonaws.com/bucket/document.pdf" base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( mock_response, pdf_url ) - + assert content_type == "application/pdf" assert base64_bytes == base64.b64encode(pdf_content).decode("utf-8") @@ -822,12 +822,12 @@ def test_bedrock_image_processor_content_type_document_formats(): Test that _post_call_image_processing handles various document formats """ import base64 - + # Create mock response mock_response = MagicMock() mock_response.headers.get.return_value = "application/octet-stream" mock_response.content = b"\x00" * 100 - + # Test various document formats test_cases = [ ("https://example.com/doc.pdf", "application/pdf"), @@ -837,7 +837,7 @@ def test_bedrock_image_processor_content_type_document_formats(): ("https://example.com/page.html", "text/html"), ("https://example.com/readme.txt", "text/plain"), ] - + for url, expected_mime in test_cases: _, content_type = BedrockImageProcessor._post_call_image_processing( mock_response, url @@ -850,21 +850,21 @@ def test_bedrock_image_processor_content_type_s3_pdf_with_query(): Test that _post_call_image_processing handles S3 PDF with query parameters """ import base64 - + # Create mock response mock_response = MagicMock() mock_response.headers.get.return_value = "binary/octet-stream" - + pdf_content = b"%PDF-1.4" + b"\x00" * 100 mock_response.content = pdf_content - + # S3 signed URL with query parameters s3_url = "https://my-bucket.s3.us-east-1.amazonaws.com/documents/report.pdf?AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Expires=1234567890&Signature=abcdef123456" - + base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( mock_response, s3_url ) - + assert content_type == "application/pdf" assert base64_bytes == base64.b64encode(pdf_content).decode("utf-8") @@ -1137,3 +1137,169 @@ def test_bedrock_create_bedrock_block_different_document_formats(): assert f"DocumentPDFmessages_" in block["document"]["name"] assert block["document"]["name"].endswith(f"_{format_type}") assert block["document"]["format"] == format_type + + +def test_convert_to_anthropic_tool_result_image_with_cache_control(): + """ + Test that cache_control is properly applied to image content in tool results. + This tests the functionality added in the uncommitted changes where + add_cache_control_to_content is called for image_url content types. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + # Test with base64 image data URI + message = { + "role": "tool", + "tool_call_id": "call_test_123", + "content": [ + { + "type": "text", + "text": "Here is the image you requested:", + }, + { + "type": "image_url", + "image_url": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQ", + "cache_control": {"type": "ephemeral"}, + }, + ], + } + + result = convert_to_anthropic_tool_result(message) + + # Verify the result structure + assert result["type"] == "tool_result" + assert result["tool_use_id"] == "call_test_123" + assert isinstance(result["content"], list) + assert len(result["content"]) == 2 + + # Verify text content + assert result["content"][0]["type"] == "text" + assert result["content"][0]["text"] == "Here is the image you requested:" + + # Verify image content with cache_control + assert result["content"][1]["type"] == "image" + assert result["content"][1]["source"]["type"] == "base64" + assert result["content"][1]["source"]["media_type"] == "image/jpeg" + assert "cache_control" in result["content"][1] + assert result["content"][1]["cache_control"]["type"] == "ephemeral" + + +def test_convert_to_anthropic_tool_result_image_without_cache_control(): + """ + Test that images without cache_control in tool results work correctly. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + message = { + "role": "tool", + "tool_call_id": "call_test_456", + "content": [ + { + "type": "image_url", + "image_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA", + }, + ], + } + + result = convert_to_anthropic_tool_result(message) + + # Verify the result structure + assert result["type"] == "tool_result" + assert result["tool_use_id"] == "call_test_456" + assert isinstance(result["content"], list) + assert len(result["content"]) == 1 + + # Verify image content without cache_control (cache_control will be None if not set) + assert result["content"][0]["type"] == "image" + assert result["content"][0]["source"]["type"] == "base64" + assert result["content"][0]["source"]["media_type"] == "image/png" + assert result["content"][0].get("cache_control") is None + + +def test_convert_to_anthropic_tool_result_mixed_content_with_cache_control(): + """ + Test tool results with mixed content types (text and image) where only some have cache_control. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + message = { + "role": "tool", + "tool_call_id": "call_test_789", + "content": [ + { + "type": "text", + "text": "First image:", + "cache_control": {"type": "ephemeral"}, + }, + { + "type": "image_url", + "image_url": "data:image/jpeg;base64,/9j/4AAQSkZJRg", + "cache_control": {"type": "ephemeral"}, + }, + { + "type": "text", + "text": "Second image (no cache):", + }, + { + "type": "image_url", + "image_url": "data:image/png;base64,iVBORw0KGgo", + }, + ], + } + + result = convert_to_anthropic_tool_result(message) + + assert result["type"] == "tool_result" + assert isinstance(result["content"], list) + assert len(result["content"]) == 4 + + # First text with cache_control + assert result["content"][0]["type"] == "text" + assert result["content"][0]["cache_control"]["type"] == "ephemeral" + + # First image with cache_control + assert result["content"][1]["type"] == "image" + assert result["content"][1]["cache_control"]["type"] == "ephemeral" + + # Second text without cache_control (cache_control will be None if not set) + assert result["content"][2]["type"] == "text" + assert result["content"][2].get("cache_control") is None + + # Second image without cache_control (cache_control will be None if not set) + assert result["content"][3]["type"] == "image" + assert result["content"][3].get("cache_control") is None + + +def test_convert_to_anthropic_tool_result_image_url_as_http(): + """ + Test that HTTP/HTTPS URLs with cache_control are handled correctly. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + message = { + "role": "tool", + "tool_call_id": "call_http_001", + "content": [ + { + "type": "image_url", + "image_url": "https://example.com/image.jpg", + "cache_control": {"type": "ephemeral"}, + }, + ], + } + + result = convert_to_anthropic_tool_result(message) + + # Verify image is passed as URL reference with cache_control + assert result["content"][0]["type"] == "image" + assert result["content"][0]["source"]["type"] == "url" + assert result["content"][0]["source"]["url"] == "https://example.com/image.jpg" + assert result["content"][0]["cache_control"]["type"] == "ephemeral" From b40a6a0f6c0f6c6998b8f9c1e4c4f0e0cd178696 Mon Sep 17 00:00:00 2001 From: Stephen Matta Date: Fri, 19 Dec 2025 00:57:28 -0500 Subject: [PATCH 08/56] [Fix] Nova model detection for Bedrock provider (#17910) Resolves issue #17910 where Amazon Nova models (like amazon.nova-pro-v1:0) were incorrectly identified as Amazon Titan models, causing requests to use textGenerationConfig instead of inferenceConfig. The fix moves the "nova" check before the initial provider check on the split model name. This ensures that models containing "nova" (like amazon.nova-pro-v1:0 or amazon.nova-2-lite-v1:0) are correctly identified as Nova models, rather than matching "amazon" first. --- litellm/llms/bedrock/base_aws_llm.py | 24 +- .../base_invoke_transformation.py | 10 +- .../test_bedrock_completion.py | 271 ++++++++++++------ 3 files changed, 203 insertions(+), 102 deletions(-) diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index 71d21001cc3..6ffe45624bc 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -314,6 +314,12 @@ class BaseAWSLLM: if model.startswith("invoke/"): model = model.replace("invoke/", "", 1) + # Special case: Check for "nova" in model name first (before "amazon") + # This handles amazon.nova-* models which would otherwise match "amazon" (Titan) + if "nova" in model.lower(): + if "nova" in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL): + return cast(BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova") + _split_model = model.split(".")[0] if _split_model in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL): return cast(BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model) @@ -323,13 +329,9 @@ class BaseAWSLLM: if provider is not None: return provider - # check if provider == "nova" - if "nova" in model: - return "nova" - else: - for provider in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL): - if provider in model: - return provider + for provider in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL): + if provider in model: + return provider return None @staticmethod @@ -364,7 +366,7 @@ class BaseAWSLLM: elif provider == "qwen3" and "qwen3/" in model_id: model_id = BaseAWSLLM._get_model_id_from_model_with_spec( model_id, spec="qwen3" - ) + ) elif provider == "stability" and "stability/" in model_id: model_id = BaseAWSLLM._get_model_id_from_model_with_spec( model_id, spec="stability" @@ -412,7 +414,7 @@ class BaseAWSLLM: if "nova" in model.lower(): if "nova" in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL): return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, "nova") - + # Handle regional models like us.twelvelabs.marengo-embed-2-7-v1:0 if "." in model: parts = model.split(".") @@ -958,7 +960,9 @@ class BaseAWSLLM: return endpoint_url, proxy_endpoint_url def _select_default_endpoint_url( - self, endpoint_type: Optional[Literal["runtime", "agent", "agentcore"]], aws_region_name: str + self, + endpoint_type: Optional[Literal["runtime", "agent", "agentcore"]], + aws_region_name: str, ) -> str: """ Select the default endpoint url based on the endpoint type diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index c602b71fe05..cf8aee6954b 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -524,6 +524,12 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): if model.startswith("invoke/"): model = model.replace("invoke/", "", 1) + # Special case: Check for "nova" in model name first (before "amazon") + # This handles amazon.nova-* models which would otherwise match "amazon" (Titan) + if "nova" in model.lower(): + if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): + return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova") + _split_model = model.split(".")[0] if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model) @@ -533,10 +539,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): if provider is not None: return provider - # check if provider == "nova" - if "nova" in model: - return "nova" - for provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): if provider in model: return provider diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 78c9f94239b..cb8b761b048 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -2841,6 +2841,34 @@ def test_bedrock_invoke_provider(): ) == "nova" ) + assert ( + litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("amazon.nova-pro-v1:0") + == "nova" + ) + assert ( + litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( + "amazon.nova-lite-v1:0" + ) + == "nova" + ) + assert ( + litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( + "amazon.nova-micro-v1:0" + ) + == "nova" + ) + assert ( + litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( + "amazon.nova-premier-v1:0" + ) + == "nova" + ) + assert ( + litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( + "amazon.nova-2-lite-v1:0" + ) + == "nova" + ) def test_bedrock_description_param(): @@ -3494,7 +3522,9 @@ def test_bedrock_openai_imported_model(): url = mock_post.call_args.kwargs["url"] print(f"URL: {url}") assert "bedrock-runtime.us-east-1.amazonaws.com" in url - assert "arn:aws:bedrock:us-east-1:117159858402:imported-model/m4gc1mrfuddy" in url + assert ( + "arn:aws:bedrock:us-east-1:117159858402:imported-model/m4gc1mrfuddy" in url + ) assert "/invoke" in url # Validate request body follows OpenAI format @@ -3523,7 +3553,9 @@ def test_bedrock_openai_imported_model(): # Check image_url content assert user_msg["content"][1]["type"] == "image_url" assert "image_url" in user_msg["content"][1] - assert user_msg["content"][1]["image_url"]["url"].startswith("data:image/jpeg;base64,") + assert user_msg["content"][1]["image_url"]["url"].startswith( + "data:image/jpeg;base64," + ) assert user_msg["content"][2]["type"] == "image_url" assert "image_url" in user_msg["content"][2] @@ -3532,21 +3564,67 @@ def test_bedrock_openai_imported_model(): assert request_body["max_tokens"] == 300 assert request_body["temperature"] == 0.5 + +def test_bedrock_nova_provider_detection(): + """ + Test that Nova models are correctly detected even when prefixed with "amazon." + Regression test for issue #17910 where models like "amazon.nova-pro-v1:0" + were incorrectly identified as "amazon" (Titan) instead of "nova". + """ + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + # Test various Nova model formats + nova_test_cases = [ + ("us.amazon.nova-pro-v1:0", "nova"), + ("us.amazon.nova-lite-v1:0", "nova"), + ("us.amazon.nova-micro-v1:0", "nova"), + ("amazon.nova-pro-v1:0", "nova"), + ("amazon.nova-lite-v1:0", "nova"), + ("amazon.nova-micro-v1:0", "nova"), + ("amazon.nova-premier-v1:0", "nova"), + ("amazon.nova-2-lite-v1:0", "nova"), + ("bedrock/amazon.nova-pro-v1:0", "nova"), + ("bedrock/invoke/amazon.nova-pro-v1:0", "nova"), + ("amazon.Nova-pro-v1:0", "nova"), + ("amazon.NOVA-pro-v1:0", "nova"), + ] + + for model, expected in nova_test_cases: + provider = BaseAWSLLM.get_bedrock_invoke_provider(model) + assert ( + provider == expected + ), f"Failed for model: {model}, expected: {expected}, got: {provider}" + + # Verify that Amazon Titan models still return "amazon" + titan_test_cases = [ + ("amazon.titan-text-express-v1", "amazon"), + ("us.amazon.titan-text-lite-v1", "amazon"), + ] + + for model, expected in titan_test_cases: + provider = BaseAWSLLM.get_bedrock_invoke_provider(model) + assert ( + provider == expected + ), f"Failed for model: {model}, expected: {expected}, got: {provider}" + + def test_bedrock_openai_provider_detection(): """ Test that the OpenAI provider is correctly detected from model strings. """ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM - + # Test various OpenAI model formats test_cases = [ "openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123", "bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/xyz789", ] - + for model in test_cases: provider = BaseAWSLLM.get_bedrock_invoke_provider(model) - assert provider == "openai", f"Failed for model: {model}, got provider: {provider}" + assert ( + provider == "openai" + ), f"Failed for model: {model}, got provider: {provider}" print(f"✓ Provider detection works for: {model}") @@ -3555,16 +3633,16 @@ def test_bedrock_openai_model_id_extraction(): Test that the model ID (ARN) is correctly extracted and encoded for OpenAI models. """ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM - - model = "openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/test-model-123" - provider = BaseAWSLLM.get_bedrock_invoke_provider(model) - - model_id = BaseAWSLLM.get_bedrock_model_id( - model=model, - provider=provider, - optional_params={} + + model = ( + "openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/test-model-123" ) - + provider = BaseAWSLLM.get_bedrock_invoke_provider(model) + + model_id = BaseAWSLLM.get_bedrock_model_id( + model=model, provider=provider, optional_params={} + ) + # The ARN should be double URL encoded assert "arn" in model_id assert "imported-model" in model_id @@ -3576,20 +3654,17 @@ def test_bedrock_openai_convert_messages_to_prompt(): Test that convert_messages_to_prompt returns empty string for OpenAI models. """ from litellm.llms.bedrock.chat.invoke_handler import BedrockLLM - + bedrock_llm = BedrockLLM() messages = [ {"role": "system", "content": "You are helpful"}, - {"role": "user", "content": "Hello"} + {"role": "user", "content": "Hello"}, ] - + prompt, chat_history = bedrock_llm.convert_messages_to_prompt( - model="test-model", - messages=messages, - provider="openai", - custom_prompt_dict={} + model="test-model", messages=messages, provider="openai", custom_prompt_dict={} ) - + # OpenAI models use messages directly, no prompt conversion assert prompt == "" assert chat_history is None @@ -3604,37 +3679,33 @@ def test_bedrock_openai_response_parsing(): from litellm import ModelResponse from unittest.mock import Mock import json - + bedrock_llm = BedrockLLM() - + # Mock OpenAI-style response openai_response = { "choices": [ { "message": { "content": "The capital of France is Paris.", - "role": "assistant" + "role": "assistant", }, "finish_reason": "stop", - "index": 0 + "index": 0, } ], - "usage": { - "prompt_tokens": 10, - "completion_tokens": 8, - "total_tokens": 18 - } + "usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18}, } - + mock_response = Mock() mock_response.json.return_value = openai_response mock_response.text = json.dumps(openai_response) mock_response.status_code = 200 mock_response.headers = {} - + model_response = ModelResponse() mock_logging = Mock() - + result = bedrock_llm.process_response( model="openai/arn:aws:bedrock:us-east-1:123:imported-model/test", response=mock_response, @@ -3646,18 +3717,18 @@ def test_bedrock_openai_response_parsing(): data={}, messages=[{"role": "user", "content": "What is the capital of France?"}], print_verbose=lambda x: None, - encoding=None + encoding=None, ) - + # Verify response content assert result.choices[0].message.content == "The capital of France is Paris." assert result.choices[0].finish_reason == "stop" - + # Verify usage assert result.usage.prompt_tokens == 10 assert result.usage.completion_tokens == 8 assert result.usage.total_tokens == 18 - + print("✓ OpenAI response parsing works correctly") @@ -3665,45 +3736,47 @@ def test_bedrock_openai_request_transformation(): """ Test that the request is correctly transformed for OpenAI models. """ - from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import AmazonInvokeConfig - + from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + AmazonInvokeConfig, + ) + config = AmazonInvokeConfig() - + model = "openai/arn:aws:bedrock:us-east-1:123:imported-model/test" messages = [ {"role": "system", "content": "You are helpful"}, - {"role": "user", "content": "Hello"} + {"role": "user", "content": "Hello"}, ] - + optional_params = { "max_tokens": 100, "temperature": 0.7, "top_p": 0.9, - "stream": False + "stream": False, } - + litellm_params = {} headers = {} - - with patch.object(config, 'get_bedrock_invoke_provider', return_value="openai"): + + with patch.object(config, "get_bedrock_invoke_provider", return_value="openai"): result = config.transform_request( model=model, messages=messages, optional_params=optional_params.copy(), litellm_params=litellm_params, - headers=headers + headers=headers, ) - + # Verify the request uses messages format (not prompt) assert "messages" in result assert len(result["messages"]) == 2 assert result["messages"][0]["role"] == "system" assert result["messages"][1]["role"] == "user" - + # Verify parameters are included assert "max_tokens" in result assert "temperature" in result - + print("✓ Request transformation works correctly") @@ -3711,20 +3784,22 @@ def test_bedrock_openai_parameter_filtering(): """ Test that only supported OpenAI parameters are included in the request. """ - from litellm.llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import AmazonBedrockOpenAIConfig - + from litellm.llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import ( + AmazonBedrockOpenAIConfig, + ) + config = AmazonBedrockOpenAIConfig() model = "test-model" - + supported_params = config.get_supported_openai_params(model=model) - + # Verify common OpenAI parameters are supported assert "max_tokens" in supported_params assert "temperature" in supported_params assert "top_p" in supported_params assert "stream" in supported_params assert "stop" in supported_params - + print(f"✓ Parameter filtering supports: {len(supported_params)} parameters") print(f" Supported params: {supported_params}") @@ -3734,12 +3809,12 @@ def test_bedrock_openai_route_detection(): Test that the OpenAI route is correctly detected. """ from litellm.llms.bedrock.common_utils import BedrockModelInfo - + test_cases = [ ("openai/arn:aws:bedrock:us-east-1:123:imported-model/test", "openai"), ("bedrock/openai/arn:aws:bedrock:us-east-1:123:imported-model/test", "openai"), ] - + for model, expected_route in test_cases: route = BedrockModelInfo.get_bedrock_route(model) assert route == expected_route, f"Failed for model: {model}, got route: {route}" @@ -3751,15 +3826,30 @@ def test_bedrock_openai_explicit_route_check(): Test the explicit OpenAI route checker helper method. """ from litellm.llms.bedrock.common_utils import BedrockModelInfo - + # Test with openai/ prefix - assert BedrockModelInfo._explicit_openai_route("openai/arn:aws:bedrock:us-east-1:123:imported-model/test") is True - assert BedrockModelInfo._explicit_openai_route("bedrock/openai/arn:aws:bedrock:us-east-1:123:imported-model/test") is True - + assert ( + BedrockModelInfo._explicit_openai_route( + "openai/arn:aws:bedrock:us-east-1:123:imported-model/test" + ) + is True + ) + assert ( + BedrockModelInfo._explicit_openai_route( + "bedrock/openai/arn:aws:bedrock:us-east-1:123:imported-model/test" + ) + is True + ) + # Test without openai/ prefix assert BedrockModelInfo._explicit_openai_route("anthropic.claude-3-sonnet") is False - assert BedrockModelInfo._explicit_openai_route("arn:aws:bedrock:us-east-1:123:imported-model/test") is False - + assert ( + BedrockModelInfo._explicit_openai_route( + "arn:aws:bedrock:us-east-1:123:imported-model/test" + ) + is False + ) + print("✓ Explicit route check works correctly") @@ -3767,16 +3857,18 @@ def test_bedrock_openai_config_initialization(): """ Test that AmazonBedrockOpenAIConfig can be properly initialized. """ - from litellm.llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import AmazonBedrockOpenAIConfig - + from litellm.llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import ( + AmazonBedrockOpenAIConfig, + ) + config = AmazonBedrockOpenAIConfig() - + # Verify it has the necessary methods - assert hasattr(config, 'get_supported_openai_params') - assert hasattr(config, 'transform_request') - assert hasattr(config, 'transform_response') - assert hasattr(config, 'map_openai_params') - + assert hasattr(config, "get_supported_openai_params") + assert hasattr(config, "transform_request") + assert hasattr(config, "transform_response") + assert hasattr(config, "map_openai_params") + print("✓ AmazonBedrockOpenAIConfig initializes correctly") @@ -3785,9 +3877,9 @@ def test_bedrock_openai_multiple_message_types(): Test that various message content types are handled correctly. """ from litellm.llms.custom_httpx.http_handler import HTTPHandler - + client = HTTPHandler() - + # Test with mixed content types messages = [ {"role": "system", "content": "You are helpful"}, @@ -3796,11 +3888,14 @@ def test_bedrock_openai_multiple_message_types(): "role": "user", "content": [ {"type": "text", "text": "Complex message with text"}, - {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,iVBORw0KGg"}} - ] - } + { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,iVBORw0KGg"}, + }, + ], + }, ] - + with patch.object(client, "post") as mock_post: try: response = completion( @@ -3811,18 +3906,18 @@ def test_bedrock_openai_multiple_message_types(): ) except Exception as e: pass - + # Verify the request was made if mock_post.called: request_body = json.loads(mock_post.call_args.kwargs["data"]) - + # Verify messages are preserved assert "messages" in request_body assert len(request_body["messages"]) == 3 - + # Verify mixed content is handled assert isinstance(request_body["messages"][2]["content"], list) - + print("✓ Multiple message types handled correctly") @@ -3835,18 +3930,18 @@ def test_bedrock_openai_error_handling(): from litellm.llms.bedrock.common_utils import BedrockError from unittest.mock import Mock import json - + bedrock_llm = BedrockLLM() - + # Mock error response mock_response = Mock() mock_response.json.side_effect = Exception("Invalid JSON") mock_response.text = "Invalid response" mock_response.status_code = 422 - + model_response = ModelResponse() mock_logging = Mock() - + with pytest.raises(BedrockError) as exc_info: bedrock_llm.process_response( model="openai/arn:aws:bedrock:us-east-1:123:imported-model/test", @@ -3859,8 +3954,8 @@ def test_bedrock_openai_error_handling(): data={}, messages=[], print_verbose=lambda x: None, - encoding=None + encoding=None, ) - + assert exc_info.value.status_code == 422 print("✓ Error handling works correctly") From 000913fa12ec41916c04ab1107cd31d196c3fb0f Mon Sep 17 00:00:00 2001 From: drorIvry Date: Wed, 7 Jan 2026 13:53:12 +0200 Subject: [PATCH 09/56] Hotfix - docs qualifire (#18724) * Hotfix - docs qualifire * Hotfix - docs qualifire * Hotfix - docs qualifire * Hotfix - docs qualifire * Hotfix - docs qualifire * Hotfix - docs qualifire * Hotfix - docs qualifire --- .../docs/proxy/guardrails/qualifire.md | 41 +- docs/my-website/sidebars.js | 1 + .../guardrail_hooks/qualifire/qualifire.py | 244 ++++++---- .../guardrail_hooks/test_qualifire.py | 433 +++++++++++++----- 4 files changed, 475 insertions(+), 244 deletions(-) diff --git a/docs/my-website/docs/proxy/guardrails/qualifire.md b/docs/my-website/docs/proxy/guardrails/qualifire.md index 66961c92d9d..850af37e47f 100644 --- a/docs/my-website/docs/proxy/guardrails/qualifire.md +++ b/docs/my-website/docs/proxy/guardrails/qualifire.md @@ -8,13 +8,7 @@ Use [Qualifire](https://qualifire.ai) to evaluate LLM outputs for quality, safet ## Quick Start -### 1. Install the Qualifire SDK - -```bash -pip install qualifire -``` - -### 2. Define Guardrails on your LiteLLM config.yaml +### 1. Define Guardrails on your LiteLLM config.yaml Define your guardrails under the `guardrails` section: @@ -61,13 +55,13 @@ guardrails: - `post_call` Run **after** LLM call, on **input & output** - `during_call` Run **during** LLM call, on **input**. Same as `pre_call` but runs in parallel as LLM call. Response not returned until guardrail check completes -### 3. Start LiteLLM Gateway +### 2. Start LiteLLM Gateway ```shell litellm --config config.yaml --detailed_debug ``` -### 4. Test request +### 3. Test request **[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** @@ -142,7 +136,7 @@ guardrails: evaluation_id: eval_abc123 # Your evaluation ID from Qualifire dashboard ``` -When `evaluation_id` is provided, LiteLLM will use `invoke_evaluation()` instead of `evaluate()`, running the pre-configured evaluation from your dashboard. +When `evaluation_id` is provided, LiteLLM will use the invoke evaluation API endpoint instead of the evaluate endpoint, running the pre-configured evaluation from your dashboard. ## Available Checks @@ -213,19 +207,19 @@ guardrails: ### Parameter Reference -| Parameter | Type | Default | Description | -| ------------------------------ | ----------- | --------------------------- | -------------------------------------------------------- | -| `api_key` | `str` | `QUALIFIRE_API_KEY` env var | Your Qualifire API key | -| `api_base` | `str` | `None` | Custom API base URL (optional) | -| `evaluation_id` | `str` | `None` | Pre-configured evaluation ID from Qualifire dashboard | -| `prompt_injections` | `bool` | `true` (if no other checks) | Enable prompt injection detection | -| `hallucinations_check` | `bool` | `None` | Enable hallucination detection | -| `grounding_check` | `bool` | `None` | Enable grounding verification | -| `pii_check` | `bool` | `None` | Enable PII detection | -| `content_moderation_check` | `bool` | `None` | Enable content moderation | -| `tool_selection_quality_check` | `bool` | `None` | Enable tool selection quality check | -| `assertions` | `List[str]` | `None` | Custom assertions to validate | -| `on_flagged` | `str` | `"block"` | Action when content is flagged: `"block"` or `"monitor"` | +| Parameter | Type | Default | Description | +| ------------------------------ | ----------- | ---------------------------- | -------------------------------------------------------- | +| `api_key` | `str` | `QUALIFIRE_API_KEY` env var | Your Qualifire API key | +| `api_base` | `str` | `https://proxy.qualifire.ai` | Custom API base URL (optional) | +| `evaluation_id` | `str` | `None` | Pre-configured evaluation ID from Qualifire dashboard | +| `prompt_injections` | `bool` | `true` (if no other checks) | Enable prompt injection detection | +| `hallucinations_check` | `bool` | `None` | Enable hallucination detection | +| `grounding_check` | `bool` | `None` | Enable grounding verification | +| `pii_check` | `bool` | `None` | Enable PII detection | +| `content_moderation_check` | `bool` | `None` | Enable content moderation | +| `tool_selection_quality_check` | `bool` | `None` | Enable tool selection quality check | +| `assertions` | `List[str]` | `None` | Custom assertions to validate | +| `on_flagged` | `str` | `"block"` | Action when content is flagged: `"block"` or `"monitor"` | ### Default Behavior @@ -261,4 +255,3 @@ This evaluates whether the LLM selected the appropriate tools and provided corre - [Qualifire Documentation](https://docs.qualifire.ai) - [Qualifire Dashboard](https://app.qualifire.ai) -- [Qualifire Python SDK](https://github.com/qualifire-dev/qualifire-python-sdk) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 5d2f096156e..004132ca05b 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -55,6 +55,7 @@ const sidebars = { "proxy/guardrails/test_playground", "proxy/guardrails/litellm_content_filter", ...[ + "proxy/guardrails/qualifire", "proxy/guardrails/aim_security", "proxy/guardrails/onyx_security", "proxy/guardrails/aporia_api", diff --git a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py index a6971b49f3b..87da11efad0 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py +++ b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py @@ -5,6 +5,7 @@ # +-------------------------------------------------------------+ # Qualifire - Evaluate LLM outputs for quality, safety, and reliability +import json import os from typing import Any, Dict, List, Literal, Optional, Type @@ -15,12 +16,17 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import ( Logging as LiteLLMLoggingObj, ) +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, +) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel from litellm.types.utils import GenericGuardrailAPIInputs GUARDRAIL_NAME = "qualifire" +DEFAULT_QUALIFIRE_API_BASE = "https://proxy.qualifire.ai" class QualifireGuardrail(CustomGuardrail): @@ -44,7 +50,7 @@ class QualifireGuardrail(CustomGuardrail): Args: api_key: API key for Qualifire (or use QUALIFIRE_API_KEY env var) - api_base: Optional custom API base URL + api_base: Optional custom API base URL (defaults to https://api.qualifire.ai) evaluation_id: Pre-configured evaluation ID from Qualifire dashboard prompt_injections: Enable prompt injection detection (default if no other checks) hallucinations_check: Enable hallucination detection @@ -64,6 +70,7 @@ class QualifireGuardrail(CustomGuardrail): api_base or get_secret_str("QUALIFIRE_BASE_URL") or os.environ.get("QUALIFIRE_BASE_URL") + or DEFAULT_QUALIFIRE_API_BASE ) self.evaluation_id = evaluation_id self.prompt_injections = prompt_injections @@ -79,7 +86,11 @@ class QualifireGuardrail(CustomGuardrail): if not self._has_any_check_enabled() and not self.evaluation_id: self.prompt_injections = True - self._client = None + # Initialize async HTTP client for direct API calls + self.async_handler = get_async_httpx_client( + llm_provider=httpxSpecialProvider.GuardrailCallback + ) + super().__init__(**kwargs) def _has_any_check_enabled(self) -> bool: @@ -96,43 +107,22 @@ class QualifireGuardrail(CustomGuardrail): ] ) - def _get_client(self): - """Lazy initialization of Qualifire client.""" - if self._client is None: - try: - from qualifire.client import Client - except ImportError: - raise ImportError( - "qualifire package is required for QualifireGuardrail. " - "Install it with: pip install qualifire" - ) - - client_kwargs: Dict[str, Any] = {} - if self.qualifire_api_key: - client_kwargs["api_key"] = self.qualifire_api_key - if self.qualifire_api_base: - client_kwargs["base_url"] = self.qualifire_api_base - - self._client = Client(**client_kwargs) - - return self._client - - def _convert_messages_to_qualifire_format( + def _convert_messages_to_api_format( self, messages: List[AllMessageValues] - ) -> List[Any]: + ) -> List[Dict[str, Any]]: """ - Convert LiteLLM messages to Qualifire's LLMMessage format. + Convert LiteLLM messages to Qualifire API format. Supports tool calls for tool_selection_quality_check. - """ - try: - from qualifire.types import LLMMessage, LLMToolCall - except ImportError: - raise ImportError( - "qualifire package is required for QualifireGuardrail. " - "Install it with: pip install qualifire" - ) - qualifire_messages = [] + Returns a list of dicts matching the API's ModelInvocationCanonicalMessage schema: + { + "role": "user" | "assistant" | "system" | "tool", + "content": "...", + "tool_call_id": "...", # optional + "tool_calls": [{"id": "...", "name": "...", "arguments": {...}}] # optional + } + """ + api_messages = [] for msg in messages: role = msg.get("role", "user") content = msg.get("content", "") @@ -147,42 +137,86 @@ class QualifireGuardrail(CustomGuardrail): text_parts.append(part) content = "\n".join(text_parts) - llm_message_kwargs: Dict[str, Any] = { + api_message: Dict[str, Any] = { "role": role, "content": content if isinstance(content, str) else str(content), } + # Handle tool_call_id for tool response messages + tool_call_id = msg.get("tool_call_id") + if tool_call_id: + api_message["tool_call_id"] = tool_call_id + # Handle tool calls if present tool_calls = msg.get("tool_calls") if tool_calls and isinstance(tool_calls, list): - qualifire_tool_calls = [] + api_tool_calls = [] for tc in tool_calls: if isinstance(tc, dict): function_info = tc.get("function", {}) # Arguments can be a string (JSON) or dict args = function_info.get("arguments", {}) if isinstance(args, str): - import json - try: args = json.loads(args) except json.JSONDecodeError: args = {} - qualifire_tool_calls.append( - LLMToolCall( - id=tc.get("id") or "", - name=function_info.get("name") or "", - arguments=args if isinstance(args, dict) else {}, - ) + api_tool_calls.append( + { + "id": tc.get("id") or "", + "name": function_info.get("name") or "", + "arguments": args if isinstance(args, dict) else {}, + } ) - if qualifire_tool_calls: - llm_message_kwargs["tool_calls"] = qualifire_tool_calls + if api_tool_calls: + api_message["tool_calls"] = api_tool_calls - qualifire_messages.append(LLMMessage(**llm_message_kwargs)) + api_messages.append(api_message) - return qualifire_messages + return api_messages - def _check_if_flagged(self, result: Any) -> bool: + def _convert_tools_to_api_format( + self, tools: Optional[List[Any]] + ) -> Optional[List[Dict[str, Any]]]: + """ + Convert OpenAI-format tools to Qualifire API format. + + Returns a list of dicts matching the API's ModelInvocationToolDefinition schema: + { + "name": "...", + "description": "...", + "parameters": {...} + } + """ + if not tools: + return None + + api_tools = [] + for tool in tools: + if isinstance(tool, dict): + # Handle OpenAI function tool format + if tool.get("type") == "function": + function_def = tool.get("function", {}) + api_tools.append( + { + "name": function_def.get("name", ""), + "description": function_def.get("description", ""), + "parameters": function_def.get("parameters", {}), + } + ) + # Handle direct tool format + elif "name" in tool: + api_tools.append( + { + "name": tool.get("name", ""), + "description": tool.get("description", ""), + "parameters": tool.get("parameters", {}), + } + ) + + return api_tools if api_tools else None + + def _check_if_flagged(self, result: Dict[str, Any]) -> bool: """ Check if the Qualifire evaluation result indicates flagged content. @@ -190,65 +224,53 @@ class QualifireGuardrail(CustomGuardrail): A high score (close to 100) indicates GOOD content, low score indicates problems. """ # Check evaluation results for any flagged items - evaluation_results = getattr(result, "evaluationResults", None) or [] - if isinstance(result, dict): - evaluation_results = result.get("evaluationResults", []) or [] + evaluation_results = result.get("evaluationResults", []) or [] for eval_result in evaluation_results: - results: List[Any] = [] - if isinstance(eval_result, dict): - results = eval_result.get("results", []) or [] - else: - results = getattr(eval_result, "results", []) or [] - + results = eval_result.get("results", []) or [] for r in results: - flagged = ( - r.get("flagged") - if isinstance(r, dict) - else getattr(r, "flagged", False) - ) - if flagged: + if r.get("flagged"): return True return False - def _build_evaluate_kwargs( + def _build_evaluate_payload( self, - qualifire_messages: List[Any], + api_messages: List[Dict[str, Any]], output: Optional[str], assertions: Optional[List[str]], - available_tools: Optional[List[Any]], + available_tools: Optional[List[Dict[str, Any]]], ) -> Dict[str, Any]: - """Build kwargs dictionary for the evaluate call.""" - kwargs: Dict[str, Any] = {"messages": qualifire_messages} + """Build payload dictionary for the /api/evaluation/evaluate endpoint.""" + payload: Dict[str, Any] = {"messages": api_messages} if output is not None: - kwargs["output"] = output + payload["output"] = output # Add enabled checks if self.prompt_injections: - kwargs["prompt_injections"] = True + payload["prompt_injections"] = True if self.hallucinations_check: - kwargs["hallucinations_check"] = True + payload["hallucinations_check"] = True if self.grounding_check: - kwargs["grounding_check"] = True + payload["grounding_check"] = True if self.pii_check: - kwargs["pii_check"] = True + payload["pii_check"] = True if self.content_moderation_check: - kwargs["content_moderation_check"] = True + payload["content_moderation_check"] = True if self.tool_selection_quality_check: # Only enable tool_selection_quality_check if available_tools is provided if available_tools: - kwargs["tool_selection_quality_check"] = True - kwargs["available_tools"] = available_tools + payload["tool_selection_quality_check"] = True + payload["available_tools"] = available_tools else: verbose_proxy_logger.debug( "Qualifire Guardrail: tool_selection_quality_check enabled but no available_tools provided, skipping this check" ) if assertions: - kwargs["assertions"] = assertions + payload["assertions"] = assertions - return kwargs + return payload async def _run_qualifire_check( self, @@ -274,11 +296,17 @@ class QualifireGuardrail(CustomGuardrail): assertions = dynamic_params.get("assertions") or self.assertions on_flagged = dynamic_params.get("on_flagged") or self.on_flagged - try: - client = self._get_client() - qualifire_messages = self._convert_messages_to_qualifire_format(messages) + # Prepare headers + headers = { + "X-Qualifire-API-Key": self.qualifire_api_key or "", + "Content-Type": "application/json", + } - # Use invoke_evaluation if evaluation_id is provided + try: + # Convert messages to API format + api_messages = self._convert_messages_to_api_format(messages) + + # Use invoke endpoint if evaluation_id is provided if evaluation_id: # For invoke_evaluation, we need to extract input/output input_text = "" @@ -291,25 +319,47 @@ class QualifireGuardrail(CustomGuardrail): input_text = content break - result = client.invoke_evaluation( - evaluation_id=evaluation_id, - input=input_text, - output=output or "", - ) + payload = { + "evaluation_id": evaluation_id, + "input": input_text, + "output": output or "", + "messages": api_messages, + } + + # Convert tools if provided + api_tools = self._convert_tools_to_api_format(available_tools) + if api_tools: + payload["available_tools"] = api_tools + + url = f"{self.qualifire_api_base}/api/evaluation/invoke" else: - # Use evaluate with individual checks - kwargs = self._build_evaluate_kwargs( - qualifire_messages=qualifire_messages, + # Use evaluate endpoint with individual checks + api_tools = self._convert_tools_to_api_format(available_tools) + payload = self._build_evaluate_payload( + api_messages=api_messages, output=output, assertions=assertions, - available_tools=available_tools, + available_tools=api_tools, ) - result = client.evaluate(**kwargs) + url = f"{self.qualifire_api_base}/api/evaluation/evaluate" - # Convert result to dict for logging + verbose_proxy_logger.debug( + f"Qualifire Guardrail: Making request to {url}" + ) + + # Make the API request + response = await self.async_handler.post( + url=url, + headers=headers, + json=payload, + ) + response.raise_for_status() + result = response.json() + + # Extract response info for logging qualifire_response = { - "score": getattr(result, "score", None), - "status": getattr(result, "status", None), + "score": result.get("score"), + "status": result.get("status"), } verbose_proxy_logger.debug( diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_qualifire.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_qualifire.py index 35ed49a84ed..fd72185d1e7 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_qualifire.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_qualifire.py @@ -2,7 +2,6 @@ Unit tests for Qualifire guardrail integration. """ -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -75,9 +74,37 @@ class TestQualifireGuardrailInit: assert guardrail.on_flagged == "monitor" + def test_init_with_default_api_base(self): + """Test that default API base is set when not provided.""" + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + DEFAULT_QUALIFIRE_API_BASE, + QualifireGuardrail, + ) + + guardrail = QualifireGuardrail( + api_key="test_key", + guardrail_name="test_guardrail", + ) + + assert guardrail.qualifire_api_base == DEFAULT_QUALIFIRE_API_BASE + + def test_init_with_custom_api_base(self): + """Test initialization with custom API base URL.""" + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) + + guardrail = QualifireGuardrail( + api_key="test_key", + api_base="https://custom.qualifire.ai", + guardrail_name="test_guardrail", + ) + + assert guardrail.qualifire_api_base == "https://custom.qualifire.ai" + class TestQualifireGuardrailMessageConversion: - """Tests for message conversion to Qualifire format.""" + """Tests for message conversion to API format.""" def test_convert_simple_messages(self): """Test conversion of simple text messages.""" @@ -95,15 +122,13 @@ class TestQualifireGuardrailMessageConversion: {"role": "assistant", "content": "Hi there!"}, ] - # Create mock LLMMessage class - mock_llm_message = MagicMock() + result = guardrail._convert_messages_to_api_format(messages) - with patch( - "litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire.QualifireGuardrail._convert_messages_to_qualifire_format" - ) as mock_convert: - mock_convert.return_value = [mock_llm_message, mock_llm_message] - result = guardrail._convert_messages_to_qualifire_format(messages) - assert len(result) == 2 + assert len(result) == 2 + assert result[0]["role"] == "user" + assert result[0]["content"] == "Hello, world!" + assert result[1]["role"] == "assistant" + assert result[1]["content"] == "Hi there!" def test_convert_multimodal_messages(self): """Test conversion of multimodal messages with text parts.""" @@ -126,112 +151,258 @@ class TestQualifireGuardrailMessageConversion: }, ] - with patch( - "litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire.QualifireGuardrail._convert_messages_to_qualifire_format" - ) as mock_convert: - mock_convert.return_value = [MagicMock()] - result = guardrail._convert_messages_to_qualifire_format(messages) - assert len(result) == 1 + result = guardrail._convert_messages_to_api_format(messages) + + assert len(result) == 1 + assert result[0]["role"] == "user" + assert result[0]["content"] == "First part\nSecond part" + + def test_convert_messages_with_tool_calls(self): + """Test conversion of messages with tool calls.""" + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) + + guardrail = QualifireGuardrail( + api_key="test_key", + guardrail_name="test_guardrail", + ) + + messages = [ + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "NYC"}', + }, + } + ], + }, + ] + + result = guardrail._convert_messages_to_api_format(messages) + + assert len(result) == 1 + assert result[0]["role"] == "assistant" + assert "tool_calls" in result[0] + assert len(result[0]["tool_calls"]) == 1 + assert result[0]["tool_calls"][0]["id"] == "call_123" + assert result[0]["tool_calls"][0]["name"] == "get_weather" + assert result[0]["tool_calls"][0]["arguments"] == {"location": "NYC"} -class TestQualifireGuardrailEvaluateKwargs: - """Tests for evaluate kwargs passed to Qualifire client.""" +class TestQualifireGuardrailToolConversion: + """Tests for tool definition conversion.""" + + def test_convert_openai_function_tools(self): + """Test conversion of OpenAI function tool format.""" + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) + + guardrail = QualifireGuardrail( + api_key="test_key", + guardrail_name="test_guardrail", + ) + + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a location", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + + result = guardrail._convert_tools_to_api_format(tools) + + assert result is not None + assert len(result) == 1 + assert result[0]["name"] == "get_weather" + assert result[0]["description"] == "Get weather for a location" + + def test_convert_empty_tools(self): + """Test that empty tools returns None.""" + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) + + guardrail = QualifireGuardrail( + api_key="test_key", + guardrail_name="test_guardrail", + ) + + result = guardrail._convert_tools_to_api_format(None) + assert result is None + + result = guardrail._convert_tools_to_api_format([]) + assert result is None + + +class TestQualifireGuardrailAPICall: + """Tests for API call with httpx client.""" @pytest.mark.asyncio async def test_evaluate_called_with_prompt_injections(self): - """Test that evaluate is called with prompt_injections enabled.""" - # Mock the qualifire module and its types - mock_qualifire_types = MagicMock() - mock_llm_message = MagicMock() - mock_llm_tool_call = MagicMock() - mock_message_instance = MagicMock() - mock_llm_message.return_value = mock_message_instance - - mock_qualifire_types.LLMMessage = mock_llm_message - mock_qualifire_types.LLMToolCall = mock_llm_tool_call - - with patch.dict('sys.modules', {'qualifire': MagicMock(), 'qualifire.types': mock_qualifire_types}): - from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( - QualifireGuardrail, - ) + """Test that evaluate endpoint is called with prompt_injections enabled.""" + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) - guardrail = QualifireGuardrail( - api_key="test_key", - prompt_injections=True, - guardrail_name="test_guardrail", - ) + guardrail = QualifireGuardrail( + api_key="test_key", + prompt_injections=True, + guardrail_name="test_guardrail", + ) - # Mock the client - mock_client = MagicMock() - mock_result = MagicMock() - mock_result.score = 100 - mock_result.status = "completed" - mock_result.evaluationResults = [] - mock_client.evaluate.return_value = mock_result - guardrail._client = mock_client + # Mock the async HTTP handler + mock_response = MagicMock() + mock_response.json.return_value = { + "score": 100, + "status": "completed", + "evaluationResults": [], + } + mock_response.raise_for_status = MagicMock() + guardrail.async_handler.post = AsyncMock(return_value=mock_response) - messages = [{"role": "user", "content": "Hello, world!"}] + messages = [{"role": "user", "content": "Hello, world!"}] - await guardrail._run_qualifire_check( - messages=messages, output=None, dynamic_params={} - ) + await guardrail._run_qualifire_check( + messages=messages, output=None, dynamic_params={} + ) - # Verify evaluate was called with correct kwargs - mock_client.evaluate.assert_called_once() - call_kwargs = mock_client.evaluate.call_args[1] - assert call_kwargs["prompt_injections"] is True - assert "messages" in call_kwargs + # Verify the API was called + guardrail.async_handler.post.assert_called_once() + call_kwargs = guardrail.async_handler.post.call_args[1] + + assert "json" in call_kwargs + payload = call_kwargs["json"] + assert payload["prompt_injections"] is True + assert "messages" in payload + assert call_kwargs["url"].endswith("/api/evaluation/evaluate") @pytest.mark.asyncio async def test_evaluate_called_with_multiple_checks(self): """Test that evaluate is called with multiple checks enabled.""" - # Mock the qualifire module and its types - mock_qualifire_types = MagicMock() - mock_llm_message = MagicMock() - mock_llm_tool_call = MagicMock() - mock_message_instance = MagicMock() - mock_llm_message.return_value = mock_message_instance - - mock_qualifire_types.LLMMessage = mock_llm_message - mock_qualifire_types.LLMToolCall = mock_llm_tool_call - - with patch.dict('sys.modules', {'qualifire': MagicMock(), 'qualifire.types': mock_qualifire_types}): - from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( - QualifireGuardrail, - ) + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) - guardrail = QualifireGuardrail( - api_key="test_key", - prompt_injections=True, - pii_check=True, - hallucinations_check=True, - assertions=["Output must be valid JSON"], - guardrail_name="test_guardrail", - ) + guardrail = QualifireGuardrail( + api_key="test_key", + prompt_injections=True, + pii_check=True, + hallucinations_check=True, + assertions=["Output must be valid JSON"], + guardrail_name="test_guardrail", + ) - # Mock the client - mock_client = MagicMock() - mock_result = MagicMock() - mock_result.score = 100 - mock_result.status = "completed" - mock_result.evaluationResults = [] - mock_client.evaluate.return_value = mock_result - guardrail._client = mock_client + # Mock the async HTTP handler + mock_response = MagicMock() + mock_response.json.return_value = { + "score": 100, + "status": "completed", + "evaluationResults": [], + } + mock_response.raise_for_status = MagicMock() + guardrail.async_handler.post = AsyncMock(return_value=mock_response) - messages = [{"role": "user", "content": "Hello, world!"}] + messages = [{"role": "user", "content": "Hello, world!"}] - await guardrail._run_qualifire_check( - messages=messages, output="Test output", dynamic_params={} - ) + await guardrail._run_qualifire_check( + messages=messages, output="Test output", dynamic_params={} + ) - # Verify evaluate was called with correct kwargs - mock_client.evaluate.assert_called_once() - call_kwargs = mock_client.evaluate.call_args[1] - assert call_kwargs["prompt_injections"] is True - assert call_kwargs["pii_check"] is True - assert call_kwargs["hallucinations_check"] is True - assert call_kwargs["assertions"] == ["Output must be valid JSON"] - assert call_kwargs["output"] == "Test output" + # Verify the API was called with correct payload + guardrail.async_handler.post.assert_called_once() + call_kwargs = guardrail.async_handler.post.call_args[1] + + payload = call_kwargs["json"] + assert payload["prompt_injections"] is True + assert payload["pii_check"] is True + assert payload["hallucinations_check"] is True + assert payload["assertions"] == ["Output must be valid JSON"] + assert payload["output"] == "Test output" + + @pytest.mark.asyncio + async def test_invoke_endpoint_used_with_evaluation_id(self): + """Test that invoke endpoint is used when evaluation_id is provided.""" + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) + + guardrail = QualifireGuardrail( + api_key="test_key", + evaluation_id="eval_123", + guardrail_name="test_guardrail", + ) + + # Mock the async HTTP handler + mock_response = MagicMock() + mock_response.json.return_value = { + "score": 100, + "status": "completed", + "evaluationResults": [], + } + mock_response.raise_for_status = MagicMock() + guardrail.async_handler.post = AsyncMock(return_value=mock_response) + + messages = [{"role": "user", "content": "Hello, world!"}] + + await guardrail._run_qualifire_check( + messages=messages, output="Test output", dynamic_params={} + ) + + # Verify the invoke endpoint was called + guardrail.async_handler.post.assert_called_once() + call_kwargs = guardrail.async_handler.post.call_args[1] + + assert call_kwargs["url"].endswith("/api/evaluation/invoke") + payload = call_kwargs["json"] + assert payload["evaluation_id"] == "eval_123" + assert payload["input"] == "Hello, world!" + assert payload["output"] == "Test output" + + @pytest.mark.asyncio + async def test_correct_headers_sent(self): + """Test that correct headers are sent with the API request.""" + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) + + guardrail = QualifireGuardrail( + api_key="my_api_key", + guardrail_name="test_guardrail", + ) + + # Mock the async HTTP handler + mock_response = MagicMock() + mock_response.json.return_value = { + "score": 100, + "status": "completed", + "evaluationResults": [], + } + mock_response.raise_for_status = MagicMock() + guardrail.async_handler.post = AsyncMock(return_value=mock_response) + + messages = [{"role": "user", "content": "Hello!"}] + + await guardrail._run_qualifire_check( + messages=messages, output=None, dynamic_params={} + ) + + call_kwargs = guardrail.async_handler.post.call_args[1] + headers = call_kwargs["headers"] + + assert headers["X-Qualifire-API-Key"] == "my_api_key" + assert headers["Content-Type"] == "application/json" class TestQualifireGuardrailCheckIfFlagged: @@ -248,12 +419,14 @@ class TestQualifireGuardrailCheckIfFlagged: guardrail_name="test_guardrail", ) - # Mock result with completed status and no flagged items - mock_result = MagicMock() - mock_result.status = "completed" - mock_result.evaluationResults = [] + # Result with completed status and no flagged items (dict format) + result = { + "status": "completed", + "score": 100, + "evaluationResults": [], + } - assert guardrail._check_if_flagged(mock_result) is False + assert guardrail._check_if_flagged(result) is False def test_check_if_flagged_returns_true_for_flagged_content(self): """Test that _check_if_flagged returns True when content is flagged.""" @@ -266,18 +439,25 @@ class TestQualifireGuardrailCheckIfFlagged: guardrail_name="test_guardrail", ) - # Mock result with flagged item - mock_inner_result = MagicMock() - mock_inner_result.flagged = True + # Result with flagged item (dict format matching API response) + result = { + "status": "completed", + "score": 15, + "evaluationResults": [ + { + "type": "prompt_injection", + "results": [ + { + "flagged": True, + "score": 0.15, + "reason": "Prompt injection detected", + } + ], + } + ], + } - mock_eval_result = MagicMock() - mock_eval_result.results = [mock_inner_result] - - mock_result = MagicMock() - mock_result.status = "completed" - mock_result.evaluationResults = [mock_eval_result] - - assert guardrail._check_if_flagged(mock_result) is True + assert guardrail._check_if_flagged(result) is True def test_check_if_flagged_returns_false_when_no_flagged_items(self): """Test that _check_if_flagged returns False when no items are flagged.""" @@ -291,17 +471,24 @@ class TestQualifireGuardrailCheckIfFlagged: ) # Result with evaluation results but nothing flagged - mock_inner_result = MagicMock() - mock_inner_result.flagged = False + result = { + "status": "completed", + "score": 95, + "evaluationResults": [ + { + "type": "prompt_injection", + "results": [ + { + "flagged": False, + "score": 0.95, + "reason": "No issues detected", + } + ], + } + ], + } - mock_eval_result = MagicMock() - mock_eval_result.results = [mock_inner_result] - - mock_result = MagicMock() - mock_result.status = "success" - mock_result.evaluationResults = [mock_eval_result] - - assert guardrail._check_if_flagged(mock_result) is False + assert guardrail._check_if_flagged(result) is False class TestQualifireGuardrailShouldRun: From 91b5c66cf2851cc9e3dd76b0299e8bbf819d1ef7 Mon Sep 17 00:00:00 2001 From: Kris Xia Date: Wed, 7 Jan 2026 23:56:47 +0800 Subject: [PATCH 10/56] fix(proxy): return json error response instead of sse format for initial streaming errors (#18757) * adding signoz integration to observability docs * Fixing build * Adding timeout for flaky test * Fixing e2e * fix(proxy): return json error response instead of sse format for initial streaming errors when the first chunk of a streaming response contains an error, return a standard json error response instead of sse format. this ensures clients receive properly formatted error responses before the stream actually begins. - rename create_streaming_response to create_response - add logic to detect error in first chunk and return JSONResponse - add _extract_error_from_sse_chunk helper function - update all call sites to use the new function name - update tests to reflect the function rename * test(proxy): add comprehensive tests for error extraction from sse chunks - Add new test class TestExtractErrorFromSSEChunk with 10 test cases - Update existing tests to verify JSONResponse returned for initial streaming errors - Add tests for error code as string, bytes input, invalid JSON, and edge cases - Verify correct error format extraction from SSE chunks --------- Co-authored-by: Goutham Karthi Co-authored-by: yuneng-jiang Co-authored-by: YutaSaito <36355491+uc4w6c@users.noreply.github.com> --- docs/my-website/docs/observability/signoz.md | 394 ++++++++++++++++++ .../proxy/anthropic_endpoints/endpoints.py | 4 +- litellm/proxy/common_request_processing.py | 75 +++- litellm/proxy/proxy_server.py | 4 +- .../proxy/test_common_request_processing.py | 176 ++++++-- .../tests/users/viewInternalUsers.spec.ts | 1 + .../ModelsAndEndpointsView.tsx | 20 +- 7 files changed, 616 insertions(+), 58 deletions(-) create mode 100644 docs/my-website/docs/observability/signoz.md diff --git a/docs/my-website/docs/observability/signoz.md b/docs/my-website/docs/observability/signoz.md new file mode 100644 index 00000000000..4b65916fdfe --- /dev/null +++ b/docs/my-website/docs/observability/signoz.md @@ -0,0 +1,394 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# SigNoz LiteLLM Integration + +For more details on setting up observability for LiteLLM, check out the [SigNoz LiteLLM observability docs](https://signoz.io/docs/litellm-observability/). + + +## Overview + +This guide walks you through setting up observability and monitoring for LiteLLM SDK and Proxy Server using [OpenTelemetry](https://opentelemetry.io/) and exporting logs, traces, and metrics to SigNoz. With this integration, you can observe various models performance, capture request/response details, and track system-level metrics in SigNoz, giving you real-time visibility into latency, error rates, and usage trends for your LiteLLM applications. + +Instrumenting LiteLLM in your AI applications with telemetry ensures full observability across your AI workflows, making it easier to debug issues, optimize performance, and understand user interactions. By leveraging SigNoz, you can analyze correlated traces, logs, and metrics in unified dashboards, configure alerts, and gain actionable insights to continuously improve reliability, responsiveness, and user experience. + +## Prerequisites + +- A [SigNoz Cloud account](https://signoz.io/teams/) with an active ingestion key +- Internet access to send telemetry data to SigNoz Cloud +- [LiteLLM](https://www.litellm.ai/) SDK or Proxy integration +- For Python: `pip` installed for managing Python packages and _(optional but recommended)_ a Python virtual environment to isolate dependencies + +## Monitoring LiteLLM + +LiteLLM can be monitored in two ways: using the **LiteLLM SDK** (directly embedded in your Python application code for programmatic LLM calls) or the **LiteLLM Proxy Server** (a standalone server that acts as a centralized gateway for managing and routing LLM requests across your infrastructure). + + + + +For more detailed info on instrumenting your LiteLLM SDK applications click [here](https://docs.litellm.ai/docs/observability/opentelemetry_integration). + + + + + +No-code auto-instrumentation is recommended for quick setup with minimal code changes. It's ideal when you want to get observability up and running without modifying your application code and are leveraging standard instrumentor libraries. + +**Step 1:** Install the necessary packages in your Python environment. + +```bash +pip install \ + opentelemetry-api \ + opentelemetry-distro \ + opentelemetry-exporter-otlp \ + httpx \ + opentelemetry-instrumentation-httpx \ + litellm +``` + +**Step 2:** Add Automatic Instrumentation + +```bash +opentelemetry-bootstrap --action=install +``` + +**Step 3:** Instrument your LiteLLM SDK application + +Initialize LiteLLM SDK instrumentation by calling `litellm.callbacks = ["otel"]`: + +```python +from litellm import litellm + +litellm.callbacks = ["otel"] +``` + +This call enables automatic tracing, logs, and metrics collection for all LiteLLM SDK calls in your application. + +> 📌 Note: Ensure this is called before any LiteLLM related calls to properly configure instrumentation of your application + +**Step 4:** Run an example + +```python +from litellm import completion, litellm + +litellm.callbacks = ["otel"] + +response = completion( + model="openai/gpt-4o", + messages=[{ "content": "What is SigNoz","role": "user"}] +) + +print(response) +``` + +> 📌 Note: LiteLLM supports a [variety of model providers](https://docs.litellm.ai/docs/providers) for LLMs. In this example, we're using OpenAI. Before running this code, ensure that you have set the environment variable `OPENAI_API_KEY` with your generated API key. + +**Step 5:** Run your application with auto-instrumentation + +```bash +OTEL_RESOURCE_ATTRIBUTES="service.name=" \ +OTEL_EXPORTER_OTLP_ENDPOINT="https://ingest..signoz.cloud:443" \ +OTEL_EXPORTER_OTLP_HEADERS="signoz-ingestion-key=" \ +OTEL_EXPORTER_OTLP_PROTOCOL=grpc \ +OTEL_TRACES_EXPORTER=otlp \ +OTEL_METRICS_EXPORTER=otlp \ +OTEL_LOGS_EXPORTER=otlp \ +OTEL_PYTHON_LOG_CORRELATION=true \ +OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true \ +OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=openai \ +opentelemetry-instrument +``` + +> 📌 Note: We're using `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=openai` in the run command to disable the OpenAI instrumentor for tracing. This avoids conflicts with LiteLLM's native telemetry/instrumentation, ensuring that telemetry is captured exclusively through LiteLLM's built-in instrumentation. + +- **``** is the name of your service +- Set the `` to match your SigNoz Cloud [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint) +- Replace `` with your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/) +- Replace `` with the actual command you would use to run your application. For example: `python main.py` + +> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted). + + + + + + +Code-based instrumentation gives you fine-grained control over your telemetry configuration. Use this approach when you need to customize resource attributes, sampling strategies, or integrate with existing observability infrastructure. + +**Step 1:** Install the necessary packages in your Python environment. + +```bash +pip install \ + opentelemetry-api \ + opentelemetry-sdk \ + opentelemetry-exporter-otlp \ + opentelemetry-instrumentation-httpx \ + opentelemetry-instrumentation-system-metrics \ + litellm +``` + +**Step 2:** Import the necessary modules in your Python application + +**Traces:** + +```python +from opentelemetry import trace +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +``` + +**Logs:** + +```python +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry._logs import set_logger_provider +import logging +``` + +**Metrics:** + +```python +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry import metrics +from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor +from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor +``` + +**Step 3:** Set up the OpenTelemetry Tracer Provider to send traces directly to SigNoz Cloud + +```python +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry import trace +import os + +resource = Resource.create({"service.name": ""}) +provider = TracerProvider(resource=resource) +span_exporter = OTLPSpanExporter( + endpoint= os.getenv("OTEL_EXPORTER_TRACES_ENDPOINT"), + headers={"signoz-ingestion-key": os.getenv("SIGNOZ_INGESTION_KEY")}, +) +processor = BatchSpanProcessor(span_exporter) +provider.add_span_processor(processor) +trace.set_tracer_provider(provider) +``` + +- **``** is the name of your service +- **`OTEL_EXPORTER_TRACES_ENDPOINT`** → SigNoz Cloud trace endpoint with appropriate [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint):`https://ingest..signoz.cloud:443/v1/traces` +- **`SIGNOZ_INGESTION_KEY`** → Your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/) + + +> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted). + + +**Step 4**: Setup Logs + +```python +import logging +from opentelemetry.sdk.resources import Resource +from opentelemetry._logs import set_logger_provider +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +import os + +resource = Resource.create({"service.name": ""}) +logger_provider = LoggerProvider(resource=resource) +set_logger_provider(logger_provider) + +otlp_log_exporter = OTLPLogExporter( + endpoint= os.getenv("OTEL_EXPORTER_LOGS_ENDPOINT"), + headers={"signoz-ingestion-key": os.getenv("SIGNOZ_INGESTION_KEY")}, +) +logger_provider.add_log_record_processor( + BatchLogRecordProcessor(otlp_log_exporter) +) +# Attach OTel logging handler to root logger +handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) +logging.basicConfig(level=logging.INFO, handlers=[handler]) + +logger = logging.getLogger(__name__) +``` + +- **``** is the name of your service +- **`OTEL_EXPORTER_LOGS_ENDPOINT`** → SigNoz Cloud endpoint with appropriate [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint):`https://ingest..signoz.cloud:443/v1/logs` +- **`SIGNOZ_INGESTION_KEY`** → Your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/) + +> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted). + + +**Step 5**: Setup Metrics + +```python +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry import metrics +from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor +import os + +resource = Resource.create({"service.name": ""}) +metric_exporter = OTLPMetricExporter( + endpoint= os.getenv("OTEL_EXPORTER_METRICS_ENDPOINT"), + headers={"signoz-ingestion-key": os.getenv("SIGNOZ_INGESTION_KEY")}, +) +reader = PeriodicExportingMetricReader(metric_exporter) +metric_provider = MeterProvider(metric_readers=[reader], resource=resource) +metrics.set_meter_provider(metric_provider) + +meter = metrics.get_meter(__name__) + +# turn on out-of-the-box metrics +SystemMetricsInstrumentor().instrument() +HTTPXClientInstrumentor().instrument() +``` + +- **``** is the name of your service +- **`OTEL_EXPORTER_METRICS_ENDPOINT`** → SigNoz Cloud endpoint with appropriate [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint):`https://ingest..signoz.cloud:443/v1/metrics` +- **`SIGNOZ_INGESTION_KEY`** → Your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/) + +> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted). + + +> 📌 Note: SystemMetricsInstrumentor provides system metrics (CPU, memory, etc.), and HTTPXClientInstrumentor provides outbound HTTP request metrics such as request duration. If you want to add custom metrics to your LiteLLM application, see [Python Custom Metrics](https://signoz.io/opentelemetry/python-custom-metrics/). + +**Step 6:** Instrument your LiteLLM application + +Initialize LiteLLM SDK instrumentation by calling `litellm.callbacks = ["otel"]`: + +```python +from litellm import litellm + +litellm.callbacks = ["otel"] +``` + +This call enables automatic tracing, logs, and metrics collection for all LiteLLM SDK calls in your application. + +> 📌 Note: Ensure this is called before any LiteLLM related calls to properly configure instrumentation of your application + +**Step 7:** Run an example + +```python +from litellm import completion, litellm + +litellm.callbacks = ["otel"] + +response = completion( + model="openai/gpt-4o", + messages=[{ "content": "What is SigNoz","role": "user"}] +) + +print(response) +``` + +> 📌 Note: LiteLLM supports a [variety of model providers](https://docs.litellm.ai/docs/providers) for LLMs. In this example, we're using OpenAI. Before running this code, ensure that you have set the environment variable `OPENAI_API_KEY` with your generated API key. + + + + +## View Traces, Logs, and Metrics in SigNoz + +Your LiteLLM commands should now automatically emit traces, logs, and metrics. + +You should be able to view traces in Signoz Cloud under the traces tab: + +![LiteLLM SDK Trace View](https://signoz.io/img/docs/llm/litellm/litellmsdk-traces.webp) + +When you click on a trace in SigNoz, you'll see a detailed view of the trace, including all associated spans, along with their events and attributes. + +![LiteLLM SDK Detailed Trace View](https://signoz.io/img/docs/llm/litellm/litellmsdk-detailed-traces.webp) + +You should be able to view logs in Signoz Cloud under the logs tab. You can also view logs by clicking on the “Related Logs” button in the trace view to see correlated logs: + +![LiteLLM SDK Logs View](https://signoz.io/img/docs/llm/litellm/litellmsdk-logs.webp) + +When you click on any of these logs in SigNoz, you'll see a detailed view of the log, including attributes: + +![LiteLLM SDK Detailed Logs View](https://signoz.io/img/docs/llm/litellm/litellmsdk-detailed-logs.webp) + +You should be able to see LiteLLM related metrics in Signoz Cloud under the metrics tab: + +![LiteLLM SDK Metrics View](https://signoz.io/img/docs/llm/litellm/litellmsdk-metrics.webp) + +When you click on any of these metrics in SigNoz, you'll see a detailed view of the metric, including attributes: + +![LiteLLM Detailed Metrics View](https://signoz.io/img/docs/llm/litellm/litellmsdk-detailed-metrics.webp) + +## Dashboard + +You can also check out our custom LiteLLM SDK dashboard [here](https://signoz.io/docs/dashboards/dashboard-templates/litellm-sdk-dashboard/) which provides specialized visualizations for monitoring your LiteLLM usage in applications. The dashboard includes pre-built charts specifically tailored for LLM usage, along with import instructions to get started quickly. + +![LiteLLM SDK Dashboard Template](https://signoz.io/img/docs/llm/litellm/litellm-sdk-dashboard.webp) + + + + + +**Step 1:** Install the necessary packages in your Python environment. + +```bash +pip install opentelemetry-api \ + opentelemetry-sdk \ + opentelemetry-exporter-otlp \ + 'litellm[proxy]' +``` + +**Step 2:** Configure otel for the LiteLLM Proxy Server + +Add the following to `config.yaml`: + +```yaml +litellm_settings: + callbacks: ['otel'] +``` + +**Step 3:** Set the following environment variables: + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT="https://ingest..signoz.cloud:443" +export OTEL_EXPORTER_OTLP_HEADERS="signoz-ingestion-key=" +export OTEL_EXPORTER_OTLP_PROTOCOL="grpc" +export OTEL_TRACES_EXPORTER="otlp" +export OTEL_METRICS_EXPORTER="otlp" +export OTEL_LOGS_EXPORTER="otlp" +``` + +- Set the `` to match your SigNoz Cloud [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint) +- Replace `` with your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/) + +> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted). + + +**Step 4:** Run the proxy server using the config file: + +```bash +litellm --config config.yaml +``` + +Now any calls made through your LiteLLM proxy server will be traced and sent to SigNoz. + +You should be able to view traces in Signoz Cloud under the traces tab: + +![LiteLLM Proxy Trace View](https://signoz.io/img/docs/llm/litellm/litellmproxy-traces.webp) + +When you click on a trace in SigNoz, you'll see a detailed view of the trace, including all associated spans, along with their events and attributes. + +![LiteLLM Proxy Detailed Trace View](https://signoz.io/img/docs/llm/litellm/litellmproxy-detailed-traces.webp) + +## Dashboard + +You can also check out our custom LiteLLM Proxy dashboard [here](https://signoz.io/docs/dashboards/dashboard-templates/litellm-proxy-dashboard/) which provides specialized visualizations for monitoring your LiteLLM Proxy usage in applications. The dashboard includes pre-built charts specifically tailored for LLM usage, along with import instructions to get started quickly. + +![LiteLLM Proxy Dashboard Template](https://signoz.io/img/docs/llm/litellm/litellm-proxy-dashboard.webp) + + + diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 334362a0271..7de6b7fccfc 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -11,7 +11,7 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, - create_streaming_response, + create_response, ) from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.types.utils import TokenCountResponse @@ -106,7 +106,7 @@ async def anthropic_response( # noqa: PLR0915 ) ) - return await create_streaming_response( + return await create_response( generator=selected_data_generator, media_type="text/event-stream", headers={}, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 537b48f06ed..95c23be5b82 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -17,7 +17,7 @@ from typing import ( import httpx import orjson from fastapi import HTTPException, Request, status -from fastapi.responses import Response, StreamingResponse +from fastapi.responses import JSONResponse, Response, StreamingResponse import litellm from litellm._logging import verbose_proxy_logger @@ -96,16 +96,55 @@ async def _parse_event_data_for_error(event_line: Union[str, bytes]) -> Optional return None -async def create_streaming_response( +def _extract_error_from_sse_chunk(event_line: Union[str, bytes]) -> dict: + """ + Extract error dictionary from SSE format chunk. + + Args: + event_line: SSE format event line, e.g. "data: {"error": {...}}\n\n" + + Returns: + Error dictionary in OpenAI API format + """ + event_line = ( + event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line + ) + + # Default error format + default_error = { + "message": "Unknown error", + "type": "internal_server_error", + "param": None, + "code": "500", + } + + if event_line.startswith("data: "): + json_str = event_line[len("data: ") :].strip() + if not json_str or json_str == "[DONE]": + return default_error + + try: + data = orjson.loads(json_str) + if isinstance(data, dict) and "error" in data: + error_obj = data["error"] + if isinstance(error_obj, dict): + return error_obj + except (orjson.JSONDecodeError, json.JSONDecodeError): + pass + + return default_error + + +async def create_response( generator: AsyncGenerator[str, None], media_type: str, headers: dict, default_status_code: int = status.HTTP_200_OK, -) -> StreamingResponse: +) -> Union[StreamingResponse, JSONResponse]: """ - Creates a StreamingResponse by inspecting the first chunk for an error code. - The entire original generator content is streamed, but the HTTP status code - of the response is set based on the first chunk if it's a recognized error. + Create streaming response, checking if the first chunk is an error. + If the first chunk is an error, return a standard JSON error response. + Otherwise, return StreamingResponse and stream all content. """ first_chunk_value: Optional[str] = None final_status_code = default_status_code @@ -124,9 +163,27 @@ async def create_streaming_response( first_chunk_value ) if error_code_from_chunk is not None: + # First chunk is an error, stream hasn't really started yet + # Should return standard JSON error response instead of SSE format final_status_code = error_code_from_chunk verbose_proxy_logger.debug( - f"Error detected in first stream chunk. Status code set to: {final_status_code}" + f"Error detected in first stream chunk. Returning JSON error response with status code: {final_status_code}" + ) + + # Parse error content + error_dict = _extract_error_from_sse_chunk(first_chunk_value) + + # Consume and close generator (avoid resource leak) + try: + await generator.aclose() + except Exception: + pass + + # Return JSON format error response + return JSONResponse( + status_code=final_status_code, + content={"error": error_dict}, + headers=headers, ) except Exception as e: verbose_proxy_logger.debug(f"Error parsing first chunk value: {e}") @@ -647,7 +704,7 @@ class ProxyBaseLLMRequestProcessing: proxy_logging_obj=proxy_logging_obj, ) ) - return await create_streaming_response( + return await create_response( generator=selected_data_generator, media_type="text/event-stream", headers=custom_headers, @@ -658,7 +715,7 @@ class ProxyBaseLLMRequestProcessing: user_api_key_dict=user_api_key_dict, request_data=self.data, ) - return await create_streaming_response( + return await create_response( generator=selected_data_generator, media_type="text/event-stream", headers=custom_headers, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 06525e39133..9e314a77c37 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -229,7 +229,7 @@ from litellm.proxy.batches_endpoints.endpoints import router as batches_router from litellm.proxy.caching_routes import router as caching_router from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, - create_streaming_response, + create_response, ) from litellm.proxy.common_utils.callback_utils import initialize_callbacks_on_proxy from litellm.proxy.common_utils.debug_utils import init_verbose_loggers @@ -6736,7 +6736,7 @@ async def run_thread( if ( "stream" in data and data["stream"] is True ): # use generate_responses to stream responses - return await create_streaming_response( + return await create_response( generator=async_assistants_data_generator( user_api_key_dict=user_api_key_dict, response=response, diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index b5d44385698..ed1c29f5dce 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -3,7 +3,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import Request, status -from fastapi.responses import StreamingResponse +from fastapi.responses import JSONResponse, StreamingResponse import litellm from litellm._uuid import uuid @@ -11,9 +11,10 @@ from litellm.integrations.opentelemetry import UserAPIKeyAuth from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, ProxyConfig, + _extract_error_from_sse_chunk, _get_cost_breakdown_from_logging_obj, _parse_event_data_for_error, - create_streaming_response, + create_response, ) from litellm.proxy.utils import ProxyLogging @@ -602,21 +603,27 @@ class TestCommonRequestProcessingHelpers: assert await _parse_event_data_for_error(event_line) == expected_code async def test_create_streaming_response_first_chunk_is_error(self): + """ + Test that when the first chunk is an error, a JSON error response is returned + instead of an SSE streaming response + """ async def mock_generator(): yield 'data: {"error": {"code": 403, "message": "forbidden"}}\n\n' yield 'data: {"content": "more data"}\n\n' yield "data: [DONE]\n\n" - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {} ) + # Should return JSONResponse instead of StreamingResponse + assert isinstance(response, JSONResponse) assert response.status_code == status.HTTP_403_FORBIDDEN - content = await self.consume_stream(response) - assert content == [ - 'data: {"error": {"code": 403, "message": "forbidden"}}\n\n', - 'data: {"content": "more data"}\n\n', - "data: [DONE]\n\n", - ] + # Verify the response is in standard JSON error format + import json + body = json.loads(response.body.decode()) + assert "error" in body + assert body["error"]["code"] == 403 + assert body["error"]["message"] == "forbidden" async def test_create_streaming_response_first_chunk_not_error(self): async def mock_generator(): @@ -624,7 +631,7 @@ class TestCommonRequestProcessingHelpers: yield 'data: {"content": "second part"}\n\n' yield "data: [DONE]\n\n" - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {} ) assert response.status_code == status.HTTP_200_OK @@ -641,7 +648,7 @@ class TestCommonRequestProcessingHelpers: yield # Implicitly raises StopAsyncIteration - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {} ) assert response.status_code == status.HTTP_200_OK @@ -654,7 +661,7 @@ class TestCommonRequestProcessingHelpers: mock_gen = AsyncMock() mock_gen.__anext__.side_effect = StopAsyncIteration - response = await create_streaming_response(mock_gen, "text/event-stream", {}) + response = await create_response(mock_gen, "text/event-stream", {}) assert response.status_code == status.HTTP_200_OK content = await self.consume_stream(response) assert content == [] @@ -665,7 +672,7 @@ class TestCommonRequestProcessingHelpers: mock_gen = AsyncMock() mock_gen.__anext__.side_effect = ValueError("Test error from generator") - response = await create_streaming_response(mock_gen, "text/event-stream", {}) + response = await create_response(mock_gen, "text/event-stream", {}) assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR content = await self.consume_stream(response) expected_error_data = { @@ -682,19 +689,24 @@ class TestCommonRequestProcessingHelpers: assert content[1] == "data: [DONE]\n\n" async def test_create_streaming_response_first_chunk_error_string_code(self): + """ + Test that when the first chunk contains a string error code, a JSON error response is returned + """ async def mock_generator(): yield 'data: {"error": {"code": "429", "message": "too many requests"}}\n\n' yield "data: [DONE]\n\n" - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {} ) + assert isinstance(response, JSONResponse) assert response.status_code == status.HTTP_429_TOO_MANY_REQUESTS - content = await self.consume_stream(response) - assert content == [ - 'data: {"error": {"code": "429", "message": "too many requests"}}\n\n', - "data: [DONE]\n\n", - ] + # Verify the response is in standard JSON error format + import json + body = json.loads(response.body.decode()) + assert "error" in body + assert body["error"]["code"] == "429" + assert body["error"]["message"] == "too many requests" async def test_create_streaming_response_custom_headers(self): async def mock_generator(): @@ -702,7 +714,7 @@ class TestCommonRequestProcessingHelpers: yield "data: [DONE]\n\n" custom_headers = {"X-Custom-Header": "TestValue"} - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", custom_headers ) assert response.headers["x-custom-header"] == "TestValue" @@ -712,7 +724,7 @@ class TestCommonRequestProcessingHelpers: yield 'data: {"content": "data"}\n\n' yield "data: [DONE]\n\n" - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {}, @@ -729,7 +741,7 @@ class TestCommonRequestProcessingHelpers: async def mock_generator(): yield "data: [DONE]\n\n" - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {} ) assert response.status_code == status.HTTP_200_OK # Default status @@ -742,7 +754,7 @@ class TestCommonRequestProcessingHelpers: yield 'data: {"content": "actual data"}\n\n' yield "data: [DONE]\n\n" - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {} ) assert response.status_code == status.HTTP_200_OK # Default status @@ -773,7 +785,7 @@ class TestCommonRequestProcessingHelpers: # Patch the tracer in the common_request_processing module with patch("litellm.proxy.common_request_processing.tracer", mock_tracer): - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {} ) @@ -810,7 +822,10 @@ class TestCommonRequestProcessingHelpers: ), f"Call {i} should have operation name 'streaming.chunk.yield', got {args[0]}" async def test_create_streaming_response_dd_trace_with_error_chunk(self): - """Test that dd trace is applied even when the first chunk contains an error""" + """ + Test that when the first chunk contains an error, JSONResponse is returned + and tracing is not triggered (since it's not a streaming response) + """ from unittest.mock import patch # Create a mock tracer @@ -827,28 +842,107 @@ class TestCommonRequestProcessingHelpers: # Patch the tracer in the common_request_processing module with patch("litellm.proxy.common_request_processing.tracer", mock_tracer): - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {} ) - # Even with error, status should be set to error code but tracing should still work + # Should return JSONResponse instead of StreamingResponse + assert isinstance(response, JSONResponse) assert response.status_code == 400 - # Consume the stream to trigger the tracer calls - content = await self.consume_stream(response) + # Verify the response is in standard JSON error format + import json + body = json.loads(response.body.decode()) + assert "error" in body + assert body["error"]["code"] == 400 + assert body["error"]["message"] == "bad request" - # Verify all chunks are present - assert len(content) == 3 + # Since JSONResponse is returned instead of StreamingResponse, streaming tracing should not be triggered + # tracer.trace should not be called + assert mock_tracer.trace.call_count == 0 - # Verify that tracer.trace was called for each chunk - assert mock_tracer.trace.call_count == 3 - # Verify that each call was made with the correct operation name - actual_calls = mock_tracer.trace.call_args_list - assert len(actual_calls) == 3 +class TestExtractErrorFromSSEChunk: + """Tests for _extract_error_from_sse_chunk function""" + + def test_extract_error_from_sse_chunk_with_valid_error(self): + """Test extracting error information from a standard SSE chunk""" + chunk = 'data: {"error": {"code": 403, "message": "forbidden", "type": "auth_error", "param": "api_key"}}\n\n' + error = _extract_error_from_sse_chunk(chunk) + + assert error["code"] == 403 + assert error["message"] == "forbidden" + assert error["type"] == "auth_error" + assert error["param"] == "api_key" + + def test_extract_error_from_sse_chunk_with_string_code(self): + """Test error code as string type""" + chunk = 'data: {"error": {"code": "429", "message": "too many requests"}}\n\n' + error = _extract_error_from_sse_chunk(chunk) + + assert error["code"] == "429" + assert error["message"] == "too many requests" + + def test_extract_error_from_sse_chunk_with_bytes(self): + """Test input as bytes type""" + chunk = b'data: {"error": {"code": 500, "message": "internal error"}}\n\n' + error = _extract_error_from_sse_chunk(chunk) + + assert error["code"] == 500 + assert error["message"] == "internal error" + + def test_extract_error_from_sse_chunk_with_done(self): + """Test [DONE] marker should return default error""" + chunk = "data: [DONE]\n\n" + error = _extract_error_from_sse_chunk(chunk) + + assert error["message"] == "Unknown error" + assert error["type"] == "internal_server_error" + assert error["code"] == "500" + assert error["param"] is None + + def test_extract_error_from_sse_chunk_without_error_field(self): + """Test missing error field should return default error""" + chunk = 'data: {"content": "some content"}\n\n' + error = _extract_error_from_sse_chunk(chunk) + + assert error["message"] == "Unknown error" + assert error["type"] == "internal_server_error" + assert error["code"] == "500" + + def test_extract_error_from_sse_chunk_with_invalid_json(self): + """Test invalid JSON should return default error""" + chunk = 'data: {invalid json}\n\n' + error = _extract_error_from_sse_chunk(chunk) + + assert error["message"] == "Unknown error" + assert error["type"] == "internal_server_error" + assert error["code"] == "500" + + def test_extract_error_from_sse_chunk_without_data_prefix(self): + """Test missing 'data:' prefix should return default error""" + chunk = '{"error": {"code": 400, "message": "bad request"}}\n\n' + error = _extract_error_from_sse_chunk(chunk) + + assert error["message"] == "Unknown error" + assert error["type"] == "internal_server_error" + assert error["code"] == "500" + + def test_extract_error_from_sse_chunk_with_empty_string(self): + """Test empty string should return default error""" + chunk = "" + error = _extract_error_from_sse_chunk(chunk) + + assert error["message"] == "Unknown error" + assert error["type"] == "internal_server_error" + assert error["code"] == "500" + + def test_extract_error_from_sse_chunk_with_minimal_error(self): + """Test minimal error object""" + chunk = 'data: {"error": {"message": "error occurred"}}\n\n' + error = _extract_error_from_sse_chunk(chunk) + + assert error["message"] == "error occurred" + # Other fields should be obtained from the original error object (if exists) + - for i, call in enumerate(actual_calls): - args, kwargs = call - assert ( - args[0] == "streaming.chunk.yield" - ), f"Call {i} should have operation name 'streaming.chunk.yield', got {args[0]}" diff --git a/ui/litellm-dashboard/e2e_tests/tests/users/viewInternalUsers.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/users/viewInternalUsers.spec.ts index 980c7233e42..65797028edc 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/users/viewInternalUsers.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/users/viewInternalUsers.spec.ts @@ -43,6 +43,7 @@ test.describe("Internal Users Page", () => { await expect(prevButton).toBeDisabled(); } + await page.waitForTimeout(1000); // Check if there are more pages const hasMorePages = infoText.includes("of") && !infoText.endsWith("25 of 25"); if (hasMorePages) { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx index a8b1d2cddc9..1cce704467a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx @@ -279,12 +279,13 @@ const ModelsAndEndpointsView: React.FC = ({ premiumUser, te {/* Missing Provider Banner */}
- +

Missing a provider?

- The LiteLLM engineering team is constantly adding support for new LLM models, providers, endpoints. If you don't see the one you need, let us know and we'll prioritize it. + The LiteLLM engineering team is constantly adding support for new LLM models, providers, endpoints. If + you don't see the one you need, let us know and we'll prioritize it.

= ({ premiumUser, te className="flex-shrink-0 inline-flex items-center gap-2 px-4 py-2 bg-[#6366f1] hover:bg-[#5558e3] text-white text-sm font-medium rounded-lg transition-colors" > Request Provider - - + +
From 92f7789f1000c29519f3270442cc4ccd16cb989e Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Wed, 7 Jan 2026 21:29:04 +0530 Subject: [PATCH 11/56] feat(prometheus): add caching metrics (#18755) --- litellm/integrations/prometheus.py | 101 +++++++-- litellm/types/integrations/prometheus.py | 21 +- .../test_prometheus_cache_metrics.py | 211 ++++++++++++++++++ 3 files changed, 318 insertions(+), 15 deletions(-) create mode 100644 tests/test_litellm/integrations/test_prometheus_cache_metrics.py diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index c01f7481277..2ec2f41b3bb 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -14,6 +14,7 @@ from typing import ( Literal, Optional, Tuple, + Union, cast, ) @@ -44,6 +45,7 @@ def _get_cached_end_user_id_for_cost_tracking(): global _get_end_user_id_for_cost_tracking if _get_end_user_id_for_cost_tracking is None: from litellm.utils import get_end_user_id_for_cost_tracking + _get_end_user_id_for_cost_tracking = get_end_user_id_for_cost_tracking return _get_end_user_id_for_cost_tracking @@ -329,6 +331,25 @@ class PrometheusLogger(CustomLogger): labelnames=self.get_labels_for_metric("litellm_requests_metric"), ) + # Cache metrics + self.litellm_cache_hits_metric = self._counter_factory( + name="litellm_cache_hits_metric", + documentation="Total number of LiteLLM cache hits", + labelnames=self.get_labels_for_metric("litellm_cache_hits_metric"), + ) + + self.litellm_cache_misses_metric = self._counter_factory( + name="litellm_cache_misses_metric", + documentation="Total number of LiteLLM cache misses", + labelnames=self.get_labels_for_metric("litellm_cache_misses_metric"), + ) + + self.litellm_cached_tokens_metric = self._counter_factory( + name="litellm_cached_tokens_metric", + documentation="Total tokens served from LiteLLM cache", + labelnames=self.get_labels_for_metric("litellm_cached_tokens_metric"), + ) + except Exception as e: print_verbose(f"Got exception on init prometheus client {str(e)}") raise e @@ -795,7 +816,7 @@ class PrometheusLogger(CustomLogger): litellm_params = kwargs.get("litellm_params", {}) or {} _metadata = litellm_params.get("metadata", {}) get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking() - + end_user_id = get_end_user_id_for_cost_tracking( litellm_params, service_type="prometheus" ) @@ -815,7 +836,7 @@ class PrometheusLogger(CustomLogger): user_api_key_auth_metadata: Optional[dict] = standard_logging_payload[ "metadata" ].get("user_api_key_auth_metadata") - + # Include top-level metadata fields (excluding nested dictionaries) # This allows accessing fields like requester_ip_address from top-level metadata top_level_metadata = standard_logging_payload.get("metadata", {}) @@ -826,7 +847,7 @@ class PrometheusLogger(CustomLogger): for k, v in top_level_metadata.items() if not isinstance(v, dict) # Exclude nested dicts to avoid conflicts } - + combined_metadata: Dict[str, Any] = { **top_level_fields, # Include top-level fields first **(_requester_metadata if _requester_metadata else {}), @@ -945,6 +966,12 @@ class PrometheusLogger(CustomLogger): kwargs, start_time, end_time, enum_values, output_tokens ) + # cache metrics + self._increment_cache_metrics( + standard_logging_payload=standard_logging_payload, # type: ignore + enum_values=enum_values, + ) + if ( standard_logging_payload["stream"] is True ): # log successful streaming requests from logging event hook. @@ -1014,6 +1041,54 @@ class PrometheusLogger(CustomLogger): standard_logging_payload["completion_tokens"] ) + def _increment_cache_metrics( + self, + standard_logging_payload: StandardLoggingPayload, + enum_values: UserAPIKeyLabelValues, + ): + """ + Increment cache-related Prometheus metrics based on cache hit/miss status. + + Args: + standard_logging_payload: Contains cache_hit field (True/False/None) + enum_values: Label values for Prometheus metrics + """ + cache_hit = standard_logging_payload.get("cache_hit") + + # Only track if cache_hit has a definite value (True or False) + if cache_hit is None: + return + + if cache_hit is True: + # Increment cache hits counter + _labels = prometheus_label_factory( + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_cache_hits_metric" + ), + enum_values=enum_values, + ) + self.litellm_cache_hits_metric.labels(**_labels).inc() + + # Increment cached tokens counter + total_tokens = standard_logging_payload.get("total_tokens", 0) + if total_tokens > 0: + _labels = prometheus_label_factory( + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_cached_tokens_metric" + ), + enum_values=enum_values, + ) + self.litellm_cached_tokens_metric.labels(**_labels).inc(total_tokens) + else: + # cache_hit is False - increment cache misses counter + _labels = prometheus_label_factory( + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_cache_misses_metric" + ), + enum_values=enum_values, + ) + self.litellm_cache_misses_metric.labels(**_labels).inc() + async def _increment_remaining_budget_metrics( self, user_api_team: Optional[str], @@ -1196,7 +1271,7 @@ class PrometheusLogger(CustomLogger): ) litellm_params = kwargs.get("litellm_params", {}) or {} get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking() - + end_user_id = get_end_user_id_for_cost_tracking( litellm_params, service_type="prometheus" ) @@ -1398,7 +1473,6 @@ class PrometheusLogger(CustomLogger): api_provider=llm_provider or "", ) if exception is not None: - _labels = prometheus_label_factory( supported_enum_labels=self.get_labels_for_metric( metric_name="litellm_deployment_failure_responses" @@ -1431,12 +1505,11 @@ class PrometheusLogger(CustomLogger): enum_values: UserAPIKeyLabelValues, output_tokens: float = 1.0, ): - try: verbose_logger.debug("setting remaining tokens requests metric") - standard_logging_payload: Optional[StandardLoggingPayload] = ( - request_kwargs.get("standard_logging_object") - ) + standard_logging_payload: Optional[ + StandardLoggingPayload + ] = request_kwargs.get("standard_logging_object") if standard_logging_payload is None: return @@ -2208,10 +2281,10 @@ class PrometheusLogger(CustomLogger): from litellm.constants import PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES from litellm.integrations.custom_logger import CustomLogger - prometheus_loggers: List[CustomLogger] = ( - litellm.logging_callback_manager.get_custom_loggers_for_type( - callback_type=PrometheusLogger - ) + prometheus_loggers: List[ + CustomLogger + ] = litellm.logging_callback_manager.get_custom_loggers_for_type( + callback_type=PrometheusLogger ) # we need to get the initialized prometheus logger instance(s) and call logger.initialize_remaining_budget_metrics() on them verbose_logger.debug("found %s prometheus loggers", len(prometheus_loggers)) @@ -2283,7 +2356,7 @@ def prometheus_label_factory( if UserAPIKeyLabelNames.END_USER.value in filtered_labels: get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking() - + filtered_labels["end_user"] = get_end_user_id_for_cost_tracking( litellm_params={"user_api_key_end_user_id": enum_values.end_user}, service_type="prometheus", diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index 6a254fc8252..fb439a9541b 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -1,7 +1,7 @@ import re from dataclasses import dataclass from enum import Enum -from typing import Dict, List, Literal, Optional, Tuple, Union +from typing import Dict, List, Literal, Optional, Tuple from pydantic import BaseModel, Field from typing_extensions import Annotated @@ -185,6 +185,10 @@ DEFINED_PROMETHEUS_METRICS = Literal[ "litellm_redis_daily_spend_update_queue_size", "litellm_in_memory_spend_update_queue_size", "litellm_redis_spend_update_queue_size", + # Cache metrics + "litellm_cache_hits_metric", + "litellm_cache_misses_metric", + "litellm_cached_tokens_metric", ] @@ -436,6 +440,21 @@ class PrometheusMetricLabels: litellm_redis_spend_update_queue_size: List[str] = [] + # Cache metrics - track cache hits, misses, and tokens served from cache + _cache_metric_labels = [ + UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value, + UserAPIKeyLabelNames.API_KEY_HASH.value, + UserAPIKeyLabelNames.API_KEY_ALIAS.value, + UserAPIKeyLabelNames.TEAM.value, + UserAPIKeyLabelNames.TEAM_ALIAS.value, + UserAPIKeyLabelNames.END_USER.value, + UserAPIKeyLabelNames.USER.value, + ] + + litellm_cache_hits_metric = _cache_metric_labels + litellm_cache_misses_metric = _cache_metric_labels + litellm_cached_tokens_metric = _cache_metric_labels + @staticmethod def get_labels(label_name: DEFINED_PROMETHEUS_METRICS) -> List[str]: default_labels = getattr(PrometheusMetricLabels, label_name) diff --git a/tests/test_litellm/integrations/test_prometheus_cache_metrics.py b/tests/test_litellm/integrations/test_prometheus_cache_metrics.py new file mode 100644 index 00000000000..660757673f6 --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_cache_metrics.py @@ -0,0 +1,211 @@ +""" +Unit tests for cache Prometheus metrics. + +Run with: poetry run pytest tests/test_litellm/integrations/test_prometheus_cache_metrics.py -v +""" +import pytest +from unittest.mock import MagicMock, patch +from litellm.types.integrations.prometheus import UserAPIKeyLabelValues + + +class TestPrometheusCacheMetrics: + """Tests for cache-related Prometheus metrics""" + + @pytest.fixture + def sample_enum_values(self): + """Create sample enum values for labels""" + return UserAPIKeyLabelValues( + end_user="test-end-user", + hashed_api_key="test-key-hash", + api_key_alias="test-key-alias", + team="test-team", + team_alias="test-team-alias", + user="test-user", + model="gpt-3.5-turbo", + ) + + def test_cache_metrics_defined_in_types(self): + """Test that cache metrics are defined in DEFINED_PROMETHEUS_METRICS""" + from litellm.types.integrations.prometheus import DEFINED_PROMETHEUS_METRICS + from typing import get_args + + defined_metrics = get_args(DEFINED_PROMETHEUS_METRICS) + + assert "litellm_cache_hits_metric" in defined_metrics + assert "litellm_cache_misses_metric" in defined_metrics + assert "litellm_cached_tokens_metric" in defined_metrics + + def test_cache_metric_labels_defined(self): + """Test that cache metric labels are properly defined""" + from litellm.types.integrations.prometheus import PrometheusMetricLabels + + # Verify labels are defined for each cache metric + assert hasattr(PrometheusMetricLabels, "litellm_cache_hits_metric") + assert hasattr(PrometheusMetricLabels, "litellm_cache_misses_metric") + assert hasattr(PrometheusMetricLabels, "litellm_cached_tokens_metric") + + # Verify labels include expected keys + expected_labels = [ + "model", + "hashed_api_key", + "api_key_alias", + "team", + "team_alias", + "end_user", + "user", + ] + for label in expected_labels: + assert label in PrometheusMetricLabels.litellm_cache_hits_metric + assert label in PrometheusMetricLabels.litellm_cache_misses_metric + assert label in PrometheusMetricLabels.litellm_cached_tokens_metric + + def test_increment_cache_metrics_on_cache_hit(self, sample_enum_values): + """Test that cache hit increments the correct metrics""" + # Create mock for PrometheusLogger instance + mock_logger = MagicMock() + + # Import the method directly and bind it to our mock + from litellm.integrations.prometheus import PrometheusLogger + + # Create a mock standard logging payload with cache_hit=True + standard_logging_payload = { + "cache_hit": True, + "total_tokens": 100, + "prompt_tokens": 50, + "completion_tokens": 50, + "model_group": "openai", + "request_tags": [], + } + + # Create mock metrics + mock_logger.litellm_cache_hits_metric = MagicMock() + mock_logger.litellm_cache_misses_metric = MagicMock() + mock_logger.litellm_cached_tokens_metric = MagicMock() + mock_logger.get_labels_for_metric = MagicMock( + return_value=[ + "model", + "hashed_api_key", + "api_key_alias", + "team", + "team_alias", + "end_user", + "user", + ] + ) + + # Call the method using unbound method approach + PrometheusLogger._increment_cache_metrics( + mock_logger, + standard_logging_payload=standard_logging_payload, + enum_values=sample_enum_values, + ) + + # Verify cache hits metric was incremented + mock_logger.litellm_cache_hits_metric.labels.assert_called() + mock_logger.litellm_cache_hits_metric.labels().inc.assert_called_once() + + # Verify cached tokens metric was incremented with total_tokens + mock_logger.litellm_cached_tokens_metric.labels.assert_called() + mock_logger.litellm_cached_tokens_metric.labels().inc.assert_called_once_with( + 100 + ) + + # Verify cache misses metric was NOT called + mock_logger.litellm_cache_misses_metric.labels.assert_not_called() + + def test_increment_cache_metrics_on_cache_miss(self, sample_enum_values): + """Test that cache miss increments the correct metrics""" + # Create mock for PrometheusLogger instance + mock_logger = MagicMock() + + from litellm.integrations.prometheus import PrometheusLogger + + # Create a mock standard logging payload with cache_hit=False + standard_logging_payload = { + "cache_hit": False, + "total_tokens": 100, + "prompt_tokens": 50, + "completion_tokens": 50, + "model_group": "openai", + "request_tags": [], + } + + # Create mock metrics + mock_logger.litellm_cache_hits_metric = MagicMock() + mock_logger.litellm_cache_misses_metric = MagicMock() + mock_logger.litellm_cached_tokens_metric = MagicMock() + mock_logger.get_labels_for_metric = MagicMock( + return_value=[ + "model", + "hashed_api_key", + "api_key_alias", + "team", + "team_alias", + "end_user", + "user", + ] + ) + + # Call the method + PrometheusLogger._increment_cache_metrics( + mock_logger, + standard_logging_payload=standard_logging_payload, + enum_values=sample_enum_values, + ) + + # Verify cache misses metric was incremented + mock_logger.litellm_cache_misses_metric.labels.assert_called() + mock_logger.litellm_cache_misses_metric.labels().inc.assert_called_once() + + # Verify cache hits and cached tokens metrics were NOT called + mock_logger.litellm_cache_hits_metric.labels.assert_not_called() + mock_logger.litellm_cached_tokens_metric.labels.assert_not_called() + + def test_increment_cache_metrics_when_cache_hit_is_none(self, sample_enum_values): + """Test that no metrics are incremented when cache_hit is None""" + # Create mock for PrometheusLogger instance + mock_logger = MagicMock() + + from litellm.integrations.prometheus import PrometheusLogger + + # Create a mock standard logging payload with cache_hit=None + standard_logging_payload = { + "cache_hit": None, + "total_tokens": 100, + "prompt_tokens": 50, + "completion_tokens": 50, + "model_group": "openai", + "request_tags": [], + } + + # Create mock metrics + mock_logger.litellm_cache_hits_metric = MagicMock() + mock_logger.litellm_cache_misses_metric = MagicMock() + mock_logger.litellm_cached_tokens_metric = MagicMock() + mock_logger.get_labels_for_metric = MagicMock( + return_value=[ + "model", + "hashed_api_key", + "api_key_alias", + "team", + "team_alias", + "end_user", + "user", + ] + ) + + # Call the method + PrometheusLogger._increment_cache_metrics( + mock_logger, + standard_logging_payload=standard_logging_payload, + enum_values=sample_enum_values, + ) + + # Verify NO metrics were called + mock_logger.litellm_cache_hits_metric.labels.assert_not_called() + mock_logger.litellm_cache_misses_metric.labels.assert_not_called() + mock_logger.litellm_cached_tokens_metric.labels.assert_not_called() + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 1b8708fccc30dc5337c92709ba146bb96aa7ac8e Mon Sep 17 00:00:00 2001 From: kothamah <104782493+kothamah@users.noreply.github.com> Date: Wed, 7 Jan 2026 11:10:36 -0500 Subject: [PATCH 12/56] Litellm embeddings calltype fix for guardrail precallhook (#18740) * adding signoz integration to observability docs * Fixing build * Adding timeout for flaky test * Fixing e2e * add team member budget duration in team/update * Reusable Duration Select and update team member budget UI * feat: allow configuring project name for OpenTelemetry service name * docs: sets ARIZE_PROJECT_NAME * added valid callType for bedrock guardrail pre hook This is to resolve the error when bedrock guardrails are enabled and invoke the embedding models. {"error":{"message":"'embeddings' is not a valid CallTypes","type":"None","param":"None","code":"500"}}* * updated the test case to reflect valid callType --------- Co-authored-by: Goutham Karthi Co-authored-by: yuneng-jiang Co-authored-by: YutaSaito <36355491+uc4w6c@users.noreply.github.com> Co-authored-by: Yuta Saito --- .../docs/observability/arize_integration.md | 1 + litellm/integrations/arize/arize.py | 2 + litellm/integrations/opentelemetry.py | 88 +++-- litellm/litellm_core_utils/litellm_logging.py | 1 + litellm/proxy/_types.py | 1 + litellm/proxy/common_request_processing.py | 4 +- .../management_endpoints/team_endpoints.py | 15 +- litellm/types/integrations/arize.py | 1 + tests/local_testing/test_arize_ai.py | 3 + .../arize/test_arize_health_check.py | 10 +- .../integrations/test_custom_guardrail.py | 2 +- .../integrations/test_opentelemetry.py | 52 +-- .../test_team_endpoints.py | 366 +++++++++++++++++- .../common_components/DurationSelect.test.tsx | 49 +++ .../common_components/DurationSelect.tsx | 17 + .../src/components/team/team_info.tsx | 12 + 16 files changed, 552 insertions(+), 72 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx create mode 100644 ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx diff --git a/docs/my-website/docs/observability/arize_integration.md b/docs/my-website/docs/observability/arize_integration.md index 0b457f08687..b3ccf98ea3b 100644 --- a/docs/my-website/docs/observability/arize_integration.md +++ b/docs/my-website/docs/observability/arize_integration.md @@ -68,6 +68,7 @@ environment_variables: ARIZE_API_KEY: "141a****" ARIZE_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize GRPC api endpoint ARIZE_HTTP_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize HTTP api endpoint. Set either this or ARIZE_ENDPOINT or Neither (defaults to https://otlp.arize.com/v1 on grpc) + ARIZE_PROJECT_NAME: "my-litellm-project" # OPTIONAL - sets the arize project name ``` 2. Start the proxy diff --git a/litellm/integrations/arize/arize.py b/litellm/integrations/arize/arize.py index 4d1aa80dcce..9c2f0d95d4d 100644 --- a/litellm/integrations/arize/arize.py +++ b/litellm/integrations/arize/arize.py @@ -51,6 +51,7 @@ class ArizeLogger(OpenTelemetry): space_id = os.environ.get("ARIZE_SPACE_ID") space_key = os.environ.get("ARIZE_SPACE_KEY") api_key = os.environ.get("ARIZE_API_KEY") + project_name = os.environ.get("ARIZE_PROJECT_NAME") grpc_endpoint = os.environ.get("ARIZE_ENDPOINT") http_endpoint = os.environ.get("ARIZE_HTTP_ENDPOINT") @@ -74,6 +75,7 @@ class ArizeLogger(OpenTelemetry): api_key=api_key, protocol=protocol, endpoint=endpoint, + project_name=project_name, ) async def async_service_success_hook( diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index a7d2326d938..7e0cfab617b 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -54,38 +54,6 @@ RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request" LITELLM_REQUEST_SPAN_NAME = "litellm_request" -def _get_litellm_resource(): - """ - Create a proper OpenTelemetry Resource that respects OTEL_RESOURCE_ATTRIBUTES - while maintaining backward compatibility with LiteLLM-specific environment variables. - """ - from opentelemetry.sdk.resources import OTELResourceDetector, Resource - - # Create base resource attributes with LiteLLM-specific defaults - # These will be overridden by OTEL_RESOURCE_ATTRIBUTES if present - base_attributes: Dict[str, Optional[str]] = { - "service.name": os.getenv("OTEL_SERVICE_NAME", "litellm"), - "deployment.environment": os.getenv("OTEL_ENVIRONMENT_NAME", "production"), - # Fix the model_id to use proper environment variable or default to service name - "model_id": os.getenv( - "OTEL_MODEL_ID", os.getenv("OTEL_SERVICE_NAME", "litellm") - ), - } - - # Create base resource with LiteLLM-specific defaults - base_resource = Resource.create(base_attributes) # type: ignore - - # Create resource from OTEL_RESOURCE_ATTRIBUTES using the detector - otel_resource_detector = OTELResourceDetector() - env_resource = otel_resource_detector.detect() - - # Merge the resources: env_resource takes precedence over base_resource - # This ensures OTEL_RESOURCE_ATTRIBUTES overrides LiteLLM defaults - merged_resource = base_resource.merge(env_resource) - - return merged_resource - - @dataclass class OpenTelemetryConfig: exporter: Union[str, SpanExporter] = "console" @@ -93,6 +61,19 @@ class OpenTelemetryConfig: headers: Optional[str] = None enable_metrics: bool = False enable_events: bool = False + service_name: Optional[str] = None + deployment_environment: Optional[str] = None + model_id: Optional[str] = None + + def __post_init__(self) -> None: + if not self.service_name: + self.service_name = os.getenv("OTEL_SERVICE_NAME", "litellm") + if not self.deployment_environment: + self.deployment_environment = os.getenv( + "OTEL_ENVIRONMENT_NAME", "production" + ) + if not self.model_id: + self.model_id = os.getenv("OTEL_MODEL_ID", self.service_name) @classmethod def from_env(cls): @@ -122,6 +103,9 @@ class OpenTelemetryConfig: os.getenv("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", "false").lower() == "true" ) + service_name = os.getenv("OTEL_SERVICE_NAME", "litellm") + deployment_environment = os.getenv("OTEL_ENVIRONMENT_NAME", "production") + model_id = os.getenv("OTEL_MODEL_ID", service_name) if exporter == "in_memory": return cls(exporter=InMemorySpanExporter()) @@ -131,6 +115,9 @@ class OpenTelemetryConfig: headers=headers, # example: OTEL_HEADERS=x-honeycomb-team=B85YgLm96***" enable_metrics=enable_metrics, enable_events=enable_events, + service_name=service_name, + deployment_environment=deployment_environment, + model_id=model_id, ) @@ -174,6 +161,22 @@ class OpenTelemetry(CustomLogger): self._init_logs(logger_provider) self._init_otel_logger_on_litellm_proxy() + @staticmethod + def _get_litellm_resource(config: OpenTelemetryConfig): + """Create an OpenTelemetry Resource using config-driven defaults.""" + from opentelemetry.sdk.resources import OTELResourceDetector, Resource + + base_attributes: Dict[str, Optional[str]] = { + "service.name": config.service_name, + "deployment.environment": config.deployment_environment, + "model_id": config.model_id or config.service_name, + } + + base_resource = Resource.create(base_attributes) # type: ignore[arg-type] + otel_resource_detector = OTELResourceDetector() + env_resource = otel_resource_detector.detect() + return base_resource.merge(env_resource) + def _init_otel_logger_on_litellm_proxy(self): """ Initializes OpenTelemetry for litellm proxy server @@ -266,7 +269,7 @@ class OpenTelemetry(CustomLogger): from opentelemetry.trace import SpanKind def create_tracer_provider(): - provider = TracerProvider(resource=_get_litellm_resource()) + provider = TracerProvider(resource=self._get_litellm_resource(self.config)) provider.add_span_processor(self._get_span_processor()) return provider @@ -300,7 +303,8 @@ class OpenTelemetry(CustomLogger): def create_meter_provider(): metric_reader = self._get_metric_reader() return MeterProvider( - metric_readers=[metric_reader], resource=_get_litellm_resource() + metric_readers=[metric_reader], + resource=self._get_litellm_resource(self.config), ) meter_provider = self._get_or_create_provider( @@ -355,7 +359,9 @@ class OpenTelemetry(CustomLogger): from opentelemetry.sdk._logs.export import BatchLogRecordProcessor def create_logger_provider(): - provider = OTLoggerProvider(resource=_get_litellm_resource()) + provider = OTLoggerProvider( + resource=self._get_litellm_resource(self.config) + ) log_exporter = self._get_log_exporter() provider.add_log_record_processor( BatchLogRecordProcessor(log_exporter) # type: ignore[arg-type] @@ -606,7 +612,7 @@ class OpenTelemetry(CustomLogger): from opentelemetry.sdk.trace import TracerProvider # Create a temporary tracer provider with dynamic headers - temp_provider = TracerProvider(resource=_get_litellm_resource()) + temp_provider = TracerProvider(resource=self._get_litellm_resource(self.config)) temp_provider.add_span_processor( self._get_span_processor(dynamic_headers=dynamic_headers) ) @@ -987,9 +993,9 @@ class OpenTelemetry(CustomLogger): # Get the resource from the logger provider logger_provider = get_logger_provider() - resource = ( - getattr(logger_provider, "_resource", None) or _get_litellm_resource() - ) + resource = getattr( + logger_provider, "_resource", None + ) or self._get_litellm_resource(self.config) parent_ctx = span.get_span_context() provider = (kwargs.get("litellm_params") or {}).get( @@ -1910,7 +1916,9 @@ class OpenTelemetry(CustomLogger): ) _split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS) - normalized_endpoint = self._normalize_otel_endpoint(self.OTEL_ENDPOINT, "metrics") + normalized_endpoint = self._normalize_otel_endpoint( + self.OTEL_ENDPOINT, "metrics" + ) if self.OTEL_EXPORTER == "console": exporter = ConsoleMetricExporter() diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index cd324935562..5448fe7c771 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3630,6 +3630,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 otel_config = OpenTelemetryConfig( exporter=arize_config.protocol, endpoint=arize_config.endpoint, + service_name=arize_config.project_name, ) os.environ[ diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 954c26e2cb2..cff3bee1ca4 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1532,6 +1532,7 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase): guardrails: Optional[List[str]] = None object_permission: Optional[LiteLLM_ObjectPermissionBase] = None team_member_budget: Optional[float] = None + team_member_budget_duration: Optional[str] = None team_member_rpm_limit: Optional[int] = None team_member_tpm_limit: Optional[int] = None team_member_key_duration: Optional[str] = None diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 95c23be5b82..4e0c4f23811 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -957,11 +957,11 @@ class ProxyBaseLLMRequestProcessing: @staticmethod def _get_pre_call_type( route_type: Literal["acompletion", "aembedding", "aresponses", "allm_passthrough_route"], - ) -> Literal["completion", "embeddings", "responses", "allm_passthrough_route"]: + ) -> Literal["completion", "embedding", "responses", "allm_passthrough_route"]: if route_type == "acompletion": return "completion" elif route_type == "aembedding": - return "embeddings" + return "embedding" elif route_type == "aresponses": return "responses" elif route_type == "allm_passthrough_route": diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 76c607f5c49..920105edc16 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -112,6 +112,7 @@ class TeamMemberBudgetHandler: team_member_budget: Optional[float] = None, team_member_rpm_limit: Optional[int] = None, team_member_tpm_limit: Optional[int] = None, + team_member_budget_duration: Optional[str] = None, ) -> bool: """Check if any team member limits are provided""" return any( @@ -119,6 +120,7 @@ class TeamMemberBudgetHandler: team_member_budget is not None, team_member_rpm_limit is not None, team_member_tpm_limit is not None, + team_member_budget_duration is not None, ] ) @@ -130,6 +132,7 @@ class TeamMemberBudgetHandler: team_member_budget: Optional[float] = None, team_member_rpm_limit: Optional[int] = None, team_member_tpm_limit: Optional[int] = None, + team_member_budget_duration: Optional[str] = None, ) -> dict: """Create team member budget table with provided limits""" from litellm.proxy._types import BudgetNewRequest @@ -147,7 +150,7 @@ class TeamMemberBudgetHandler: # Create budget request with all provided limits budget_request = BudgetNewRequest( budget_id=budget_id, - budget_duration=data.budget_duration, + budget_duration=data.budget_duration or team_member_budget_duration, ) if team_member_budget is not None: @@ -156,6 +159,8 @@ class TeamMemberBudgetHandler: budget_request.rpm_limit = team_member_rpm_limit if team_member_tpm_limit is not None: budget_request.tpm_limit = team_member_tpm_limit + if team_member_budget_duration is not None: + budget_request.budget_duration = team_member_budget_duration team_member_budget_table = await new_budget( budget_obj=budget_request, @@ -182,6 +187,7 @@ class TeamMemberBudgetHandler: team_member_budget: Optional[float] = None, team_member_rpm_limit: Optional[int] = None, team_member_tpm_limit: Optional[int] = None, + team_member_budget_duration: Optional[str] = None, ) -> dict: """Upsert team member budget table with provided limits""" from litellm.proxy._types import BudgetNewRequest @@ -203,6 +209,8 @@ class TeamMemberBudgetHandler: budget_request.rpm_limit = team_member_rpm_limit if team_member_tpm_limit is not None: budget_request.tpm_limit = team_member_tpm_limit + if team_member_budget_duration is not None: + budget_request.budget_duration = team_member_budget_duration budget_row = await update_budget( budget_obj=budget_request, @@ -223,6 +231,7 @@ class TeamMemberBudgetHandler: team_member_budget=team_member_budget, team_member_rpm_limit=team_member_rpm_limit, team_member_tpm_limit=team_member_tpm_limit, + team_member_budget_duration=team_member_budget_duration, ) # Remove team member fields from updated_kv @@ -233,6 +242,7 @@ class TeamMemberBudgetHandler: def _clean_team_member_fields(data_dict: dict) -> None: """Remove team member fields from data dictionary""" data_dict.pop("team_member_budget", None) + data_dict.pop("team_member_budget_duration", None) data_dict.pop("team_member_rpm_limit", None) data_dict.pop("team_member_tpm_limit", None) @@ -1214,6 +1224,7 @@ async def update_team( # noqa: PLR0915 - disable_global_guardrails: Optional[bool] - Whether to disable global guardrails for the key. - object_permission: Optional[LiteLLM_ObjectPermissionBase] - team-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"], "agents": ["agent_1", "agent_2"], "agent_access_groups": ["dev_group"]}. IF null or {} then no object permission. - team_member_budget: Optional[float] - The maximum budget allocated to an individual team member. + - team_member_budget_duration: Optional[str] - The duration of the budget for the team member. Doc [here](https://docs.litellm.ai/docs/proxy/team_budgets) - team_member_rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for individual team members. - team_member_tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for individual team members. - team_member_key_duration: Optional[str] - The duration for a team member's key. e.g. "1d", "1w", "1mo" @@ -1349,6 +1360,7 @@ async def update_team( # noqa: PLR0915 team_member_budget=data.team_member_budget, team_member_rpm_limit=data.team_member_rpm_limit, team_member_tpm_limit=data.team_member_tpm_limit, + team_member_budget_duration=data.team_member_budget_duration, ): updated_kv = await TeamMemberBudgetHandler.upsert_team_member_budget_table( team_table=existing_team_row, @@ -1357,6 +1369,7 @@ async def update_team( # noqa: PLR0915 team_member_budget=data.team_member_budget, team_member_rpm_limit=data.team_member_rpm_limit, team_member_tpm_limit=data.team_member_tpm_limit, + team_member_budget_duration=data.team_member_budget_duration, ) else: TeamMemberBudgetHandler._clean_team_member_fields(updated_kv) diff --git a/litellm/types/integrations/arize.py b/litellm/types/integrations/arize.py index be4df30e794..248fdac3b3a 100644 --- a/litellm/types/integrations/arize.py +++ b/litellm/types/integrations/arize.py @@ -14,3 +14,4 @@ class ArizeConfig(BaseModel): api_key: Optional[str] = None protocol: Protocol endpoint: str + project_name: Optional[str] = None diff --git a/tests/local_testing/test_arize_ai.py b/tests/local_testing/test_arize_ai.py index 6a773521435..3b497d638ae 100644 --- a/tests/local_testing/test_arize_ai.py +++ b/tests/local_testing/test_arize_ai.py @@ -71,6 +71,7 @@ def test_get_arize_config(mock_env_vars): assert config.api_key == "test_api_key" assert config.endpoint == "https://otlp.arize.com/v1" assert config.protocol == "otlp_grpc" + assert config.project_name is None def test_get_arize_config_with_endpoints(mock_env_vars, monkeypatch): @@ -79,10 +80,12 @@ def test_get_arize_config_with_endpoints(mock_env_vars, monkeypatch): """ monkeypatch.setenv("ARIZE_ENDPOINT", "grpc://test.endpoint") monkeypatch.setenv("ARIZE_HTTP_ENDPOINT", "http://test.endpoint") + monkeypatch.setenv("ARIZE_PROJECT_NAME", "custom-project") config = ArizeLogger.get_arize_config() assert config.endpoint == "grpc://test.endpoint" assert config.protocol == "otlp_grpc" + assert config.project_name == "custom-project" @pytest.mark.skip( diff --git a/tests/test_litellm/integrations/arize/test_arize_health_check.py b/tests/test_litellm/integrations/arize/test_arize_health_check.py index 91d0b42d48d..8d86b7dc097 100644 --- a/tests/test_litellm/integrations/arize/test_arize_health_check.py +++ b/tests/test_litellm/integrations/arize/test_arize_health_check.py @@ -123,7 +123,8 @@ class TestArizeIntegrationWithProxy: with patch.dict(os.environ, { "ARIZE_SPACE_KEY": "test-space-123", "ARIZE_API_KEY": "test-api-456", - "ARIZE_ENDPOINT": "https://custom.arize.com/v1" + "ARIZE_ENDPOINT": "https://custom.arize.com/v1", + "ARIZE_PROJECT_NAME": "custom-project", }): config = ArizeLogger.get_arize_config() @@ -131,13 +132,15 @@ class TestArizeIntegrationWithProxy: assert config.api_key == "test-api-456" assert config.endpoint == "https://custom.arize.com/v1" assert config.protocol == "otlp_grpc" + assert config.project_name == "custom-project" def test_arize_get_config_defaults(self): """Test ArizeLogger.get_arize_config() with default endpoint.""" with patch.dict(os.environ, { "ARIZE_SPACE_KEY": "test-space-default", - "ARIZE_API_KEY": "test-api-default" + "ARIZE_API_KEY": "test-api-default", + "ARIZE_PROJECT_NAME": "default-project", }, clear=True): config = ArizeLogger.get_arize_config() @@ -145,6 +148,7 @@ class TestArizeIntegrationWithProxy: assert config.api_key == "test-api-default" assert config.endpoint == "https://otlp.arize.com/v1" # Default endpoint assert config.protocol == "otlp_grpc" # Default protocol + assert config.project_name == "default-project" def test_arize_construct_dynamic_headers(self): """Test dynamic OTEL headers construction for team/key logging.""" @@ -180,4 +184,4 @@ class TestArizeIntegrationWithProxy: if __name__ == "__main__": - pytest.main([__file__, "-v"]) \ No newline at end of file + pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py index a322dfe9a2b..d7d7720ff43 100644 --- a/tests/test_litellm/integrations/test_custom_guardrail.py +++ b/tests/test_litellm/integrations/test_custom_guardrail.py @@ -530,7 +530,7 @@ class TestPassthroughCallTypeHandling: ) assert ( ProxyBaseLLMRequestProcessing._get_pre_call_type(route_type="aembedding") - == "embeddings" + == "embedding" ) assert ( ProxyBaseLLMRequestProcessing._get_pre_call_type(route_type="aresponses") diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 6c17570e135..55b65fbb92a 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -258,6 +258,22 @@ class TestOpenTelemetry(unittest.TestCase): MODEL = "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0" HERE = os.path.dirname(__file__) + @patch.dict(os.environ, {}, clear=True) + def test_open_telemetry_config_manual_defaults(self): + """Manual OpenTelemetryConfig creation should populate default identifiers.""" + config = OpenTelemetryConfig(exporter="console", endpoint="http://collector") + self.assertEqual(config.service_name, "litellm") + self.assertEqual(config.deployment_environment, "production") + self.assertEqual(config.model_id, "litellm") + + @patch.dict(os.environ, {}, clear=True) + def test_open_telemetry_config_custom_service_name(self): + """Model ID should inherit provided service name when not explicitly set.""" + config = OpenTelemetryConfig(service_name="custom-service", exporter="console") + self.assertEqual(config.service_name, "custom-service") + self.assertEqual(config.deployment_environment, "production") + self.assertEqual(config.model_id, "custom-service") + def wait_for_spans(self, exporter: InMemorySpanExporter, prefix: str): """Poll until we see at least one span with an attribute key starting with `prefix`.""" deadline = time.time() + self.POLL_TIMEOUT @@ -504,8 +520,6 @@ class TestOpenTelemetry(unittest.TestCase): self, mock_detector_cls, mock_resource_create ): """Test _get_litellm_resource with default values when no environment variables are set.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - # Mock the Resource.create method mock_base_resource = MagicMock() mock_resource_create.return_value = mock_base_resource @@ -520,8 +534,8 @@ class TestOpenTelemetry(unittest.TestCase): mock_merged_resource = MagicMock() mock_base_resource.merge.return_value = mock_merged_resource - # Call the function - result = _get_litellm_resource() + config = OpenTelemetryConfig() + result = OpenTelemetry._get_litellm_resource(config) # Verify Resource.create was called with correct default attributes expected_attributes = { @@ -549,8 +563,6 @@ class TestOpenTelemetry(unittest.TestCase): self, mock_detector_cls, mock_resource_create ): """Test _get_litellm_resource with LiteLLM-specific environment variables.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - # Mock the Resource.create method mock_base_resource = MagicMock() mock_resource_create.return_value = mock_base_resource @@ -565,8 +577,8 @@ class TestOpenTelemetry(unittest.TestCase): mock_merged_resource = MagicMock() mock_base_resource.merge.return_value = mock_merged_resource - # Call the function - result = _get_litellm_resource() + config = OpenTelemetryConfig.from_env() + result = OpenTelemetry._get_litellm_resource(config) # Verify Resource.create was called with environment variable values expected_attributes = { @@ -593,8 +605,6 @@ class TestOpenTelemetry(unittest.TestCase): self, mock_detector_cls, mock_resource_create ): """Test _get_litellm_resource with OTEL_RESOURCE_ATTRIBUTES environment variable.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - # Mock the Resource.create method to simulate the actual behavior # In reality, Resource.create() would parse OTEL_RESOURCE_ATTRIBUTES and merge it mock_base_resource = MagicMock() @@ -610,8 +620,8 @@ class TestOpenTelemetry(unittest.TestCase): mock_merged_resource = MagicMock() mock_base_resource.merge.return_value = mock_merged_resource - # Call the function - result = _get_litellm_resource() + config = OpenTelemetryConfig.from_env() + result = OpenTelemetry._get_litellm_resource(config) # Verify Resource.create was called with the base attributes # The actual OTEL_RESOURCE_ATTRIBUTES parsing is handled by OpenTelemetry SDK @@ -628,10 +638,8 @@ class TestOpenTelemetry(unittest.TestCase): @patch.dict(os.environ, {}, clear=True) def test_get_litellm_resource_integration_with_real_resource(self): """Integration test to verify _get_litellm_resource works with actual OpenTelemetry Resource.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - - # This test uses the real OpenTelemetry Resource.create() method - result = _get_litellm_resource() + config = OpenTelemetryConfig() + result = OpenTelemetry._get_litellm_resource(config) # Verify the result is a Resource instance from opentelemetry.sdk.resources import Resource @@ -653,10 +661,8 @@ class TestOpenTelemetry(unittest.TestCase): ) def test_get_litellm_resource_real_otel_resource_attributes(self): """Integration test to verify OTEL_RESOURCE_ATTRIBUTES is properly handled.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - - # This test uses the real OpenTelemetry Resource.create() method - result = _get_litellm_resource() + config = OpenTelemetryConfig.from_env() + result = OpenTelemetry._get_litellm_resource(config) print("RESULT", result) @@ -683,10 +689,8 @@ class TestOpenTelemetry(unittest.TestCase): ) def test_get_litellm_resource_precedence(self): """Test that OTEL_SERVICE_NAME takes precedence over OTEL_RESOURCE_ATTRIBUTES according to OpenTelemetry spec.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - - # This test verifies the OpenTelemetry standard behavior - result = _get_litellm_resource() + config = OpenTelemetryConfig.from_env() + result = OpenTelemetry._get_litellm_resource(config) # Verify the result is a Resource instance from opentelemetry.sdk.resources import Resource diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 6cf8f745e07..57064586afb 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -1279,7 +1279,7 @@ async def test_update_team_team_member_budget_not_passed_to_db(): # Mock budget upsert to return updated_kv without team_member_budget def mock_upsert_side_effect( - team_table, user_api_key_dict, updated_kv, team_member_budget=None, team_member_rpm_limit=None, team_member_tpm_limit=None + team_table, user_api_key_dict, updated_kv, team_member_budget=None, team_member_rpm_limit=None, team_member_tpm_limit=None, team_member_budget_duration=None ): # Remove team_member_budget from updated_kv as the real function does result_kv = updated_kv.copy() @@ -1376,6 +1376,370 @@ async def test_update_team_team_member_budget_not_passed_to_db(): ) +def test_clean_team_member_fields(): + """ + Test that _clean_team_member_fields removes all team member fields from a dictionary. + """ + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + data_dict = { + "team_id": "test_team", + "team_alias": "Test Team", + "team_member_budget": 100.0, + "team_member_budget_duration": "30d", + "team_member_rpm_limit": 50, + "team_member_tpm_limit": 1000, + "other_field": "should_remain", + } + + TeamMemberBudgetHandler._clean_team_member_fields(data_dict) + + assert "team_member_budget" not in data_dict + assert "team_member_budget_duration" not in data_dict + assert "team_member_rpm_limit" not in data_dict + assert "team_member_tpm_limit" not in data_dict + assert data_dict["team_id"] == "test_team" + assert data_dict["team_alias"] == "Test Team" + assert data_dict["other_field"] == "should_remain" + + +def test_clean_team_member_fields_with_missing_fields(): + """ + Test that _clean_team_member_fields handles dictionaries without team member fields gracefully. + """ + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + data_dict = { + "team_id": "test_team", + "team_alias": "Test Team", + } + + TeamMemberBudgetHandler._clean_team_member_fields(data_dict) + + assert data_dict["team_id"] == "test_team" + assert data_dict["team_alias"] == "Test Team" + + +@pytest.mark.asyncio +async def test_create_team_member_budget_table(): + """ + Test that create_team_member_budget_table creates a budget and adds it to metadata. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.proxy._types import LitellmUserRoles, NewTeamRequest, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id" + ) + + data = NewTeamRequest( + team_id="test_team_id", + team_alias="Test Team", + budget_duration="1mo", + ) + new_team_data_json = { + "team_id": "test_team_id", + "team_alias": "Test Team", + "team_member_budget": 100.0, + "team_member_budget_duration": "30d", + "team_member_rpm_limit": 50, + "team_member_tpm_limit": 1000, + } + + mock_budget_response = MagicMock() + mock_budget_response.budget_id = "budget_123" + + with patch( + "litellm.proxy.management_endpoints.budget_management_endpoints.new_budget", + new_callable=AsyncMock + ) as mock_new_budget: + mock_new_budget.return_value = mock_budget_response + + result = await TeamMemberBudgetHandler.create_team_member_budget_table( + data=data, + new_team_data_json=new_team_data_json, + user_api_key_dict=mock_user_api_key_dict, + team_member_budget=100.0, + team_member_rpm_limit=50, + team_member_tpm_limit=1000, + team_member_budget_duration="30d", + ) + + assert mock_new_budget.called + call_args = mock_new_budget.call_args + budget_request = call_args[1]["budget_obj"] + + assert budget_request.max_budget == 100.0 + assert budget_request.rpm_limit == 50 + assert budget_request.tpm_limit == 1000 + assert budget_request.budget_duration == "30d" + assert budget_request.budget_id is not None + assert "team-" in budget_request.budget_id + + assert "team_member_budget_id" in result["metadata"] + assert result["metadata"]["team_member_budget_id"] == "budget_123" + + assert "team_member_budget" not in result + assert "team_member_budget_duration" not in result + assert "team_member_rpm_limit" not in result + assert "team_member_tpm_limit" not in result + + +@pytest.mark.asyncio +async def test_create_team_member_budget_table_without_team_alias(): + """ + Test that create_team_member_budget_table generates budget_id correctly when team_alias is None. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.proxy._types import LitellmUserRoles, NewTeamRequest, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id" + ) + + data = NewTeamRequest(team_id="test_team_id") + new_team_data_json = { + "team_id": "test_team_id", + "team_member_budget": 100.0, + } + + mock_budget_response = MagicMock() + mock_budget_response.budget_id = "budget_123" + + with patch( + "litellm.proxy.management_endpoints.budget_management_endpoints.new_budget", + new_callable=AsyncMock + ) as mock_new_budget: + mock_new_budget.return_value = mock_budget_response + + result = await TeamMemberBudgetHandler.create_team_member_budget_table( + data=data, + new_team_data_json=new_team_data_json, + user_api_key_dict=mock_user_api_key_dict, + team_member_budget=100.0, + ) + + assert mock_new_budget.called + call_args = mock_new_budget.call_args + budget_request = call_args[1]["budget_obj"] + + assert budget_request.budget_id is not None + assert budget_request.budget_id.startswith("team-budget-") + + +@pytest.mark.asyncio +async def test_upsert_team_member_budget_table_existing_budget(): + """ + Test that upsert_team_member_budget_table updates an existing budget when team_member_budget_id exists. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.proxy._types import LitellmUserRoles, LiteLLM_TeamTable, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id" + ) + + team_table = MagicMock(spec=LiteLLM_TeamTable) + team_table.metadata = {"team_member_budget_id": "existing_budget_123"} + + updated_kv = { + "team_id": "test_team_id", + "team_member_budget": 200.0, + "team_member_budget_duration": "60d", + "team_member_rpm_limit": 100, + } + + mock_budget_response = MagicMock() + mock_budget_response.budget_id = "existing_budget_123" + + with patch( + "litellm.proxy.management_endpoints.budget_management_endpoints.update_budget", + new_callable=AsyncMock + ) as mock_update_budget: + mock_update_budget.return_value = mock_budget_response + + result = await TeamMemberBudgetHandler.upsert_team_member_budget_table( + team_table=team_table, + user_api_key_dict=mock_user_api_key_dict, + updated_kv=updated_kv, + team_member_budget=200.0, + team_member_budget_duration="60d", + team_member_rpm_limit=100, + ) + + assert mock_update_budget.called + call_args = mock_update_budget.call_args + budget_request = call_args[1]["budget_obj"] + + assert budget_request.budget_id == "existing_budget_123" + assert budget_request.max_budget == 200.0 + assert budget_request.budget_duration == "60d" + assert budget_request.rpm_limit == 100 + + assert "team_member_budget_id" in result["metadata"] + assert result["metadata"]["team_member_budget_id"] == "existing_budget_123" + + assert "team_member_budget" not in result + assert "team_member_budget_duration" not in result + assert "team_member_rpm_limit" not in result + + +@pytest.mark.asyncio +async def test_upsert_team_member_budget_table_no_existing_budget(): + """ + Test that upsert_team_member_budget_table creates a new budget when team_member_budget_id does not exist. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.proxy._types import LitellmUserRoles, LiteLLM_TeamTable, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id" + ) + + team_table = MagicMock(spec=LiteLLM_TeamTable) + team_table.metadata = {} + team_table.team_alias = "Test Team" + team_table.budget_duration = None + + updated_kv = { + "team_id": "test_team_id", + "team_member_budget": 150.0, + "team_member_budget_duration": "45d", + } + + mock_budget_response = MagicMock() + mock_budget_response.budget_id = "new_budget_456" + + with patch( + "litellm.proxy.management_endpoints.budget_management_endpoints.new_budget", + new_callable=AsyncMock + ) as mock_new_budget: + mock_new_budget.return_value = mock_budget_response + + result = await TeamMemberBudgetHandler.upsert_team_member_budget_table( + team_table=team_table, + user_api_key_dict=mock_user_api_key_dict, + updated_kv=updated_kv, + team_member_budget=150.0, + team_member_budget_duration="45d", + ) + + assert mock_new_budget.called + assert "team_member_budget_id" in result["metadata"] + assert result["metadata"]["team_member_budget_id"] == "new_budget_456" + + assert "team_member_budget" not in result + assert "team_member_budget_duration" not in result + + +@pytest.mark.asyncio +async def test_update_team_with_team_member_budget_duration(): + """ + Test that team/update endpoint properly handles team_member_budget_duration. + """ + from unittest.mock import AsyncMock, MagicMock, Mock, patch + + from fastapi import Request + + from litellm.proxy._types import LitellmUserRoles, UpdateTeamRequest, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import update_team + + mock_request = Mock(spec=Request) + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id" + ) + + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma_client, patch( + "litellm.proxy.proxy_server.llm_router" + ) as mock_llm_router, patch( + "litellm.proxy.proxy_server.user_api_key_cache" + ) as mock_cache, patch( + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as mock_logging, patch( + "litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin" + ), patch( + "litellm.proxy.auth.auth_checks._cache_team_object" + ) as mock_cache_team, patch( + "litellm.proxy.management_endpoints.team_endpoints.TeamMemberBudgetHandler.upsert_team_member_budget_table" + ) as mock_upsert_budget: + + mock_existing_team = MagicMock() + mock_existing_team.model_dump.return_value = { + "team_id": "test_team_id", + "team_alias": "test_team", + "metadata": {"team_member_budget_id": "budget_123"}, + } + mock_existing_team.metadata = {"team_member_budget_id": "budget_123"} + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=mock_existing_team + ) + + mock_updated_team = MagicMock() + mock_updated_team.team_id = "test_team_id" + mock_updated_team.model_dump.return_value = {"team_id": "test_team_id"} + mock_prisma_client.db.litellm_teamtable.update = AsyncMock( + return_value=mock_updated_team + ) + mock_prisma_client.jsonify_team_object = MagicMock( + side_effect=lambda db_data: db_data + ) + + def mock_upsert_side_effect( + team_table, user_api_key_dict, updated_kv, team_member_budget=None, team_member_rpm_limit=None, team_member_tpm_limit=None, team_member_budget_duration=None + ): + result_kv = updated_kv.copy() + result_kv.pop("team_member_budget", None) + result_kv.pop("team_member_budget_duration", None) + return result_kv + + mock_upsert_budget.side_effect = mock_upsert_side_effect + + update_request = UpdateTeamRequest( + team_id="test_team_id", + team_alias="updated_alias", + team_member_budget=100.0, + team_member_budget_duration="30d", + ) + + result = await update_team( + data=update_request, + http_request=mock_request, + user_api_key_dict=mock_user_api_key_dict, + ) + + assert mock_upsert_budget.called + call_args = mock_upsert_budget.call_args + assert call_args[1]["team_member_budget"] == 100.0 + assert call_args[1]["team_member_budget_duration"] == "30d" + + assert mock_prisma_client.db.litellm_teamtable.update.called + update_call_args = mock_prisma_client.db.litellm_teamtable.update.call_args + update_data = update_call_args[1]["data"] + + assert "team_member_budget" not in update_data + assert "team_member_budget_duration" not in update_data + + @pytest.mark.asyncio async def test_bulk_team_member_add_success(): """ diff --git a/ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx b/ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx new file mode 100644 index 00000000000..296ef1ae632 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx @@ -0,0 +1,49 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, it, expect, vi } from "vitest"; +import DurationSelect from "./DurationSelect"; + +describe("DurationSelect", () => { + it("should render", () => { + render(); + expect(screen.getByRole("combobox")).toBeInTheDocument(); + }); + + it("should render all three duration options", async () => { + const user = userEvent.setup(); + render(); + + const select = screen.getByRole("combobox"); + await user.click(select); + + expect(screen.getByText("Daily")).toBeInTheDocument(); + expect(screen.getByText("Weekly")).toBeInTheDocument(); + expect(screen.getByText("Monthly")).toBeInTheDocument(); + }); + + it("should apply className prop", () => { + render(); + const select = screen.getByRole("combobox"); + expect(select.closest(".test-class")).toBeInTheDocument(); + }); + + it("should call onChange when an option is selected", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + render(); + + const select = screen.getByRole("combobox"); + await user.click(select); + + const dailyOption = screen.getByText("Daily"); + await user.click(dailyOption); + + expect(onChange).toHaveBeenCalledWith("24h", expect.any(Object)); + }); + + it("should accept and pass value prop to Select", () => { + render(); + const select = screen.getByRole("combobox"); + expect(select).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx b/ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx new file mode 100644 index 00000000000..a84e8aeb110 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx @@ -0,0 +1,17 @@ +import { Select } from "antd"; + +interface DurationSelectProps { + className?: string; + value?: string; + onChange?: (value: string) => void; +} + +export default function DurationSelect({ className, value, onChange }: DurationSelectProps) { + return ( + + ); +} diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx index d2d1c885931..fd7994aa1da 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.tsx @@ -48,6 +48,7 @@ import EditLoggingSettings from "./EditLoggingSettings"; import MemberModal from "./EditMembership"; import MemberPermissions from "./member_permissions"; import TeamMembersComponent from "./team_member_view"; +import DurationSelect from "../common_components/DurationSelect"; export interface TeamMembership { user_id: string; @@ -413,6 +414,7 @@ const TeamInfoView: React.FC = ({ }; updateData.max_budget = mapEmptyStringToNull(updateData.max_budget); + updateData.team_member_budget_duration = values.team_member_budget_duration; if (values.team_member_budget !== undefined) { updateData.team_member_budget = Number(values.team_member_budget); @@ -650,6 +652,8 @@ const TeamInfoView: React.FC = ({ budget_duration: info.budget_duration, team_member_tpm_limit: info.team_member_budget_table?.tpm_limit, team_member_rpm_limit: info.team_member_budget_table?.rpm_limit, + team_member_budget: info.team_member_budget_table?.max_budget, + team_member_budget_duration: info.team_member_budget_table?.budget_duration, guardrails: info.metadata?.guardrails || [], disable_global_guardrails: info.metadata?.disable_global_guardrails || false, metadata: info.metadata @@ -747,6 +751,13 @@ const TeamInfoView: React.FC = ({ + + form.setFieldValue("team_member_budget_duration", value)} + value={form.getFieldValue("team_member_budget_duration")} + /> + + = ({
Max Budget: {info.team_member_budget_table?.max_budget || "No Limit"}
+
Budget Duration: {info.team_member_budget_table?.budget_duration || "No Limit"}
Key Duration: {info.metadata?.team_member_key_duration || "No Limit"}
TPM Limit: {info.team_member_budget_table?.tpm_limit || "No Limit"}
RPM Limit: {info.team_member_budget_table?.rpm_limit || "No Limit"}
From f7212d84d5ae7ac2f7bf469709bea03080acb6b7 Mon Sep 17 00:00:00 2001 From: tianduo-fh <153234738+Tianduo16@users.noreply.github.com> Date: Wed, 7 Jan 2026 11:12:27 -0500 Subject: [PATCH 13/56] fix: prevent duplicate User-Agent tags in request_tags (#18723) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `_get_request_tags` function was returning a reference to the original tags list from metadata, then mutating it with `.extend()`. This caused duplicate User-Agent tags when the function was called multiple times during a single request lifecycle (e.g., by logging, prometheus, and guardrails). The fix uses `.copy()` to create a new list before extending, ensuring the original metadata tags are not mutated. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Tianduo Zhai Co-authored-by: Claude Opus 4.5 --- litellm/litellm_core_utils/litellm_logging.py | 4 +- .../test_litellm_logging.py | 57 +++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 5448fe7c771..d9161f1db41 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4824,9 +4824,9 @@ class StandardLoggingPayloadSetup: metadata = litellm_params.get("metadata") or {} litellm_metadata = litellm_params.get("litellm_metadata") or {} if metadata.get("tags", []): - request_tags = metadata.get("tags", []) + request_tags = metadata.get("tags", []).copy() elif litellm_metadata.get("tags", []): - request_tags = litellm_metadata.get("tags", []) + request_tags = litellm_metadata.get("tags", []).copy() else: request_tags = [] user_agent_tags = StandardLoggingPayloadSetup._get_user_agent_tags( diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 9b150fd89f4..bae0e5bbb4f 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -393,6 +393,63 @@ def test_get_request_tags_from_metadata_and_litellm_metadata(): assert "User-Agent: litellm/1.0.0" in tags +def test_get_request_tags_does_not_mutate_original_tags(): + """ + Test that _get_request_tags does not mutate the original tags list in metadata. + + This is a regression test for a bug where calling _get_request_tags multiple times + would cause User-Agent tags to be duplicated because the function was mutating + the original tags list instead of creating a copy. + """ + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + # Create metadata with original tags + original_tags = ["custom-tag-1", "custom-tag-2"] + metadata = {"tags": original_tags} + litellm_params = {"metadata": metadata} + proxy_server_request = { + "headers": { + "user-agent": "AsyncOpenAI/Python 1.99.9", + } + } + + # Call _get_request_tags multiple times (simulating multiple callbacks) + tags1 = StandardLoggingPayloadSetup._get_request_tags( + litellm_params=litellm_params, + proxy_server_request=proxy_server_request, + ) + tags2 = StandardLoggingPayloadSetup._get_request_tags( + litellm_params=litellm_params, + proxy_server_request=proxy_server_request, + ) + tags3 = StandardLoggingPayloadSetup._get_request_tags( + litellm_params=litellm_params, + proxy_server_request=proxy_server_request, + ) + + # Verify the original tags list was NOT mutated + assert original_tags == ["custom-tag-1", "custom-tag-2"], ( + f"Original tags list was mutated: {original_tags}" + ) + assert metadata["tags"] == ["custom-tag-1", "custom-tag-2"], ( + f"metadata['tags'] was mutated: {metadata['tags']}" + ) + + # Verify each returned list has exactly 2 User-Agent tags (not duplicated) + user_agent_count_1 = len([t for t in tags1 if t.startswith("User-Agent:")]) + user_agent_count_2 = len([t for t in tags2 if t.startswith("User-Agent:")]) + user_agent_count_3 = len([t for t in tags3 if t.startswith("User-Agent:")]) + + assert user_agent_count_1 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_1}" + assert user_agent_count_2 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_2}" + assert user_agent_count_3 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_3}" + + # Verify all returned lists are independent (different objects) + assert tags1 is not tags2 + assert tags2 is not tags3 + assert tags1 is not original_tags + + def test_get_extra_header_tags(): """Test the _get_extra_header_tags method with various scenarios.""" import litellm From 8ce4eea88f977a5c0e31358babed40c468885f67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20Marc=C3=ADlio=20J=C3=BAnior?= Date: Wed, 7 Jan 2026 13:13:55 -0300 Subject: [PATCH 14/56] make base_connection_pool_limit default value the same (#18721) --- litellm/proxy/_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index cff3bee1ca4..bb6b56d5090 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1937,7 +1937,7 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): description="connect to a postgres db - needed for generating temporary keys + tracking spend / key", ) database_connection_pool_limit: Optional[int] = Field( - 100, + 10, description="default connection pool for prisma client connecting to postgres db", ) database_connection_timeout: Optional[float] = Field( From dc4ce7c5a21615504b739d445d9c3ffd35aaeb38 Mon Sep 17 00:00:00 2001 From: Abliteration AI Date: Wed, 7 Jan 2026 08:16:54 -0800 Subject: [PATCH 15/56] feat: Add abliteration.ai provider (#18678) * feat: Add abliteration.ai provider * adding signoz integration to observability docs * Fixing build * Adding timeout for flaky test * Fixing e2e * add team member budget duration in team/update * Reusable Duration Select and update team member budget UI --------- Co-authored-by: Goutham Karthi Co-authored-by: yuneng-jiang Co-authored-by: YutaSaito <36355491+uc4w6c@users.noreply.github.com> --- README.md | 2 +- .../my-website/docs/providers/abliteration.md | 109 ++++++++++++++++++ docs/my-website/sidebars.js | 13 ++- litellm/llms/openai_like/providers.json | 4 + provider_endpoints_support.json | 17 +++ .../openai_like/test_abliteration_provider.py | 50 ++++++++ 6 files changed, 188 insertions(+), 7 deletions(-) create mode 100644 docs/my-website/docs/providers/abliteration.md create mode 100644 tests/litellm/llms/openai_like/test_abliteration_provider.py diff --git a/README.md b/README.md index a020bd80898..75a23faa5c1 100644 --- a/README.md +++ b/README.md @@ -262,6 +262,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature | Provider | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` | |-------------------------------------------------------------------------------------|---------------------|-------------|--------------|---------------|----------------------|-------------------------|-----------------|----------------|-----------|-----------| +| [Abliteration (`abliteration`)](https://docs.litellm.ai/docs/providers/abliteration) | ✅ | | | | | | | | | | | [AI/ML API (`aiml`)](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | [AI21 (`ai21`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | | | [AI21 Chat (`ai21_chat`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | | @@ -455,4 +456,3 @@ All these checks must pass before your PR can be merged. - diff --git a/docs/my-website/docs/providers/abliteration.md b/docs/my-website/docs/providers/abliteration.md new file mode 100644 index 00000000000..a0fc7f39310 --- /dev/null +++ b/docs/my-website/docs/providers/abliteration.md @@ -0,0 +1,109 @@ +# Abliteration + +## Overview + +| Property | Details | +|-------|-------| +| Description | Abliteration provides an OpenAI-compatible `/chat/completions` endpoint. | +| Provider Route on LiteLLM | `abliteration/` | +| Link to Provider Doc | [Abliteration](https://abliteration.ai) | +| Base URL | `https://api.abliteration.ai/v1` | +| Supported Operations | [`/chat/completions`](#sample-usage) | + +
+ +## Required Variables + +```python showLineNumbers title="Environment Variables" +os.environ["ABLITERATION_API_KEY"] = "" # your Abliteration API key +``` + +## Sample Usage + +```python showLineNumbers title="Abliteration Completion" +import os +from litellm import completion + +os.environ["ABLITERATION_API_KEY"] = "" + +response = completion( + model="abliteration/abliterated-model", + messages=[{"role": "user", "content": "Hello from LiteLLM"}], +) + +print(response) +``` + +## Sample Usage - Streaming + +```python showLineNumbers title="Abliteration Streaming Completion" +import os +from litellm import completion + +os.environ["ABLITERATION_API_KEY"] = "" + +response = completion( + model="abliteration/abliterated-model", + messages=[{"role": "user", "content": "Stream a short reply"}], + stream=True, +) + +for chunk in response: + print(chunk) +``` + +## Usage with LiteLLM Proxy Server + +1. Add the model to your proxy config: + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: abliteration-chat + litellm_params: + model: abliteration/abliterated-model + api_key: os.environ/ABLITERATION_API_KEY +``` + +2. Start the proxy: + +```bash +litellm --config /path/to/config.yaml +``` + +## Direct API Usage (Bearer Token) + +Use the environment variable as a Bearer token against the OpenAI-compatible endpoint: +`https://api.abliteration.ai/v1/chat/completions`. + +```bash showLineNumbers title="cURL" +export ABLITERATION_API_KEY="" +curl https://api.abliteration.ai/v1/chat/completions \ + -H "Authorization: Bearer ${ABLITERATION_API_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "abliterated-model", + "messages": [{"role": "user", "content": "Hello from Abliteration"}] + }' +``` + +```python showLineNumbers title="Python (requests)" +import os +import requests + +api_key = os.environ["ABLITERATION_API_KEY"] + +response = requests.post( + "https://api.abliteration.ai/v1/chat/completions", + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + json={ + "model": "abliterated-model", + "messages": [{"role": "user", "content": "Hello from Abliteration"}], + }, + timeout=60, +) + +print(response.json()) +``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 004132ca05b..4c50449afcc 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -683,12 +683,13 @@ const sidebars = { "providers/bedrock_writer", "providers/bedrock_batches", "providers/aws_polly", - "providers/bedrock_vector_store", - ] - }, - "providers/litellm_proxy", - "providers/ai21", - "providers/aiml", + "providers/bedrock_vector_store", + ] + }, + "providers/litellm_proxy", + "providers/abliteration", + "providers/ai21", + "providers/aiml", "providers/aleph_alpha", "providers/amazon_nova", "providers/anyscale", diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 206aee1359d..bda3684a8a8 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -61,6 +61,10 @@ "max_completion_tokens": "max_tokens" } }, + "abliteration": { + "base_url": "https://api.abliteration.ai/v1", + "api_key_env": "ABLITERATION_API_KEY" + }, "llamagate": { "base_url": "https://api.llamagate.dev/v1", "api_key_env": "LLAMAGATE_API_KEY", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index bc5dea7b97c..2521d71b5c0 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -34,6 +34,23 @@ } }, "providers": { + "abliteration": { + "display_name": "Abliteration (`abliteration`)", + "url": "https://docs.litellm.ai/docs/providers/abliteration", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "aiml": { "display_name": "AI/ML API (`aiml`)", "url": "https://docs.litellm.ai/docs/providers/aiml", diff --git a/tests/litellm/llms/openai_like/test_abliteration_provider.py b/tests/litellm/llms/openai_like/test_abliteration_provider.py new file mode 100644 index 00000000000..8b8d443fc44 --- /dev/null +++ b/tests/litellm/llms/openai_like/test_abliteration_provider.py @@ -0,0 +1,50 @@ +""" +Unit tests for the Abliteration OpenAI-like provider. +""" + +import os +import sys + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..")) +) + +from litellm.llms.openai_like.dynamic_config import create_config_class +from litellm.llms.openai_like.json_loader import JSONProviderRegistry + +ABLITERATION_BASE_URL = "https://api.abliteration.ai/v1" + + +def _get_config(): + provider = JSONProviderRegistry.get("abliteration") + assert provider is not None + config_class = create_config_class(provider) + return config_class() + + +def test_abliteration_provider_registered(): + provider = JSONProviderRegistry.get("abliteration") + assert provider is not None + assert provider.base_url == ABLITERATION_BASE_URL + assert provider.api_key_env == "ABLITERATION_API_KEY" + + +def test_abliteration_resolves_env_api_key(monkeypatch): + config = _get_config() + monkeypatch.setenv("ABLITERATION_API_KEY", "test-key") + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == ABLITERATION_BASE_URL + assert api_key == "test-key" + + +def test_abliteration_complete_url_appends_endpoint(): + config = _get_config() + url = config.get_complete_url( + api_base=ABLITERATION_BASE_URL, + api_key="test-key", + model="abliteration/abliterated-model", + optional_params={}, + litellm_params={}, + stream=False, + ) + assert url == f"{ABLITERATION_BASE_URL}/chat/completions" From 5a4242e1882517cf850849721897bf39f46e2cf9 Mon Sep 17 00:00:00 2001 From: Wen-Tien Chang Date: Thu, 8 Jan 2026 03:25:59 +0800 Subject: [PATCH 16/56] fix(braintrust): pass span_attributes in async logging and skip tags on non-root spans (#18409) * fix(braintrust): handle tags and span attributes for non-root spans * fix(braintrust): refactor span attributes handling and remove duplicate metrics assignment --- litellm/integrations/braintrust_logging.py | 35 ++++++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index 364fa3f5def..585de510e8b 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -225,10 +225,13 @@ class BraintrustLogger(CustomLogger): "id": litellm_call_id, "input": prompt["messages"], "metadata": standard_logging_object, - "tags": tags, "span_attributes": {"name": span_name, "type": "llm"}, } - + + # Braintrust cannot specify 'tags' for non-root spans + if dynamic_metadata.get("root_span_id") is None: + request_data["tags"] = tags + # Only add those that are not None (or falsy) for key, value in span_attributes.items(): if value: @@ -351,14 +354,37 @@ class BraintrustLogger(CustomLogger): # Allow metadata override for span name span_name = dynamic_metadata.get("span_name", "Chat Completion") + # Span parents is a special case + span_parents = dynamic_metadata.get("span_parents") + + # Convert comma-separated string to list if present + if span_parents: + span_parents = [s.strip() for s in span_parents.split(",") if s.strip()] + + # Add optional span attributes only if present + span_attributes = { + "span_id": dynamic_metadata.get("span_id"), + "root_span_id": dynamic_metadata.get("root_span_id"), + "span_parents": span_parents, + } + request_data = { "id": litellm_call_id, "input": prompt["messages"], "output": output, "metadata": standard_logging_object, - "tags": tags, "span_attributes": {"name": span_name, "type": "llm"}, } + + # Braintrust cannot specify 'tags' for non-root spans + if dynamic_metadata.get("root_span_id") is None: + request_data["tags"] = tags + + # Only add those that are not None (or falsy) + for key, value in span_attributes.items(): + if value: + request_data[key] = value + if choices is not None: request_data["output"] = [choice.dict() for choice in choices] else: @@ -367,9 +393,6 @@ class BraintrustLogger(CustomLogger): if metrics is not None: request_data["metrics"] = metrics - if metrics is not None: - request_data["metrics"] = metrics - try: await self.global_braintrust_http_handler.post( url=f"{self.api_base}/project_logs/{project_id}/insert", From bae625bdc6e9f9608c48bc5251134ff2515cb84e Mon Sep 17 00:00:00 2001 From: Elkhan Eminov Date: Wed, 7 Jan 2026 19:27:31 +0000 Subject: [PATCH 17/56] OpenRouter embeddings API support (#18391) * support for OpenRouter embeddings * add bearer * add content header --- .../openrouter/embedding/transformation.py | 182 ++++++++++++++++++ litellm/main.py | 45 +++++ litellm/utils.py | 5 + provider_endpoints_support.json | 2 +- tests/llm_translation/test_openrouter.py | 15 ++ ...est_openrouter_embedding_transformation.py | 132 +++++++++++++ 6 files changed, 380 insertions(+), 1 deletion(-) create mode 100644 litellm/llms/openrouter/embedding/transformation.py create mode 100644 tests/test_litellm/llms/openrouter/test_openrouter_embedding_transformation.py diff --git a/litellm/llms/openrouter/embedding/transformation.py b/litellm/llms/openrouter/embedding/transformation.py new file mode 100644 index 00000000000..d1d0e911d16 --- /dev/null +++ b/litellm/llms/openrouter/embedding/transformation.py @@ -0,0 +1,182 @@ +""" +OpenRouter Embedding API Configuration. + +This module provides the configuration for OpenRouter's Embedding API. +OpenRouter is OpenAI-compatible and supports embeddings via the /v1/embeddings endpoint. + +Docs: https://openrouter.ai/docs +""" +from typing import TYPE_CHECKING, Any, Optional + +import httpx + +from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig +from litellm.types.llms.openai import AllEmbeddingInputValues +from litellm.types.utils import EmbeddingResponse +from litellm.utils import convert_to_model_response_object + +from ..common_utils import OpenRouterException + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class OpenrouterEmbeddingConfig(BaseEmbeddingConfig): + """ + Configuration for OpenRouter's Embedding API. + + Reference: https://openrouter.ai/docs + """ + + def validate_environment( + self, + headers: dict, + model: str, + messages: list, + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate environment and set up headers for OpenRouter API. + + OpenRouter requires: + - Authorization header with Bearer token + - HTTP-Referer header (site URL) + - X-Title header (app name) + """ + from litellm import get_secret + + # Get OpenRouter-specific headers + openrouter_site_url = get_secret("OR_SITE_URL") or "https://litellm.ai" + openrouter_app_name = get_secret("OR_APP_NAME") or "liteLLM" + + openrouter_headers = { + "HTTP-Referer": openrouter_site_url, + "X-Title": openrouter_app_name, + "Content-Type": "application/json", + } + + # Add Authorization header if api_key is provided + if api_key: + openrouter_headers["Authorization"] = f"Bearer {api_key}" + + # Merge with existing headers (user's extra_headers take priority) + merged_headers = {**openrouter_headers, **headers} + + return merged_headers + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for OpenRouter Embedding API endpoint. + """ + # api_base is already set to https://openrouter.ai/api/v1 in main.py + # Remove trailing slashes + if api_base: + api_base = api_base.rstrip("/") + else: + api_base = "https://openrouter.ai/api/v1" + + # Return the embeddings endpoint + return f"{api_base}/embeddings" + + def transform_embedding_request( + self, + model: str, + input: AllEmbeddingInputValues, + optional_params: dict, + headers: dict, + ) -> dict: + """ + Transform embedding request to OpenRouter format (OpenAI-compatible). + """ + # Ensure input is a list + if isinstance(input, str): + input = [input] + + # OpenRouter expects the full model name (e.g., google/gemini-embedding-001) + # Strip 'openrouter/' prefix if present + if model.startswith("openrouter/"): + model = model.replace("openrouter/", "", 1) + + return { + "model": model, + "input": input, + **optional_params, + } + + def transform_embedding_response( + self, + model: str, + raw_response: httpx.Response, + model_response: EmbeddingResponse, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + request_data: dict, + optional_params: dict, + litellm_params: dict, + ) -> EmbeddingResponse: + """ + Transform embedding response from OpenRouter format (OpenAI-compatible). + """ + logging_obj.post_call(original_response=raw_response.text) + + # OpenRouter returns standard OpenAI-compatible embedding response + response_json = raw_response.json() + + return convert_to_model_response_object( + response_object=response_json, + model_response_object=model_response, + response_type="embedding", + ) + + def get_supported_openai_params(self, model: str) -> list: + """ + Get list of supported OpenAI parameters for OpenRouter embeddings. + """ + return [ + "timeout", + "dimensions", + "encoding_format", + "user", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Map OpenAI parameters to OpenRouter format. + """ + for param, value in non_default_params.items(): + if param in self.get_supported_openai_params(model): + optional_params[param] = value + return optional_params + + def get_error_class( + self, error_message: str, status_code: int, headers: Any + ) -> Any: + """ + Get the error class for OpenRouter errors. + """ + return OpenRouterException( + message=error_message, + status_code=status_code, + headers=headers, + ) diff --git a/litellm/main.py b/litellm/main.py index e8a8b504d96..6905d8f6a87 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4701,6 +4701,51 @@ def embedding( # noqa: PLR0915 litellm_params=litellm_params_dict, headers=headers, ) + elif custom_llm_provider == "openrouter": + api_base = ( + api_base + or litellm.api_base + or get_secret_str("OPENROUTER_API_BASE") + or "https://openrouter.ai/api/v1" + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.openrouter_key + or get_secret("OPENROUTER_API_KEY") + or get_secret("OR_API_KEY") + ) + + openrouter_site_url = get_secret("OR_SITE_URL") or "https://litellm.ai" + openrouter_app_name = get_secret("OR_APP_NAME") or "liteLLM" + + openrouter_headers = { + "HTTP-Referer": openrouter_site_url, + "X-Title": openrouter_app_name, + } + + _headers = headers or litellm.headers + if _headers: + openrouter_headers.update(_headers) + + headers = openrouter_headers + + response = base_llm_http_handler.embedding( + model=model, + input=input, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + logging_obj=logging, + timeout=timeout, + model_response=EmbeddingResponse(), + optional_params=optional_params, + client=client, + aembedding=aembedding, + litellm_params=litellm_params_dict, + headers=headers, + ) elif custom_llm_provider == "huggingface": api_key = ( api_key diff --git a/litellm/utils.py b/litellm/utils.py index fbbaa94f7a1..fb01337bc50 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7718,6 +7718,11 @@ class ProviderConfigManager: return litellm.CometAPIEmbeddingConfig() elif litellm.LlmProviders.GITHUB_COPILOT == provider: return litellm.GithubCopilotEmbeddingConfig() + elif litellm.LlmProviders.OPENROUTER == provider: + from litellm.llms.openrouter.embedding.transformation import ( + OpenrouterEmbeddingConfig, + ) + return OpenrouterEmbeddingConfig() elif litellm.LlmProviders.GIGACHAT == provider: return litellm.GigaChatEmbeddingConfig() elif litellm.LlmProviders.SAGEMAKER == provider: diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 2521d71b5c0..a29b66f6c54 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1565,7 +1565,7 @@ "chat_completions": true, "messages": true, "responses": true, - "embeddings": false, + "embeddings": true, "image_generations": false, "audio_transcriptions": false, "audio_speech": false, diff --git a/tests/llm_translation/test_openrouter.py b/tests/llm_translation/test_openrouter.py index 839d08e12bf..105b05d3449 100644 --- a/tests/llm_translation/test_openrouter.py +++ b/tests/llm_translation/test_openrouter.py @@ -32,3 +32,18 @@ def test_completion_openrouter_image_generation(): .message.images[0]["image_url"]["url"] .startswith("data:image/png;base64,") ) + + +def test_openrouter_embedding(): + """Test OpenRouter embeddings support.""" + litellm._turn_on_debug() + resp = litellm.embedding( + model="openrouter/openai/text-embedding-3-small", + input=["Hello world", "How are you?"], + ) + print(resp) + assert resp is not None + assert len(resp.data) == 2 + assert resp.data[0]["embedding"] is not None + assert isinstance(resp.data[0]["embedding"], list) + assert len(resp.data[0]["embedding"]) > 0 diff --git a/tests/test_litellm/llms/openrouter/test_openrouter_embedding_transformation.py b/tests/test_litellm/llms/openrouter/test_openrouter_embedding_transformation.py new file mode 100644 index 00000000000..714adc346db --- /dev/null +++ b/tests/test_litellm/llms/openrouter/test_openrouter_embedding_transformation.py @@ -0,0 +1,132 @@ +""" +Unit tests for OpenRouter embedding transformation logic. +""" +from litellm.llms.openrouter.embedding.transformation import ( + OpenrouterEmbeddingConfig, +) + + +def test_openrouter_embedding_supported_params(): + """Test that supported OpenAI params are correctly defined.""" + config = OpenrouterEmbeddingConfig() + supported = config.get_supported_openai_params("test-model") + + assert "timeout" in supported + assert "dimensions" in supported + assert "encoding_format" in supported + assert "user" in supported + + +def test_openrouter_embedding_transform_request(): + """Test request transformation logic.""" + config = OpenrouterEmbeddingConfig() + + # Test with string input + result = config.transform_embedding_request( + model="openrouter/google/text-embedding-004", + input="Hello world", + optional_params={}, + headers={}, + ) + + assert result["model"] == "google/text-embedding-004" + assert result["input"] == ["Hello world"] + + # Test with list input + result = config.transform_embedding_request( + model="google/text-embedding-004", + input=["Hello", "World"], + optional_params={"dimensions": 512}, + headers={}, + ) + + assert result["model"] == "google/text-embedding-004" + assert result["input"] == ["Hello", "World"] + assert result["dimensions"] == 512 + + +def test_openrouter_embedding_validate_environment(): + """Test environment validation and header setup.""" + config = OpenrouterEmbeddingConfig() + + # Test with API key + headers = config.validate_environment( + headers={"Custom-Header": "value"}, + model="test-model", + messages=[], + optional_params={}, + litellm_params={}, + api_key="test-api-key", + ) + + # Should include OpenRouter-specific headers + assert "HTTP-Referer" in headers + assert "X-Title" in headers + # Should include Content-Type header + assert "Content-Type" in headers + assert headers["Content-Type"] == "application/json" + # Should include Authorization header + assert "Authorization" in headers + assert headers["Authorization"] == "Bearer test-api-key" + # Should preserve custom headers + assert headers["Custom-Header"] == "value" + + # Test without API key + headers_no_key = config.validate_environment( + headers={}, + model="test-model", + messages=[], + optional_params={}, + litellm_params={}, + api_key=None, + ) + + # Should still include OpenRouter headers but not Authorization + assert "HTTP-Referer" in headers_no_key + assert "X-Title" in headers_no_key + assert "Content-Type" in headers_no_key + assert "Authorization" not in headers_no_key + + +def test_openrouter_embedding_get_complete_url(): + """Test URL construction.""" + config = OpenrouterEmbeddingConfig() + + url = config.get_complete_url( + api_base="https://openrouter.ai/api/v1", + api_key="test-key", + model="test-model", + optional_params={}, + litellm_params={}, + ) + + assert url == "https://openrouter.ai/api/v1/embeddings" + + # Test with trailing slash + url = config.get_complete_url( + api_base="https://openrouter.ai/api/v1/", + api_key="test-key", + model="test-model", + optional_params={}, + litellm_params={}, + ) + + assert url == "https://openrouter.ai/api/v1/embeddings" + + +def test_openrouter_embedding_map_params(): + """Test parameter mapping.""" + config = OpenrouterEmbeddingConfig() + + result = config.map_openai_params( + non_default_params={"dimensions": 512, "timeout": 30, "unsupported": "value"}, + optional_params={}, + model="test-model", + drop_params=False, + ) + + # Supported params should be included + assert result["dimensions"] == 512 + assert result["timeout"] == 30 + # Unsupported params should not be included + assert "unsupported" not in result From ad501048f391ee473fe1fa2e6b69095a076f4630 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 15:22:14 +0530 Subject: [PATCH 18/56] Add support for Vertex AI API keys --- docs/my-website/docs/providers/vertex.md | 45 +++++- litellm/llms/gemini/common_utils.py | 9 ++ litellm/llms/vertex_ai/vertex_llm_base.py | 40 +++-- litellm/main.py | 10 +- .../llms/vertex_ai/test_vertex_llm_base.py | 137 ++++++++++++++++++ 5 files changed, 226 insertions(+), 15 deletions(-) diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 33ebf535d29..f46608aa57c 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -35,6 +35,8 @@ import json # !gcloud auth application-default login - run this to add vertex credentials to your env ## OR ## file_path = 'path/to/vertex_ai_service_account.json' +## OR ## +export VERTEXAI_API_KEY="your-api-key" # Load the JSON file with open(file_path, 'r') as file: @@ -47,7 +49,7 @@ vertex_credentials_json = json.dumps(vertex_credentials) response = completion( model="vertex_ai/gemini-2.5-pro", messages=[{ "content": "Hello, how are you?","role": "user"}], - vertex_credentials=vertex_credentials_json + vertex_credentials=vertex_credentials_json # Can remove this is added VERTEXAI_API_KEY in env ) ``` @@ -1329,15 +1331,41 @@ Here's how to use Vertex AI with the LiteLLM Proxy Server ## Authentication - vertex_project, vertex_location, etc. +LiteLLM supports two authentication methods for Vertex AI: + +1. **API Key Authentication** (Recommended for getting started) +2. **Service Account Credentials** (Recommended for production) + Set your vertex credentials via: - dynamic params OR - env vars +### **Authentication Method 1: -### **Dynamic Params** +The simplest way to authenticate with Vertex AI. You can set: +- `api_key` (str) - Your Vertex AI API key -You can set: +**Environment Variables:** +```bash +export VERTEXAI_API_KEY="your-api-key" +``` + +**Or pass as parameters:** +```python +from litellm import completion + +response = completion( + model="vertex_ai/gemini-2.0-flash-exp", + messages=[{"role": "user", "content": "Hello!"}], + api_key="your-vertex-api-key", + +) +``` + +### **Authentication Method 2: Service Account Credentials** + +For production environments with fine-grained access control. You can set: - `vertex_credentials` (str) - can be a json string or filepath to your vertex ai service account.json - `vertex_location` (str) - place where vertex model is deployed (us-central1, asia-southeast1, etc.). Some models support the global location, please see [Vertex AI documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations#supported_models) - `vertex_project` Optional[str] - use if vertex project different from the one in vertex_credentials @@ -1392,7 +1420,16 @@ model_list: ### **Environment Variables** -You can set: +#### For API Key Authentication: + +- `VERTEXAI_API_KEY` or `VERTEX_API_KEY` - Your Vertex AI API key + +```bash +export VERTEXAI_API_KEY="your-vertex-api-key" +``` + +#### For Service Account Authentication: + - `GOOGLE_APPLICATION_CREDENTIALS` - store the filepath for your service_account.json in here (used by vertex sdk directly). - VERTEXAI_LOCATION - place where vertex model is deployed (us-central1, asia-southeast1, etc.) - VERTEXAI_PROJECT - Optional[str] - use if vertex project different from the one in vertex_credentials diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index e53829d3329..30c5b4f17c5 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -150,6 +150,15 @@ def get_api_key_from_env() -> Optional[str]: return get_secret_str("GOOGLE_API_KEY") or get_secret_str("GEMINI_API_KEY") +def get_vertex_api_key_from_env() -> Optional[str]: + """ + Get API key from environment for Vertex AI. + Checks VERTEXAI_API_KEY and VERTEX_API_KEY environment variables. + This allows using Vertex AI with API keys instead of service account credentials. + """ + return get_secret_str("VERTEXAI_API_KEY") or get_secret_str("VERTEX_API_KEY") + + class GoogleAIStudioTokenCounter(BaseTokenCounter): """Token counter implementation for Google AI Studio provider.""" def should_use_token_counting_api( diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 826f151df35..a3606ff9deb 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -388,6 +388,10 @@ class VertexBase: Internal function. Returns the token and url for the call. Handles logic if it's google ai studio vs. vertex ai. + + For Vertex AI: + - If gemini_api_key is provided, use API key authentication (x-goog-api-key header) + - Otherwise, use service account credentials (OAuth2 Bearer token) Returns token, url @@ -400,7 +404,7 @@ class VertexBase: stream=stream, gemini_api_key=gemini_api_key, ) - auth_header = None # this field is not used for gemin + auth_header = None # this field is not used for gemini else: vertex_location = self.get_vertex_region( vertex_region=vertex_location, @@ -409,14 +413,32 @@ class VertexBase: ### SET RUNTIME ENDPOINT ### version = "v1beta1" if should_use_v1beta1_features is True else "v1" - url, endpoint = _get_vertex_url( - mode=mode, - model=model, - stream=stream, - vertex_project=vertex_project, - vertex_location=vertex_location, - vertex_api_version=version, - ) + + # Check if using API key authentication for Vertex AI + if gemini_api_key and not vertex_credentials: + # When using API key with Vertex AI, use the Google AI Studio endpoint + # This is because Vertex AI API keys work with generativelanguage.googleapis.com + verbose_logger.debug( + f"Using Vertex AI API key authentication for model: {model} - routing to Google AI Studio endpoint" + ) + url, endpoint = _get_gemini_url( + mode=mode, + model=model, + stream=stream, + gemini_api_key=gemini_api_key, + ) + # API key is already included in the URL by _get_gemini_url + auth_header = None + else: + # Use OAuth2 Bearer token authentication (traditional Vertex AI) + url, endpoint = _get_vertex_url( + mode=mode, + model=model, + stream=stream, + vertex_project=vertex_project, + vertex_location=vertex_location, + vertex_api_version=version, + ) return self._check_custom_proxy( api_base=api_base, diff --git a/litellm/main.py b/litellm/main.py index e8a8b504d96..0264d0a7a71 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -189,7 +189,7 @@ from .llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from .llms.custom_llm import CustomLLM, custom_chat_llm_router from .llms.databricks.embed.handler import DatabricksEmbeddingHandler from .llms.deprecated_providers import aleph_alpha, palm -from .llms.gemini.common_utils import get_api_key_from_env +from .llms.gemini.common_utils import get_api_key_from_env, get_vertex_api_key_from_env from .llms.groq.chat.handler import GroqChatCompletion from .llms.heroku.chat.transformation import HerokuChatConfig from .llms.huggingface.embedding.handler import HuggingFaceEmbedding @@ -3230,6 +3230,12 @@ def completion( # type: ignore # noqa: PLR0915 or get_secret("VERTEXAI_CREDENTIALS") ) + vertex_api_key = ( + api_key + or get_vertex_api_key_from_env() + or litellm.api_key + ) + api_base = api_base or litellm.api_base or get_secret("VERTEXAI_API_BASE") new_params = safe_deep_copy(optional_params or {}) @@ -3271,7 +3277,7 @@ def completion( # type: ignore # noqa: PLR0915 vertex_location=vertex_ai_location, vertex_project=vertex_ai_project, vertex_credentials=vertex_credentials, - gemini_api_key=None, + gemini_api_key=vertex_api_key, # Support for Vertex AI API Key logging_obj=logging, acompletion=acompletion, timeout=timeout, diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py b/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py index 389c8446135..80d65991acb 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py @@ -13,6 +13,7 @@ sys.path.insert( import litellm from litellm.llms.vertex_ai.vertex_llm_base import VertexBase +from litellm.llms.vertex_ai.common_utils import _get_gemini_url def run_sync(coro): @@ -1048,3 +1049,139 @@ class TestVertexBase: MockCredentials.from_info.assert_called_once_with(json_obj) mock_creds.with_scopes.assert_called_once_with(scopes) assert result == "scoped_creds" + + def test_get_token_and_url_with_api_key(self): + """Test that API key authentication routes to Google AI Studio endpoint""" + vertex_base = VertexBase() + + # Test with API key and no credentials - should use Google AI Studio endpoint + auth_header, url = vertex_base._get_token_and_url( + model="gemini-2.0-flash-exp", + auth_header=None, + gemini_api_key="test-api-key-123", + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=None, # No service account credentials + stream=False, + custom_llm_provider="vertex_ai", + api_base=None, + should_use_v1beta1_features=False, + mode="chat", + ) + + # Should route to Google AI Studio endpoint + assert "generativelanguage.googleapis.com" in url + assert "gemini-2.0-flash-exp" in url + assert "key=test-api-key-123" in url + assert auth_header is None # API key is in URL, not header + + def test_get_token_and_url_with_credentials(self): + """Test that service account credentials route to Vertex AI endpoint""" + vertex_base = VertexBase() + + mock_creds = MagicMock() + mock_creds.token = "mock-bearer-token" + mock_creds.expired = False + + with patch.object( + vertex_base, "_ensure_access_token", return_value=("mock-bearer-token", "test-project") + ): + # Test with credentials - should use Vertex AI endpoint + auth_header, url = vertex_base._get_token_and_url( + model="gemini-2.0-flash-exp", + auth_header="mock-bearer-token", + gemini_api_key=None, + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials={"type": "service_account"}, + stream=False, + custom_llm_provider="vertex_ai", + api_base=None, + should_use_v1beta1_features=False, + mode="chat", + ) + + # Should route to Vertex AI endpoint + assert "aiplatform.googleapis.com" in url + assert "projects/test-project" in url + assert "locations/us-central1" in url + assert auth_header == "mock-bearer-token" + + def test_get_token_and_url_api_key_with_streaming(self): + """Test API key authentication with streaming enabled""" + vertex_base = VertexBase() + + auth_header, url = vertex_base._get_token_and_url( + model="gemini-2.0-flash-exp", + auth_header=None, + gemini_api_key="test-api-key-456", + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=None, + stream=True, # Streaming enabled + custom_llm_provider="vertex_ai", + api_base=None, + should_use_v1beta1_features=False, + mode="chat", + ) + + # Should route to Google AI Studio endpoint with streaming + assert "generativelanguage.googleapis.com" in url + assert "streamGenerateContent" in url + assert "key=test-api-key-456" in url + assert "alt=sse" in url + assert auth_header is None + + def test_get_token_and_url_api_key_priority(self): + """Test that credentials take priority over API key when both are provided""" + vertex_base = VertexBase() + + # When both API key and credentials are provided, credentials take priority + mock_creds = MagicMock() + mock_creds.token = "mock-bearer-token" + mock_creds.expired = False + + with patch.object( + vertex_base, "_ensure_access_token", return_value=("mock-bearer-token", "test-project") + ): + auth_header, url = vertex_base._get_token_and_url( + model="gemini-2.0-flash-exp", + auth_header="mock-bearer-token", + gemini_api_key="test-api-key-789", + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials={"type": "service_account"}, # Credentials provided + stream=False, + custom_llm_provider="vertex_ai", + api_base=None, + should_use_v1beta1_features=False, + mode="chat", + ) + + # Should use Vertex AI endpoint with Bearer token (credentials take priority) + assert "aiplatform.googleapis.com" in url + assert auth_header == "mock-bearer-token" + + def test_get_token_and_url_with_embedding_mode(self): + """Test API key authentication with embedding mode""" + vertex_base = VertexBase() + + auth_header, url = vertex_base._get_token_and_url( + model="text-embedding-004", + auth_header=None, + gemini_api_key="test-embedding-key", + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=None, + stream=False, + custom_llm_provider="vertex_ai", + api_base=None, + should_use_v1beta1_features=False, + mode="embedding", + ) + + # Should route to Google AI Studio endpoint for embeddings + assert "generativelanguage.googleapis.com" in url + assert "embedContent" in url + assert "key=test-embedding-key" in url + assert auth_header is None \ No newline at end of file From cfda03ebe1229d5e0da1a48e9c23792172e76c52 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Thu, 8 Jan 2026 14:55:36 -0300 Subject: [PATCH 19/56] fix(gemini): support snake_case for google_search tool parameters (#18451) * fix(gemini): support snake_case for google_search tool parameters Add snake_case aliases for Gemini tool names to match the pattern already used by other tools (url_context, google_maps, code_execution): - google_search -> googleSearch - google_search_retrieval -> googleSearchRetrieval - enterprise_web_search -> enterpriseWebSearch * test(gemini): add tests for snake_case google_search tool aliases * refactor(gemini): simplify get_tool_value calls formatting --- .../vertex_and_google_ai_studio_gemini.py | 27 ++++---- .../vertex_ai/gemini/test_transformation.py | 64 ++++++++++++++++++- 2 files changed, 77 insertions(+), 14 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index ba1788a217f..3c93b1943e4 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -480,20 +480,21 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): or tool_name == VertexToolName.CODE_EXECUTION.value ): # code_execution maintained for backwards compatibility code_execution = self.get_tool_value(tool, "codeExecution") - elif tool_name and tool_name == VertexToolName.GOOGLE_SEARCH.value: - googleSearch = self.get_tool_value( - tool, VertexToolName.GOOGLE_SEARCH.value - ) - elif ( - tool_name and tool_name == VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value + elif tool_name and ( + tool_name == VertexToolName.GOOGLE_SEARCH.value + or tool_name == "google_search" ): - googleSearchRetrieval = self.get_tool_value( - tool, VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value - ) - elif tool_name and tool_name == VertexToolName.ENTERPRISE_WEB_SEARCH.value: - enterpriseWebSearch = self.get_tool_value( - tool, VertexToolName.ENTERPRISE_WEB_SEARCH.value - ) + googleSearch = self.get_tool_value(tool, tool_name) + elif tool_name and ( + tool_name == VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value + or tool_name == "google_search_retrieval" + ): + googleSearchRetrieval = self.get_tool_value(tool, tool_name) + elif tool_name and ( + tool_name == VertexToolName.ENTERPRISE_WEB_SEARCH.value + or tool_name == "enterprise_web_search" + ): + enterpriseWebSearch = self.get_tool_value(tool, tool_name) elif tool_name and ( tool_name == VertexToolName.URL_CONTEXT.value or tool_name == "urlContext" diff --git a/tests/litellm/llms/vertex_ai/gemini/test_transformation.py b/tests/litellm/llms/vertex_ai/gemini/test_transformation.py index 6d005af28ac..20f48b6f393 100644 --- a/tests/litellm/llms/vertex_ai/gemini/test_transformation.py +++ b/tests/litellm/llms/vertex_ai/gemini/test_transformation.py @@ -7,6 +7,7 @@ sys.path.insert( 0, os.path.abspath("../../../../..") ) # Adds the parent directory to the system path from litellm.llms.vertex_ai.gemini import transformation +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig from litellm.types.llms import openai from litellm.types import completion from litellm.types.llms.vertex_ai import RequestBody @@ -225,4 +226,65 @@ async def test__transform_request_body_image_config_with_image_size(): assert "generationConfig" in rb assert "imageConfig" in rb["generationConfig"] assert rb["generationConfig"]["imageConfig"]["aspectRatio"] == "16:9" - assert rb["generationConfig"]["imageConfig"]["imageSize"] == "4K" \ No newline at end of file + assert rb["generationConfig"]["imageConfig"]["imageSize"] == "4K" + + +def test_map_function_google_search_snake_case(): + """ + Test that google_search tool (snake_case) is properly mapped to googleSearch. + Fixes issue where tools=[{"google_search": {}}] was being stripped. + """ + config = VertexGeminiConfig() + optional_params = {} + + # Test snake_case google_search + tools = [{"google_search": {}}] + result = config._map_function(tools, optional_params) + + assert len(result) == 1 + assert "googleSearch" in result[0] + assert result[0]["googleSearch"] == {} + + +def test_map_function_google_search_camel_case(): + """ + Test that googleSearch tool (camelCase) still works. + """ + config = VertexGeminiConfig() + optional_params = {} + + # Test camelCase googleSearch + tools = [{"googleSearch": {}}] + result = config._map_function(tools, optional_params) + + assert len(result) == 1 + assert "googleSearch" in result[0] + assert result[0]["googleSearch"] == {} + + +def test_map_function_google_search_retrieval_snake_case(): + """ + Test that google_search_retrieval tool (snake_case) is properly mapped. + """ + config = VertexGeminiConfig() + optional_params = {} + + tools = [{"google_search_retrieval": {"dynamic_retrieval_config": {"mode": "MODE_DYNAMIC"}}}] + result = config._map_function(tools, optional_params) + + assert len(result) == 1 + assert "googleSearchRetrieval" in result[0] + + +def test_map_function_enterprise_web_search_snake_case(): + """ + Test that enterprise_web_search tool (snake_case) is properly mapped. + """ + config = VertexGeminiConfig() + optional_params = {} + + tools = [{"enterprise_web_search": {}}] + result = config._map_function(tools, optional_params) + + assert len(result) == 1 + assert "enterpriseWebSearch" in result[0] \ No newline at end of file From 3ebec39b740395efe6d97b175fc8119006a67607 Mon Sep 17 00:00:00 2001 From: Constantine Date: Thu, 8 Jan 2026 20:56:46 +0300 Subject: [PATCH 20/56] fix(proxy): use async anthropic client to prevent event loop blocking (#18435) Fixes #16716. Previously, synchronous Anthropic client was used for token counting, which blocked the event loop. This change switches to AsyncAnthropic and caches the client instance. --- litellm/proxy/utils.py | 10 ++++-- tests/test_litellm/test_utils_custom.py | 42 +++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/test_utils_custom.py diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index d1a78534dae..bd44cef9541 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -131,6 +131,7 @@ else: unified_guardrail = UnifiedLLMGuardrails() +_anthropic_async_clients = {} def print_verbose(print_statement): """ @@ -4254,11 +4255,16 @@ async def count_tokens_with_anthropic_api( if anthropic_api_key and messages: # Call Anthropic API directly for more accurate token counting - client = anthropic.Anthropic(api_key=anthropic_api_key) + + # Use cached client if available to avoid socket exhaustion + if anthropic_api_key not in _anthropic_async_clients: + _anthropic_async_clients[anthropic_api_key] = anthropic.AsyncAnthropic(api_key=anthropic_api_key) + + client = _anthropic_async_clients[anthropic_api_key] # Call with explicit parameters to satisfy type checking # Type ignore for now since messages come from generic dict input - response = client.beta.messages.count_tokens( + response = await client.beta.messages.count_tokens( model=model_to_use, messages=messages, # type: ignore betas=["token-counting-2024-11-01"], diff --git a/tests/test_litellm/test_utils_custom.py b/tests/test_litellm/test_utils_custom.py new file mode 100644 index 00000000000..292da4132b9 --- /dev/null +++ b/tests/test_litellm/test_utils_custom.py @@ -0,0 +1,42 @@ +import pytest +from unittest.mock import MagicMock, patch, AsyncMock +from litellm.proxy.utils import count_tokens_with_anthropic_api, _anthropic_async_clients + +@pytest.mark.asyncio +async def test_count_tokens_caching(): + """ + Test that count_tokens_with_anthropic_api caches the client. + """ + # Clear cache + _anthropic_async_clients.clear() + + api_key = "sk-ant-test-key" + messages = [{"role": "user", "content": "hello"}] + model = "claude-3-opus-20240229" + + # Mock anthropic + with patch("anthropic.AsyncAnthropic") as mock_cls: + mock_client = MagicMock() + mock_cls.return_value = mock_client + + # Mock response + mock_response = MagicMock() + mock_response.input_tokens = 10 + + # Setup async return for count_tokens + mock_client.beta.messages.count_tokens = AsyncMock(return_value=mock_response) + + # First call + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": api_key}): + await count_tokens_with_anthropic_api(model, messages) + + assert api_key in _anthropic_async_clients + assert _anthropic_async_clients[api_key] == mock_client + mock_cls.assert_called_once() # Should be called once + + # Second call + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": api_key}): + await count_tokens_with_anthropic_api(model, messages) + + # Should still be called once (cached) + mock_cls.assert_called_once() From 516e4f8b9652cb6199b5e504de97835a51f07592 Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Thu, 8 Jan 2026 23:53:36 +0530 Subject: [PATCH 21/56] fix: proactive RDS IAM token refresh to prevent 15-min connection failed (#18795) * fix: proactive RDS IAM token refresh to prevent 15-min connection failures (#16220) * fix: add noqa for PLR0915 in proxy_startup_event --- litellm/proxy/db/prisma_client.py | 309 +++++++++++++++--- litellm/proxy/proxy_server.py | 97 +++--- .../proxy/db/test_rds_iam_token_expiry.py | 275 ++++++++++++++++ 3 files changed, 602 insertions(+), 79 deletions(-) create mode 100644 tests/test_litellm/proxy/db/test_rds_iam_token_expiry.py diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index 406ddceabf5..c9c0cfe8f68 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -17,50 +17,141 @@ from litellm.secret_managers.main import str_to_bool class PrismaWrapper: + """ + Wrapper around Prisma client that handles RDS IAM token authentication. + + When iam_token_db_auth is enabled, this wrapper: + 1. Proactively refreshes IAM tokens before they expire (background task) + 2. Falls back to synchronous refresh if a token is found expired + 3. Uses proper locking to prevent race conditions during reconnection + + RDS IAM tokens are valid for 15 minutes. This wrapper refreshes them + 3 minutes before expiration to ensure uninterrupted database connectivity. + """ + + # Buffer time in seconds before token expiration to trigger refresh + # Refresh 3 minutes (180 seconds) before the token expires + TOKEN_REFRESH_BUFFER_SECONDS = 180 + + # Fallback refresh interval if token parsing fails (10 minutes) + FALLBACK_REFRESH_INTERVAL_SECONDS = 600 + def __init__(self, original_prisma: Any, iam_token_db_auth: bool): self._original_prisma = original_prisma self.iam_token_db_auth = iam_token_db_auth + # Background token refresh task management + self._token_refresh_task: Optional[asyncio.Task] = None + self._reconnection_lock = asyncio.Lock() + self._last_refresh_time: Optional[datetime] = None + + def _extract_token_from_db_url(self, db_url: Optional[str]) -> Optional[str]: + """ + Extract the token (password) from the DATABASE_URL. + + The token contains the AWS signature with X-Amz-Date and X-Amz-Expires parameters. + + Important: We must parse the URL while it's still encoded to preserve structure, + then decode the password portion. Otherwise the '?' in the token breaks URL parsing. + """ + if db_url is None: + return None + try: + # Parse URL while still encoded to preserve structure + parsed = urllib.parse.urlparse(db_url) + if parsed.password: + # Now decode just the password/token + return urllib.parse.unquote(parsed.password) + return None + except Exception: + return None + + def _parse_token_expiration(self, token: Optional[str]) -> Optional[datetime]: + """ + Parse the token to extract its expiration time. + + Returns the datetime when the token expires, or None if parsing fails. + """ + if token is None: + return None + + try: + # Token format: ...?X-Amz-Date=YYYYMMDDTHHMMSSZ&X-Amz-Expires=900&... + if "?" not in token: + return None + + query_string = token.split("?", 1)[1] + params = urllib.parse.parse_qs(query_string) + + expires_str = params.get("X-Amz-Expires", [None])[0] + date_str = params.get("X-Amz-Date", [None])[0] + + if not expires_str or not date_str: + return None + + token_created = datetime.strptime(date_str, "%Y%m%dT%H%M%SZ") + expires_in = int(expires_str) + + return token_created + timedelta(seconds=expires_in) + except Exception as e: + verbose_proxy_logger.debug(f"Failed to parse token expiration: {e}") + return None + + def _calculate_seconds_until_refresh(self) -> float: + """ + Calculate exactly how many seconds until we need to refresh the token. + + Uses precise timing: sleeps until (token_expiration - buffer_seconds). + For a 15-minute (900s) token with 180s buffer, this returns ~720s (12 min). + + Returns: + Number of seconds to sleep before the next refresh. + Returns 0 if token should be refreshed immediately. + Returns FALLBACK_REFRESH_INTERVAL_SECONDS if parsing fails. + """ + db_url = os.getenv("DATABASE_URL") + token = self._extract_token_from_db_url(db_url) + expiration_time = self._parse_token_expiration(token) + + if expiration_time is None: + # If we can't parse the token, use fallback interval + verbose_proxy_logger.debug( + f"Could not parse token expiration, using fallback interval of " + f"{self.FALLBACK_REFRESH_INTERVAL_SECONDS}s" + ) + return self.FALLBACK_REFRESH_INTERVAL_SECONDS + + # Calculate when we should refresh (expiration - buffer) + refresh_at = expiration_time - timedelta( + seconds=self.TOKEN_REFRESH_BUFFER_SECONDS + ) + + # How long until refresh time? + now = datetime.utcnow() + seconds_until_refresh = (refresh_at - now).total_seconds() + + # If already past refresh time, return 0 (refresh immediately) + return max(0, seconds_until_refresh) + def is_token_expired(self, token_url: Optional[str]) -> bool: + """Check if the token in the given URL is expired.""" if token_url is None: return True - # Decode the token URL to handle URL-encoded characters - decoded_url = urllib.parse.unquote(token_url) - # Parse the token URL - parsed_url = urllib.parse.urlparse(decoded_url) + token = self._extract_token_from_db_url(token_url) + expiration_time = self._parse_token_expiration(token) - # Parse the query parameters from the path component (if they exist there) - query_params = urllib.parse.parse_qs(parsed_url.query) + if expiration_time is None: + # If we can't parse the token, assume it's expired to trigger refresh + verbose_proxy_logger.debug( + "Could not parse token expiration, treating as expired" + ) + return True - # Get expiration time from the query parameters - expires = query_params.get("X-Amz-Expires", [None])[0] - if expires is None: - raise ValueError("X-Amz-Expires parameter is missing or invalid.") - - expires_int = int(expires) - - # Get the token's creation time from the X-Amz-Date parameter - token_time_str = query_params.get("X-Amz-Date", [""])[0] - if not token_time_str: - raise ValueError("X-Amz-Date parameter is missing or invalid.") - - # Ensure the token time string is parsed correctly - try: - token_time = datetime.strptime(token_time_str, "%Y%m%dT%H%M%SZ") - except ValueError as e: - raise ValueError(f"Invalid X-Amz-Date format: {e}") - - # Calculate the expiration time - expiration_time = token_time + timedelta(seconds=expires_int) - - # Current time in UTC - current_time = datetime.utcnow() - - # Check if the token is expired - return current_time > expiration_time + return datetime.utcnow() > expiration_time def get_rds_iam_token(self) -> Optional[str]: + """Generate a new RDS IAM token and update DATABASE_URL.""" if self.iam_token_db_auth: from litellm.proxy.auth.rds_iam_token import generate_iam_auth_token @@ -74,7 +165,6 @@ class PrismaWrapper: db_host=db_host, db_port=db_port, db_user=db_user ) - # print(f"token: {token}") _db_url = f"postgresql://{db_user}:{token}@{db_host}:{db_port}/{db_name}" if db_schema: _db_url += f"?schema={db_schema}" @@ -86,6 +176,7 @@ class PrismaWrapper: async def recreate_prisma_client( self, new_db_url: str, http_client: Optional[Any] = None ): + """Disconnect and reconnect the Prisma client with a new database URL.""" from prisma import Prisma # type: ignore try: @@ -100,21 +191,159 @@ class PrismaWrapper: await self._original_prisma.connect() + async def start_token_refresh_task(self) -> None: + """ + Start the background token refresh task. + + This task proactively refreshes RDS IAM tokens before they expire, + preventing connection failures. Should be called after the initial + Prisma client connection is established. + """ + if not self.iam_token_db_auth: + verbose_proxy_logger.debug( + "IAM token auth not enabled, skipping token refresh task" + ) + return + + if self._token_refresh_task is not None: + verbose_proxy_logger.debug("Token refresh task already running") + return + + self._token_refresh_task = asyncio.create_task(self._token_refresh_loop()) + verbose_proxy_logger.info( + "Started RDS IAM token proactive refresh background task" + ) + + async def stop_token_refresh_task(self) -> None: + """ + Stop the background token refresh task gracefully. + + Should be called during application shutdown to clean up resources. + """ + if self._token_refresh_task is None: + return + + self._token_refresh_task.cancel() + try: + await self._token_refresh_task + except asyncio.CancelledError: + pass + self._token_refresh_task = None + verbose_proxy_logger.info("Stopped RDS IAM token refresh background task") + + async def _token_refresh_loop(self) -> None: + """ + Background loop that proactively refreshes RDS IAM tokens before expiration. + + Uses precise timing: calculates the exact sleep duration until the token + needs to be refreshed (expiration - 3 minute buffer), then refreshes. + This is more efficient than polling, requiring only 1 wake-up per token cycle. + """ + verbose_proxy_logger.info( + f"RDS IAM token refresh loop started. " + f"Tokens will be refreshed {self.TOKEN_REFRESH_BUFFER_SECONDS}s before expiration." + ) + + while True: + try: + # Calculate exactly how long to sleep until next refresh + sleep_seconds = self._calculate_seconds_until_refresh() + + if sleep_seconds > 0: + verbose_proxy_logger.info( + f"RDS IAM token refresh scheduled in {sleep_seconds:.0f} seconds " + f"({sleep_seconds / 60:.1f} minutes)" + ) + await asyncio.sleep(sleep_seconds) + + # Refresh the token + verbose_proxy_logger.info("Proactively refreshing RDS IAM token...") + await self._safe_refresh_token() + + except asyncio.CancelledError: + verbose_proxy_logger.info("RDS IAM token refresh loop cancelled") + break + except Exception as e: + verbose_proxy_logger.error( + f"Error in RDS IAM token refresh loop: {e}. " + f"Retrying in {self.FALLBACK_REFRESH_INTERVAL_SECONDS}s..." + ) + # On error, wait before retrying to avoid tight error loops + try: + await asyncio.sleep(self.FALLBACK_REFRESH_INTERVAL_SECONDS) + except asyncio.CancelledError: + break + + async def _safe_refresh_token(self) -> None: + """ + Refresh the RDS IAM token with proper locking to prevent race conditions. + + Uses an asyncio lock to ensure only one refresh operation happens at a time, + preventing multiple concurrent reconnection attempts. + """ + async with self._reconnection_lock: + new_db_url = self.get_rds_iam_token() + if new_db_url: + await self.recreate_prisma_client(new_db_url) + self._last_refresh_time = datetime.utcnow() + verbose_proxy_logger.info( + "RDS IAM token refreshed successfully. New token valid for ~15 minutes." + ) + else: + verbose_proxy_logger.error( + "Failed to generate new RDS IAM token during proactive refresh" + ) + def __getattr__(self, name: str): + """ + Proxy attribute access to the underlying Prisma client. + + If IAM token auth is enabled and the token is expired, this method + provides a synchronous fallback to refresh the token. However, this + should rarely be needed since the background task proactively refreshes + tokens before they expire. + + FIXED: Now properly waits for reconnection to complete before returning, + instead of the previous fire-and-forget pattern that caused the bug. + """ original_attr = getattr(self._original_prisma, name) + if self.iam_token_db_auth: db_url = os.getenv("DATABASE_URL") - if self.is_token_expired(db_url): - db_url = self.get_rds_iam_token() - loop = asyncio.get_event_loop() - if db_url: + # Check if token is expired (should be rare if background task is running) + if self.is_token_expired(db_url): + verbose_proxy_logger.warning( + "RDS IAM token expired in __getattr__ - proactive refresh may have failed. " + "Triggering synchronous fallback refresh..." + ) + + new_db_url = self.get_rds_iam_token() + if new_db_url: + loop = asyncio.get_event_loop() + if loop.is_running(): - asyncio.run_coroutine_threadsafe( - self.recreate_prisma_client(db_url), loop + # FIXED: Actually wait for the reconnection to complete! + # The previous code used fire-and-forget which caused the bug. + future = asyncio.run_coroutine_threadsafe( + self.recreate_prisma_client(new_db_url), loop ) + try: + # Wait up to 30 seconds for reconnection + future.result(timeout=30) + verbose_proxy_logger.info( + "Synchronous token refresh completed successfully" + ) + except Exception as e: + verbose_proxy_logger.error( + f"Failed to refresh token synchronously: {e}" + ) + raise else: - asyncio.run(self.recreate_prisma_client(db_url)) + asyncio.run(self.recreate_prisma_client(new_db_url)) + + # Get the NEW attribute after reconnection + original_attr = getattr(self._original_prisma, name) else: raise ValueError("Failed to get RDS IAM token") diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 06525e39133..a56a6379cb7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -533,9 +533,9 @@ except ImportError: server_root_path = os.getenv("SERVER_ROOT_PATH", "") _license_check = LicenseCheck() premium_user: bool = _license_check.is_premium() -premium_user_data: Optional["EnterpriseLicenseData"] = ( - _license_check.airgapped_license_data -) +premium_user_data: Optional[ + "EnterpriseLicenseData" +] = _license_check.airgapped_license_data global_max_parallel_request_retries_env: Optional[str] = os.getenv( "LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES" ) @@ -658,7 +658,7 @@ async def _initialize_shared_aiohttp_session(): @asynccontextmanager -async def proxy_startup_event(app: FastAPI): +async def proxy_startup_event(app: FastAPI): # noqa: PLR0915 global prisma_client, master_key, use_background_health_checks, llm_router, llm_model_list, general_settings, proxy_budget_rescheduler_min_time, proxy_budget_rescheduler_max_time, litellm_proxy_admin_name, db_writer_client, store_model_in_db, premium_user, _license_check, proxy_batch_polling_interval, shared_aiohttp_session import json @@ -788,6 +788,17 @@ async def proxy_startup_event(app: FastAPI): except Exception as e: verbose_proxy_logger.error(f"Error closing shared aiohttp session: {e}") + # Shutdown event - stop RDS IAM token refresh background task + if ( + prisma_client is not None + and hasattr(prisma_client, "db") + and hasattr(prisma_client.db, "stop_token_refresh_task") + ): + try: + await prisma_client.db.stop_token_refresh_task() + except Exception as e: + verbose_proxy_logger.error(f"Error stopping token refresh task: {e}") + await proxy_shutdown_event() # type: ignore[reportGeneralTypeIssues] @@ -1083,9 +1094,7 @@ try: # In non-root Docker, we restructure in /var/lib/litellm/ui. try: _restructure_ui_html_files(ui_path) - verbose_proxy_logger.info( - f"Restructured UI directory: {ui_path}" - ) + verbose_proxy_logger.info(f"Restructured UI directory: {ui_path}") except PermissionError as e: verbose_proxy_logger.exception( f"Permission error while restructuring UI directory {ui_path}: {e}" @@ -1171,9 +1180,9 @@ master_key: Optional[str] = None config_agents: Optional[List[AgentConfig]] = None otel_logging = False prisma_client: Optional[PrismaClient] = None -shared_aiohttp_session: Optional["ClientSession"] = ( - None # Global shared session for connection reuse -) +shared_aiohttp_session: Optional[ + "ClientSession" +] = None # Global shared session for connection reuse user_api_key_cache = DualCache( default_in_memory_ttl=UserAPIKeyCacheTTLEnum.in_memory_cache_ttl.value ) @@ -1181,9 +1190,9 @@ model_max_budget_limiter = _PROXY_VirtualKeyModelMaxBudgetLimiter( dual_cache=user_api_key_cache ) litellm.logging_callback_manager.add_litellm_callback(model_max_budget_limiter) -redis_usage_cache: Optional[RedisCache] = ( - None # redis cache used for tracking spend, tpm/rpm limits -) +redis_usage_cache: Optional[ + RedisCache +] = None # redis cache used for tracking spend, tpm/rpm limits polling_via_cache_enabled: Union[Literal["all"], List[str], bool] = False polling_cache_ttl: int = 3600 # Default 1 hour TTL for polling cache user_custom_auth = None @@ -1522,9 +1531,9 @@ async def update_cache( # noqa: PLR0915 _id = "team_id:{}".format(team_id) try: # Fetch the existing cost for the given user - existing_spend_obj: Optional[LiteLLM_TeamTable] = ( - await user_api_key_cache.async_get_cache(key=_id) - ) + existing_spend_obj: Optional[ + LiteLLM_TeamTable + ] = await user_api_key_cache.async_get_cache(key=_id) if existing_spend_obj is None: # do nothing if team not in api key cache return @@ -1876,7 +1885,6 @@ class ProxyConfig: "environment_variables" in config_to_save and config_to_save["environment_variables"] ): - # decrypt the environment_variables - in case a caller function has already encrypted the environment_variables decrypted_env_vars = self._decrypt_and_set_db_env_variables( environment_variables=config_to_save["environment_variables"], @@ -2794,21 +2802,21 @@ class ProxyConfig: verbose_proxy_logger.debug(f"_alerting_callbacks: {general_settings}") if _alerting_callbacks is None: return - + # Ensure proxy_logging_obj.alerting is set for all alerting types _alerting_value = general_settings.get("alerting", None) - verbose_proxy_logger.debug(f"_load_alerting_settings: Calling update_values with alerting={_alerting_value}") + verbose_proxy_logger.debug( + f"_load_alerting_settings: Calling update_values with alerting={_alerting_value}" + ) proxy_logging_obj.update_values( alerting=_alerting_value, alerting_threshold=general_settings.get("alerting_threshold", 600), alert_types=general_settings.get("alert_types", None), - alert_to_webhook_url=general_settings.get( - "alert_to_webhook_url", None - ), + alert_to_webhook_url=general_settings.get("alert_to_webhook_url", None), alerting_args=general_settings.get("alerting_args", None), redis_cache=redis_usage_cache, ) - + for _alert in _alerting_callbacks: if _alert == "slack": # [OLD] v0 implementation - already handled by update_values above @@ -3279,7 +3287,7 @@ class ProxyConfig: proxy_logging_obj: ProxyLogging """ _general_settings = config_data.get("general_settings", {}) - + if _general_settings is not None and "alerting" in _general_settings: if ( general_settings is not None @@ -3294,7 +3302,8 @@ class ProxyConfig: _merged_alerting = list(_yaml_alerting.union(_db_alerting)) # Preserve order: YAML values first, then DB values _merged_alerting = list(general_settings["alerting"]) + [ - item for item in _general_settings["alerting"] + item + for item in _general_settings["alerting"] if item not in general_settings["alerting"] ] verbose_proxy_logger.debug( @@ -3605,7 +3614,6 @@ class ProxyConfig: await self._init_vector_stores_in_db(prisma_client=prisma_client) if self._should_load_db_object(object_type="vector_store_indexes"): - await self._init_vector_store_indexes_in_db(prisma_client=prisma_client) if self._should_load_db_object(object_type="mcp"): @@ -3804,10 +3812,10 @@ class ProxyConfig: ) try: - guardrails_in_db: List[Guardrail] = ( - await GuardrailRegistry.get_all_guardrails_from_db( - prisma_client=prisma_client - ) + guardrails_in_db: List[ + Guardrail + ] = await GuardrailRegistry.get_all_guardrails_from_db( + prisma_client=prisma_client ) verbose_proxy_logger.debug( "guardrails from the DB %s", str(guardrails_in_db) @@ -4134,9 +4142,9 @@ async def initialize( # noqa: PLR0915 user_api_base = api_base dynamic_config[user_model]["api_base"] = api_base if api_version: - os.environ["AZURE_API_VERSION"] = ( - api_version # set this for azure - litellm can read this from the env - ) + os.environ[ + "AZURE_API_VERSION" + ] = api_version # set this for azure - litellm can read this from the env if max_tokens: # model-specific param dynamic_config[user_model]["max_tokens"] = max_tokens if temperature: # model-specific param @@ -4654,10 +4662,14 @@ class ProxyStartupEvent: replace_existing=True, misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, ) - verbose_proxy_logger.info("Responses cost check job scheduled successfully") + verbose_proxy_logger.info( + "Responses cost check job scheduled successfully" + ) except Exception as e: - verbose_proxy_logger.debug(f"Failed to setup responses cost checking: {e}") + verbose_proxy_logger.debug( + f"Failed to setup responses cost checking: {e}" + ) verbose_proxy_logger.debug( "Checking responses cost for LiteLLM Managed Files is an Enterprise Feature. Skipping..." ) @@ -4826,6 +4838,14 @@ class ProxyStartupEvent: await prisma_client.connect() + ## Start RDS IAM token refresh background task if enabled ## + # This proactively refreshes IAM tokens before they expire, + # preventing the 15-minute connection failure bug (#16220) + if hasattr(prisma_client, "db") and hasattr( + prisma_client.db, "start_token_refresh_task" + ): + await prisma_client.db.start_token_refresh_task() + ## Add necessary views to proxy ## asyncio.create_task( prisma_client.check_view_exists() @@ -5937,7 +5957,6 @@ async def realtime_websocket_endpoint( ), user_api_key_dict=Depends(user_api_key_auth_websocket), ): - await websocket.accept() # Only use explicit parameters, not all query params @@ -9525,9 +9544,9 @@ async def get_config_list( hasattr(sub_field_info, "description") and sub_field_info.description is not None ): - nested_fields[idx].field_description = ( - sub_field_info.description - ) + nested_fields[ + idx + ].field_description = sub_field_info.description idx += 1 _stored_in_db = None diff --git a/tests/test_litellm/proxy/db/test_rds_iam_token_expiry.py b/tests/test_litellm/proxy/db/test_rds_iam_token_expiry.py new file mode 100644 index 00000000000..1492acb0794 --- /dev/null +++ b/tests/test_litellm/proxy/db/test_rds_iam_token_expiry.py @@ -0,0 +1,275 @@ +""" +Tests for the RDS IAM token proactive refresh implementation. + +Tests for GitHub Issue #16220: RDS IAM authentication connection failures after 15 minutes. + +The fix implements: +1. Proactive background token refresh (refreshes 3 min before expiration) +2. Precise sleep timing (1 wake-up per token cycle instead of polling) +3. Proper locking during reconnection +4. Fixed __getattr__ fallback that now waits for reconnection + +Run these tests: + poetry run pytest tests/test_litellm/proxy/db/test_rds_iam_token_expiry.py -v -s +""" + +import asyncio +import os +import urllib.parse +from datetime import datetime, timedelta +from unittest.mock import MagicMock, patch + +import pytest + + +class TestPrismaWrapperTokenRefresh: + """Tests for the PrismaWrapper RDS IAM token refresh implementation.""" + + @pytest.fixture + def setup_env(self): + """Setup environment variables for testing.""" + os.environ["DATABASE_HOST"] = "test-host.rds.amazonaws.com" + os.environ["DATABASE_PORT"] = "5432" + os.environ["DATABASE_USER"] = "test_user" + os.environ["DATABASE_NAME"] = "test_db" + os.environ["IAM_TOKEN_DB_AUTH"] = "True" + yield + # Cleanup + for key in [ + "DATABASE_HOST", + "DATABASE_PORT", + "DATABASE_USER", + "DATABASE_NAME", + "DATABASE_URL", + "IAM_TOKEN_DB_AUTH", + "DATABASE_SCHEMA", + ]: + os.environ.pop(key, None) + + def _generate_mock_token(self, expires_in_seconds: int = 900) -> str: + """Generate a mock IAM token with expiration info.""" + now = datetime.utcnow() + date_str = now.strftime("%Y%m%dT%H%M%SZ") + # Build the token like AWS does + token = f"mock-token?X-Amz-Date={date_str}&X-Amz-Expires={expires_in_seconds}&X-Amz-Signature=abc123" + return urllib.parse.quote(token, safe="") + + def _set_database_url_with_token(self, expires_in_seconds: int = 900): + """Set DATABASE_URL with a mock token.""" + token = self._generate_mock_token(expires_in_seconds) + os.environ[ + "DATABASE_URL" + ] = f"postgresql://test_user:{token}@test-host:5432/test_db" + + @pytest.mark.asyncio + async def test_is_token_expired_fresh(self, setup_env): + """Test that fresh token is not detected as expired.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + mock_prisma = MagicMock() + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=True) + + self._set_database_url_with_token(expires_in_seconds=900) + db_url = os.getenv("DATABASE_URL") + + assert wrapper.is_token_expired(db_url) is False + + @pytest.mark.asyncio + async def test_is_token_expired_old(self, setup_env): + """Test that old token is detected as expired.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + mock_prisma = MagicMock() + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=True) + + # Create an expired token + old_date = datetime.utcnow() - timedelta(seconds=901) + date_str = old_date.strftime("%Y%m%dT%H%M%SZ") + token = ( + f"mock-token?X-Amz-Date={date_str}&X-Amz-Expires=900&X-Amz-Signature=abc" + ) + encoded_token = urllib.parse.quote(token, safe="") + db_url = f"postgresql://test_user:{encoded_token}@test-host:5432/test_db" + + assert wrapper.is_token_expired(db_url) is True + + @pytest.mark.asyncio + async def test_start_stop_token_refresh_task(self, setup_env): + """Test that token refresh task starts and stops correctly.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + mock_prisma = MagicMock() + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=True) + + # Set a valid token + self._set_database_url_with_token(expires_in_seconds=900) + + # Start the task + await wrapper.start_token_refresh_task() + assert wrapper._token_refresh_task is not None + assert not wrapper._token_refresh_task.done() + + # Stop the task + await wrapper.stop_token_refresh_task() + assert wrapper._token_refresh_task is None + + @pytest.mark.asyncio + async def test_start_task_not_enabled(self, setup_env): + """Test that task doesn't start when IAM auth is not enabled.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + mock_prisma = MagicMock() + # IAM auth disabled + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=False) + + await wrapper.start_token_refresh_task() + assert wrapper._token_refresh_task is None + + @pytest.mark.asyncio + async def test_is_token_expired_null(self, setup_env): + """Test that None token is treated as expired.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + mock_prisma = MagicMock() + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=True) + + assert wrapper.is_token_expired(None) is True + + +class TestTokenExpirationParsing: + """Tests for token expiration parsing utilities.""" + + def test_parse_token_expiration_valid(self): + """Test parsing expiration from a valid token.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + mock_prisma = MagicMock() + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=True) + + # Create a token with known expiration + token = "mock-token?X-Amz-Date=20240101T120000Z&X-Amz-Expires=900&X-Amz-Signature=abc" + + expiration = wrapper._parse_token_expiration(token) + + assert expiration is not None + expected = datetime(2024, 1, 1, 12, 0, 0) + timedelta(seconds=900) + assert expiration == expected + + def test_parse_token_expiration_invalid(self): + """Test that invalid token returns None.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + mock_prisma = MagicMock() + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=True) + + # Invalid tokens + assert wrapper._parse_token_expiration(None) is None + assert wrapper._parse_token_expiration("no-query-params") is None + assert wrapper._parse_token_expiration("?missing=params") is None + + +class TestBackgroundRefreshLoop: + """Tests for the background refresh loop timing.""" + + @pytest.fixture + def setup_env(self): + """Setup environment variables for testing.""" + os.environ["DATABASE_HOST"] = "test-host.rds.amazonaws.com" + os.environ["DATABASE_PORT"] = "5432" + os.environ["DATABASE_USER"] = "test_user" + os.environ["DATABASE_NAME"] = "test_db" + yield + # Cleanup + for key in [ + "DATABASE_HOST", + "DATABASE_PORT", + "DATABASE_USER", + "DATABASE_NAME", + "DATABASE_URL", + ]: + os.environ.pop(key, None) + + @pytest.mark.asyncio + async def test_calculate_seconds_fallback_when_no_url(self, setup_env): + """Test that fallback is used when DATABASE_URL is not set.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + mock_prisma = MagicMock() + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=True) + + # Don't set DATABASE_URL + seconds = wrapper._calculate_seconds_until_refresh() + + # Should return fallback interval + assert seconds == wrapper.FALLBACK_REFRESH_INTERVAL_SECONDS + + +# ============================================================================ +# DEMONSTRATION SCRIPT +# ============================================================================ + + +async def demonstrate_fix(): + """ + Demonstrates the fix for the RDS IAM token expiration bug. + + Shows how the proactive refresh prevents the 15-minute connection failure. + """ + # Import the actual implementation + try: + from litellm.proxy.db.prisma_client import PrismaWrapper + except ImportError: + return + + # Setup mock environment + os.environ["DATABASE_HOST"] = "mock-rds.region.rds.amazonaws.com" + os.environ["DATABASE_PORT"] = "5432" + os.environ["DATABASE_USER"] = "iam_user" + os.environ["DATABASE_NAME"] = "litellm" + + # Create initial token (expires in 10 seconds for demo) + now = datetime.utcnow() + date_str = now.strftime("%Y%m%dT%H%M%SZ") + token = f"mock-token?X-Amz-Date={date_str}&X-Amz-Expires=10&X-Amz-Signature=abc123" + encoded_token = urllib.parse.quote(token, safe="") + os.environ[ + "DATABASE_URL" + ] = f"postgresql://iam_user:{encoded_token}@mock-rds:5432/litellm" + + # Create mock prisma client + mock_prisma = MagicMock() + + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=True) + + # Override buffer for faster demo + wrapper.TOKEN_REFRESH_BUFFER_SECONDS = 3 + wrapper.FALLBACK_REFRESH_INTERVAL_SECONDS = 5 + _ = wrapper._calculate_seconds_until_refresh() # Verify calculation works + db_url = os.getenv("DATABASE_URL") + is_expired = wrapper.is_token_expired(db_url) + assert is_expired is False, "Fresh token should not be expired!" + + # Mock the _token_refresh_loop to prevent it from actually running + async def mock_loop(): + try: + await asyncio.sleep(1000) + except asyncio.CancelledError: + pass + + with patch.object(wrapper, "_token_refresh_loop", side_effect=mock_loop): + await wrapper.start_token_refresh_task() + await wrapper.stop_token_refresh_task() + + # Cleanup + for key in [ + "DATABASE_HOST", + "DATABASE_PORT", + "DATABASE_USER", + "DATABASE_NAME", + "DATABASE_URL", + ]: + os.environ.pop(key, None) + + +if __name__ == "__main__": + asyncio.run(demonstrate_fix()) From 8ff4cb89e15c0876a2acb371fb0644e53b6ff24a Mon Sep 17 00:00:00 2001 From: Dror Ivry Date: Fri, 9 Jan 2026 00:03:26 +0200 Subject: [PATCH 22/56] feat: added qualifire eval webhook --- .../observability/qualifire_integration.md | 131 ++++++++++++++++++ .../generic_api_compatible_callbacks.json | 59 ++++---- 2 files changed, 165 insertions(+), 25 deletions(-) create mode 100644 docs/my-website/docs/observability/qualifire_integration.md diff --git a/docs/my-website/docs/observability/qualifire_integration.md b/docs/my-website/docs/observability/qualifire_integration.md new file mode 100644 index 00000000000..8f66353f2c6 --- /dev/null +++ b/docs/my-website/docs/observability/qualifire_integration.md @@ -0,0 +1,131 @@ +import Image from '@theme/IdealImage'; + +# Qualifire - LLM Evaluation, Guardrails & Observability + +[Qualifire](https://qualifire.ai/) provides real-time Agentic evaluations, guardrails and observability for production AI applications. + + + +**Key Features:** + +- **Evaluation** - Systematically assess AI behavior to detect hallucinations, jailbreaks, policy breaches, and other vulnerabilities +- **Guardrails** - Real-time interventions to prevent risks like brand damage, data leaks, and compliance breaches +- **Observability** - Complete tracing and logging for RAG pipelines, chatbots, and AI agents +- **Prompt Management** - Centralized prompt management with versioning and no-code studio + +:::tip + +Looking for Qualifire Guardrails? Check out the [Qualifire Guardrails Integration](../proxy/guardrails/qualifire.md) for real-time content moderation, prompt injection detection, PII checks, and more. + +::: + +## Pre-Requisites + +1. Create an account on [Qualifire](https://qualifire.ai/) +2. Get your API key and webhook URL from the Qualifire dashboard + +```bash +pip install litellm +``` + +## Quick Start + +Use just 2 lines of code to instantly log your responses **across all providers** with Qualifire. + +```python +litellm.callbacks = ["qualifire_eval"] +``` + +```python +import litellm +import os + +# Set Qualifire credentials +os.environ["QUALIFIRE_API_KEY"] = "your-qualifire-api-key" +os.environ["QUALIFIRE_WEBHOOK_URL"] = "https://your-qualifire-webhook-url" + +# LLM API Keys +os.environ['OPENAI_API_KEY'] = "your-openai-api-key" + +# Set qualifire_eval as a callback & LiteLLM will send the data to Qualifire +litellm.callbacks = ["qualifire_eval"] + +# OpenAI call +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[ + {"role": "user", "content": "Hi 👋 - i'm openai"} + ] +) +``` + +## Using with LiteLLM Proxy + +1. Setup config.yaml + +```yaml +model_list: + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY + +litellm_settings: + callbacks: ["qualifire_eval"] + +general_settings: + master_key: "sk-1234" + +environment_variables: + QUALIFIRE_API_KEY: "your-qualifire-api-key" + QUALIFIRE_WEBHOOK_URL: "https://your-qualifire-webhook-url" +``` + +2. Start the proxy + +```bash +litellm --config config.yaml +``` + +3. Test it! + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ "model": "gpt-4o", "messages": [{"role": "user", "content": "Hi 👋 - i'm openai"}]}' +``` + +## Environment Variables + +| Variable | Description | +| ----------------------- | ------------------------------------------------------ | +| `QUALIFIRE_API_KEY` | Your Qualifire API key for authentication | +| `QUALIFIRE_WEBHOOK_URL` | The Qualifire webhook endpoint URL from your dashboard | + +## What Gets Logged? + +The [LiteLLM Standard Logging Payload](https://docs.litellm.ai/docs/proxy/logging_spec) is sent to your Qualifire endpoint on each successful LLM API call. + +This includes: + +- Request messages and parameters +- Response content and metadata +- Token usage statistics +- Latency metrics +- Model information +- Cost data + +Once data is in Qualifire, you can: + +- Run evaluations to detect hallucinations, toxicity, and policy violations +- Set up guardrails to block or modify responses in real-time +- View traces across your entire AI pipeline +- Track performance and quality metrics over time + +## Support & Talk to Founders + +- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) +- [Community Discord 💭](https://discord.gg/wuPM9dRgDw) +- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ +- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/litellm/integrations/generic_api/generic_api_compatible_callbacks.json b/litellm/integrations/generic_api/generic_api_compatible_callbacks.json index 12dc4ae643c..13fe79ae671 100644 --- a/litellm/integrations/generic_api/generic_api_compatible_callbacks.json +++ b/litellm/integrations/generic_api/generic_api_compatible_callbacks.json @@ -1,28 +1,37 @@ { - "sample_callback": { - "event_types": ["llm_api_success", "llm_api_failure"], - "endpoint": "{{environment_variables.SAMPLE_CALLBACK_URL}}", - "headers": { - "Content-Type": "application/json", - "Authorization": "Bearer {{environment_variables.SAMPLE_CALLBACK_API_KEY}}" - }, - "environment_variables": ["SAMPLE_CALLBACK_URL", "SAMPLE_CALLBACK_API_KEY"] + "sample_callback": { + "event_types": ["llm_api_success", "llm_api_failure"], + "endpoint": "{{environment_variables.SAMPLE_CALLBACK_URL}}", + "headers": { + "Content-Type": "application/json", + "Authorization": "Bearer {{environment_variables.SAMPLE_CALLBACK_API_KEY}}" }, - "rubrik": { - "event_types": ["llm_api_success"], - "endpoint": "{{environment_variables.RUBRIK_WEBHOOK_URL}}", - "headers": { - "Content-Type": "application/json", - "Authorization": "Bearer {{environment_variables.RUBRIK_API_KEY}}" - }, - "environment_variables": ["RUBRIK_API_KEY", "RUBRIK_WEBHOOK_URL"] + "environment_variables": ["SAMPLE_CALLBACK_URL", "SAMPLE_CALLBACK_API_KEY"] + }, + "rubrik": { + "event_types": ["llm_api_success"], + "endpoint": "{{environment_variables.RUBRIK_WEBHOOK_URL}}", + "headers": { + "Content-Type": "application/json", + "Authorization": "Bearer {{environment_variables.RUBRIK_API_KEY}}" }, - "sumologic": { - "endpoint": "{{environment_variables.SUMOLOGIC_WEBHOOK_URL}}", - "headers": { - "Content-Type": "application/json" - }, - "environment_variables": ["SUMOLOGIC_WEBHOOK_URL"], - "log_format": "ndjson" - } -} \ No newline at end of file + "environment_variables": ["RUBRIK_API_KEY", "RUBRIK_WEBHOOK_URL"] + }, + "sumologic": { + "endpoint": "{{environment_variables.SUMOLOGIC_WEBHOOK_URL}}", + "headers": { + "Content-Type": "application/json" + }, + "environment_variables": ["SUMOLOGIC_WEBHOOK_URL"], + "log_format": "ndjson" + }, + "qualifire_eval": { + "event_types": ["llm_api_success"], + "endpoint": "{{environment_variables.QUALIFIRE_WEBHOOK_URL}}", + "headers": { + "Content-Type": "application/json", + "X-Qualifire-API-Key": "{{environment_variables.QUALIFIRE_API_KEY}}" + }, + "environment_variables": ["QUALIFIRE_API_KEY", "QUALIFIRE_WEBHOOK_URL"] + } +} From 5793f8b86663bc9bb501b4479bcf340630bad0d1 Mon Sep 17 00:00:00 2001 From: Dror Ivry Date: Fri, 9 Jan 2026 00:08:03 +0200 Subject: [PATCH 23/56] docs --- .../docs/observability/qualifire_integration.md | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/docs/my-website/docs/observability/qualifire_integration.md b/docs/my-website/docs/observability/qualifire_integration.md index 8f66353f2c6..9838bf6b16a 100644 --- a/docs/my-website/docs/observability/qualifire_integration.md +++ b/docs/my-website/docs/observability/qualifire_integration.md @@ -4,8 +4,6 @@ import Image from '@theme/IdealImage'; [Qualifire](https://qualifire.ai/) provides real-time Agentic evaluations, guardrails and observability for production AI applications. - - **Key Features:** - **Evaluation** - Systematically assess AI behavior to detect hallucinations, jailbreaks, policy breaches, and other vulnerabilities @@ -52,7 +50,7 @@ litellm.callbacks = ["qualifire_eval"] # OpenAI call response = litellm.completion( - model="gpt-3.5-turbo", + model="gpt-5", messages=[ {"role": "user", "content": "Hi 👋 - i'm openai"} ] @@ -78,7 +76,7 @@ general_settings: environment_variables: QUALIFIRE_API_KEY: "your-qualifire-api-key" - QUALIFIRE_WEBHOOK_URL: "https://your-qualifire-webhook-url" + QUALIFIRE_WEBHOOK_URL: "https://app.qualifire.ai/api/v1/webhooks/evaluations" ``` 2. Start the proxy @@ -122,10 +120,3 @@ Once data is in Qualifire, you can: - Set up guardrails to block or modify responses in real-time - View traces across your entire AI pipeline - Track performance and quality metrics over time - -## Support & Talk to Founders - -- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) -- [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ -- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai From e6c41c8f47c9ac8c804c894d72c4738d723559b0 Mon Sep 17 00:00:00 2001 From: Dror Ivry Date: Fri, 9 Jan 2026 00:08:47 +0200 Subject: [PATCH 24/56] docs --- docs/my-website/docs/observability/qualifire_integration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/observability/qualifire_integration.md b/docs/my-website/docs/observability/qualifire_integration.md index 9838bf6b16a..cf866f467bf 100644 --- a/docs/my-website/docs/observability/qualifire_integration.md +++ b/docs/my-website/docs/observability/qualifire_integration.md @@ -19,7 +19,7 @@ Looking for Qualifire Guardrails? Check out the [Qualifire Guardrails Integratio ## Pre-Requisites -1. Create an account on [Qualifire](https://qualifire.ai/) +1. Create an account on [Qualifire](https://app.qualifire.ai/) 2. Get your API key and webhook URL from the Qualifire dashboard ```bash From c38294dc1690e1b2cd56bd767d5bbd1c2f2621de Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Fri, 9 Jan 2026 07:55:55 +0900 Subject: [PATCH 25/56] docs: add focus --- docs/my-website/docs/observability/focus.md | 93 +++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 docs/my-website/docs/observability/focus.md diff --git a/docs/my-website/docs/observability/focus.md b/docs/my-website/docs/observability/focus.md new file mode 100644 index 00000000000..c282f4a220c --- /dev/null +++ b/docs/my-website/docs/observability/focus.md @@ -0,0 +1,93 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Focus Export (Experimental) + +:::caution Experimental feature +Focus Format export is under active development and currently considered experimental. +Interfaces, schema mappings, and configuration options may change as we iterate based on user feedback. +Please treat this integration as a preview and report any issues or suggestions to help us stabilize and improve the workflow. +::: + +LiteLLM can emit usage data in the [FinOps FOCUS format](https://focus.finops.org/focus-specification/v1-2/) and push artifacts (for example Parquet files) to destinations such as Amazon S3. This enables downstream cost-analysis tooling to ingest a standardised dataset directly from LiteLLM. + +LiteLLM currently conforms to the FinOps FOCUS v1.2 specification when emitting this dataset. + +## Overview + +| Property | Details | +|----------|---------| +| Destination | Export LiteLLM usage data in FOCUS format to managed storage (currently S3) | +| Callback name | `focus` | +| Supported operations | Automatic scheduled export | +| Data format | FOCUS Normalised Dataset (Parquet) | + +## Environment Variables + +### Common settings + +| Variable | Required | Description | +|----------|----------|-------------| +| `FOCUS_PROVIDER` | No | Destination provider (defaults to `s3`). | +| `FOCUS_FORMAT` | No | Output format (currently only `parquet`). | +| `FOCUS_FREQUENCY` | No | Export cadence. Prefer `hourly` or `daily` for production; `interval` is intended for short test loops. Defaults to `hourly`. | +| `FOCUS_CRON_OFFSET` | No | Minute offset used for hourly/daily cron triggers. Defaults to `5`. | +| `FOCUS_INTERVAL_SECONDS` | No | Interval (seconds) when `FOCUS_FREQUENCY="interval"`. | +| `FOCUS_PREFIX` | No | Object key prefix/folder. Defaults to `focus_exports`. | + +### S3 destination + +| Variable | Required | Description | +|----------|----------|-------------| +| `FOCUS_S3_BUCKET_NAME` | Yes | Destination bucket for exported files. | +| `FOCUS_S3_REGION_NAME` | No | AWS region for the bucket. | +| `FOCUS_S3_ENDPOINT_URL` | No | Custom endpoint (useful for S3-compatible storage). | +| `FOCUS_S3_ACCESS_KEY` | Yes | AWS access key for uploads. | +| `FOCUS_S3_SECRET_KEY` | Yes | AWS secret key for uploads. | +| `FOCUS_S3_SESSION_TOKEN` | No | AWS session token if using temporary credentials. | + +## Setup via Config + +### Configure environment variables + +```bash +export FOCUS_PROVIDER="s3" +export FOCUS_PREFIX="focus_exports" + +# S3 example +export FOCUS_S3_BUCKET_NAME="my-litellm-focus-bucket" +export FOCUS_S3_REGION_NAME="us-east-1" +export FOCUS_S3_ACCESS_KEY="AKIA..." +export FOCUS_S3_SECRET_KEY="..." +``` + +### Update LiteLLM config + +```yaml +model_list: + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + api_key: sk-your-key + +litellm_settings: + callbacks: ["focus"] +``` + +### Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +During boot LiteLLM registers the Focus logger and a background job that runs according to the configured frequency. + +## Planned Enhancements +- Add "Setup on UI" flow alongside the current configuration-based setup. +- Add GCS / Azure Blob to the Destination options. +- Support CSV output alongside Parquet. + +## Related Links + +- [Focus](https://focus.finops.org/) + From f129f598a050ab657d798fe4cbe34ca194af1c94 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Fri, 9 Jan 2026 10:52:48 +0900 Subject: [PATCH 26/56] fix: how to execute cloudzero sql --- litellm/integrations/cloudzero/database.py | 182 ++---------------- .../integrations/cloudzero/test_database.py | 57 ++++++ 2 files changed, 75 insertions(+), 164 deletions(-) create mode 100644 tests/test_litellm/integrations/cloudzero/test_database.py diff --git a/litellm/integrations/cloudzero/database.py b/litellm/integrations/cloudzero/database.py index 2128b55bf83..71929398103 100644 --- a/litellm/integrations/cloudzero/database.py +++ b/litellm/integrations/cloudzero/database.py @@ -19,7 +19,7 @@ """Database connection and data extraction for LiteLLM.""" from datetime import datetime -from typing import Any, Dict, Optional +from typing import Any, Optional, List import polars as pl @@ -46,19 +46,9 @@ class LiteLLMDatabase: """Retrieve usage data from LiteLLM daily user spend table.""" client = self._ensure_prisma_client() - # Build WHERE clause for time filtering - where_conditions = [] - if start_time_utc: - where_conditions.append(f"dus.updated_at >= '{start_time_utc.isoformat()}'") - if end_time_utc: - where_conditions.append(f"dus.updated_at <= '{end_time_utc.isoformat()}'") - - where_clause = "" - if where_conditions: - where_clause = "WHERE " + " AND ".join(where_conditions) - - # Query to get user spend data with team information - query = f""" + # Query to get user spend data with team information. Use parameter binding to + # avoid SQL injection from user-supplied timestamps or limits. + query = """ SELECT dus.id, dus.date, @@ -85,163 +75,27 @@ class LiteLLMDatabase: LEFT JOIN "LiteLLM_VerificationToken" vt ON dus.api_key = vt.token LEFT JOIN "LiteLLM_TeamTable" tt ON vt.team_id = tt.team_id LEFT JOIN "LiteLLM_UserTable" ut ON dus.user_id = ut.user_id - {where_clause} + WHERE ($1::timestamptz IS NULL OR dus.updated_at >= $1::timestamptz) + AND ($2::timestamptz IS NULL OR dus.updated_at <= $2::timestamptz) ORDER BY dus.date DESC, dus.created_at DESC """ - if limit: - query += f" LIMIT {limit}" + params: List[Any] = [ + start_time_utc, + end_time_utc, + ] + + if limit is not None: + try: + params.append(int(limit)) + except (TypeError, ValueError): + raise ValueError("limit must be an integer") + query += " LIMIT $3" try: - db_response = await client.db.query_raw(query) + db_response = await client.db.query_raw(query, *params) # Convert the response to polars DataFrame with full schema inference # This prevents schema mismatch errors when data types vary across rows return pl.DataFrame(db_response, infer_schema_length=None) except Exception as e: raise Exception(f"Error retrieving usage data: {str(e)}") - - async def get_table_info(self) -> Dict[str, Any]: - """Get information about the daily user spend table.""" - client = self._ensure_prisma_client() - - try: - # Get row count from user spend table - user_count = await self._get_table_row_count("LiteLLM_DailyUserSpend") - - # Get column structure from user spend table - query = """ - SELECT column_name, data_type, is_nullable - FROM information_schema.columns - WHERE table_name = 'LiteLLM_DailyUserSpend' - ORDER BY ordinal_position; - """ - columns_response = await client.db.query_raw(query) - - return { - "columns": columns_response, - "row_count": user_count, - "table_name": "LiteLLM_DailyUserSpend", - } - except Exception as e: - raise Exception(f"Error getting table info: {str(e)}") - - async def _get_table_row_count(self, table_name: str) -> int: - """Get row count from specified table.""" - client = self._ensure_prisma_client() - - try: - query = f'SELECT COUNT(*) as count FROM "{table_name}"' - response = await client.db.query_raw(query) - - if response and len(response) > 0: - return response[0].get("count", 0) - return 0 - except Exception: - return 0 - - async def discover_all_tables(self) -> Dict[str, Any]: - """Discover all tables in the LiteLLM database and their schemas.""" - client = self._ensure_prisma_client() - - try: - # Get all LiteLLM tables - litellm_tables_query = """ - SELECT table_name - FROM information_schema.tables - WHERE table_schema = 'public' - AND table_name LIKE 'LiteLLM_%' - ORDER BY table_name; - """ - tables_response = await client.db.query_raw(litellm_tables_query) - table_names = [row["table_name"] for row in tables_response] - - # Get detailed schema for each table - tables_info = {} - for table_name in table_names: - # Get column information - columns_query = """ - SELECT - column_name, - data_type, - is_nullable, - column_default, - character_maximum_length, - numeric_precision, - numeric_scale, - ordinal_position - FROM information_schema.columns - WHERE table_name = $1 - AND table_schema = 'public' - ORDER BY ordinal_position; - """ - columns_response = await client.db.query_raw(columns_query, table_name) - - # Get primary key information - pk_query = """ - SELECT a.attname - FROM pg_index i - JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) - WHERE i.indrelid = $1::regclass AND i.indisprimary; - """ - pk_response = await client.db.query_raw(pk_query, f'"{table_name}"') - primary_keys = ( - [row["attname"] for row in pk_response] if pk_response else [] - ) - - # Get foreign key information - fk_query = """ - SELECT - tc.constraint_name, - kcu.column_name, - ccu.table_name AS foreign_table_name, - ccu.column_name AS foreign_column_name - FROM information_schema.table_constraints AS tc - JOIN information_schema.key_column_usage AS kcu - ON tc.constraint_name = kcu.constraint_name - JOIN information_schema.constraint_column_usage AS ccu - ON ccu.constraint_name = tc.constraint_name - WHERE tc.constraint_type = 'FOREIGN KEY' - AND tc.table_name = $1; - """ - fk_response = await client.db.query_raw(fk_query, table_name) - foreign_keys = fk_response if fk_response else [] - - # Get indexes - indexes_query = """ - SELECT - i.relname AS index_name, - array_agg(a.attname ORDER BY a.attnum) AS column_names, - ix.indisunique AS is_unique - FROM pg_class t - JOIN pg_index ix ON t.oid = ix.indrelid - JOIN pg_class i ON i.oid = ix.indexrelid - JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(ix.indkey) - WHERE t.relname = $1 - AND t.relkind = 'r' - GROUP BY i.relname, ix.indisunique - ORDER BY i.relname; - """ - indexes_response = await client.db.query_raw(indexes_query, table_name) - indexes = indexes_response if indexes_response else [] - - # Get row count - try: - row_count = await self._get_table_row_count(table_name) - except Exception: - row_count = 0 - - tables_info[table_name] = { - "columns": columns_response, - "primary_keys": primary_keys, - "foreign_keys": foreign_keys, - "indexes": indexes, - "row_count": row_count, - } - - return { - "tables": tables_info, - "table_count": len(table_names), - "table_names": table_names, - } - except Exception as e: - raise Exception(f"Error discovering tables: {str(e)}") diff --git a/tests/test_litellm/integrations/cloudzero/test_database.py b/tests/test_litellm/integrations/cloudzero/test_database.py new file mode 100644 index 00000000000..89a5028011c --- /dev/null +++ b/tests/test_litellm/integrations/cloudzero/test_database.py @@ -0,0 +1,57 @@ +"""Tests for LiteLLM CloudZero database helper.""" + +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from litellm.integrations.cloudzero.database import LiteLLMDatabase + + +def _setup_db(monkeypatch: pytest.MonkeyPatch, query_return): + """Return a database instance with prisma client mocked out.""" + query_mock = AsyncMock(return_value=query_return) + mock_client = SimpleNamespace(db=SimpleNamespace(query_raw=query_mock)) + db = LiteLLMDatabase() + monkeypatch.setattr(db, "_ensure_prisma_client", lambda: mock_client) + return db, query_mock + + +@pytest.mark.asyncio +async def test_get_usage_data_parameterized(monkeypatch: pytest.MonkeyPatch): + """Start/end filters and limit should be parameterized via placeholders.""" + start = datetime(2024, 5, 1, tzinfo=timezone.utc) + end = datetime(2024, 5, 2, tzinfo=timezone.utc) + db, query_mock = _setup_db(monkeypatch, []) + + await db.get_usage_data(limit=10, start_time_utc=start, end_time_utc=end) + + query_text, *params = query_mock.await_args.args + assert "dus.updated_at >= $1::timestamptz" in query_text + assert "dus.updated_at <= $2::timestamptz" in query_text + assert "LIMIT $3" in query_text + assert params == [start, end, 10] + + +@pytest.mark.asyncio +async def test_get_usage_data_handles_missing_filters(monkeypatch: pytest.MonkeyPatch): + """When no filters provided the params should be None placeholders.""" + db, query_mock = _setup_db(monkeypatch, []) + + await db.get_usage_data() + + query_text, *params = query_mock.await_args.args + assert "LIMIT $3" not in query_text + assert params == [None, None] + + +@pytest.mark.asyncio +async def test_get_usage_data_rejects_invalid_limit(monkeypatch: pytest.MonkeyPatch): + """limit must coerce to int or raise ValueError before hitting the DB.""" + db, query_mock = _setup_db(monkeypatch, []) + + with pytest.raises(ValueError): + await db.get_usage_data(limit="invalid") + + assert query_mock.await_count == 0 From 48bc5ccb4f003dee859aec4c6721ded4d89c0426 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Fri, 9 Jan 2026 11:23:47 +0900 Subject: [PATCH 27/56] fix: test --- .../integrations/cloudzero/test_cloudzero.py | 79 ++++++++++++++++--- ...database.py => test_cloudzero_database.py} | 0 ...est_database.py => test_focus_database.py} | 0 3 files changed, 70 insertions(+), 9 deletions(-) rename tests/test_litellm/integrations/cloudzero/{test_database.py => test_cloudzero_database.py} (100%) rename tests/test_litellm/integrations/focus/{test_database.py => test_focus_database.py} (100%) diff --git a/tests/test_litellm/integrations/cloudzero/test_cloudzero.py b/tests/test_litellm/integrations/cloudzero/test_cloudzero.py index 31a2f6cbf51..b0aac17e7d9 100644 --- a/tests/test_litellm/integrations/cloudzero/test_cloudzero.py +++ b/tests/test_litellm/integrations/cloudzero/test_cloudzero.py @@ -66,16 +66,77 @@ class TestCloudZeroHourlyExport: fake_client = MagicMock() fake_db = MagicMock() - async def query_raw_mock(query: str): - sql_context = pl.SQLContext( - LiteLLM_DailyUserSpend=spend_mock_data, - LiteLLM_VerificationToken=verification_mock_data, - LiteLLM_TeamTable=team_mock_data, - LiteLLM_UserTable=user_mock_data, - ) - result = sql_context.execute(query).collect() + async def query_raw_mock(query: str, *params): + start_time_utc = params[0] if len(params) > 0 else None + end_time_utc = params[1] if len(params) > 1 else None + limit = params[2] if len(params) > 2 else None - return result + spend_df = spend_mock_data.collect() + verification_df = verification_mock_data.collect().rename( + {"key_alias": "api_key_alias"} + ) + team_df = team_mock_data.collect() + user_df = user_mock_data.collect() + + joined = ( + spend_df.join( + verification_df, left_on="api_key", right_on="token", how="left" + ) + .join( + team_df, + left_on="team_id", + right_on="team_id", + how="left", + suffix="_team", + ) + .join( + user_df, + left_on="user_id", + right_on="user_id", + how="left", + suffix="_user", + ) + ) + + for duplicate_column in ("team_id_team", "user_id_user"): + if duplicate_column in joined.columns: + joined = joined.drop(duplicate_column) + + if start_time_utc is not None: + joined = joined.filter(pl.col("updated_at") >= start_time_utc) + if end_time_utc is not None: + joined = joined.filter(pl.col("updated_at") <= end_time_utc) + + joined = joined.select( + [ + "id", + "date", + "user_id", + "api_key", + "model", + "model_group", + "custom_llm_provider", + "prompt_tokens", + "completion_tokens", + "spend", + "api_requests", + "successful_requests", + "failed_requests", + "cache_creation_input_tokens", + "cache_read_input_tokens", + "created_at", + "updated_at", + "team_id", + "api_key_alias", + "team_alias", + "user_email", + ] + ).sort(["date", "created_at"], descending=[True, True]) + + if limit is not None: + joined = joined.head(int(limit)) + + return joined fake_db.query_raw = AsyncMock(side_effect=query_raw_mock) fake_client.db = fake_db diff --git a/tests/test_litellm/integrations/cloudzero/test_database.py b/tests/test_litellm/integrations/cloudzero/test_cloudzero_database.py similarity index 100% rename from tests/test_litellm/integrations/cloudzero/test_database.py rename to tests/test_litellm/integrations/cloudzero/test_cloudzero_database.py diff --git a/tests/test_litellm/integrations/focus/test_database.py b/tests/test_litellm/integrations/focus/test_focus_database.py similarity index 100% rename from tests/test_litellm/integrations/focus/test_database.py rename to tests/test_litellm/integrations/focus/test_focus_database.py From bcbf8d1de3df4d1b945187411f9f140946f80ffe Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 8 Jan 2026 20:23:54 -0800 Subject: [PATCH 28/56] Adding unit test to expand unit testing coverage --- .../chat_ui/ChatImageUtils.test.tsx | 187 ++++++++++ .../chat_ui/CodeInterpreterOutput.test.tsx | 326 ++++++++++++++++++ .../components/UnifiedSelector.test.tsx | 161 +++++++++ .../compareUI/endpoint_config.test.ts | 104 ++++++ 4 files changed, 778 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/playground/chat_ui/ChatImageUtils.test.tsx create mode 100644 ui/litellm-dashboard/src/components/playground/chat_ui/CodeInterpreterOutput.test.tsx create mode 100644 ui/litellm-dashboard/src/components/playground/compareUI/components/UnifiedSelector.test.tsx create mode 100644 ui/litellm-dashboard/src/components/playground/compareUI/endpoint_config.test.ts diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatImageUtils.test.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatImageUtils.test.tsx new file mode 100644 index 00000000000..ecc4914b0ab --- /dev/null +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatImageUtils.test.tsx @@ -0,0 +1,187 @@ +import { describe, expect, it, vi, beforeEach } from "vitest"; +import { + convertImageToBase64, + createChatMultimodalMessage, + createChatDisplayMessage, + shouldShowChatAttachedImage, +} from "./ChatImageUtils"; +import { MessageType } from "./types"; + +describe("ChatImageUtils", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe("convertImageToBase64", () => { + it("should convert file to base64 data URI", async () => { + const file = new File(["test content"], "test.png", { type: "image/png" }); + const result = await convertImageToBase64(file); + expect(result).toMatch(/^data:image\/png;base64,/); + }); + + it("should handle different file types", async () => { + const jpegFile = new File(["jpeg content"], "test.jpg", { type: "image/jpeg" }); + const result = await convertImageToBase64(jpegFile); + expect(result).toMatch(/^data:image\/jpeg;base64,/); + }); + + it("should reject on file read error", async () => { + const file = new File(["test"], "test.png", { type: "image/png" }); + const originalReadAsDataURL = FileReader.prototype.readAsDataURL; + + FileReader.prototype.readAsDataURL = vi.fn(function (this: FileReader) { + setTimeout(() => { + if (this.onerror) { + this.onerror(new Error("Read error") as any); + } + }, 0); + }); + + await expect(convertImageToBase64(file)).rejects.toThrow(); + + FileReader.prototype.readAsDataURL = originalReadAsDataURL; + }); + }); + + describe("createChatMultimodalMessage", () => { + it("should create multimodal message with text and image", async () => { + const file = new File(["test content"], "test.png", { type: "image/png" }); + const inputMessage = "What is in this image?"; + + const result = await createChatMultimodalMessage(inputMessage, file); + + expect(result.role).toBe("user"); + expect(result.content).toHaveLength(2); + expect(result.content[0]).toEqual({ type: "text", text: inputMessage }); + expect(result.content[1]).toMatchObject({ + type: "image_url", + image_url: { + url: expect.stringMatching(/^data:image\/png;base64,/), + }, + }); + }); + + it("should include base64 data URI in image_url", async () => { + const file = new File(["test content"], "test.png", { type: "image/png" }); + const result = await createChatMultimodalMessage("test", file); + + const imageContent = result.content[1]; + expect(imageContent.type).toBe("image_url"); + if ("image_url" in imageContent && imageContent.image_url) { + expect(imageContent.image_url.url).toMatch(/^data:/); + } + }); + }); + + describe("createChatDisplayMessage", () => { + it("should create display message without file", () => { + const result = createChatDisplayMessage("Hello world", false); + + expect(result.role).toBe("user"); + expect(result.content).toBe("Hello world"); + expect(result.imagePreviewUrl).toBeUndefined(); + }); + + it("should create display message with PDF file", () => { + const filePreviewUrl = "blob:test-url"; + const result = createChatDisplayMessage("Read this", true, filePreviewUrl, "document.pdf"); + + expect(result.content).toBe("Read this [PDF attached]"); + expect(result.imagePreviewUrl).toBe(filePreviewUrl); + }); + + it("should create display message with image file", () => { + const filePreviewUrl = "blob:test-url"; + const result = createChatDisplayMessage("Look at this", true, filePreviewUrl, "photo.jpg"); + + expect(result.content).toBe("Look at this [Image attached]"); + expect(result.imagePreviewUrl).toBe(filePreviewUrl); + }); + + it("should create display message with file but no fileName", () => { + const filePreviewUrl = "blob:test-url"; + const result = createChatDisplayMessage("Check this", true, filePreviewUrl); + + expect(result.content).toBe("Check this "); + expect(result.imagePreviewUrl).toBe(filePreviewUrl); + }); + + it("should create display message with file but no preview URL", () => { + const result = createChatDisplayMessage("See this", true, undefined, "image.png"); + + expect(result.content).toBe("See this [Image attached]"); + expect(result.imagePreviewUrl).toBeUndefined(); + }); + }); + + describe("shouldShowChatAttachedImage", () => { + it("should return true for user message with image attachment", () => { + const message: MessageType = { + role: "user", + content: "Check this [Image attached]", + imagePreviewUrl: "blob:test-url", + }; + + expect(shouldShowChatAttachedImage(message)).toBe(true); + }); + + it("should return true for user message with PDF attachment", () => { + const message: MessageType = { + role: "user", + content: "Read this [PDF attached]", + imagePreviewUrl: "blob:test-url", + }; + + expect(shouldShowChatAttachedImage(message)).toBe(true); + }); + + it("should return false for assistant message", () => { + const message: MessageType = { + role: "assistant", + content: "Here is the image [Image attached]", + imagePreviewUrl: "blob:test-url", + }; + + expect(shouldShowChatAttachedImage(message)).toBe(false); + }); + + it("should return false when content is not a string", () => { + const message: MessageType = { + role: "user", + content: [{ type: "input_text", text: "test" }], + imagePreviewUrl: "blob:test-url", + }; + + expect(shouldShowChatAttachedImage(message)).toBe(false); + }); + + it("should return false when content does not include attachment marker", () => { + const message: MessageType = { + role: "user", + content: "Just regular text", + imagePreviewUrl: "blob:test-url", + }; + + expect(shouldShowChatAttachedImage(message)).toBe(false); + }); + + it("should return false when imagePreviewUrl is missing", () => { + const message: MessageType = { + role: "user", + content: "Check this [Image attached]", + }; + + expect(shouldShowChatAttachedImage(message)).toBe(false); + }); + + it("should return false when imagePreviewUrl is empty string", () => { + const message: MessageType = { + role: "user", + content: "Check this [Image attached]", + imagePreviewUrl: "", + }; + + expect(shouldShowChatAttachedImage(message)).toBe(false); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/CodeInterpreterOutput.test.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/CodeInterpreterOutput.test.tsx new file mode 100644 index 00000000000..d87a74f4641 --- /dev/null +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/CodeInterpreterOutput.test.tsx @@ -0,0 +1,326 @@ +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; +import CodeInterpreterOutput from "./CodeInterpreterOutput"; + +vi.mock("@/components/networking", () => ({ + getProxyBaseUrl: vi.fn(() => "https://example.com"), +})); + +global.fetch = vi.fn(); + +describe("CodeInterpreterOutput", () => { + beforeEach(() => { + vi.clearAllMocks(); + URL.createObjectURL = vi.fn((blob) => `blob:${blob}`); + URL.revokeObjectURL = vi.fn(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("should render", () => { + render(); + + expect(screen.getByText("Python Code Executed")).toBeInTheDocument(); + }); + + it("should display code in syntax highlighter", async () => { + const user = userEvent.setup(); + const code = "print('hello world')"; + const { container } = render(); + + expect(screen.getByText("Python Code Executed")).toBeInTheDocument(); + + const collapseHeader = screen.getByRole("button"); + await user.click(collapseHeader); + + await waitFor(() => { + const codeElement = container.querySelector("code.language-python"); + expect(codeElement).toBeInTheDocument(); + expect(codeElement?.textContent).toContain(code); + }); + }); + + it("should fetch and display images from annotations", async () => { + const mockBlob = new Blob(["image data"], { type: "image/png" }); + const mockResponse = { + ok: true, + blob: vi.fn().mockResolvedValue(mockBlob), + }; + + (global.fetch as any).mockResolvedValue(mockResponse); + + const annotations = [ + { + type: "container_file_citation" as const, + container_id: "container-1", + file_id: "file-1", + filename: "chart.png", + start_index: 0, + end_index: 10, + }, + ]; + + render( + , + ); + + await waitFor(() => { + expect(global.fetch).toHaveBeenCalledWith( + "https://example.com/v1/containers/container-1/files/file-1/content", + expect.objectContaining({ + headers: { + Authorization: "Bearer test-token", + }, + }), + ); + }); + + await waitFor(() => { + expect(screen.getByText("chart.png")).toBeInTheDocument(); + }); + }); + + it("should show loading state while fetching images", async () => { + const mockBlob = new Blob(["image data"], { type: "image/png" }); + let resolveBlob: (value: Blob) => void; + const blobPromise = new Promise((resolve) => { + resolveBlob = resolve; + }); + + const mockResponse = { + ok: true, + blob: vi.fn().mockReturnValue(blobPromise), + }; + + (global.fetch as any).mockResolvedValue(mockResponse); + + const annotations = [ + { + type: "container_file_citation" as const, + container_id: "container-1", + file_id: "file-1", + filename: "chart.png", + start_index: 0, + end_index: 10, + }, + ]; + + render( + , + ); + + await waitFor(() => { + expect(screen.getByText("Loading image...")).toBeInTheDocument(); + }); + + resolveBlob!(mockBlob); + await waitFor(() => { + expect(screen.queryByText("Loading image...")).not.toBeInTheDocument(); + }); + }); + + it("should handle download for image files", async () => { + const user = userEvent.setup(); + const mockBlob = new Blob(["image data"], { type: "image/png" }); + const mockResponse = { + ok: true, + blob: vi.fn().mockResolvedValue(mockBlob), + }; + + (global.fetch as any).mockResolvedValue(mockResponse); + + const annotations = [ + { + type: "container_file_citation" as const, + container_id: "container-1", + file_id: "file-1", + filename: "chart.png", + start_index: 0, + end_index: 10, + }, + ]; + + const createElementSpy = vi.spyOn(document, "createElement"); + const appendChildSpy = vi.spyOn(document.body, "appendChild"); + const removeChildSpy = vi.spyOn(document.body, "removeChild"); + + render( + , + ); + + await waitFor(() => { + expect(screen.getByText("chart.png")).toBeInTheDocument(); + }); + + const downloadButton = screen.getByText("Download"); + await user.click(downloadButton); + + await waitFor(() => { + expect(global.fetch).toHaveBeenCalledWith( + "https://example.com/v1/containers/container-1/files/file-1/content", + expect.objectContaining({ + headers: { + Authorization: "Bearer test-token", + }, + }), + ); + }); + + createElementSpy.mockRestore(); + appendChildSpy.mockRestore(); + removeChildSpy.mockRestore(); + }); + + it("should handle download for non-image files", async () => { + const user = userEvent.setup(); + const mockBlob = new Blob(["file data"], { type: "text/plain" }); + const mockResponse = { + ok: true, + blob: vi.fn().mockResolvedValue(mockBlob), + }; + + (global.fetch as any).mockResolvedValue(mockResponse); + + const annotations = [ + { + type: "container_file_citation" as const, + container_id: "container-1", + file_id: "file-1", + filename: "data.csv", + start_index: 0, + end_index: 10, + }, + ]; + + render(); + + await waitFor(() => { + expect(screen.getByText("data.csv")).toBeInTheDocument(); + }); + + const downloadButton = screen.getByText("data.csv").closest("button"); + expect(downloadButton).toBeInTheDocument(); + if (downloadButton) { + await user.click(downloadButton); + } + + await waitFor(() => { + expect(global.fetch).toHaveBeenCalledWith( + "https://example.com/v1/containers/container-1/files/file-1/content", + expect.objectContaining({ + headers: { + Authorization: "Bearer test-token", + }, + }), + ); + }); + }); + + it("should return null when no code and no annotations", () => { + const { container } = render(); + + expect(container.firstChild).toBeNull(); + }); + + it("should handle multiple image formats", async () => { + const mockBlob = new Blob(["image data"], { type: "image/png" }); + const mockResponse = { + ok: true, + blob: vi.fn().mockResolvedValue(mockBlob), + }; + + (global.fetch as any).mockResolvedValue(mockResponse); + + const annotations = [ + { + type: "container_file_citation" as const, + container_id: "container-1", + file_id: "file-1", + filename: "image.png", + start_index: 0, + end_index: 10, + }, + { + type: "container_file_citation" as const, + container_id: "container-1", + file_id: "file-2", + filename: "image.jpg", + start_index: 0, + end_index: 10, + }, + { + type: "container_file_citation" as const, + container_id: "container-1", + file_id: "file-3", + filename: "image.jpeg", + start_index: 0, + end_index: 10, + }, + { + type: "container_file_citation" as const, + container_id: "container-1", + file_id: "file-4", + filename: "image.gif", + start_index: 0, + end_index: 10, + }, + ]; + + render( + , + ); + + await waitFor(() => { + expect(global.fetch).toHaveBeenCalledTimes(4); + }); + }); + + it("should handle fetch errors gracefully", async () => { + const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + (global.fetch as any).mockRejectedValue(new Error("Network error")); + + const annotations = [ + { + type: "container_file_citation" as const, + container_id: "container-1", + file_id: "file-1", + filename: "chart.png", + start_index: 0, + end_index: 10, + }, + ]; + + render( + , + ); + + await waitFor(() => { + expect(consoleErrorSpy).toHaveBeenCalled(); + }); + + consoleErrorSpy.mockRestore(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/playground/compareUI/components/UnifiedSelector.test.tsx b/ui/litellm-dashboard/src/components/playground/compareUI/components/UnifiedSelector.test.tsx new file mode 100644 index 00000000000..1c6951fc421 --- /dev/null +++ b/ui/litellm-dashboard/src/components/playground/compareUI/components/UnifiedSelector.test.tsx @@ -0,0 +1,161 @@ +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import { UnifiedSelector } from "./UnifiedSelector"; +import { EndpointId, ENDPOINT_CONFIGS } from "../endpoint_config"; + +describe("UnifiedSelector", () => { + it("should render", () => { + const onChange = vi.fn(); + const options = [ + { value: "option1", label: "Option 1" }, + { value: "option2", label: "Option 2" }, + ]; + const config = ENDPOINT_CONFIGS[EndpointId.CHAT_COMPLETIONS]; + + render(); + + const select = screen.getByRole("combobox"); + expect(select).toBeInTheDocument(); + }); + + it("should display placeholder when not loading", () => { + const onChange = vi.fn(); + const options = [{ value: "option1", label: "Option 1" }]; + const config = ENDPOINT_CONFIGS[EndpointId.CHAT_COMPLETIONS]; + + const { container } = render( + , + ); + + const placeholder = container.querySelector(".ant-select-selection-placeholder"); + expect(placeholder).toHaveTextContent(config.selectorPlaceholder); + }); + + it("should display loading placeholder when loading", () => { + const onChange = vi.fn(); + const options = [{ value: "option1", label: "Option 1" }]; + const config = ENDPOINT_CONFIGS[EndpointId.CHAT_COMPLETIONS]; + + const { container } = render( + , + ); + + const placeholder = container.querySelector(".ant-select-selection-placeholder"); + expect(placeholder).toHaveTextContent(`Loading ${config.selectorLabel.toLowerCase()}s...`); + }); + + it("should call onChange when option is selected", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + const options = [ + { value: "option1", label: "Option 1" }, + { value: "option2", label: "Option 2" }, + ]; + const config = ENDPOINT_CONFIGS[EndpointId.CHAT_COMPLETIONS]; + + render(); + + const select = screen.getByRole("combobox"); + await user.click(select); + + await waitFor(() => { + const option = screen.getByText("Option 1"); + expect(option).toBeInTheDocument(); + }); + + const option = screen.getByText("Option 1"); + await user.click(option); + + await waitFor(() => { + expect(onChange).toHaveBeenCalled(); + }); + const callArgs = onChange.mock.calls[0]; + expect(callArgs[0]).toBe("option1"); + }); + + it("should display selected value", () => { + const onChange = vi.fn(); + const options = [ + { value: "option1", label: "Option 1" }, + { value: "option2", label: "Option 2" }, + ]; + const config = ENDPOINT_CONFIGS[EndpointId.CHAT_COMPLETIONS]; + + const { container } = render( + , + ); + + const selectedValue = container.querySelector(".ant-select-selection-item"); + expect(selectedValue).toHaveTextContent("Option 1"); + }); + + it("should filter options by search input", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + const options = [ + { value: "option1", label: "Option One" }, + { value: "option2", label: "Option Two" }, + { value: "option3", label: "Different" }, + ]; + const config = ENDPOINT_CONFIGS[EndpointId.CHAT_COMPLETIONS]; + + render(); + + const select = screen.getByRole("combobox"); + await user.click(select); + await user.type(select, "One"); + + await waitFor(() => { + expect(screen.getByText("Option One")).toBeInTheDocument(); + expect(screen.queryByText("Option Two")).not.toBeInTheDocument(); + expect(screen.queryByText("Different")).not.toBeInTheDocument(); + }); + }); + + it("should show loading spinner in notFoundContent when loading", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + const options: { value: string; label: string }[] = []; + const config = ENDPOINT_CONFIGS[EndpointId.CHAT_COMPLETIONS]; + + render(); + + const select = screen.getByRole("combobox"); + await user.click(select); + + await waitFor(() => { + const spin = document.querySelector(".ant-spin"); + expect(spin).toBeInTheDocument(); + }); + }); + + it("should show no options message when not loading and no options", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + const options: { value: string; label: string }[] = []; + const config = ENDPOINT_CONFIGS[EndpointId.CHAT_COMPLETIONS]; + + render(); + + const select = screen.getByRole("combobox"); + await user.click(select); + + await waitFor(() => { + expect(screen.getByText(`No ${config.selectorLabel.toLowerCase()}s available`)).toBeInTheDocument(); + }); + }); + + it("should work with agent endpoint config", () => { + const onChange = vi.fn(); + const options = [{ value: "agent1", label: "Agent One" }]; + const config = ENDPOINT_CONFIGS[EndpointId.A2A_AGENTS]; + + const { container } = render( + , + ); + + const placeholder = container.querySelector(".ant-select-selection-placeholder"); + expect(placeholder).toHaveTextContent(config.selectorPlaceholder); + }); +}); diff --git a/ui/litellm-dashboard/src/components/playground/compareUI/endpoint_config.test.ts b/ui/litellm-dashboard/src/components/playground/compareUI/endpoint_config.test.ts new file mode 100644 index 00000000000..67ecf32fc2a --- /dev/null +++ b/ui/litellm-dashboard/src/components/playground/compareUI/endpoint_config.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from "vitest"; +import { + EndpointId, + ENDPOINT_CONFIGS, + getAvailableEndpoints, + getEndpointConfig, + isAgentEndpoint, + isModelEndpoint, + modelOptionsToSelectorOptions, + agentOptionsToSelectorOptions, + getSelectionFieldName, + getComparisonSelection, + hasValidSelection, +} from "./endpoint_config"; +import { Agent } from "../llm_calls/fetch_agents"; + +describe("endpoint_config", () => { + it("should export EndpointId constants", () => { + expect(EndpointId.CHAT_COMPLETIONS).toBe("/v1/chat/completions"); + expect(EndpointId.A2A_AGENTS).toBe("/a2a"); + }); + + it("should have endpoint configs for all endpoint IDs", () => { + expect(ENDPOINT_CONFIGS[EndpointId.CHAT_COMPLETIONS]).toBeDefined(); + expect(ENDPOINT_CONFIGS[EndpointId.A2A_AGENTS]).toBeDefined(); + expect(ENDPOINT_CONFIGS[EndpointId.CHAT_COMPLETIONS].selectorType).toBe("model"); + expect(ENDPOINT_CONFIGS[EndpointId.A2A_AGENTS].selectorType).toBe("agent"); + }); + + it("should get available endpoints", () => { + const endpoints = getAvailableEndpoints(); + expect(endpoints).toHaveLength(2); + expect(endpoints).toContainEqual({ + value: EndpointId.CHAT_COMPLETIONS, + label: "/v1/chat/completions", + }); + expect(endpoints).toContainEqual({ + value: EndpointId.A2A_AGENTS, + label: "/a2a (Agents)", + }); + }); + + it("should get endpoint config by ID", () => { + const config = getEndpointConfig(EndpointId.CHAT_COMPLETIONS); + expect(config.id).toBe(EndpointId.CHAT_COMPLETIONS); + expect(config.selectorType).toBe("model"); + expect(config.selectorLabel).toBe("Model"); + }); + + it("should check if endpoint is agent endpoint", () => { + expect(isAgentEndpoint(EndpointId.A2A_AGENTS)).toBe(true); + expect(isAgentEndpoint(EndpointId.CHAT_COMPLETIONS)).toBe(false); + }); + + it("should check if endpoint is model endpoint", () => { + expect(isModelEndpoint(EndpointId.CHAT_COMPLETIONS)).toBe(true); + expect(isModelEndpoint(EndpointId.A2A_AGENTS)).toBe(false); + }); + + it("should convert model options to selector options", () => { + const models = ["gpt-4", "gpt-3.5-turbo", "claude-3"]; + const options = modelOptionsToSelectorOptions(models); + expect(options).toHaveLength(3); + expect(options[0]).toEqual({ value: "gpt-4", label: "gpt-4" }); + expect(options[1]).toEqual({ value: "gpt-3.5-turbo", label: "gpt-3.5-turbo" }); + expect(options[2]).toEqual({ value: "claude-3", label: "claude-3" }); + }); + + it("should convert agent options to selector options", () => { + const agents: Agent[] = [ + { agent_id: "agent-1", agent_name: "Agent One" }, + { agent_id: "agent-2", agent_name: "Agent Two" }, + { agent_id: "agent-3", agent_name: undefined as any }, + ]; + const options = agentOptionsToSelectorOptions(agents); + expect(options).toHaveLength(3); + expect(options[0]).toEqual({ value: "Agent One", label: "Agent One" }); + expect(options[1]).toEqual({ value: "Agent Two", label: "Agent Two" }); + expect(options[2]).toEqual({ value: undefined, label: "agent-3" }); + }); + + it("should get selection field name based on endpoint", () => { + expect(getSelectionFieldName(EndpointId.CHAT_COMPLETIONS)).toBe("model"); + expect(getSelectionFieldName(EndpointId.A2A_AGENTS)).toBe("agent"); + }); + + it("should get comparison selection based on endpoint", () => { + const comparison = { model: "gpt-4", agent: "agent-1" }; + expect(getComparisonSelection(comparison, EndpointId.CHAT_COMPLETIONS)).toBe("gpt-4"); + expect(getComparisonSelection(comparison, EndpointId.A2A_AGENTS)).toBe("agent-1"); + }); + + it("should check if comparison has valid selection", () => { + const comparisonWithModel = { model: "gpt-4", agent: "" }; + const comparisonWithAgent = { model: "", agent: "agent-1" }; + const comparisonEmpty = { model: "", agent: "" }; + const comparisonWhitespace = { model: " ", agent: "" }; + + expect(hasValidSelection(comparisonWithModel, EndpointId.CHAT_COMPLETIONS)).toBe(true); + expect(hasValidSelection(comparisonWithAgent, EndpointId.A2A_AGENTS)).toBe(true); + expect(hasValidSelection(comparisonEmpty, EndpointId.CHAT_COMPLETIONS)).toBe(false); + expect(hasValidSelection(comparisonWhitespace, EndpointId.CHAT_COMPLETIONS)).toBe(false); + }); +}); From 4d28359608045a03d17fcfd9a0cff972e09704fa Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 8 Jan 2026 21:50:19 -0800 Subject: [PATCH 29/56] e2e refactoring --- .../e2e_tests/fixtures/menuMappings.ts | 38 +++++++++++++++++++ .../e2e_tests/fixtures/pages.ts | 33 ++++++++++++++++ .../e2e_tests/helpers/navigation.ts | 12 ++++++ .../tests/modelsPage/addModel.spec.ts | 2 +- .../tests/navigation/sidebar.spec.ts | 36 +++++++++++++++--- 5 files changed, 114 insertions(+), 7 deletions(-) create mode 100644 ui/litellm-dashboard/e2e_tests/fixtures/menuMappings.ts create mode 100644 ui/litellm-dashboard/e2e_tests/fixtures/pages.ts create mode 100644 ui/litellm-dashboard/e2e_tests/helpers/navigation.ts diff --git a/ui/litellm-dashboard/e2e_tests/fixtures/menuMappings.ts b/ui/litellm-dashboard/e2e_tests/fixtures/menuMappings.ts new file mode 100644 index 00000000000..4a4bb64c8ed --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/fixtures/menuMappings.ts @@ -0,0 +1,38 @@ +import { Page } from "./pages"; + +/** + * Maps sidebar menu item labels to their corresponding page enum values. + * This mapping is for the admin role. + */ +export const menuLabelToPage: Record = { + "Virtual Keys": Page.ApiKeys, + Playground: Page.LlmPlayground, + Models: Page.Models, + "Models + Endpoints": Page.Models, + Usage: Page.NewUsage, + Teams: Page.Teams, + "Internal Users": Page.Users, + "Internal User": Page.Users, // Legacy label support + Organizations: Page.Organizations, + "API Reference": Page.ApiRef, + "AI Hub": Page.ModelHubTable, + "Model Hub": Page.ModelHubTable, + Logs: Page.Logs, + Guardrails: Page.Guardrails, + // Settings submenu items + "Router Settings": Page.RouterSettings, + "Logging & Alerts": Page.LoggingAndAlerts, + "Admin Settings": Page.AdminPanel, + "Cost Tracking": Page.CostTracking, + "UI Theme": Page.UiTheme, + // Experimental submenu items + Caching: Page.Caching, + Prompts: Page.Prompts, + Budgets: Page.Budgets, + "API Playground": Page.TransformRequest, + "Tag Management": Page.TagManagement, + "Old Usage": Page.Usage, + // Tools submenu items + "MCP Servers": Page.McpServers, + "Vector Stores": Page.VectorStores, +}; diff --git a/ui/litellm-dashboard/e2e_tests/fixtures/pages.ts b/ui/litellm-dashboard/e2e_tests/fixtures/pages.ts new file mode 100644 index 00000000000..3ea37718ab5 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/fixtures/pages.ts @@ -0,0 +1,33 @@ +/** + * Enum for all page query parameters supported in the app. + * These values correspond to the `page` query parameter used in the URL. + */ +export enum Page { + ApiKeys = "api-keys", + Models = "models", + LlmPlayground = "llm-playground", + Users = "users", + Teams = "teams", + Organizations = "organizations", + AdminPanel = "admin-panel", + ApiRef = "api_ref", + LoggingAndAlerts = "logging-and-alerts", + Budgets = "budgets", + Guardrails = "guardrails", + Agents = "agents", + Prompts = "prompts", + TransformRequest = "transform-request", + RouterSettings = "router-settings", + UiTheme = "ui-theme", + CostTracking = "cost-tracking", + ModelHubTable = "model-hub-table", + Caching = "caching", + PassThroughSettings = "pass-through-settings", + Logs = "logs", + McpServers = "mcp-servers", + SearchTools = "search-tools", + TagManagement = "tag-management", + VectorStores = "vector-stores", + NewUsage = "new_usage", + Usage = "usage", +} diff --git a/ui/litellm-dashboard/e2e_tests/helpers/navigation.ts b/ui/litellm-dashboard/e2e_tests/helpers/navigation.ts new file mode 100644 index 00000000000..919e516b35b --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/helpers/navigation.ts @@ -0,0 +1,12 @@ +import { Page } from "../fixtures/pages"; +import { Page as PlaywrightPage } from "@playwright/test"; + +/** + * Navigates to a specific page using the page query parameter. + * Uses relative path which will be resolved against the baseURL configured in playwright.config.ts + * @param page - The Playwright page object + * @param pageEnum - The page enum value to navigate to + */ +export async function navigateToPage(page: PlaywrightPage, pageEnum: Page): Promise { + await page.goto(`/ui?page=${pageEnum}`); +} diff --git a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts index c0619cfa845..2ab782d5678 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts @@ -14,7 +14,7 @@ test.describe("Add Model", () => { await providerInputDropdown.fill("Anthropic"); await page.waitForTimeout(1000); await providerInputDropdown.press("Enter"); - await page.waitForTimeout(1000); + await page.waitForTimeout(2000); const providerModelsDropdown = page.locator(".ant-select-selection-overflow").first(); await providerModelsDropdown.click(); diff --git a/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts index c90be698ae1..ce07cc2b83d 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts @@ -1,6 +1,9 @@ import test, { expect } from "@playwright/test"; import { Role } from "../../fixtures/roles"; import { ADMIN_STORAGE_PATH } from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { menuLabelToPage } from "../../fixtures/menuMappings"; +import { navigateToPage } from "../../helpers/navigation"; const sidebarButtons = { [Role.ProxyAdmin]: [ @@ -9,9 +12,7 @@ const sidebarButtons = { "Models", "Usage", "Teams", - "Internal User", - "Settings", - "Experimental", + "Internal Users", "API Reference", "AI Hub", ], @@ -23,13 +24,36 @@ for (const { role, storage } of roles) { test.describe(`${role} sidebar`, () => { test.use({ storageState: storage }); - test("can see and navigate all sidebar buttons", async ({ page }) => { + test("should navigate to correct URL when clicking sidebar menu items from homepage", async ({ page }) => { await page.goto("/ui"); - for (const button of sidebarButtons[role as keyof typeof sidebarButtons]) { - const tab = page.getByRole("menuitem", { name: button }); + + for (const buttonLabel of sidebarButtons[role as keyof typeof sidebarButtons]) { + const expectedPage = menuLabelToPage[buttonLabel]; + + if (!expectedPage) { + throw new Error(`No page mapping found for menu label: ${buttonLabel}`); + } + + const tab = page.getByRole("menuitem", { name: buttonLabel }); await expect(tab).toBeVisible(); + await tab.click(); + + // Verify URL contains the correct page query parameter + await expect(page).toHaveURL(new RegExp(`[?&]page=${expectedPage}(&|$)`)); } }); + + test("should navigate directly to page using navigation helper", async ({ page }) => { + // Test direct navigation to verify the helper function works + await navigateToPage(page, Page.ApiKeys); + await expect(page).toHaveURL(new RegExp(`[?&]page=${Page.ApiKeys}(&|$)`)); + + await navigateToPage(page, Page.Models); + await expect(page).toHaveURL(new RegExp(`[?&]page=${Page.Models}(&|$)`)); + + await navigateToPage(page, Page.LlmPlayground); + await expect(page).toHaveURL(new RegExp(`[?&]page=${Page.LlmPlayground}(&|$)`)); + }); }); } From 022db6c9edb62c648c70d12ab037de46dd8b140c Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Fri, 9 Jan 2026 15:07:39 +0900 Subject: [PATCH 30/56] feat: add mcp registry --- docs/my-website/docs/mcp_control.md | 13 +++ .../mcp_management_endpoints.py | 109 ++++++++++++++++-- .../test_mcp_management_endpoints.py | 73 +++++++++++- 3 files changed, 184 insertions(+), 11 deletions(-) diff --git a/docs/my-website/docs/mcp_control.md b/docs/my-website/docs/mcp_control.md index a7d66a6b7fc..96c71ef9278 100644 --- a/docs/my-website/docs/mcp_control.md +++ b/docs/my-website/docs/mcp_control.md @@ -649,3 +649,16 @@ general_settings: ``` This is useful when you want discoverability for MCP offerings without granting additional execution privileges. + + +## Publish MCP Registry + +If you want other systems—for example external agent frameworks such as MCP-capable IDEs running outside your network—to automatically discover the MCP servers hosted on LiteLLM, you can expose a Model Context Protocol Registry endpoint. This registry lists the built-in LiteLLM MCP server and every server you have configured, using the [official MCP Registry spec](https://github.com/modelcontextprotocol/registry). + +1. Set `enable_mcp_registry: true` under `general_settings` in your proxy config (or DB settings) and restart the proxy. +2. LiteLLM will serve the registry at `GET /v1/mcp/registry.json`. +3. Each entry points to either `/mcp` (built-in server) or `/{mcp_server_name}/mcp` for your custom servers, so clients can connect directly using the advertised Streamable HTTP URL. + +:::note Permissions still apply +The registry only advertises server URLs. Actual access control is still enforced by LiteLLM when the client connects to `/mcp` or `/{server}/mcp`, so publishing the registry does not bypass per-key permissions. +::: diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 47793c8fc8e..d8816df010a 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -36,12 +36,19 @@ from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._uuid import uuid from litellm.constants import LITELLM_PROXY_ADMIN_NAME from litellm.proxy._experimental.mcp_server.utils import ( + get_server_prefix, validate_and_normalize_mcp_server_payload, ) router = APIRouter(prefix="/v1/mcp", tags=["mcp"]) + MCP_AVAILABLE: bool = True + TEMPORARY_MCP_SERVER_TTL_SECONDS = 300 +DEFAULT_MCP_REGISTRY_VERSION = "1.0.0" +LITELLM_MCP_SERVER_NAME = "litellm-mcp-server" +LITELLM_MCP_SERVER_DESCRIPTION = "MCP Server for LiteLLM" + try: importlib.import_module("mcp") except ImportError as e: @@ -57,6 +64,7 @@ if MCP_AVAILABLE: update_mcp_server, ) from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + get_request_base_url, authorize_with_server, exchange_token_with_server, register_client_with_server, @@ -89,6 +97,66 @@ if MCP_AVAILABLE: server: MCPServer expires_at: datetime + def _is_public_registry_enabled() -> bool: + from litellm.proxy.proxy_server import ( + general_settings as proxy_general_settings, + ) + + return bool(proxy_general_settings.get("enable_mcp_registry")) + + def _build_registry_remote_url(base_url: str, path: str) -> str: + normalized_base = base_url.rstrip("/") + normalized_path = path if path.startswith("/") else f"/{path}" + return f"{normalized_base}{normalized_path}" + + def _build_mcp_registry_server_name(server: MCPServer) -> str: + if server.alias: + return server.alias + if server.server_name: + return server.server_name + return server.server_id + + def _build_mcp_registry_entry_for_server( + server: MCPServer, base_url: str + ) -> Dict[str, Any]: + server_name = _build_mcp_registry_server_name(server) + title = server_name + description = server_name + version = DEFAULT_MCP_REGISTRY_VERSION + + server_prefix = get_server_prefix(server) + if not server_prefix: + raise ValueError("MCP server prefix is missing") + remote_url = _build_registry_remote_url(base_url, f"/{server_prefix}/mcp") + + return { + "name": server_name, + "title": title, + "description": description, + "version": version, + "remotes": [ + { + "type": "streamable-http", + "url": remote_url, + } + ], + } + + def _build_builtin_registry_entry(base_url: str) -> Dict[str, Any]: + remote_url = _build_registry_remote_url(base_url, "/mcp") + return { + "name": LITELLM_MCP_SERVER_NAME, + "title": LITELLM_MCP_SERVER_NAME, + "description": LITELLM_MCP_SERVER_DESCRIPTION, + "version": DEFAULT_MCP_REGISTRY_VERSION, + "remotes": [ + { + "type": "streamable-http", + "url": remote_url, + } + ], + } + _temporary_mcp_servers: Dict[str, _TemporaryMCPServerEntry] = {} def _prune_expired_temporary_mcp_servers() -> None: @@ -302,15 +370,42 @@ if MCP_AVAILABLE: access_groups_list = sorted(list(access_groups)) return {"access_groups": access_groups_list} + @router.get( + "/registry.json", + tags=["mcp"], + description="MCP registry endpoint. Spec: https://github.com/modelcontextprotocol/registry", + ) + async def get_mcp_registry(request: Request): + if not _is_public_registry_enabled(): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="MCP registry is not enabled", + ) + + base_url = get_request_base_url(request) + registry_servers: List[Dict[str, Any]] = [] + registry_servers.append({"server": _build_builtin_registry_entry(base_url)}) + + registered_servers = list(global_mcp_server_manager.get_registry().values()) + registered_servers.sort(key=_build_mcp_registry_server_name) + + for server in registered_servers: + try: + entry = _build_mcp_registry_entry_for_server(server, base_url) + except Exception as e: + verbose_proxy_logger.debug( + f"Skipping MCP server {getattr(server, 'server_id', 'unknown')} in registry: {e}" + ) + continue + registry_servers.append({"server": entry}) + + return {"servers": registry_servers} + ## FastAPI Routes def _get_user_mcp_management_mode() -> UserMCPManagementMode: - proxy_general_settings: dict = {} - try: - from litellm.proxy.proxy_server import ( - general_settings as proxy_general_settings, - ) - except Exception: - pass + from litellm.proxy.proxy_server import ( + general_settings as proxy_general_settings, + ) mode = proxy_general_settings.get("user_mcp_management_mode") if mode == "view_all": diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index f2bae2cb14a..bc223d15d5f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -1,20 +1,20 @@ import json import os import sys -from litellm._uuid import uuid +import types from datetime import datetime, timedelta -from typing import List +from typing import List, Optional from unittest.mock import AsyncMock, MagicMock, patch import pytest +from fastapi import FastAPI from fastapi.testclient import TestClient +from litellm._uuid import uuid sys.path.insert( 0, os.path.abspath("../../../..") ) # Adds the parent directory to the system path -from typing import Optional - from litellm.proxy._types import ( LiteLLM_MCPServerTable, LitellmUserRoles, @@ -118,6 +118,22 @@ def setup_mock_prisma_client( return mock_prisma_client +def create_mcp_router_test_client() -> TestClient: + from litellm.proxy.management_endpoints.mcp_management_endpoints import router + + app = FastAPI() + app.include_router(router) + return TestClient(app) + + +def patch_proxy_general_settings(settings: dict): + fake_proxy_server_module = types.SimpleNamespace(general_settings=settings) + return patch.dict( + sys.modules, + {"litellm.proxy.proxy_server": fake_proxy_server_module}, + ) + + class TestListMCPServers: """Test suite for list MCP servers functionality""" @@ -1082,6 +1098,55 @@ class TestHealthCheckServers: assert result[1]["server_id"] == "server-2" assert result[1]["status"] == "unhealthy" + +class TestMCPRegistryEndpoint: + def test_registry_returns_404_when_flag_missing(self): + client = create_mcp_router_test_client() + + with patch_proxy_general_settings({}): + response = client.get("/v1/mcp/registry.json") + + assert response.status_code == 404 + + def test_registry_returns_404_when_flag_false(self): + client = create_mcp_router_test_client() + + with patch_proxy_general_settings({"enable_mcp_registry": False}): + response = client.get("/v1/mcp/registry.json") + + assert response.status_code == 404 + + def test_registry_returns_entries_when_enabled(self): + client = create_mcp_router_test_client() + + mock_server = generate_mock_mcp_server_config_record( + server_id="server-123", + name="zapier", + url="https://zapier.example.com/mcp", + transport="http", + ) + + mock_manager = MagicMock() + mock_manager.get_registry.return_value = {mock_server.server_id: mock_server} + + with patch_proxy_general_settings({"enable_mcp_registry": True}), patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ): + response = client.get("/v1/mcp/registry.json") + + assert response.status_code == 200 + data = response.json() + assert len(data["servers"]) == 2 # built-in + custom server + + builtin_entry = data["servers"][0]["server"] + assert builtin_entry["name"] == "litellm-mcp-server" + assert builtin_entry["remotes"][0]["url"].endswith("/mcp") + + custom_entry = data["servers"][1]["server"] + assert custom_entry["name"] == "zapier" + assert custom_entry["remotes"][0]["url"].endswith("/zapier/mcp") + @pytest.mark.asyncio async def test_health_check_specific_servers(self): """ From c0ee5da44476da0ba973e1ec6e7febb6eff94b15 Mon Sep 17 00:00:00 2001 From: Justas Brazauskas Date: Fri, 9 Jan 2026 10:03:45 +0200 Subject: [PATCH 31/56] Fix: Add thought_signatures to VertexGeminiConfig and test --- .../vertex_and_google_ai_studio_gemini.py | 1 + .../test_vertex_gemini_unbound_local_error.py | 38 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 tests/test_litellm/llms/vertex_ai/gemini/test_vertex_gemini_unbound_local_error.py diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index ba1788a217f..b9f46b540a8 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -1811,6 +1811,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): functions: Optional[ChatCompletionToolCallFunctionChunk] = None thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None reasoning_content: Optional[str] = None + thought_signatures: Optional[Any] = None for idx, candidate in enumerate(_candidates): if "content" not in candidate: diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_gemini_unbound_local_error.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_gemini_unbound_local_error.py new file mode 100644 index 00000000000..0a1ac7e2a54 --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_gemini_unbound_local_error.py @@ -0,0 +1,38 @@ +import pytest +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig +from litellm import ModelResponse + +def test_process_candidates_unbound_local_error_fix(): + # Setup + candidates = [ + { + "content": { + "role": "model" + # "parts" is missing intentionally to trigger the issue + }, + "finishReason": "STOP" + } + ] + model_response = ModelResponse() + + # Execution + try: + VertexGeminiConfig._process_candidates( + _candidates=candidates, + model_response=model_response, + standard_optional_params={}, + cumulative_tool_call_index=0 + ) + except UnboundLocalError as e: + pytest.fail(f"UnboundLocalError raised: {e}") + except Exception as e: + # Other exceptions might be okay if they are not UnboundLocalError, + # but ideally it should pass without error or raise a specific error if parts are required. + # However, the goal is to verify thought_signatures doesn't crash. + pass + + # Verify that we didn't crash with UnboundLocalError + +if __name__ == "__main__": + test_process_candidates_unbound_local_error_fix() + print("Test passed!") From 8c11ddfc5bf9770bbc46ae2168cf132a50340e46 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Fri, 9 Jan 2026 17:12:17 +0900 Subject: [PATCH 32/56] fix: periodically refresh MCP registry across instances via scheduler job --- .../mcp_server/mcp_server_manager.py | 28 +++++++++++++++--- litellm/proxy/proxy_server.py | 29 +++++++++++++++++++ .../types/mcp_server/mcp_server_manager.py | 2 ++ 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 3a548e203c5..6e7db26a95c 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -551,6 +551,7 @@ class MCPServerManager: allowed_tools=getattr(mcp_server, "allowed_tools", None), disallowed_tools=getattr(mcp_server, "disallowed_tools", None), allow_all_keys=mcp_server.allow_all_keys, + updated_at=getattr(mcp_server, "updated_at", None), ) return new_server @@ -2074,15 +2075,34 @@ class MCPServerManager: db_mcp_servers = await get_all_mcp_servers(prisma_client) verbose_logger.info(f"Found {len(db_mcp_servers)} MCP servers in database") - # ensure the global_mcp_server_manager is up to date with the db + previous_registry = self.registry + new_registry: Dict[str, MCPServer] = {} + for server in db_mcp_servers: + existing_server = previous_registry.get(server.server_id) + + if ( + existing_server is not None + and existing_server.updated_at is not None + and server.updated_at is not None + and existing_server.updated_at == server.updated_at + ): + # Re-use existing server instance to avoid re-running build_mcp_server_from_table() + # which can perform network discovery for OAuth2 servers. + new_registry[server.server_id] = existing_server + continue + verbose_logger.debug( - f"Adding server to registry: {server.server_id} ({server.server_name})" + f"Building server from DB: {server.server_id} ({server.server_name})" ) - await self.add_server(server) + new_registry[server.server_id] = await self.build_mcp_server_from_table( + server + ) + + self.registry = new_registry verbose_logger.debug( - f"Registry now contains {len(self.get_registry())} servers" + "MCP registry refreshed (%s servers in registry)", len(new_registry) ) def get_mcp_servers_from_ids(self, server_ids: List[str]) -> List[MCPServer]: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0e58ae7c643..77a0f5a1c09 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4109,6 +4109,23 @@ class ProxyConfig: return [] +async def _reload_mcp_servers_job(): + """Background job entrypoint for MCP registry refreshes.""" + if proxy_config._should_load_db_object(object_type="mcp") is False: + return + + try: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + await global_mcp_server_manager._add_mcp_servers_from_db_to_in_memory_registry() # noqa: SLF001 + except Exception as e: + verbose_proxy_logger.exception( + "Failed to reload MCP servers from database: %s", str(e) + ) + + proxy_config = ProxyConfig() @@ -4646,6 +4663,18 @@ class ProxyStartupEvent: misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, ) await proxy_config.get_credentials(prisma_client=prisma_client) + + from litellm.proxy._experimental.mcp_server.utils import is_mcp_available + + if is_mcp_available(): + scheduler.add_job( + _reload_mcp_servers_job, + "interval", + seconds=30, + id="reload_mcp_servers_job", + replace_existing=True, + misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, + ) await cls._initialize_slack_alerting_jobs( scheduler=scheduler, general_settings=general_settings, diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index 96fd79f466b..94f33ffb297 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -1,3 +1,4 @@ +from datetime import datetime from typing import Any, Dict, List, Optional from pydantic import BaseModel, ConfigDict @@ -50,4 +51,5 @@ class MCPServer(BaseModel): env: Optional[Dict[str, str]] = None access_groups: Optional[List[str]] = None allow_all_keys: bool = False + updated_at: Optional[datetime] = None model_config = ConfigDict(arbitrary_types_allowed=True) From 8b90e5f4dd53766694c9fe833b1601c695b8df56 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Fri, 9 Jan 2026 17:16:38 +0900 Subject: [PATCH 33/56] refactor: expose MCP registry reload helper --- .../_experimental/mcp_server/mcp_server_manager.py | 10 ++-------- litellm/proxy/proxy_server.py | 4 ++-- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 6e7db26a95c..fb4943100ad 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -2060,7 +2060,8 @@ class MCPServerManager: return None - async def _add_mcp_servers_from_db_to_in_memory_registry(self): + async def reload_servers_from_database(self): + """Re-synchronize the in-memory MCP server registry with the database.""" from litellm.proxy._experimental.mcp_server.db import get_all_mcp_servers from litellm.proxy.management_endpoints.mcp_management_endpoints import ( get_prisma_client_or_throw, @@ -2389,13 +2390,6 @@ class MCPServerManager: servers.append(self._build_mcp_server_table(server)) return servers - async def reload_servers_from_database(self): - """ - Public method to reload all MCP servers from database into registry. - This can be called from management endpoints to ensure registry is up to date. - """ - await self._add_mcp_servers_from_db_to_in_memory_registry() - async def get_all_mcp_servers_with_health_unfiltered( self, server_ids: Optional[List[str]] = None ) -> List[LiteLLM_MCPServerTable]: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 77a0f5a1c09..4a44586f0d4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3972,7 +3972,7 @@ class ProxyConfig: ) try: - await global_mcp_server_manager._add_mcp_servers_from_db_to_in_memory_registry() + await global_mcp_server_manager.reload_servers_from_database() except Exception as e: verbose_proxy_logger.exception( "litellm.proxy.proxy_server.py::ProxyConfig:_init_mcp_servers_in_db - {}".format( @@ -4119,7 +4119,7 @@ async def _reload_mcp_servers_job(): global_mcp_server_manager, ) - await global_mcp_server_manager._add_mcp_servers_from_db_to_in_memory_registry() # noqa: SLF001 + await global_mcp_server_manager.reload_servers_from_database() except Exception as e: verbose_proxy_logger.exception( "Failed to reload MCP servers from database: %s", str(e) From 5927a557fbd78e03549bb0f9d25d840b42678698 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Fri, 9 Jan 2026 17:25:21 +0900 Subject: [PATCH 34/56] tests: add test --- .../mcp_server/test_mcp_server.py | 104 +++++++++++++++++- 1 file changed, 103 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index 8062243dfdd..f1558ac5791 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -1,4 +1,5 @@ import asyncio +from datetime import datetime, timedelta from typing import Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -7,7 +8,12 @@ from fastapi import HTTPException from mcp import ReadResourceResult, Resource from mcp.types import Prompt, ResourceTemplate, TextResourceContents -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import ( + LiteLLM_MCPServerTable, + MCPTransport, + UserAPIKeyAuth, +) +from litellm.types.mcp_server.mcp_server_manager import MCPServer @pytest.mark.asyncio @@ -1688,3 +1694,99 @@ def test_filter_tools_by_allowed_tools(): assert len(filtered_tools) == 2 assert filtered_tools[0].name == "my_api_mcp-getpetbyid" assert filtered_tools[1].name == "my_api_mcp-findpetsbystatus" + + +def _make_db_mcp_server(server_id: str, updated_at: datetime) -> LiteLLM_MCPServerTable: + return LiteLLM_MCPServerTable( + server_id=server_id, + server_name="server", + alias="server", + url="https://example.com", + transport=MCPTransport.http, + created_at=updated_at, + updated_at=updated_at, + mcp_info={}, + ) + + +class TestMCPServerManagerReload: + @pytest.mark.asyncio + async def test_reuses_existing_server_when_updated_at_matches(self): + try: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, + ) + except ImportError: + pytest.skip("MCP server not available") + + manager = MCPServerManager() + timestamp = datetime.utcnow() + existing_server = MCPServer( + server_id="server-1", + name="server", + transport=MCPTransport.http, + updated_at=timestamp, + ) + manager.registry = {existing_server.server_id: existing_server} + + db_row = _make_db_mcp_server("server-1", timestamp) + + with patch( + "litellm.proxy._experimental.mcp_server.db.get_all_mcp_servers", + new=AsyncMock(return_value=[db_row]), + ) as mock_get_all, patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=object(), + ), patch.object( + manager, "build_mcp_server_from_table", AsyncMock() + ) as mock_build: + await manager.reload_servers_from_database() + + mock_get_all.assert_awaited_once() + mock_build.assert_not_awaited() + assert manager.registry["server-1"] is existing_server + + @pytest.mark.asyncio + async def test_rebuilds_server_when_updated_at_changes(self): + try: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, + ) + except ImportError: + pytest.skip("MCP server not available") + + manager = MCPServerManager() + timestamp = datetime.utcnow() + existing_server = MCPServer( + server_id="server-1", + name="server", + transport=MCPTransport.http, + updated_at=timestamp, + ) + manager.registry = {existing_server.server_id: existing_server} + + new_timestamp = timestamp + timedelta(minutes=5) + db_row = _make_db_mcp_server("server-1", new_timestamp) + rebuilt_server = MCPServer( + server_id="server-1", + name="server", + transport=MCPTransport.http, + updated_at=new_timestamp, + ) + + with patch( + "litellm.proxy._experimental.mcp_server.db.get_all_mcp_servers", + new=AsyncMock(return_value=[db_row]), + ) as mock_get_all, patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=object(), + ), patch.object( + manager, + "build_mcp_server_from_table", + AsyncMock(return_value=rebuilt_server), + ) as mock_build: + await manager.reload_servers_from_database() + + mock_get_all.assert_awaited_once() + mock_build.assert_awaited_once_with(db_row) + assert manager.registry["server-1"] is rebuilt_server From d4483d8422e5b92de414f8d248113a50e54eef51 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Fri, 9 Jan 2026 17:31:30 +0900 Subject: [PATCH 35/56] fix: formatter --- litellm/proxy/_experimental/mcp_server/mcp_server_manager.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index fb4943100ad..1029f2241a1 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -698,9 +698,7 @@ class MCPServerManager: results = await asyncio.gather(*tasks) # Flatten results into single list - list_tools_result: List[MCPTool] = [ - tool for tools in results for tool in tools - ] + list_tools_result: List[MCPTool] = [tool for tools in results for tool in tools] verbose_logger.info( f"Successfully fetched {len(list_tools_result)} tools total from all servers" From ba78194ff10e83e028d76aa6c5aff43fa15d2668 Mon Sep 17 00:00:00 2001 From: Raghav Jhavar Date: Fri, 9 Jan 2026 17:08:07 +0700 Subject: [PATCH 36/56] add support for bedrock in token counting api --- litellm/llms/bedrock/common_utils.py | 142 ++++++++------ .../count_tokens/bedrock_token_counter.py | 87 +++++++++ litellm/llms/bedrock/count_tokens/handler.py | 3 + .../bedrock/count_tokens/transformation.py | 4 +- litellm/utils.py | 9 +- .../test_bedrock_common_utils.py | 182 ++++++++++++++++++ 6 files changed, 363 insertions(+), 64 deletions(-) create mode 100644 litellm/llms/bedrock/count_tokens/bedrock_token_counter.py create mode 100644 tests/llm_translation/test_bedrock_common_utils.py diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 9edfe320fb2..59abfae813e 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -15,7 +15,7 @@ import litellm from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) -from litellm.llms.base_llm.base_utils import BaseLLMModelInfo +from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret @@ -359,6 +359,74 @@ def get_bedrock_tool_name(response_tool_name: str) -> str: return response_tool_name +# ============================================================================ +# Standalone model name utility functions +# ============================================================================ + +# Cache the global regions list at module level +_BEDROCK_GLOBAL_REGIONS: Optional[List[str]] = None + + +def _get_all_bedrock_regions() -> List[str]: + """Get all Bedrock regions, cached at module level.""" + global _BEDROCK_GLOBAL_REGIONS + if _BEDROCK_GLOBAL_REGIONS is None: + _BEDROCK_GLOBAL_REGIONS = AmazonBedrockGlobalConfig().get_all_regions() + return _BEDROCK_GLOBAL_REGIONS + + +def get_bedrock_cross_region_inference_regions() -> List[str]: + """Abbreviations of regions AWS Bedrock supports for cross region inference.""" + return ["global", "us", "eu", "apac", "jp", "au", "us-gov"] + + +def extract_model_name_from_bedrock_arn(model: str) -> str: + """ + Extract the model name from an AWS Bedrock ARN. + Returns the string after the last '/' if 'arn' is in the input string. + """ + if "arn" in model.lower(): + return model.split("/")[-1] + return model + + +def strip_bedrock_routing_prefix(model: str) -> str: + """Strip LiteLLM routing prefixes from model name.""" + for prefix in ["bedrock/", "converse/", "invoke/", "openai/"]: + if model.startswith(prefix): + model = model.split("/", 1)[1] + return model + + +def get_bedrock_base_model(model: str) -> str: + """ + Get the base model from the given model name. + + Handle model names like: + - "us.meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1" + - "bedrock/converse/model" -> "model" + """ + model = strip_bedrock_routing_prefix(model) + model = extract_model_name_from_bedrock_arn(model) + + potential_region = model.split(".", 1)[0] + alt_potential_region = model.split("/", 1)[0] + + if potential_region in get_bedrock_cross_region_inference_regions(): + return model.split(".", 1)[1] + elif ( + alt_potential_region in _get_all_bedrock_regions() + and len(model.split("/", 1)) > 1 + ): + return model.split("/", 1)[1] + + return model + + +# Import after standalone functions to avoid circular imports +from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter + + class BedrockModelInfo(BaseLLMModelInfo): global_config = AmazonBedrockGlobalConfig() all_global_regions = global_config.get_all_regions() @@ -394,76 +462,34 @@ class BedrockModelInfo(BaseLLMModelInfo): ) -> List[str]: return [] - @staticmethod - def extract_model_name_from_arn(model: str) -> str: + def get_token_counter(self) -> Optional[BaseTokenCounter]: """ - Extract the model name from an AWS Bedrock ARN. - Returns the string after the last '/' if 'arn' is in the input string. - - Args: - arn (str): The ARN string to parse + Factory method to create a Bedrock token counter. Returns: - str: The extracted model name if 'arn' is in the string, - otherwise returns the original string + BedrockTokenCounter instance for this provider. """ - if "arn" in model.lower(): - return model.split("/")[-1] - return model + return BedrockTokenCounter() + + @staticmethod + def extract_model_name_from_arn(model: str) -> str: + """Wrapper for standalone function. See extract_model_name_from_bedrock_arn().""" + return extract_model_name_from_bedrock_arn(model) @staticmethod def get_non_litellm_routing_model_name(model: str) -> str: - if model.startswith("bedrock/"): - model = model.split("/", 1)[1] - - if model.startswith("converse/"): - model = model.split("/", 1)[1] - - if model.startswith("invoke/"): - model = model.split("/", 1)[1] - - if model.startswith("openai/"): - model = model.split("/", 1)[1] - - return model + """Wrapper for standalone function. See strip_bedrock_routing_prefix().""" + return strip_bedrock_routing_prefix(model) @staticmethod def get_base_model(model: str) -> str: - """ - Get the base model from the given model name. - - Handle model names like - "us.meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1" - AND "meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1" - """ - - model = BedrockModelInfo.get_non_litellm_routing_model_name(model=model) - model = BedrockModelInfo.extract_model_name_from_arn(model) - - potential_region = model.split(".", 1)[0] - - alt_potential_region = model.split("/", 1)[ - 0 - ] # in model cost map we store regional information like `/us-west-2/bedrock-model` - - if ( - potential_region - in BedrockModelInfo._supported_cross_region_inference_region() - ): - return model.split(".", 1)[1] - elif ( - alt_potential_region in BedrockModelInfo.all_global_regions - and len(model.split("/", 1)) > 1 - ): - return model.split("/", 1)[1] - - return model + """Wrapper for standalone function. See get_bedrock_base_model().""" + return get_bedrock_base_model(model) @staticmethod def _supported_cross_region_inference_region() -> List[str]: - """ - Abbreviations of regions AWS Bedrock supports for cross region inference - """ - return ["global", "us", "eu", "apac", "jp", "au", "us-gov"] + """Wrapper for standalone function. See get_bedrock_cross_region_inference_regions().""" + return get_bedrock_cross_region_inference_regions() @staticmethod def get_bedrock_route( diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py new file mode 100644 index 00000000000..b680bd046ef --- /dev/null +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -0,0 +1,87 @@ +""" +Bedrock Token Counter implementation using the CountTokens API. +""" + +from typing import Any, Dict, List, Optional + +from litellm._logging import verbose_logger +from litellm.llms.base_llm.base_utils import BaseTokenCounter +from litellm.llms.bedrock.common_utils import get_bedrock_base_model +from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler +from litellm.types.utils import LlmProviders, TokenCountResponse + + +class BedrockTokenCounter(BaseTokenCounter): + """Token counter implementation for AWS Bedrock provider using the CountTokens API.""" + + def should_use_token_counting_api( + self, + custom_llm_provider: Optional[str] = None, + ) -> bool: + """ + Returns True if we should use the Bedrock CountTokens API for token counting. + """ + return custom_llm_provider == LlmProviders.BEDROCK.value + + async def count_tokens( + self, + model_to_use: str, + messages: Optional[List[Dict[str, Any]]], + contents: Optional[List[Dict[str, Any]]], + deployment: Optional[Dict[str, Any]] = None, + request_model: str = "", + ) -> Optional[TokenCountResponse]: + """ + Count tokens using AWS Bedrock's CountTokens API. + + This method calls the existing BedrockCountTokensHandler to make an API call + to Bedrock's token counting endpoint, bypassing the local tiktoken-based counting. + + Args: + model_to_use: The model identifier + messages: The messages to count tokens for + contents: Alternative content format (not used for Bedrock) + deployment: Deployment configuration containing litellm_params + request_model: The original request model name + + Returns: + TokenCountResponse with token count, or None if counting fails + """ + if not messages: + return None + + deployment = deployment or {} + litellm_params = deployment.get("litellm_params", {}) + + # Build request data in the format expected by BedrockCountTokensHandler + request_data = { + "model": model_to_use, + "messages": messages, + } + + # Get the resolved model (strip prefixes like bedrock/, converse/, etc.) + resolved_model = get_bedrock_base_model(model_to_use) + + try: + handler = BedrockCountTokensHandler() + result = await handler.handle_count_tokens_request( + request_data=request_data, + litellm_params=litellm_params, + resolved_model=resolved_model, + ) + + # Transform response to TokenCountResponse + if result is not None: + return TokenCountResponse( + total_tokens=result.get("input_tokens", 0), + request_model=request_model, + model_used=model_to_use, + tokenizer_type="bedrock_api", + original_response=result, + ) + except Exception as e: + verbose_logger.warning( + f"Error calling Bedrock CountTokens API: {e}, falling back to default tokenizer" + ) + + return None diff --git a/litellm/llms/bedrock/count_tokens/handler.py b/litellm/llms/bedrock/count_tokens/handler.py index d4355c0c360..60ace7f3369 100644 --- a/litellm/llms/bedrock/count_tokens/handler.py +++ b/litellm/llms/bedrock/count_tokens/handler.py @@ -70,6 +70,8 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): verbose_logger.debug(f"Making request to: {endpoint_url}") # Use existing _sign_request method from BaseAWSLLM + # Extract api_key for bearer token auth if provided + api_key = litellm_params.get("api_key", None) headers = {"Content-Type": "application/json"} signed_headers, signed_body = self._sign_request( service_name="bedrock", @@ -78,6 +80,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): request_data=bedrock_request, api_base=endpoint_url, model=resolved_model, + api_key=api_key, ) async_client = get_async_httpx_client(llm_provider=litellm.LlmProviders.BEDROCK) diff --git a/litellm/llms/bedrock/count_tokens/transformation.py b/litellm/llms/bedrock/count_tokens/transformation.py index d46ed3aa452..b313cc9df3c 100644 --- a/litellm/llms/bedrock/count_tokens/transformation.py +++ b/litellm/llms/bedrock/count_tokens/transformation.py @@ -8,7 +8,7 @@ to AWS Bedrock's CountTokens API format and vice versa. from typing import Any, Dict, List from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM -from litellm.llms.bedrock.common_utils import BedrockModelInfo +from litellm.llms.bedrock.common_utils import get_bedrock_base_model class BedrockCountTokensConfig(BaseAWSLLM): @@ -141,7 +141,7 @@ class BedrockCountTokensConfig(BaseAWSLLM): Complete endpoint URL for CountTokens API """ # Use existing LiteLLM function to get the base model ID (removes region prefix) - model_id = BedrockModelInfo.get_base_model(model) + model_id = get_bedrock_base_model(model) # Remove bedrock/ prefix if present if model_id.startswith("bedrock/"): diff --git a/litellm/utils.py b/litellm/utils.py index 2260b2c7ba5..d5d4e61fff4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -47,7 +47,6 @@ from tiktoken import Encoding from tokenizers import Tokenizer import litellm - import litellm.litellm_core_utils # audio_utils.utils is lazy-loaded - only imported when needed for transcription calls import litellm.litellm_core_utils.json_validation_rule @@ -291,7 +290,7 @@ if TYPE_CHECKING: from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig from litellm.llms.base_llm.search.transformation import BaseSearchConfig from litellm.llms.base_llm.text_to_speech.transformation import BaseTextToSpeechConfig - from litellm.llms.bedrock.common_utils import BedrockModelInfo + from litellm.llms.bedrock.bedrock_model_info import BedrockModelInfo from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.mistral.ocr.transformation import MistralOCRConfig # Type stubs for lazy-loaded functions and classes @@ -4954,7 +4953,7 @@ def _get_base_bedrock_model(model_name) -> str: Handle model names like - "us.meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1" AND "meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1" """ - from litellm.llms.bedrock.common_utils import BedrockModelInfo + from litellm.llms.bedrock.bedrock_model_info import BedrockModelInfo return BedrockModelInfo.get_base_model(model_name) @@ -7779,7 +7778,7 @@ class ProviderConfigManager: # The 'BEDROCK' provider corresponds to Amazon's implementation of Anthropic Claude v3. # This mapping ensures that the correct configuration is returned for BEDROCK. elif litellm.LlmProviders.BEDROCK == provider: - from litellm.llms.bedrock.common_utils import BedrockModelInfo + from litellm.llms.bedrock.bedrock_model_info import BedrockModelInfo return BedrockModelInfo.get_bedrock_provider_config_for_messages_api(model) elif litellm.LlmProviders.VERTEX_AI == provider: @@ -7941,6 +7940,8 @@ class ProviderConfigManager: return litellm.LemonadeChatConfig() elif LlmProviders.CLARIFAI == provider: return litellm.ClarifaiConfig() + elif LlmProviders.BEDROCK == provider: + return litellm.llms.bedrock.common_utils.BedrockModelInfo() return None @staticmethod diff --git a/tests/llm_translation/test_bedrock_common_utils.py b/tests/llm_translation/test_bedrock_common_utils.py new file mode 100644 index 00000000000..7b6a05b6988 --- /dev/null +++ b/tests/llm_translation/test_bedrock_common_utils.py @@ -0,0 +1,182 @@ +""" +Unit tests for litellm/llms/bedrock/common_utils.py + +Tests the standalone model name utility functions and BedrockTokenCounter. +""" + +import pytest + +from litellm.llms.bedrock.common_utils import ( + BedrockModelInfo, + extract_model_name_from_bedrock_arn, + get_bedrock_base_model, + get_bedrock_cross_region_inference_regions, + strip_bedrock_routing_prefix, +) +from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter + + +class TestStripBedrockRoutingPrefix: + """Tests for strip_bedrock_routing_prefix function.""" + + def test_strips_bedrock_prefix(self): + assert strip_bedrock_routing_prefix("bedrock/claude-3-sonnet") == "claude-3-sonnet" + + def test_strips_converse_prefix(self): + assert strip_bedrock_routing_prefix("converse/claude-3-sonnet") == "claude-3-sonnet" + + def test_strips_invoke_prefix(self): + assert strip_bedrock_routing_prefix("invoke/claude-3-sonnet") == "claude-3-sonnet" + + def test_strips_openai_prefix(self): + assert strip_bedrock_routing_prefix("openai/gpt-4") == "gpt-4" + + def test_strips_all_known_prefixes(self): + # Function strips all known prefixes iteratively + # bedrock/converse/model -> converse/model -> model + assert strip_bedrock_routing_prefix("bedrock/converse/claude-3") == "claude-3" + + def test_no_prefix_unchanged(self): + assert strip_bedrock_routing_prefix("claude-3-sonnet") == "claude-3-sonnet" + + def test_model_with_dots_unchanged(self): + assert ( + strip_bedrock_routing_prefix("anthropic.claude-3-sonnet-20240229-v1:0") + == "anthropic.claude-3-sonnet-20240229-v1:0" + ) + + +class TestExtractModelNameFromBedrockArn: + """Tests for extract_model_name_from_bedrock_arn function.""" + + def test_extracts_from_provisioned_model_arn(self): + arn = "arn:aws:bedrock:us-east-1:123456789012:provisioned-model/my-model-id" + assert extract_model_name_from_bedrock_arn(arn) == "my-model-id" + + def test_extracts_from_foundation_model_arn(self): + arn = "arn:aws:bedrock:us-west-2:123456789012:foundation-model/anthropic.claude-v2" + assert extract_model_name_from_bedrock_arn(arn) == "anthropic.claude-v2" + + def test_non_arn_unchanged(self): + model = "anthropic.claude-3-sonnet-20240229-v1:0" + assert extract_model_name_from_bedrock_arn(model) == model + + def test_case_insensitive_arn_detection(self): + arn = "ARN:aws:bedrock:us-east-1:123456789012:model/my-model" + assert extract_model_name_from_bedrock_arn(arn) == "my-model" + + +class TestGetBedrockCrossRegionInferenceRegions: + """Tests for get_bedrock_cross_region_inference_regions function.""" + + def test_returns_expected_regions(self): + regions = get_bedrock_cross_region_inference_regions() + assert "us" in regions + assert "eu" in regions + assert "global" in regions + assert "apac" in regions + + def test_returns_list(self): + regions = get_bedrock_cross_region_inference_regions() + assert isinstance(regions, list) + + +class TestGetBedrockBaseModel: + """Tests for get_bedrock_base_model function.""" + + def test_strips_bedrock_prefix(self): + assert get_bedrock_base_model("bedrock/claude-3-sonnet") == "claude-3-sonnet" + + def test_strips_converse_prefix(self): + assert get_bedrock_base_model("bedrock/converse/claude-3-sonnet") == "claude-3-sonnet" + + def test_strips_us_region_prefix(self): + # us.anthropic.model -> anthropic.model + assert ( + get_bedrock_base_model("us.anthropic.claude-3-sonnet-20240229-v1:0") + == "anthropic.claude-3-sonnet-20240229-v1:0" + ) + + def test_strips_eu_region_prefix(self): + assert ( + get_bedrock_base_model("eu.anthropic.claude-3-sonnet-20240229-v1:0") + == "anthropic.claude-3-sonnet-20240229-v1:0" + ) + + def test_extracts_from_arn(self): + arn = "arn:aws:bedrock:us-east-1:123456789012:provisioned-model/my-model" + assert get_bedrock_base_model(arn) == "my-model" + + def test_model_without_prefix_unchanged(self): + model = "anthropic.claude-3-sonnet-20240229-v1:0" + assert get_bedrock_base_model(model) == model + + def test_combined_bedrock_and_region_prefix(self): + # bedrock/us.anthropic.model -> anthropic.model + assert ( + get_bedrock_base_model("bedrock/us.anthropic.claude-3-sonnet-20240229-v1:0") + == "anthropic.claude-3-sonnet-20240229-v1:0" + ) + + +class TestBedrockModelInfoWrappers: + """Tests that BedrockModelInfo methods correctly wrap standalone functions.""" + + def test_get_base_model_matches_standalone(self): + test_cases = [ + "bedrock/claude-3-sonnet", + "us.anthropic.claude-3-sonnet-20240229-v1:0", + "arn:aws:bedrock:us-east-1:123:model/my-model", + ] + for model in test_cases: + assert BedrockModelInfo.get_base_model(model) == get_bedrock_base_model(model) + + def test_extract_model_name_from_arn_matches_standalone(self): + arn = "arn:aws:bedrock:us-east-1:123456789012:provisioned-model/my-model" + assert ( + BedrockModelInfo.extract_model_name_from_arn(arn) + == extract_model_name_from_bedrock_arn(arn) + ) + + def test_get_non_litellm_routing_model_name_matches_standalone(self): + model = "bedrock/converse/claude-3" + assert ( + BedrockModelInfo.get_non_litellm_routing_model_name(model) + == strip_bedrock_routing_prefix(model) + ) + + +class TestBedrockTokenCounter: + """Tests for BedrockTokenCounter class.""" + + def test_should_use_token_counting_api_for_bedrock(self): + counter = BedrockTokenCounter() + assert counter.should_use_token_counting_api("bedrock") is True + + def test_should_not_use_token_counting_api_for_other_providers(self): + counter = BedrockTokenCounter() + assert counter.should_use_token_counting_api("openai") is False + assert counter.should_use_token_counting_api("anthropic") is False + assert counter.should_use_token_counting_api(None) is False + + def test_get_token_counter_returns_bedrock_token_counter(self): + model_info = BedrockModelInfo() + token_counter = model_info.get_token_counter() + assert isinstance(token_counter, BedrockTokenCounter) + + @pytest.mark.asyncio + async def test_count_tokens_returns_none_for_empty_messages(self): + counter = BedrockTokenCounter() + result = await counter.count_tokens( + model_to_use="anthropic.claude-3-sonnet", + messages=None, + contents=None, + ) + assert result is None + + result = await counter.count_tokens( + model_to_use="anthropic.claude-3-sonnet", + messages=[], + contents=None, + ) + assert result is None From 0bdc411fb71e49884b4795d30e87e44f873139af Mon Sep 17 00:00:00 2001 From: Raghav Jhavar Date: Fri, 9 Jan 2026 17:24:33 +0700 Subject: [PATCH 37/56] remove comment --- litellm/llms/bedrock/common_utils.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 59abfae813e..5cb51cf994f 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -359,10 +359,6 @@ def get_bedrock_tool_name(response_tool_name: str) -> str: return response_tool_name -# ============================================================================ -# Standalone model name utility functions -# ============================================================================ - # Cache the global regions list at module level _BEDROCK_GLOBAL_REGIONS: Optional[List[str]] = None From f5ee89dcf6ca55c9da6445de0cb25c73a5fd9f67 Mon Sep 17 00:00:00 2001 From: Raghav Jhavar Date: Fri, 9 Jan 2026 17:43:23 +0700 Subject: [PATCH 38/56] fix broken imports --- litellm/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index d5d4e61fff4..4f990bb5f4b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -290,7 +290,7 @@ if TYPE_CHECKING: from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig from litellm.llms.base_llm.search.transformation import BaseSearchConfig from litellm.llms.base_llm.text_to_speech.transformation import BaseTextToSpeechConfig - from litellm.llms.bedrock.bedrock_model_info import BedrockModelInfo + from litellm.llms.bedrock.common_utils import BedrockModelInfo from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.mistral.ocr.transformation import MistralOCRConfig # Type stubs for lazy-loaded functions and classes @@ -4953,7 +4953,7 @@ def _get_base_bedrock_model(model_name) -> str: Handle model names like - "us.meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1" AND "meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1" """ - from litellm.llms.bedrock.bedrock_model_info import BedrockModelInfo + from litellm.llms.bedrock.common_utils import BedrockModelInfo return BedrockModelInfo.get_base_model(model_name) @@ -7778,7 +7778,7 @@ class ProviderConfigManager: # The 'BEDROCK' provider corresponds to Amazon's implementation of Anthropic Claude v3. # This mapping ensures that the correct configuration is returned for BEDROCK. elif litellm.LlmProviders.BEDROCK == provider: - from litellm.llms.bedrock.bedrock_model_info import BedrockModelInfo + from litellm.llms.bedrock.common_utils import BedrockModelInfo return BedrockModelInfo.get_bedrock_provider_config_for_messages_api(model) elif litellm.LlmProviders.VERTEX_AI == provider: From ffa0d6706c05555735a1a2e1aa958b01dfc384a3 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 9 Jan 2026 16:53:35 +0530 Subject: [PATCH 39/56] Fix: response_format leaking into extra_body --- litellm/utils.py | 1 + .../test_azure_image_generation_init.py | 87 +++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/litellm/utils.py b/litellm/utils.py index 2260b2c7ba5..eb2ea46bac4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2895,6 +2895,7 @@ def get_optional_params_image_gen( litellm.drop_params is True or drop_params is True ) and k not in supported_params: # drop the unsupported non-default values non_default_params.pop(k, None) + passed_params.pop(k, None) elif k not in supported_params: raise UnsupportedParamsError( status_code=500, diff --git a/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py b/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py index 998510efcd9..987eb5bf998 100644 --- a/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py +++ b/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py @@ -164,3 +164,90 @@ def test_azure_image_generation_headers_without_api_key(): # Verify api-key is added when api_key is valid assert "api-key" in default_headers_with_key assert default_headers_with_key["api-key"] == "valid-key-123" + + +def test_azure_image_generation_drop_params_response_format(): + """ + Test that unsupported params like response_format are dropped when drop_params=True. + + Azure gpt-image-1.5 doesn't support response_format parameter. When drop_params=True, + this parameter should be completely removed and not appear in the final request body, + including not being added to extra_body. + + This test verifies the fix where: + 1. Unsupported params are removed from non_default_params in _check_valid_arg + 2. Unsupported params are also removed from passed_params to prevent them from + being re-added via extra_body in add_provider_specific_params_to_optional_params + + Without the fix, response_format would be added to extra_body and cause Azure to + return a 400 Bad Request error due to strict schema validation. + """ + from litellm.llms.openai.image_generation.gpt_transformation import ( + GPTImageGenerationConfig, + ) + + # Test with gpt-image-1.5 which doesn't support response_format + config = GPTImageGenerationConfig() + supported_params = config.get_supported_openai_params(model="gpt-image-1.5") + + # Verify response_format is NOT in supported params for gpt-image-1.5 + assert "response_format" not in supported_params + assert "n" in supported_params + assert "size" in supported_params + + # Test get_optional_params_image_gen with drop_params=True + optional_params = get_optional_params_image_gen( + model="gpt-image-1.5", + n=1, + size="1024x1024", + response_format="b64_json", # This should be dropped + custom_llm_provider="azure", + provider_config=config, + drop_params=True, + ) + + # Verify response_format is NOT in optional_params + assert "response_format" not in optional_params, ( + "response_format should be dropped from optional_params" + ) + + # Verify response_format is NOT in extra_body either + if "extra_body" in optional_params: + assert "response_format" not in optional_params["extra_body"], ( + "response_format should not be in extra_body" + ) + + # Verify supported params ARE in optional_params + assert "n" in optional_params + assert optional_params["n"] == 1 + assert "size" in optional_params + assert optional_params["size"] == "1024x1024" + + +def test_azure_image_generation_drop_params_false_raises_error(): + """ + Test that unsupported params raise an error when drop_params=False. + + This verifies that the error handling still works correctly when drop_params + is not enabled. + """ + from litellm.exceptions import UnsupportedParamsError + from litellm.llms.openai.image_generation.gpt_transformation import ( + GPTImageGenerationConfig, + ) + + config = GPTImageGenerationConfig() + + # Test that passing unsupported param with drop_params=False raises error + with pytest.raises(UnsupportedParamsError) as exc_info: + optional_params = get_optional_params_image_gen( + model="gpt-image-1.5", + n=1, + response_format="b64_json", # Unsupported param + custom_llm_provider="azure", + provider_config=config, + drop_params=False, + ) + + # Verify the error message mentions the unsupported parameter + assert "response_format" in str(exc_info.value) From 819468554f9e3908838e0301e34a3a0aa9f0fa3d Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Fri, 9 Jan 2026 22:27:39 +0530 Subject: [PATCH 40/56] fix(security): prevent expired key plaintext leak in error response (#18860) --- litellm/proxy/auth/user_api_key_auth.py | 33 ++++++++----------- .../proxy/auth/test_user_api_key_auth.py | 12 +++++++ 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 9b53d9a3a80..401aa7fd443 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -138,7 +138,7 @@ def _apply_budget_limits_to_end_user_params( ) -> None: """ Helper function to apply budget limits to end user parameters. - + Args: end_user_params: Dictionary to update with budget parameters budget_info: Budget table object containing limits @@ -146,16 +146,14 @@ def _apply_budget_limits_to_end_user_params( """ if budget_info.tpm_limit is not None: end_user_params["end_user_tpm_limit"] = budget_info.tpm_limit - + if budget_info.rpm_limit is not None: end_user_params["end_user_rpm_limit"] = budget_info.rpm_limit - + if budget_info.max_budget is not None: end_user_params["end_user_max_budget"] = budget_info.max_budget - - verbose_proxy_logger.debug( - f"Applied budget limits to end user {end_user_id}" - ) + + verbose_proxy_logger.debug(f"Applied budget limits to end user {end_user_id}") async def user_api_key_auth_websocket(websocket: WebSocket): @@ -170,12 +168,10 @@ async def user_api_key_auth_websocket(websocket: WebSocket): model = query_params.get("model") - async def return_body(): return _realtime_request_body(model) - - request.body = return_body # type: ignore + request.body = return_body # type: ignore authorization = websocket.headers.get("authorization") # If no Authorization header, try the api-key header @@ -586,7 +582,9 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 if team_membership is not None else None ), - team_metadata=team_object.metadata if team_object is not None else None, + team_metadata=team_object.metadata + if team_object is not None + else None, ) # run through common checks _ = await common_checks( @@ -669,9 +667,9 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 route=route, ) if _end_user_object is not None: - end_user_params["allowed_model_region"] = ( - _end_user_object.allowed_model_region - ) + end_user_params[ + "allowed_model_region" + ] = _end_user_object.allowed_model_region if _end_user_object.litellm_budget_table is not None: _apply_budget_limits_to_end_user_params( end_user_params=end_user_params, @@ -753,7 +751,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}", type=ProxyErrorTypes.expired_key, code=400, - param=api_key, + param=abbreviate_api_key(api_key=api_key), ) valid_token = update_valid_token_with_end_user_params( valid_token=valid_token, end_user_params=end_user_params @@ -994,7 +992,6 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 # Check 3. Check if user is in their team budget if valid_token.team_member_spend is not None: - if prisma_client is not None: _cache_key = f"{valid_token.team_id}_{valid_token.user_id}" @@ -1055,7 +1052,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}", type=ProxyErrorTypes.expired_key, code=400, - param=api_key, + param=abbreviate_api_key(api_key=api_key), ) # Check 4. Token Spend is under budget @@ -1216,8 +1213,6 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 ) - - @tracer.wrap() async def user_api_key_auth( request: Request, diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index fcc8c1f0f2e..5f49db66089 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -338,6 +338,17 @@ async def test_proxy_admin_expired_key_from_cache(): f"Exception message should mention 'Expired Key', got: {exc_info.value.message}" ) + # Verify that the param field does NOT leak the full API key (Issue #18731) + # The param should be abbreviated like "sk-...XXXX" not the full plaintext key + assert exc_info.value.param is not None, "Exception should have 'param' attribute" + assert exc_info.value.param != api_key, ( + f"SECURITY: Full API key should NOT be in param field! " + f"Got: {exc_info.value.param}, Expected abbreviated format like 'sk-...XXXX'" + ) + assert exc_info.value.param.startswith("sk-..."), ( + f"Param should be abbreviated to 'sk-...XXXX' format. Got: {exc_info.value.param}" + ) + # Verify that cache deletion was called mock_delete_cache.assert_called_once() call_args = mock_delete_cache.call_args @@ -347,3 +358,4 @@ async def test_proxy_admin_expired_key_from_cache(): finally: # Clean up - restore original values if needed pass + From 153fd7ad0ac237ee410872797f144370f29378e8 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Fri, 9 Jan 2026 14:26:32 -0300 Subject: [PATCH 41/56] fix: prevent Prisma migration workflow from running in forks (#18863) - Add repository check to only run workflow in BerriAI/litellm - Prevents workflow failures and wasted resources in forked repositories - Avoids confusion for external contributors --- .github/workflows/publish-migrations.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-migrations.yml b/.github/workflows/publish-migrations.yml index 8e5a67bcf85..a5187cb2f55 100644 --- a/.github/workflows/publish-migrations.yml +++ b/.github/workflows/publish-migrations.yml @@ -13,6 +13,7 @@ on: jobs: publish-migrations: + if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest services: postgres: From 9768eca33e121da04e9e3691161b4cdcf39b66cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s?= Date: Fri, 9 Jan 2026 18:27:50 +0100 Subject: [PATCH 42/56] fix(azure): add logprobs support for Azure OpenAI GPT-5.2 model (#18856) * fix(azure): add logprobs support for Azure OpenAI GPT-5 models Azure OpenAI GPT-5 models (including gpt-5.2) support logprobs parameters, unlike OpenAI's GPT-5 reasoning models. This fix overrides the parent class restriction to enable logprobs for Azure. Changes: - Override get_supported_openai_params() in AzureOpenAIGPT5Config - Add "logprobs" and "top_logprobs" to supported params - Add comprehensive tests for logprobs functionality Testing: - Verified with direct Azure API calls to gpt-5.2 - API version: 2025-01-01-preview - Successfully returns logprobs data Related: #7974, #4022 * refactor: restrict logprobs support to gpt-5.2 only Only gpt-5.2 has been verified to support logprobs on Azure. Other gpt-5 variants (gpt-5, gpt-5.1) have not been tested. Changes: - Add conditional check for is_model_gpt_5_2_model() - Update tests to be specific to gpt-5.2 - Add negative tests for gpt-5 and gpt-5.1 - Update documentation to reflect gpt-5.2 specificity --- .../llms/azure/chat/gpt_5_transformation.py | 19 +++++- .../chat/test_azure_gpt5_transformation.py | 59 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/litellm/llms/azure/chat/gpt_5_transformation.py b/litellm/llms/azure/chat/gpt_5_transformation.py index 87f81d117f0..506b7fdfe5e 100644 --- a/litellm/llms/azure/chat/gpt_5_transformation.py +++ b/litellm/llms/azure/chat/gpt_5_transformation.py @@ -25,7 +25,24 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): return "gpt-5" in model or "gpt5_series" in model def get_supported_openai_params(self, model: str) -> List[str]: - return OpenAIGPT5Config.get_supported_openai_params(self, model=model) + """Get supported parameters for Azure OpenAI GPT-5 models. + + Azure OpenAI GPT-5.2 models support logprobs, unlike OpenAI's GPT-5. + This overrides the parent class to add logprobs support back for gpt-5.2. + + Reference: + - Tested with Azure OpenAI GPT-5.2 (api-version: 2025-01-01-preview) + - Azure returns logprobs successfully despite Microsoft's general + documentation stating reasoning models don't support it. + """ + params = OpenAIGPT5Config.get_supported_openai_params(self, model=model) + + # Only gpt-5.2 has been verified to support logprobs on Azure + if self.is_model_gpt_5_2_model(model): + azure_supported_params = ["logprobs", "top_logprobs"] + params.extend(azure_supported_params) + + return params def map_openai_params( self, diff --git a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py index 91d664c3216..199a16d8590 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py @@ -204,3 +204,62 @@ def test_azure_gpt5_reasoning_effort_none_dropped(config: AzureOpenAIGPT5Config) ) assert "reasoning_effort" not in params or params.get("reasoning_effort") != "none" + +# Logprobs support tests for Azure GPT-5.2 +def test_azure_gpt5_2_supports_logprobs(config: AzureOpenAIGPT5Config): + """Test that Azure GPT-5.2 models support logprobs parameters. + + Only Azure OpenAI GPT-5.2 supports logprobs, unlike OpenAI's GPT-5 or Azure's gpt-5/gpt-5.1. + Tested with gpt-5.2 on api-version 2025-01-01-preview. + """ + supported_params = config.get_supported_openai_params(model="gpt-5.2") + assert "logprobs" in supported_params + assert "top_logprobs" in supported_params + + +def test_azure_gpt5_2_with_prefix_supports_logprobs(config: AzureOpenAIGPT5Config): + """Test that Azure GPT-5.2 with azure/ prefix supports logprobs parameters.""" + supported_params = config.get_supported_openai_params(model="azure/gpt-5.2") + assert "logprobs" in supported_params + assert "top_logprobs" in supported_params + + +def test_azure_gpt5_2_series_supports_logprobs(config: AzureOpenAIGPT5Config): + """Test that Azure GPT-5.2 with gpt5_series prefix supports logprobs.""" + supported_params = config.get_supported_openai_params(model="gpt5_series/gpt-5.2") + assert "logprobs" in supported_params + assert "top_logprobs" in supported_params + + +def test_azure_gpt5_2_logprobs_params_passed_through(config: AzureOpenAIGPT5Config): + """Test that logprobs parameters are correctly passed through to the API for gpt-5.2.""" + params = config.map_openai_params( + non_default_params={"logprobs": True, "top_logprobs": 5}, + optional_params={}, + model="azure/gpt-5.2", + drop_params=False, + api_version="2025-01-01-preview", + ) + assert params["logprobs"] is True + assert params["top_logprobs"] == 5 + + +def test_azure_gpt5_base_does_not_support_logprobs(config: AzureOpenAIGPT5Config): + """Test that Azure GPT-5 (non-5.2) does not support logprobs parameters. + + Only gpt-5.2 has been verified to support logprobs on Azure. + """ + supported_params = config.get_supported_openai_params(model="gpt-5") + assert "logprobs" not in supported_params + assert "top_logprobs" not in supported_params + + +def test_azure_gpt5_1_does_not_support_logprobs(config: AzureOpenAIGPT5Config): + """Test that Azure GPT-5.1 does not support logprobs parameters. + + Only gpt-5.2 has been verified to support logprobs on Azure. + """ + supported_params = config.get_supported_openai_params(model="gpt-5.1") + assert "logprobs" not in supported_params + assert "top_logprobs" not in supported_params + From dd087c8cca877afe1588ea1c1c19c05332720d3e Mon Sep 17 00:00:00 2001 From: Martin Gauthier Date: Fri, 9 Jan 2026 13:27:18 -0500 Subject: [PATCH 43/56] =?UTF-8?q?=F0=9F=90=9B=20fix:=20propagate=20headers?= =?UTF-8?q?=20in=20router=20embedding=20calls=20(#18844)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix router embedding methods to properly propagate proxy model configuration headers to LLM API calls by calling _update_kwargs_before_fallbacks() just like completion() does. Previously, router.embedding() and router.aembedding() manually set num_retries and metadata but didn't call _update_kwargs_before_fallbacks(), which meant default_litellm_params (including headers) were not propagated correctly. Changes: - Replace manual kwargs setup with _update_kwargs_before_fallbacks() in _embedding method (litellm/router.py:3318) - Apply Black formatting to router.py for consistency - Add comprehensive unit tests for header propagation - Add integration tests for various router configurations Tests verify: - Headers from default_litellm_params are included in embedding calls - Metadata (model_group) is properly set - Consistency between completion() and embedding() behavior - Support for deployment-specific headers, fallbacks, and retries --- litellm/router.py | 119 +++--- .../test_router_embedding_headers.py | 372 ++++++++++++++++++ .../test_router_embedding_integration.py | 355 +++++++++++++++++ 3 files changed, 790 insertions(+), 56 deletions(-) create mode 100644 tests/router_unit_tests/test_router_embedding_headers.py create mode 100644 tests/router_unit_tests/test_router_embedding_integration.py diff --git a/litellm/router.py b/litellm/router.py index 88b4087c1ee..364e6719300 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -386,9 +386,9 @@ class Router: ) # names of models under litellm_params. ex. azure/chatgpt-v-2 self.deployment_latency_map = {} ### CACHING ### - cache_type: Literal["local", "redis", "redis-semantic", "s3", "disk"] = ( - "local" # default to an in-memory cache - ) + cache_type: Literal[ + "local", "redis", "redis-semantic", "s3", "disk" + ] = "local" # default to an in-memory cache redis_cache = None cache_config: Dict[str, Any] = {} @@ -430,9 +430,9 @@ class Router: self.default_max_parallel_requests = default_max_parallel_requests self.provider_default_deployment_ids: List[str] = [] self.pattern_router = PatternMatchRouter() - self.team_pattern_routers: Dict[str, PatternMatchRouter] = ( - {} - ) # {"TEAM_ID": PatternMatchRouter} + self.team_pattern_routers: Dict[ + str, PatternMatchRouter + ] = {} # {"TEAM_ID": PatternMatchRouter} self.auto_routers: Dict[str, "AutoRouter"] = {} # Initialize model_group_alias early since it's used in set_model_list @@ -613,9 +613,9 @@ class Router: ) ) - self.model_group_retry_policy: Optional[Dict[str, RetryPolicy]] = ( - model_group_retry_policy - ) + self.model_group_retry_policy: Optional[ + Dict[str, RetryPolicy] + ] = model_group_retry_policy self.allowed_fails_policy: Optional[AllowedFailsPolicy] = None if allowed_fails_policy is not None: @@ -722,7 +722,10 @@ class Router: valid_strategy_strings = ["simple-shuffle"] + [s.value for s in RoutingStrategy] if routing_strategy is not None: - is_valid_string = isinstance(routing_strategy, str) and routing_strategy in valid_strategy_strings + is_valid_string = ( + isinstance(routing_strategy, str) + and routing_strategy in valid_strategy_strings + ) is_valid_enum = isinstance(routing_strategy, RoutingStrategy) if not is_valid_string and not is_valid_enum: raise ValueError( @@ -1071,7 +1074,7 @@ class Router: self.delete_container = self.factory_function( delete_container, call_type="delete_container" ) - + # Auto-register JSON-generated container file endpoints for name, func in container_file_endpoints.items(): setattr(self, name, self.factory_function(func, call_type=name)) # type: ignore[arg-type] @@ -1500,10 +1503,7 @@ class Router: async def _acompletion( self, model: str, messages: List[Dict[str, str]], **kwargs - ) -> Union[ - ModelResponse, - CustomStreamWrapper, - ]: + ) -> Union[ModelResponse, CustomStreamWrapper,]: """ - Get an available deployment - call it with a semaphore over the call @@ -3021,7 +3021,9 @@ class Router: kwargs["original_generic_function"] = original_function kwargs["original_function"] = self._aguardrail_helper self._update_kwargs_before_fallbacks( - model=guardrail_name, kwargs=kwargs, metadata_variable_name="litellm_metadata" + model=guardrail_name, + kwargs=kwargs, + metadata_variable_name="litellm_metadata", ) verbose_router_logger.debug( f"Inside aguardrail() - guardrail_name: {guardrail_name}; kwargs: {kwargs}" @@ -3314,8 +3316,7 @@ class Router: kwargs["model"] = model kwargs["input"] = input kwargs["original_function"] = self._embedding - kwargs["num_retries"] = kwargs.get("num_retries", self.num_retries) - kwargs.setdefault("metadata", {}).update({"model_group": model}) + self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs) response = self.function_with_fallbacks(**kwargs) return response except Exception as e: @@ -3617,9 +3618,9 @@ class Router: healthy_deployments=healthy_deployments, responses=responses ) returned_response = cast(OpenAIFileObject, responses[0]) - returned_response._hidden_params["model_file_id_mapping"] = ( - model_file_id_mapping - ) + returned_response._hidden_params[ + "model_file_id_mapping" + ] = model_file_id_mapping return returned_response except Exception as e: verbose_router_logger.exception( @@ -4366,11 +4367,11 @@ class Router: if isinstance(e, litellm.ContextWindowExceededError): if context_window_fallbacks is not None: - context_window_fallback_model_group: Optional[List[str]] = ( - self._get_fallback_model_group_from_fallbacks( - fallbacks=context_window_fallbacks, - model_group=model_group, - ) + context_window_fallback_model_group: Optional[ + List[str] + ] = self._get_fallback_model_group_from_fallbacks( + fallbacks=context_window_fallbacks, + model_group=model_group, ) if context_window_fallback_model_group is None: raise original_exception @@ -4402,11 +4403,11 @@ class Router: e.message += "\n{}".format(error_message) elif isinstance(e, litellm.ContentPolicyViolationError): if content_policy_fallbacks is not None: - content_policy_fallback_model_group: Optional[List[str]] = ( - self._get_fallback_model_group_from_fallbacks( - fallbacks=content_policy_fallbacks, - model_group=model_group, - ) + content_policy_fallback_model_group: Optional[ + List[str] + ] = self._get_fallback_model_group_from_fallbacks( + fallbacks=content_policy_fallbacks, + model_group=model_group, ) if content_policy_fallback_model_group is None: raise original_exception @@ -5681,26 +5682,26 @@ class Router: """ from litellm.router_strategy.auto_router.auto_router import AutoRouter - auto_router_config_path: Optional[str] = ( - deployment.litellm_params.auto_router_config_path - ) + auto_router_config_path: Optional[ + str + ] = deployment.litellm_params.auto_router_config_path auto_router_config: Optional[str] = deployment.litellm_params.auto_router_config if auto_router_config_path is None and auto_router_config is None: raise ValueError( "auto_router_config_path or auto_router_config is required for auto-router deployments. Please set it in the litellm_params" ) - default_model: Optional[str] = ( - deployment.litellm_params.auto_router_default_model - ) + default_model: Optional[ + str + ] = deployment.litellm_params.auto_router_default_model if default_model is None: raise ValueError( "auto_router_default_model is required for auto-router deployments. Please set it in the litellm_params" ) - embedding_model: Optional[str] = ( - deployment.litellm_params.auto_router_embedding_model - ) + embedding_model: Optional[ + str + ] = deployment.litellm_params.auto_router_embedding_model if embedding_model is None: raise ValueError( "auto_router_embedding_model is required for auto-router deployments. Please set it in the litellm_params" @@ -6247,9 +6248,9 @@ class Router: # Add custom_llm_provider if deployment.litellm_params.custom_llm_provider: - credentials["custom_llm_provider"] = ( - deployment.litellm_params.custom_llm_provider - ) + credentials[ + "custom_llm_provider" + ] = deployment.litellm_params.custom_llm_provider elif "/" in deployment.litellm_params.model: # Extract provider from "provider/model" format credentials["custom_llm_provider"] = deployment.litellm_params.model.split( @@ -6943,42 +6944,44 @@ class Router: """ return candidate_id in self.model_id_to_deployment_index_map - def resolve_model_name_from_model_id(self, model_id: Optional[str]) -> Optional[str]: + def resolve_model_name_from_model_id( + self, model_id: Optional[str] + ) -> Optional[str]: """ Resolve model_name from model_id. - + This method attempts to find the correct model_name to use with the router so that litellm_params can be automatically injected from the model config. - + Strategy: 1. First, check if model_id directly matches a model_name or deployment ID 2. If not, search through router's model_list to find a match by litellm_params.model 3. Return the model_name if found, None otherwise - + Args: model_id: The model_id extracted from decoded video_id (could be model_name or litellm_params.model value) - + Returns: model_name if found, None otherwise. If None, the request will fall through to normal flow using environment variables. """ if not model_id: return None - + # Strategy 1: Check if model_id directly matches a model_name or deployment ID if model_id in self.model_names or self.has_model_id(model_id): return model_id - + # Strategy 2: Search through router's model_list to find by litellm_params.model all_models = self.get_model_list(model_name=None) if not all_models: return None - + for deployment in all_models: litellm_params = deployment.get("litellm_params", {}) actual_model = litellm_params.get("model") - + # Match by exact match or by checking if actual_model ends with /model_id or :model_id # e.g., model_id="veo-2.0-generate-001" matches actual_model="vertex_ai/veo-2.0-generate-001" matches = ( @@ -6986,12 +6989,12 @@ class Router: or (actual_model and actual_model.endswith(f"/{model_id}")) or (actual_model and actual_model.endswith(f":{model_id}")) ) - + if matches: model_name = deployment.get("model_name") if model_name: return model_name - + # No match found return None @@ -7785,14 +7788,18 @@ class Router: request_kwargs=request_kwargs, ) - verbose_router_logger.debug(f"healthy_deployments after team filter: {healthy_deployments}") + verbose_router_logger.debug( + f"healthy_deployments after team filter: {healthy_deployments}" + ) healthy_deployments = filter_web_search_deployments( healthy_deployments=healthy_deployments, request_kwargs=request_kwargs, ) - verbose_router_logger.debug(f"healthy_deployments after web search filter: {healthy_deployments}") + verbose_router_logger.debug( + f"healthy_deployments after web search filter: {healthy_deployments}" + ) if isinstance(healthy_deployments, dict): return healthy_deployments diff --git a/tests/router_unit_tests/test_router_embedding_headers.py b/tests/router_unit_tests/test_router_embedding_headers.py new file mode 100644 index 00000000000..6d480792b7c --- /dev/null +++ b/tests/router_unit_tests/test_router_embedding_headers.py @@ -0,0 +1,372 @@ +""" +Test suite for router embedding method header propagation. + +This tests the fix for the issue where the embedding method was not +propagating proxy model configuration headers to the LLM API calls. + +The fix ensures that router.embedding() calls _update_kwargs_before_fallbacks() +just like router.completion() does, which properly sets up metadata and allows +default_litellm_params (including headers) to be propagated. +""" +import os +import sys +from unittest.mock import MagicMock, patch, AsyncMock + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +from litellm import Router + + +class TestRouterEmbeddingHeaders: + """Test that embedding methods properly propagate headers from router configuration.""" + + def test_embedding_calls_update_kwargs_before_fallbacks(self): + """ + Test that router.embedding() calls _update_kwargs_before_fallbacks. + + This ensures that metadata is properly set up before the fallback mechanism, + which is necessary for header propagation to work correctly. + """ + model_list = [ + { + "model_name": "text-embedding-ada-002", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "fake-key", + }, + } + ] + + router = Router(model_list=model_list) + + # Mock the _update_kwargs_before_fallbacks method to verify it's called + with patch.object( + router, + "_update_kwargs_before_fallbacks", + wraps=router._update_kwargs_before_fallbacks, + ) as mock_update: + with patch("litellm.embedding") as mock_litellm_embedding: + mock_litellm_embedding.return_value = MagicMock( + data=[{"embedding": [0.1, 0.2, 0.3]}] + ) + + router.embedding(model="text-embedding-ada-002", input=["test input"]) + + # Verify _update_kwargs_before_fallbacks was called + mock_update.assert_called_once() + call_kwargs = mock_update.call_args[1] + assert call_kwargs["model"] == "text-embedding-ada-002" + assert "kwargs" in call_kwargs + + @pytest.mark.asyncio + async def test_aembedding_calls_update_kwargs_before_fallbacks(self): + """ + Test that router.aembedding() calls _update_kwargs_before_fallbacks. + + This ensures consistency between sync and async embedding methods. + """ + model_list = [ + { + "model_name": "text-embedding-ada-002", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "fake-key", + }, + } + ] + + router = Router(model_list=model_list) + + # Mock the _update_kwargs_before_fallbacks method to verify it's called + with patch.object( + router, + "_update_kwargs_before_fallbacks", + wraps=router._update_kwargs_before_fallbacks, + ) as mock_update: + with patch( + "litellm.aembedding", new_callable=AsyncMock + ) as mock_litellm_aembedding: + mock_litellm_aembedding.return_value = MagicMock( + data=[{"embedding": [0.1, 0.2, 0.3]}] + ) + + await router.aembedding( + model="text-embedding-ada-002", input=["test input"] + ) + + # Verify _update_kwargs_before_fallbacks was called + mock_update.assert_called_once() + call_kwargs = mock_update.call_args[1] + assert call_kwargs["model"] == "text-embedding-ada-002" + assert "kwargs" in call_kwargs + + def test_embedding_propagates_default_litellm_params(self): + """ + Test that embedding calls properly propagate default_litellm_params including headers. + + This is the main fix - ensuring that headers set in default_litellm_params + are included in the embedding request. + """ + custom_headers = {"X-Custom-Header": "test-value", "X-API-Version": "v2"} + + model_list = [ + { + "model_name": "text-embedding-ada-002", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "fake-key", + }, + } + ] + + # Create router with default_litellm_params containing headers + router = Router( + model_list=model_list, + default_litellm_params={ + "headers": custom_headers, + "metadata": {"test_key": "test_value"}, + }, + ) + + with patch("litellm.embedding") as mock_litellm_embedding: + mock_litellm_embedding.return_value = MagicMock( + data=[{"embedding": [0.1, 0.2, 0.3]}] + ) + + router.embedding(model="text-embedding-ada-002", input=["test input"]) + + # Verify that litellm.embedding was called with the headers + mock_litellm_embedding.assert_called_once() + call_kwargs = mock_litellm_embedding.call_args[1] + + # Check that headers were included + assert "headers" in call_kwargs + assert call_kwargs["headers"] == custom_headers + + # Check that metadata was properly set up + assert "metadata" in call_kwargs + assert "model_group" in call_kwargs["metadata"] + assert call_kwargs["metadata"]["model_group"] == "text-embedding-ada-002" + + @pytest.mark.asyncio + async def test_aembedding_propagates_default_litellm_params(self): + """ + Test that async embedding calls properly propagate default_litellm_params including headers. + """ + custom_headers = {"X-Custom-Header": "test-value", "X-API-Version": "v2"} + + model_list = [ + { + "model_name": "text-embedding-ada-002", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "fake-key", + }, + } + ] + + # Create router with default_litellm_params containing headers + router = Router( + model_list=model_list, + default_litellm_params={ + "headers": custom_headers, + "metadata": {"test_key": "test_value"}, + }, + ) + + with patch( + "litellm.aembedding", new_callable=AsyncMock + ) as mock_litellm_aembedding: + mock_litellm_aembedding.return_value = MagicMock( + data=[{"embedding": [0.1, 0.2, 0.3]}] + ) + + await router.aembedding( + model="text-embedding-ada-002", input=["test input"] + ) + + # Verify that litellm.aembedding was called with the headers + mock_litellm_aembedding.assert_called_once() + call_kwargs = mock_litellm_aembedding.call_args[1] + + # Check that headers were included + assert "headers" in call_kwargs + assert call_kwargs["headers"] == custom_headers + + # Check that metadata was properly set up + assert "metadata" in call_kwargs + assert "model_group" in call_kwargs["metadata"] + assert call_kwargs["metadata"]["model_group"] == "text-embedding-ada-002" + + def test_embedding_metadata_includes_model_group(self): + """ + Test that embedding calls include model_group in metadata. + + The _update_kwargs_before_fallbacks method should set this up. + """ + model_list = [ + { + "model_name": "test-embedding-model", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "fake-key", + }, + } + ] + + router = Router(model_list=model_list) + + with patch("litellm.embedding") as mock_litellm_embedding: + mock_litellm_embedding.return_value = MagicMock( + data=[{"embedding": [0.1, 0.2, 0.3]}] + ) + + router.embedding(model="test-embedding-model", input=["test input"]) + + call_kwargs = mock_litellm_embedding.call_args[1] + + # Verify metadata contains model_group + assert "metadata" in call_kwargs + assert "model_group" in call_kwargs["metadata"] + assert call_kwargs["metadata"]["model_group"] == "test-embedding-model" + + def test_embedding_sets_num_retries_from_router(self): + """ + Test that embedding calls inherit num_retries from router configuration. + + This is set by _update_kwargs_before_fallbacks. + """ + model_list = [ + { + "model_name": "text-embedding-ada-002", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "fake-key", + }, + } + ] + + # Create router with num_retries set + router = Router(model_list=model_list, num_retries=3) + + with patch("litellm.embedding") as mock_litellm_embedding: + mock_litellm_embedding.return_value = MagicMock( + data=[{"embedding": [0.1, 0.2, 0.3]}] + ) + + router.embedding(model="text-embedding-ada-002", input=["test input"]) + + # Verify num_retries was not set in the call (it's handled by function_with_fallbacks) + # The important thing is that it was set in kwargs before being passed to function_with_fallbacks + # We verify this indirectly by checking that _update_kwargs_before_fallbacks was called + mock_litellm_embedding.assert_called_once() + + def test_embedding_sets_litellm_trace_id(self): + """ + Test that embedding calls include a litellm_trace_id. + + This is generated and set by _update_kwargs_before_fallbacks. + """ + model_list = [ + { + "model_name": "text-embedding-ada-002", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "fake-key", + }, + } + ] + + router = Router(model_list=model_list) + + with patch("litellm.embedding") as mock_litellm_embedding: + mock_litellm_embedding.return_value = MagicMock( + data=[{"embedding": [0.1, 0.2, 0.3]}] + ) + + router.embedding(model="text-embedding-ada-002", input=["test input"]) + + call_kwargs = mock_litellm_embedding.call_args[1] + + # Verify litellm_trace_id was set + assert "litellm_trace_id" in call_kwargs + assert isinstance(call_kwargs["litellm_trace_id"], str) + assert len(call_kwargs["litellm_trace_id"]) > 0 + + def test_embedding_consistency_with_completion(self): + """ + Test that embedding and completion methods handle kwargs similarly. + + Both should call _update_kwargs_before_fallbacks to ensure consistent behavior. + """ + custom_headers = {"X-Test": "value"} + + model_list = [ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "fake-key", + }, + }, + { + "model_name": "text-embedding-ada-002", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "fake-key", + }, + }, + ] + + router = Router( + model_list=model_list, default_litellm_params={"headers": custom_headers} + ) + + # Test completion + with patch("litellm.completion") as mock_completion: + mock_completion.return_value = MagicMock() + + router.completion( + model="gpt-3.5-turbo", messages=[{"role": "user", "content": "test"}] + ) + + completion_kwargs = mock_completion.call_args[1] + + # Test embedding + with patch("litellm.embedding") as mock_embedding: + mock_embedding.return_value = MagicMock( + data=[{"embedding": [0.1, 0.2, 0.3]}] + ) + + router.embedding(model="text-embedding-ada-002", input=["test input"]) + + embedding_kwargs = mock_embedding.call_args[1] + + # Both should have headers from default_litellm_params + assert "headers" in completion_kwargs + assert "headers" in embedding_kwargs + assert completion_kwargs["headers"] == custom_headers + assert embedding_kwargs["headers"] == custom_headers + + # Both should have metadata with model_group + assert "metadata" in completion_kwargs + assert "metadata" in embedding_kwargs + assert "model_group" in completion_kwargs["metadata"] + assert "model_group" in embedding_kwargs["metadata"] + + # Both should have litellm_trace_id + assert "litellm_trace_id" in completion_kwargs + assert "litellm_trace_id" in embedding_kwargs + + +if __name__ == "__main__": + # Run a simple test + test = TestRouterEmbeddingHeaders() + test.test_embedding_calls_update_kwargs_before_fallbacks() + test.test_embedding_propagates_default_litellm_params() + test.test_embedding_metadata_includes_model_group() + test.test_embedding_sets_litellm_trace_id() + test.test_embedding_consistency_with_completion() + print("All tests passed!") # noqa: T201 diff --git a/tests/router_unit_tests/test_router_embedding_integration.py b/tests/router_unit_tests/test_router_embedding_integration.py new file mode 100644 index 00000000000..ab2071714a9 --- /dev/null +++ b/tests/router_unit_tests/test_router_embedding_integration.py @@ -0,0 +1,355 @@ +""" +Integration tests for router embedding method with various configurations. + +These tests simulate real-world scenarios where headers and configuration +need to be properly propagated through the router to the LLM API. +""" +import os +import sys +from unittest.mock import MagicMock, patch, AsyncMock + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +from litellm import Router + + +class TestRouterEmbeddingIntegration: + """Integration tests for embedding with router configuration.""" + + def test_embedding_with_deployment_specific_headers(self): + """ + Test that deployment-specific headers are propagated. + + This simulates a scenario where different deployments have + different header requirements (e.g., different API versions). + """ + model_list = [ + { + "model_name": "embedding-deployment-1", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "key-1", + "headers": {"X-Deployment": "deployment-1"}, + }, + }, + { + "model_name": "embedding-deployment-2", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "key-2", + "headers": {"X-Deployment": "deployment-2"}, + }, + }, + ] + + router = Router(model_list=model_list) + + # Test first deployment + with patch("litellm.embedding") as mock_embedding: + mock_embedding.return_value = MagicMock(data=[{"embedding": [0.1, 0.2]}]) + + router.embedding(model="embedding-deployment-1", input=["test"]) + + call_kwargs = mock_embedding.call_args[1] + assert call_kwargs["api_key"] == "key-1" + + # Test second deployment + with patch("litellm.embedding") as mock_embedding: + mock_embedding.return_value = MagicMock(data=[{"embedding": [0.1, 0.2]}]) + + router.embedding(model="embedding-deployment-2", input=["test"]) + + call_kwargs = mock_embedding.call_args[1] + assert call_kwargs["api_key"] == "key-2" + + def test_embedding_with_router_and_deployment_headers_merge(self): + """ + Test that router-level headers are propagated. + + When no request headers are provided, router default headers should be used. + """ + model_list = [ + { + "model_name": "test-embedding", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "test-key", + }, + } + ] + + router = Router( + model_list=model_list, + default_litellm_params={ + "headers": { + "X-Router-Header": "router-value", + "X-Common-Header": "router-common", + } + }, + ) + + # Test: No request headers - router headers should be used + with patch("litellm.embedding") as mock_embedding: + mock_embedding.return_value = MagicMock(data=[{"embedding": [0.1, 0.2]}]) + + router.embedding( + model="test-embedding", + input=["test"], + ) + + call_kwargs = mock_embedding.call_args[1] + + # Router headers should be present + assert "headers" in call_kwargs + assert call_kwargs["headers"]["X-Router-Header"] == "router-value" + assert call_kwargs["headers"]["X-Common-Header"] == "router-common" + + def test_embedding_metadata_propagation(self): + """ + Test that metadata is properly set up and propagated. + + This is important for logging, tracking, and debugging. + """ + model_list = [ + { + "model_name": "test-embedding", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "test-key", + }, + } + ] + + router = Router( + model_list=model_list, + default_litellm_params={ + "metadata": {"environment": "test", "service": "embedding-service"} + }, + ) + + with patch("litellm.embedding") as mock_embedding: + mock_embedding.return_value = MagicMock(data=[{"embedding": [0.1, 0.2]}]) + + router.embedding( + model="test-embedding", + input=["test"], + metadata={"request_id": "req-123"}, # Additional metadata from request + ) + + call_kwargs = mock_embedding.call_args[1] + + # Check metadata contains all expected fields + assert "metadata" in call_kwargs + metadata = call_kwargs["metadata"] + + # From _update_kwargs_before_fallbacks + assert "model_group" in metadata + assert metadata["model_group"] == "test-embedding" + + # From default_litellm_params + assert "environment" in metadata + assert metadata["environment"] == "test" + assert "service" in metadata + assert metadata["service"] == "embedding-service" + + # From request + assert "request_id" in metadata + assert metadata["request_id"] == "req-123" + + @pytest.mark.asyncio + async def test_async_embedding_with_multiple_retries(self): + """ + Test that async embedding properly uses num_retries from router config. + + This ensures the fix works with the retry mechanism. + """ + model_list = [ + { + "model_name": "test-embedding", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "test-key", + }, + } + ] + + router = Router(model_list=model_list, num_retries=2) + + with patch("litellm.aembedding", new_callable=AsyncMock) as mock_aembedding: + mock_aembedding.return_value = MagicMock(data=[{"embedding": [0.1, 0.2]}]) + + await router.aembedding(model="test-embedding", input=["test"]) + + # The call should succeed + mock_aembedding.assert_called_once() + + def test_embedding_with_timeout_from_router(self): + """ + Test that timeout settings from router config are propagated. + """ + model_list = [ + { + "model_name": "test-embedding", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "test-key", + }, + } + ] + + router = Router(model_list=model_list, timeout=30.0) + + with patch("litellm.embedding") as mock_embedding: + mock_embedding.return_value = MagicMock(data=[{"embedding": [0.1, 0.2]}]) + + router.embedding(model="test-embedding", input=["test"]) + + call_kwargs = mock_embedding.call_args[1] + + # Timeout should be set from router config + assert "timeout" in call_kwargs + assert call_kwargs["timeout"] == 30.0 + + def test_embedding_with_multiple_deployments_load_balancing(self): + """ + Test that headers are correctly propagated when router load balances + between multiple deployments. + """ + model_list = [ + { + "model_name": "shared-embedding-model", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "key-1", + }, + }, + { + "model_name": "shared-embedding-model", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "key-2", + }, + }, + ] + + router = Router( + model_list=model_list, + default_litellm_params={"headers": {"X-Shared-Header": "shared-value"}}, + ) + + # Make multiple calls and verify headers are always present + for i in range(5): + with patch("litellm.embedding") as mock_embedding: + mock_embedding.return_value = MagicMock( + data=[{"embedding": [0.1, 0.2]}] + ) + + router.embedding(model="shared-embedding-model", input=[f"test {i}"]) + + call_kwargs = mock_embedding.call_args[1] + + # Headers should always be present regardless of which deployment is chosen + assert "headers" in call_kwargs + assert call_kwargs["headers"]["X-Shared-Header"] == "shared-value" + + @pytest.mark.asyncio + async def test_embedding_with_fallback_configuration(self): + """ + Test that headers are propagated correctly when using fallback models. + """ + model_list = [ + { + "model_name": "primary-embedding", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "primary-key", + }, + }, + { + "model_name": "fallback-embedding", + "litellm_params": { + "model": "text-embedding-ada-002", + "api_key": "fallback-key", + }, + }, + ] + + router = Router( + model_list=model_list, + fallbacks=[{"primary-embedding": ["fallback-embedding"]}], + default_litellm_params={"headers": {"X-Fallback-Test": "test-value"}}, + ) + + # Simulate primary failing, fallback succeeding + with patch("litellm.aembedding", new_callable=AsyncMock) as mock_aembedding: + call_count = 0 + + async def side_effect(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + # First call (primary) fails + raise Exception("Primary failed") + else: + # Second call (fallback) succeeds + return MagicMock(data=[{"embedding": [0.1, 0.2]}]) + + mock_aembedding.side_effect = side_effect + + await router.aembedding(model="primary-embedding", input=["test"]) + + # Both calls should have headers + assert mock_aembedding.call_count == 2 + + # Check that both calls had headers + for call_obj in mock_aembedding.call_args_list: + call_kwargs = call_obj[1] + assert "headers" in call_kwargs + assert call_kwargs["headers"]["X-Fallback-Test"] == "test-value" + + def test_embedding_with_custom_provider_headers(self): + """ + Test that provider-specific headers are correctly propagated. + + Some providers require specific headers for API versioning, features, etc. + """ + model_list = [ + { + "model_name": "azure-embedding", + "litellm_params": { + "model": "azure/text-embedding-ada-002", + "api_key": "azure-key", + "api_base": "https://example.openai.azure.com", + "api_version": "2024-02-01", + }, + } + ] + + router = Router( + model_list=model_list, + default_litellm_params={ + "headers": {"X-Custom-Azure-Header": "azure-value"} + }, + ) + + with patch("litellm.embedding") as mock_embedding: + mock_embedding.return_value = MagicMock(data=[{"embedding": [0.1, 0.2]}]) + + router.embedding(model="azure-embedding", input=["test"]) + + call_kwargs = mock_embedding.call_args[1] + + # Verify Azure-specific params are present + assert call_kwargs["api_base"] == "https://example.openai.azure.com" + assert call_kwargs["api_version"] == "2024-02-01" + + # Verify custom headers are present + assert "headers" in call_kwargs + assert call_kwargs["headers"]["X-Custom-Azure-Header"] == "azure-value" + + +if __name__ == "__main__": + # Run tests + pytest.main([__file__, "-v"]) From 3a22fa89c45c824420483b159aa0f42a3b5431ce Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Fri, 9 Jan 2026 10:40:06 -0800 Subject: [PATCH 44/56] Refactor ProviderConfigManager.get_provider_chat_config for O(1) performance (#18867) --- litellm/utils.py | 428 +++++++++++++++++++++-------------------------- 1 file changed, 191 insertions(+), 237 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 8f9b9dba09d..42d4a1ac372 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7410,15 +7410,184 @@ def validate_chat_completion_tool_choice( class ProviderConfigManager: + # Dictionary mapping for O(1) provider lookup + # Stores tuples of (factory_function, needs_model_parameter) + # This is initialized lazily on first access to avoid circular imports + _PROVIDER_CONFIG_MAP: Optional[dict[LlmProviders, tuple[Callable, bool]]] = None + + @staticmethod + def _build_provider_config_map() -> dict[LlmProviders, tuple[Callable, bool]]: + """Build the provider-to-config mapping dictionary. + + Returns a dict mapping provider to (factory_function, needs_model_parameter). + This avoids expensive inspect.signature() calls at runtime. + """ + return { + # Most common providers first for readability + # Format: (factory_function, needs_model_parameter: bool) + LlmProviders.OPENAI: (lambda: litellm.OpenAIGPTConfig(), False), + LlmProviders.ANTHROPIC: (lambda: litellm.AnthropicConfig(), False), + LlmProviders.AZURE: (lambda model: ProviderConfigManager._get_azure_config(model), True), + LlmProviders.AZURE_AI: (lambda model: ProviderConfigManager._get_azure_ai_config(model), True), + LlmProviders.VERTEX_AI: (lambda model: ProviderConfigManager._get_vertex_ai_config(model), True), + LlmProviders.BEDROCK: (lambda model: ProviderConfigManager._get_bedrock_config(model), True), + LlmProviders.COHERE: (lambda model: ProviderConfigManager._get_cohere_config(model), True), + LlmProviders.COHERE_CHAT: (lambda model: ProviderConfigManager._get_cohere_config(model), True), + # Simple provider mappings (no model parameter needed) + LlmProviders.DEEPSEEK: (lambda: litellm.DeepSeekChatConfig(), False), + LlmProviders.GROQ: (lambda: litellm.GroqChatConfig(), False), + LlmProviders.BYTEZ: (lambda: litellm.BytezChatConfig(), False), + LlmProviders.DATABRICKS: (lambda: litellm.DatabricksConfig(), False), + LlmProviders.XAI: (lambda: litellm.XAIChatConfig(), False), + LlmProviders.ZAI: (lambda: litellm.ZAIChatConfig(), False), + LlmProviders.LAMBDA_AI: (lambda: litellm.LambdaAIChatConfig(), False), + LlmProviders.LLAMA: (lambda: litellm.LlamaAPIConfig(), False), + LlmProviders.TEXT_COMPLETION_OPENAI: (lambda: litellm.OpenAITextCompletionConfig(), False), + LlmProviders.SNOWFLAKE: (lambda: litellm.SnowflakeConfig(), False), + LlmProviders.CLARIFAI: (lambda: litellm.ClarifaiConfig(), False), + LlmProviders.ANTHROPIC_TEXT: (lambda: litellm.AnthropicTextConfig(), False), + LlmProviders.VERTEX_AI_BETA: (lambda: litellm.VertexGeminiConfig(), False), + LlmProviders.CLOUDFLARE: (lambda: litellm.CloudflareChatConfig(), False), + LlmProviders.SAGEMAKER_CHAT: (lambda: litellm.SagemakerChatConfig(), False), + LlmProviders.SAGEMAKER: (lambda: litellm.SagemakerConfig(), False), + LlmProviders.FIREWORKS_AI: (lambda: litellm.FireworksAIConfig(), False), + LlmProviders.FRIENDLIAI: (lambda: litellm.FriendliaiChatConfig(), False), + LlmProviders.WATSONX: (lambda: litellm.IBMWatsonXChatConfig(), False), + LlmProviders.WATSONX_TEXT: (lambda: litellm.IBMWatsonXAIConfig(), False), + LlmProviders.EMPOWER: (lambda: litellm.EmpowerChatConfig(), False), + LlmProviders.MINIMAX: (lambda: litellm.MinimaxChatConfig(), False), + LlmProviders.GITHUB: (lambda: litellm.GithubChatConfig(), False), + LlmProviders.COMPACTIFAI: (lambda: litellm.CompactifAIChatConfig(), False), + LlmProviders.GITHUB_COPILOT: (lambda: litellm.GithubCopilotConfig(), False), + LlmProviders.GIGACHAT: (lambda: litellm.GigaChatConfig(), False), + LlmProviders.RAGFLOW: (lambda: litellm.RAGFlowConfig(), False), + LlmProviders.CUSTOM: (lambda: litellm.OpenAILikeChatConfig(), False), + LlmProviders.CUSTOM_OPENAI: (lambda: litellm.OpenAILikeChatConfig(), False), + LlmProviders.OPENAI_LIKE: (lambda: litellm.OpenAILikeChatConfig(), False), + LlmProviders.AIOHTTP_OPENAI: (lambda: litellm.AiohttpOpenAIChatConfig(), False), + LlmProviders.HOSTED_VLLM: (lambda: litellm.HostedVLLMChatConfig(), False), + LlmProviders.LLAMAFILE: (lambda: litellm.LlamafileChatConfig(), False), + LlmProviders.LM_STUDIO: (lambda: litellm.LMStudioChatConfig(), False), + LlmProviders.GALADRIEL: (lambda: litellm.GaladrielChatConfig(), False), + LlmProviders.REPLICATE: (lambda: litellm.ReplicateConfig(), False), + LlmProviders.HUGGINGFACE: (lambda: litellm.HuggingFaceChatConfig(), False), + LlmProviders.TOGETHER_AI: (lambda: litellm.TogetherAIConfig(), False), + LlmProviders.OPENROUTER: (lambda: litellm.OpenrouterConfig(), False), + LlmProviders.VERCEL_AI_GATEWAY: (lambda: litellm.VercelAIGatewayConfig(), False), + LlmProviders.COMETAPI: (lambda: litellm.CometAPIConfig(), False), + LlmProviders.DATAROBOT: (lambda: litellm.DataRobotConfig(), False), + LlmProviders.GEMINI: (lambda: litellm.GoogleAIStudioGeminiConfig(), False), + LlmProviders.AI21: (lambda: litellm.AI21ChatConfig(), False), + LlmProviders.AI21_CHAT: (lambda: litellm.AI21ChatConfig(), False), + LlmProviders.AZURE_TEXT: (lambda: litellm.AzureOpenAITextConfig(), False), + LlmProviders.NLP_CLOUD: (lambda: litellm.NLPCloudConfig(), False), + LlmProviders.OOBABOOGA: (lambda: litellm.OobaboogaConfig(), False), + LlmProviders.OLLAMA_CHAT: (lambda: litellm.OllamaChatConfig(), False), + LlmProviders.DEEPINFRA: (lambda: litellm.DeepInfraConfig(), False), + LlmProviders.PERPLEXITY: (lambda: litellm.PerplexityChatConfig(), False), + LlmProviders.MISTRAL: (lambda: litellm.MistralConfig(), False), + LlmProviders.CODESTRAL: (lambda: litellm.MistralConfig(), False), + LlmProviders.NVIDIA_NIM: (lambda: litellm.NvidiaNimConfig(), False), + LlmProviders.CEREBRAS: (lambda: litellm.CerebrasConfig(), False), + LlmProviders.BASETEN: (lambda: litellm.BasetenConfig(), False), + LlmProviders.VOLCENGINE: (lambda: litellm.VolcEngineConfig(), False), + LlmProviders.TEXT_COMPLETION_CODESTRAL: (lambda: litellm.CodestralTextCompletionConfig(), False), + LlmProviders.SAMBANOVA: (lambda: litellm.SambanovaConfig(), False), + LlmProviders.MARITALK: (lambda: litellm.MaritalkConfig(), False), + LlmProviders.VLLM: (lambda: litellm.VLLMConfig(), False), + LlmProviders.OLLAMA: (lambda: litellm.OllamaConfig(), False), + LlmProviders.PREDIBASE: (lambda: litellm.PredibaseConfig(), False), + LlmProviders.TRITON: (lambda: litellm.TritonConfig(), False), + LlmProviders.PETALS: (lambda: litellm.PetalsConfig(), False), + LlmProviders.SAP_GENERATIVE_AI_HUB: (lambda: litellm.GenAIHubOrchestrationConfig(), False), + LlmProviders.FEATHERLESS_AI: (lambda: litellm.FeatherlessAIConfig(), False), + LlmProviders.NOVITA: (lambda: litellm.NovitaConfig(), False), + LlmProviders.NEBIUS: (lambda: litellm.NebiusConfig(), False), + LlmProviders.WANDB: (lambda: litellm.WandbConfig(), False), + LlmProviders.DASHSCOPE: (lambda: litellm.DashScopeChatConfig(), False), + LlmProviders.MOONSHOT: (lambda: litellm.MoonshotChatConfig(), False), + LlmProviders.DOCKER_MODEL_RUNNER: (lambda: litellm.DockerModelRunnerChatConfig(), False), + LlmProviders.V0: (lambda: litellm.V0ChatConfig(), False), + LlmProviders.MORPH: (lambda: litellm.MorphChatConfig(), False), + LlmProviders.LITELLM_PROXY: (lambda: litellm.LiteLLMProxyChatConfig(), False), + LlmProviders.GRADIENT_AI: (lambda: litellm.GradientAIConfig(), False), + LlmProviders.NSCALE: (lambda: litellm.NscaleConfig(), False), + LlmProviders.HEROKU: (lambda: litellm.HerokuChatConfig(), False), + LlmProviders.OCI: (lambda: litellm.OCIChatConfig(), False), + LlmProviders.HYPERBOLIC: (lambda: litellm.HyperbolicChatConfig(), False), + LlmProviders.OVHCLOUD: (lambda: litellm.OVHCloudChatConfig(), False), + LlmProviders.AMAZON_NOVA: (lambda: litellm.AmazonNovaChatConfig(), False), + LlmProviders.LANGGRAPH: (lambda: ProviderConfigManager._get_langgraph_config(), False), + } + + @staticmethod + def _get_azure_config(model: str) -> BaseConfig: + """Get Azure config based on model type.""" + if litellm.AzureOpenAIO1Config().is_o_series_model(model=model): + return litellm.AzureOpenAIO1Config() + if litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model(model=model): + return litellm.AzureOpenAIGPT5Config() + return litellm.AzureOpenAIConfig() + + @staticmethod + def _get_azure_ai_config(model: str) -> BaseConfig: + """Get Azure AI config based on model type.""" + if "claude" in model.lower(): + return litellm.AzureAnthropicConfig() + return litellm.AzureAIStudioConfig() + + @staticmethod + def _get_vertex_ai_config(model: str) -> BaseConfig: + """Get Vertex AI config based on model type.""" + if "gemini" in model: + return litellm.VertexGeminiConfig() + elif "claude" in model: + return litellm.VertexAIAnthropicConfig() + elif "gpt-oss" in model: + from litellm.llms.vertex_ai.vertex_ai_partner_models.gpt_oss.transformation import ( + VertexAIGPTOSSTransformation, + ) + return VertexAIGPTOSSTransformation() + elif model in litellm.vertex_mistral_models: + if "codestral" in model: + return litellm.CodestralTextCompletionConfig() + return litellm.MistralConfig() + elif model in litellm.vertex_ai_ai21_models: + return litellm.VertexAIAi21Config() + else: + return litellm.VertexAILlama3Config() + + @staticmethod + def _get_bedrock_config(model: str) -> BaseConfig: + """Get Bedrock config based on model.""" + from litellm.llms.bedrock.common_utils import get_bedrock_chat_config + return get_bedrock_chat_config(model=model) + + @staticmethod + def _get_cohere_config(model: str) -> BaseConfig: + """Get Cohere config based on route.""" + CohereModelInfo = getattr(sys.modules[__name__], 'CohereModelInfo') + route = CohereModelInfo.get_cohere_route(model) + if route == "v2": + return litellm.CohereV2ChatConfig() + return litellm.CohereChatConfig() + + @staticmethod + def _get_langgraph_config() -> BaseConfig: + """Get LangGraph config.""" + from litellm.llms.langgraph.chat.transformation import LangGraphConfig + return LangGraphConfig() + @staticmethod def get_provider_chat_config( # noqa: PLR0915 model: str, provider: LlmProviders ) -> Optional[BaseConfig]: """ Returns the provider config for a given provider. + + Uses O(1) dictionary lookup for fast provider resolution. """ - - # Check JSON providers FIRST + # Check JSON providers FIRST (these override standard mappings) from litellm.llms.openai_like.dynamic_config import create_config_class from litellm.llms.openai_like.json_loader import JSONProviderRegistry @@ -7428,244 +7597,29 @@ class ProviderConfigManager: raise ValueError(f"Provider {provider.value} not found") return create_config_class(provider_config)() - if ( - provider == LlmProviders.OPENAI - and litellm.openaiOSeriesConfig.is_model_o_series_model(model=model) - ): - return litellm.openaiOSeriesConfig - elif ( - provider == LlmProviders.OPENAI - and litellm.OpenAIGPT5Config.is_model_gpt_5_model(model=model) - ): - return litellm.OpenAIGPT5Config() - elif litellm.LlmProviders.DEEPSEEK == provider: - return litellm.DeepSeekChatConfig() - elif litellm.LlmProviders.GROQ == provider: - return litellm.GroqChatConfig() - elif litellm.LlmProviders.BYTEZ == provider: - return litellm.BytezChatConfig() - elif litellm.LlmProviders.DATABRICKS == provider: - return litellm.DatabricksConfig() - elif litellm.LlmProviders.XAI == provider: - return litellm.XAIChatConfig() - elif litellm.LlmProviders.ZAI == provider: - return litellm.ZAIChatConfig() - elif litellm.LlmProviders.LAMBDA_AI == provider: - return litellm.LambdaAIChatConfig() - elif litellm.LlmProviders.LLAMA == provider: - return litellm.LlamaAPIConfig() - elif litellm.LlmProviders.TEXT_COMPLETION_OPENAI == provider: - return litellm.OpenAITextCompletionConfig() - elif ( - litellm.LlmProviders.COHERE_CHAT == provider - or litellm.LlmProviders.COHERE == provider - ): - CohereModelInfo = getattr(sys.modules[__name__], 'CohereModelInfo') - route = CohereModelInfo.get_cohere_route(model) - if route == "v2": - return litellm.CohereV2ChatConfig() - else: + # Handle OpenAI special cases (O-series and GPT-5 models) + if provider == LlmProviders.OPENAI: + if litellm.openaiOSeriesConfig.is_model_o_series_model(model=model): + return litellm.openaiOSeriesConfig + if litellm.OpenAIGPT5Config.is_model_gpt_5_model(model=model): + return litellm.OpenAIGPT5Config() - return litellm.CohereChatConfig() - elif litellm.LlmProviders.SNOWFLAKE == provider: - return litellm.SnowflakeConfig() - elif litellm.LlmProviders.CLARIFAI == provider: - return litellm.ClarifaiConfig() - elif litellm.LlmProviders.ANTHROPIC == provider: - return litellm.AnthropicConfig() - elif litellm.LlmProviders.ANTHROPIC_TEXT == provider: - return litellm.AnthropicTextConfig() - elif litellm.LlmProviders.VERTEX_AI_BETA == provider: - return litellm.VertexGeminiConfig() - elif litellm.LlmProviders.VERTEX_AI == provider: - if "gemini" in model: - return litellm.VertexGeminiConfig() - elif "claude" in model: - return litellm.VertexAIAnthropicConfig() - elif "gpt-oss" in model: - from litellm.llms.vertex_ai.vertex_ai_partner_models.gpt_oss.transformation import ( - VertexAIGPTOSSTransformation, - ) + # Initialize provider config map lazily (avoids circular imports) + if ProviderConfigManager._PROVIDER_CONFIG_MAP is None: + ProviderConfigManager._PROVIDER_CONFIG_MAP = ProviderConfigManager._build_provider_config_map() - return VertexAIGPTOSSTransformation() - elif model in litellm.vertex_mistral_models: - if "codestral" in model: - return litellm.CodestralTextCompletionConfig() - else: - return litellm.MistralConfig() - elif model in litellm.vertex_ai_ai21_models: - return litellm.VertexAIAi21Config() - else: # use generic openai-like param mapping - return litellm.VertexAILlama3Config() - elif litellm.LlmProviders.CLOUDFLARE == provider: - return litellm.CloudflareChatConfig() - elif litellm.LlmProviders.SAGEMAKER_CHAT == provider: - return litellm.SagemakerChatConfig() - elif litellm.LlmProviders.SAGEMAKER == provider: - return litellm.SagemakerConfig() - elif litellm.LlmProviders.FIREWORKS_AI == provider: - return litellm.FireworksAIConfig() - elif litellm.LlmProviders.FRIENDLIAI == provider: - return litellm.FriendliaiChatConfig() - elif litellm.LlmProviders.WATSONX == provider: - return litellm.IBMWatsonXChatConfig() - elif litellm.LlmProviders.WATSONX_TEXT == provider: - return litellm.IBMWatsonXAIConfig() - elif litellm.LlmProviders.EMPOWER == provider: - return litellm.EmpowerChatConfig() - elif litellm.LlmProviders.MINIMAX == provider: - return litellm.MinimaxChatConfig() - elif litellm.LlmProviders.GITHUB == provider: - return litellm.GithubChatConfig() - elif litellm.LlmProviders.COMPACTIFAI == provider: - return litellm.CompactifAIChatConfig() - elif litellm.LlmProviders.GITHUB_COPILOT == provider: - return litellm.GithubCopilotConfig() - elif litellm.LlmProviders.GIGACHAT == provider: - return litellm.GigaChatConfig() - elif litellm.LlmProviders.RAGFLOW == provider: - return litellm.RAGFlowConfig() - elif ( - litellm.LlmProviders.CUSTOM == provider - or litellm.LlmProviders.CUSTOM_OPENAI == provider - or litellm.LlmProviders.OPENAI_LIKE == provider - ): - return litellm.OpenAILikeChatConfig() - elif litellm.LlmProviders.AIOHTTP_OPENAI == provider: - return litellm.AiohttpOpenAIChatConfig() - elif litellm.LlmProviders.HOSTED_VLLM == provider: - return litellm.HostedVLLMChatConfig() - elif litellm.LlmProviders.LLAMAFILE == provider: - return litellm.LlamafileChatConfig() - elif litellm.LlmProviders.LM_STUDIO == provider: - return litellm.LMStudioChatConfig() - elif litellm.LlmProviders.GALADRIEL == provider: - return litellm.GaladrielChatConfig() - elif litellm.LlmProviders.REPLICATE == provider: - return litellm.ReplicateConfig() - elif litellm.LlmProviders.HUGGINGFACE == provider: - return litellm.HuggingFaceChatConfig() - elif litellm.LlmProviders.TOGETHER_AI == provider: - return litellm.TogetherAIConfig() - elif litellm.LlmProviders.OPENROUTER == provider: - return litellm.OpenrouterConfig() - elif litellm.LlmProviders.VERCEL_AI_GATEWAY == provider: - return litellm.VercelAIGatewayConfig() - elif litellm.LlmProviders.COMETAPI == provider: - return litellm.CometAPIConfig() - elif litellm.LlmProviders.DATAROBOT == provider: - return litellm.DataRobotConfig() - elif litellm.LlmProviders.GEMINI == provider: - return litellm.GoogleAIStudioGeminiConfig() - elif ( - litellm.LlmProviders.AI21 == provider - or litellm.LlmProviders.AI21_CHAT == provider - ): - return litellm.AI21ChatConfig() - elif litellm.LlmProviders.AZURE == provider: - if litellm.AzureOpenAIO1Config().is_o_series_model(model=model): - return litellm.AzureOpenAIO1Config() - if litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model(model=model): - return litellm.AzureOpenAIGPT5Config() - return litellm.AzureOpenAIConfig() - elif litellm.LlmProviders.AZURE_AI == provider: - if "claude" in model.lower(): - return litellm.AzureAnthropicConfig() - return litellm.AzureAIStudioConfig() - elif litellm.LlmProviders.AZURE_TEXT == provider: - return litellm.AzureOpenAITextConfig() - elif litellm.LlmProviders.HOSTED_VLLM == provider: - return litellm.HostedVLLMChatConfig() - elif litellm.LlmProviders.NLP_CLOUD == provider: - return litellm.NLPCloudConfig() - elif litellm.LlmProviders.OOBABOOGA == provider: - return litellm.OobaboogaConfig() - elif litellm.LlmProviders.OLLAMA_CHAT == provider: - return litellm.OllamaChatConfig() - elif litellm.LlmProviders.DEEPINFRA == provider: - return litellm.DeepInfraConfig() - elif litellm.LlmProviders.PERPLEXITY == provider: - return litellm.PerplexityChatConfig() - elif ( - litellm.LlmProviders.MISTRAL == provider - or litellm.LlmProviders.CODESTRAL == provider - ): - return litellm.MistralConfig() - elif litellm.LlmProviders.NVIDIA_NIM == provider: - return litellm.NvidiaNimConfig() - elif litellm.LlmProviders.CEREBRAS == provider: - return litellm.CerebrasConfig() - elif litellm.LlmProviders.BASETEN == provider: - return litellm.BasetenConfig() - elif litellm.LlmProviders.VOLCENGINE == provider: - return litellm.VolcEngineConfig() - elif litellm.LlmProviders.TEXT_COMPLETION_CODESTRAL == provider: - return litellm.CodestralTextCompletionConfig() - elif litellm.LlmProviders.SAMBANOVA == provider: - return litellm.SambanovaConfig() - elif litellm.LlmProviders.MARITALK == provider: - return litellm.MaritalkConfig() - elif litellm.LlmProviders.CLOUDFLARE == provider: - return litellm.CloudflareChatConfig() - elif litellm.LlmProviders.ANTHROPIC_TEXT == provider: - return litellm.AnthropicTextConfig() - elif litellm.LlmProviders.VLLM == provider: - return litellm.VLLMConfig() - elif litellm.LlmProviders.OLLAMA == provider: - return litellm.OllamaConfig() - elif litellm.LlmProviders.PREDIBASE == provider: - return litellm.PredibaseConfig() - elif litellm.LlmProviders.TRITON == provider: - return litellm.TritonConfig() - elif litellm.LlmProviders.PETALS == provider: - return litellm.PetalsConfig() - elif litellm.LlmProviders.SAP_GENERATIVE_AI_HUB == provider: - return litellm.GenAIHubOrchestrationConfig() - elif litellm.LlmProviders.FEATHERLESS_AI == provider: - return litellm.FeatherlessAIConfig() - elif litellm.LlmProviders.NOVITA == provider: - return litellm.NovitaConfig() - elif litellm.LlmProviders.NEBIUS == provider: - return litellm.NebiusConfig() - elif litellm.LlmProviders.WANDB == provider: - return litellm.WandbConfig() - elif litellm.LlmProviders.DASHSCOPE == provider: - return litellm.DashScopeChatConfig() - elif litellm.LlmProviders.MOONSHOT == provider: - return litellm.MoonshotChatConfig() - elif litellm.LlmProviders.DOCKER_MODEL_RUNNER == provider: - return litellm.DockerModelRunnerChatConfig() - elif litellm.LlmProviders.V0 == provider: - return litellm.V0ChatConfig() - elif litellm.LlmProviders.MORPH == provider: - return litellm.MorphChatConfig() - elif litellm.LlmProviders.BEDROCK == provider: - from litellm.llms.bedrock.common_utils import get_bedrock_chat_config + # O(1) dictionary lookup + config_entry = ProviderConfigManager._PROVIDER_CONFIG_MAP.get(provider) + if config_entry is None: + return None - return get_bedrock_chat_config(model=model) - elif litellm.LlmProviders.LITELLM_PROXY == provider: - return litellm.LiteLLMProxyChatConfig() - elif litellm.LlmProviders.OPENAI == provider: - return litellm.OpenAIGPTConfig() - elif litellm.LlmProviders.GRADIENT_AI == provider: - return litellm.GradientAIConfig() - elif litellm.LlmProviders.NSCALE == provider: - return litellm.NscaleConfig() - elif litellm.LlmProviders.HEROKU == provider: - return litellm.HerokuChatConfig() - elif litellm.LlmProviders.OCI == provider: - return litellm.OCIChatConfig() - elif litellm.LlmProviders.HYPERBOLIC == provider: - return litellm.HyperbolicChatConfig() - elif litellm.LlmProviders.OVHCLOUD == provider: - return litellm.OVHCloudChatConfig() - elif litellm.LlmProviders.AMAZON_NOVA == provider: - return litellm.AmazonNovaChatConfig() - elif litellm.LlmProviders.LANGGRAPH == provider: - from litellm.llms.langgraph.chat.transformation import LangGraphConfig - - return LangGraphConfig() - return None + # Unpack factory function and whether it needs model parameter + # This avoids expensive inspect.signature() calls at runtime + config_factory, needs_model = config_entry + if needs_model: + return config_factory(model) # type: ignore + else: + return config_factory() # type: ignore @staticmethod def get_provider_embedding_config( From 7161f41746dbe2f5501502c0afda9689f8b32da0 Mon Sep 17 00:00:00 2001 From: mel2oo Date: Sat, 10 Jan 2026 03:01:52 +0800 Subject: [PATCH 45/56] Fix: google_genai streaming adapter provider handling (#18845) --- litellm/google_genai/main.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/litellm/google_genai/main.py b/litellm/google_genai/main.py index b7523ef8c16..1dc805a6b54 100644 --- a/litellm/google_genai/main.py +++ b/litellm/google_genai/main.py @@ -130,6 +130,9 @@ class GenerateContentHelper: api_key=litellm_params.api_key, ) + if litellm_params.custom_llm_provider is None: + litellm_params.custom_llm_provider = custom_llm_provider + # get provider config generate_content_provider_config: Optional[ BaseGoogleGenAIGenerateContentConfig @@ -407,6 +410,9 @@ async def agenerate_content_stream( # Check if we should use the adapter (when provider config is None) if setup_result.generate_content_provider_config is None: + if "stream" in kwargs: + kwargs.pop("stream", None) + # Use the adapter to convert to completion format return ( await GenerateContentToCompletionHandler.async_generate_content_handler( @@ -490,6 +496,9 @@ def generate_content_stream( # Check if we should use the adapter (when provider config is None) if setup_result.generate_content_provider_config is None: + if "stream" in kwargs: + kwargs.pop("stream", None) + # Use the adapter to convert to completion format return GenerateContentToCompletionHandler.generate_content_handler( model=model, From fa2b0fb533db64e5ad735f40eaa8f2c127adda06 Mon Sep 17 00:00:00 2001 From: DominikHallab Date: Sat, 10 Jan 2026 08:02:21 +1300 Subject: [PATCH 46/56] docs: Update header to be markdown bold by removing space (#18846) --- docs/my-website/docs/proxy/logging.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 5fe8f17d7b0..a27b6dcf083 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -67,7 +67,7 @@ Set `litellm.turn_off_message_logging=True` This will prevent the messages and r -**1. Setup config.yaml ** +**1. Setup config.yaml** ```yaml model_list: - model_name: gpt-3.5-turbo From c19c97591ed720ebb050b5def772c1ae877b6e80 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Fri, 9 Jan 2026 16:07:45 -0300 Subject: [PATCH 47/56] fix: align max_tokens with max_output_tokens for consistency (#18820) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: align max_tokens with max_output_tokens for consistency Fixed inconsistent max_tokens definitions in model_prices_and_context_window.json. According to LiteLLM convention, max_tokens should equal max_output_tokens when available. Models fixed: - deepseek-chat: 131072 → 8192 (now equals max_output_tokens) - dashscope/qwen-flash: 1000000 → 32768 (now equals max_output_tokens) - databricks/databricks-gemma-3-12b: 128000 → 32000 (now equals max_output_tokens) This ensures consistency across all providers where max_tokens represents the maximum number of tokens that can be generated in the output. * fix: align max_tokens with max_output_tokens for 244 models - Fix 244 models where max_tokens != max_output_tokens - Add test to validate max_tokens consistency and prevent regressions According to model_prices_and_context_window.json spec: - max_tokens is a LEGACY parameter - Should always equal max_output_tokens when both are present This ensures consistency across all model definitions. --- model_prices_and_context_window.json | 1046 ++++++++++++++------------ tests/test_litellm/test_utils.py | 51 ++ 2 files changed, 596 insertions(+), 501 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3c2c20b4dce..72e8128d791 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -410,8 +410,8 @@ "max_input_tokens": 8172, "max_tokens": 8172, "mode": "embedding", - "input_cost_per_token": 1.35e-7, - "input_cost_per_image": 6e-5, + "input_cost_per_token": 1.35e-07, + "input_cost_per_image": 6e-05, "input_cost_per_video_per_second": 0.0007, "input_cost_per_audio_per_second": 0.00014, "output_cost_per_token": 0.0, @@ -1398,8 +1398,8 @@ "mode": "chat" }, "azure_ai/gpt-oss-120b": { - "input_cost_per_token": 1.5e-7, - "output_cost_per_token": 6e-7, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, @@ -2077,7 +2077,7 @@ "litellm_provider": "azure", "max_input_tokens": 4097, "max_output_tokens": 4096, - "max_tokens": 4097, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, "supports_function_calling": true, @@ -2090,7 +2090,7 @@ "litellm_provider": "azure", "max_input_tokens": 4097, "max_output_tokens": 4096, - "max_tokens": 4097, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, "supports_function_calling": true, @@ -2869,7 +2869,7 @@ "/v1/audio/transcriptions" ] }, - "azure/gpt-5.1-2025-11-13": { + "azure/gpt-5.1-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -2905,7 +2905,7 @@ "supports_service_tier": true, "supports_vision": true }, - "azure/gpt-5.1-chat-2025-11-13": { + "azure/gpt-5.1-chat-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -2940,7 +2940,7 @@ "supports_tool_choice": false, "supports_vision": true }, - "azure/gpt-5.1-codex-2025-11-13": { + "azure/gpt-5.1-codex-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -3298,7 +3298,7 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 400000, + "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00012, "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/concepts/models-sold-directly-by-azure?pivots=azure-openai&tabs=global-standard-aoai%2Cstandard-chat-completions%2Cglobal-standard#gpt-5", @@ -3623,7 +3623,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", - "output_cost_per_token": 1.68e-04, + "output_cost_per_token": 0.000168, "supported_endpoints": [ "/v1/batch", "/v1/responses" @@ -3654,7 +3654,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", - "output_cost_per_token": 1.68e-04, + "output_cost_per_token": 0.000168, "supported_endpoints": [ "/v1/batch", "/v1/responses" @@ -4297,13 +4297,13 @@ "output_cost_per_token": 0.0 }, "azure/speech/azure-tts": { - "input_cost_per_character": 15e-06, + "input_cost_per_character": 1.5e-05, "litellm_provider": "azure", "mode": "audio_speech", "source": "https://azure.microsoft.com/en-us/pricing/calculator/" }, "azure/speech/azure-tts-hd": { - "input_cost_per_character": 30e-06, + "input_cost_per_character": 3e-05, "litellm_provider": "azure", "mode": "audio_speech", "source": "https://azure.microsoft.com/en-us/pricing/calculator/" @@ -5197,7 +5197,7 @@ }, "azure_ai/mistral-document-ai-2505": { "litellm_provider": "azure_ai", - "ocr_cost_per_page": 3e-3, + "ocr_cost_per_page": 0.003, "mode": "ocr", "supported_endpoints": [ "/v1/ocr" @@ -5206,7 +5206,7 @@ }, "azure_ai/doc-intelligence/prebuilt-read": { "litellm_provider": "azure_ai", - "ocr_cost_per_page": 1.5e-3, + "ocr_cost_per_page": 0.0015, "mode": "ocr", "supported_endpoints": [ "/v1/ocr" @@ -5215,7 +5215,7 @@ }, "azure_ai/doc-intelligence/prebuilt-layout": { "litellm_provider": "azure_ai", - "ocr_cost_per_page": 1e-2, + "ocr_cost_per_page": 0.01, "mode": "ocr", "supported_endpoints": [ "/v1/ocr" @@ -5224,7 +5224,7 @@ }, "azure_ai/doc-intelligence/prebuilt-document": { "litellm_provider": "azure_ai", - "ocr_cost_per_page": 1e-2, + "ocr_cost_per_page": 0.01, "mode": "ocr", "supported_endpoints": [ "/v1/ocr" @@ -5298,12 +5298,12 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, - "azure_ai/deepseek-v3.2": { + "azure_ai/deepseek-v3.2": { "input_cost_per_token": 5.8e-07, "litellm_provider": "azure_ai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 8192, + "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.68e-06, "supports_assistant_prefill": true, @@ -5317,7 +5317,7 @@ "litellm_provider": "azure_ai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 8192, + "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.68e-06, "supports_assistant_prefill": true, @@ -5452,7 +5452,7 @@ "supports_web_search": true }, "azure_ai/grok-4-fast-non-reasoning": { - "input_cost_per_token": 0.43e-06, + "input_cost_per_token": 4.3e-07, "output_cost_per_token": 1.73e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -5465,7 +5465,7 @@ "supports_web_search": true }, "azure_ai/grok-4-fast-reasoning": { - "input_cost_per_token": 0.43e-06, + "input_cost_per_token": 4.3e-07, "output_cost_per_token": 1.73e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -5623,7 +5623,7 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 16384, + "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 4e-07 }, @@ -7038,7 +7038,7 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 1000000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_200k_tokens": 2.25e-05, @@ -7821,7 +7821,7 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 131072, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.7e-06, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -7842,7 +7842,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 1000000, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -7883,7 +7883,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 1000000, + "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -7913,7 +7913,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 30720, "max_output_tokens": 8192, - "max_tokens": 32768, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -7926,7 +7926,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 131072, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -7939,7 +7939,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 8192, - "max_tokens": 131072, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -7952,7 +7952,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 131072, + "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -7966,7 +7966,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 131072, + "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -7979,7 +7979,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 1000000, + "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -8010,7 +8010,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 1000000, + "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -8041,7 +8041,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 1000000, + "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -8073,7 +8073,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 131072, + "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -8087,7 +8087,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 1000000, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -8100,7 +8100,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 1000000, + "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -8114,7 +8114,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 1000000, + "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -8127,7 +8127,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 131072, + "max_tokens": 16384, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -8138,7 +8138,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 1000000, + "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -8187,7 +8187,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 1000000, + "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -8232,7 +8232,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 1000000, + "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -8281,7 +8281,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 1000000, + "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -8326,7 +8326,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 262144, + "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -8364,7 +8364,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 98304, "max_output_tokens": 8192, - "max_tokens": 131072, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -8393,7 +8393,7 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 200000, + "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -8412,7 +8412,7 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 200000, + "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -8431,7 +8431,7 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 200000, + "max_tokens": 32000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -8450,7 +8450,7 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 200000, + "max_tokens": 32000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -8469,7 +8469,7 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 200000, + "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -8488,7 +8488,7 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 200000, + "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -8507,7 +8507,7 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 200000, + "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -8526,7 +8526,7 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 200000, + "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -8545,7 +8545,7 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 1048576, + "max_tokens": 65535, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -8562,7 +8562,7 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 1048576, + "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -8594,7 +8594,7 @@ "litellm_provider": "databricks", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 400000, + "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -8609,7 +8609,7 @@ "litellm_provider": "databricks", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 400000, + "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -8624,7 +8624,7 @@ "litellm_provider": "databricks", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 400000, + "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -8639,7 +8639,7 @@ "litellm_provider": "databricks", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 400000, + "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -8747,7 +8747,7 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 200000, + "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -8846,7 +8846,7 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 16384, + "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 2e-06 }, @@ -10107,7 +10107,7 @@ "litellm_provider": "deepseek", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 8192, + "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 4e-07, "supports_assistant_prefill": true, @@ -10121,7 +10121,7 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, "max_output_tokens": 81920, - "max_tokens": 163840, + "max_tokens": 81920, "mode": "chat", "output_cost_per_token": 1.68e-06, "supports_function_calling": true, @@ -10202,14 +10202,14 @@ "mode": "search", "tiered_pricing": [ { - "input_cost_per_query": 5e-03, + "input_cost_per_query": 0.005, "max_results_range": [ 0, 25 ] }, { - "input_cost_per_query": 25e-03, + "input_cost_per_query": 0.025, "max_results_range": [ 26, 100 @@ -10222,70 +10222,70 @@ "mode": "search", "tiered_pricing": [ { - "input_cost_per_query": 1.66e-03, + "input_cost_per_query": 0.00166, "max_results_range": [ 1, 10 ] }, { - "input_cost_per_query": 3.32e-03, + "input_cost_per_query": 0.00332, "max_results_range": [ 11, 20 ] }, { - "input_cost_per_query": 4.98e-03, + "input_cost_per_query": 0.00498, "max_results_range": [ 21, 30 ] }, { - "input_cost_per_query": 6.64e-03, + "input_cost_per_query": 0.00664, "max_results_range": [ 31, 40 ] }, { - "input_cost_per_query": 8.3e-03, + "input_cost_per_query": 0.0083, "max_results_range": [ 41, 50 ] }, { - "input_cost_per_query": 9.96e-03, + "input_cost_per_query": 0.00996, "max_results_range": [ 51, 60 ] }, { - "input_cost_per_query": 11.62e-03, + "input_cost_per_query": 0.01162, "max_results_range": [ 61, 70 ] }, { - "input_cost_per_query": 13.28e-03, + "input_cost_per_query": 0.01328, "max_results_range": [ 71, 80 ] }, { - "input_cost_per_query": 14.94e-03, + "input_cost_per_query": 0.01494, "max_results_range": [ 81, 90 ] }, { - "input_cost_per_query": 16.6e-03, + "input_cost_per_query": 0.0166, "max_results_range": [ 91, 100 @@ -10297,7 +10297,7 @@ } }, "perplexity/search": { - "input_cost_per_query": 5e-03, + "input_cost_per_query": 0.005, "litellm_provider": "perplexity", "mode": "search" }, @@ -10395,7 +10395,7 @@ "supports_embedding_image_input": true }, "embed-multilingual-light-v3.0": { - "input_cost_per_token": 1e-04, + "input_cost_per_token": 0.0001, "litellm_provider": "cohere", "max_input_tokens": 1024, "max_tokens": 1024, @@ -10689,7 +10689,7 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 128000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.3e-07, "supports_function_calling": true, @@ -10700,7 +10700,7 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 128000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.9e-07, "supports_function_calling": true, @@ -10711,7 +10711,7 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 128000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -10817,14 +10817,14 @@ "litellm_provider": "featherless_ai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 32768, + "max_tokens": 4096, "mode": "chat" }, "featherless_ai/featherless-ai/Qwerky-QwQ-32B": { "litellm_provider": "featherless_ai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 32768, + "max_tokens": 4096, "mode": "chat" }, "fireworks-ai-4.1b-to-16b": { @@ -11031,7 +11031,7 @@ "supports_tool_choice": true }, "fireworks_ai/accounts/fireworks/models/glm-4p6": { - "input_cost_per_token": 0.55e-06, + "input_cost_per_token": 5.5e-07, "output_cost_per_token": 2.19e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 202800, @@ -11077,7 +11077,7 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 131072, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://fireworks.ai/models/fireworks/kimi-k2-instruct", @@ -11090,7 +11090,7 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 262144, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://app.fireworks.ai/models/fireworks/kimi-k2-instruct-0905", @@ -11337,7 +11337,7 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 16384, + "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 2e-07 @@ -11348,7 +11348,7 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 16384, + "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 1e-06 @@ -11638,7 +11638,7 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 8192, "max_output_tokens": 2048, - "max_tokens": 8192, + "max_tokens": 2048, "mode": "chat", "output_cost_per_character": 3.75e-07, "output_cost_per_token": 1.5e-06, @@ -11655,7 +11655,7 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 8192, "max_output_tokens": 2048, - "max_tokens": 8192, + "max_tokens": 2048, "mode": "chat", "output_cost_per_character": 3.75e-07, "output_cost_per_token": 1.5e-06, @@ -12585,10 +12585,10 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 65536, + "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, - "output_cost_per_image_token": 1.2e-04, + "output_cost_per_image_token": 0.00012, "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -14112,7 +14112,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 65536, "max_pdf_size_mb": 30, - "max_tokens": 8192, + "max_tokens": 65536, "max_video_length": 1, "max_videos_per_prompt": 10, "mode": "chat", @@ -14162,7 +14162,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 65536, "max_pdf_size_mb": 30, - "max_tokens": 8192, + "max_tokens": 65536, "max_video_length": 1, "max_videos_per_prompt": 10, "mode": "chat", @@ -14388,10 +14388,10 @@ "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 65536, + "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, - "output_cost_per_image_token": 1.2e-04, + "output_cost_per_image_token": 0.00012, "output_cost_per_token": 1.2e-05, "rpm": 1000, "tpm": 4000000, @@ -15524,7 +15524,7 @@ "max_input_tokens": 1024, "max_tokens": 1024, "mode": "video_generation", - "output_cost_per_second": 0.40, + "output_cost_per_second": 0.4, "source": "https://ai.google.dev/gemini-api/docs/video", "supported_modalities": [ "text" @@ -15552,7 +15552,7 @@ "max_input_tokens": 1024, "max_tokens": 1024, "mode": "video_generation", - "output_cost_per_second": 0.40, + "output_cost_per_second": 0.4, "source": "https://ai.google.dev/gemini-api/docs/video", "supported_modalities": [ "text" @@ -16056,11 +16056,11 @@ "supports_vision": true }, "gpt-3.5-turbo": { - "input_cost_per_token": 0.5e-06, + "input_cost_per_token": 5e-07, "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4097, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -16073,7 +16073,7 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 16385, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -16087,7 +16087,7 @@ "litellm_provider": "openai", "max_input_tokens": 4097, "max_output_tokens": 4096, - "max_tokens": 4097, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, "supports_prompt_caching": true, @@ -16099,7 +16099,7 @@ "litellm_provider": "openai", "max_input_tokens": 4097, "max_output_tokens": 4096, - "max_tokens": 4097, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, "supports_function_calling": true, @@ -16113,7 +16113,7 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 16385, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, "supports_function_calling": true, @@ -16127,7 +16127,7 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 16385, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-06, "supports_prompt_caching": true, @@ -16139,7 +16139,7 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 16385, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-06, "supports_prompt_caching": true, @@ -17191,7 +17191,7 @@ "supports_pdf_input": true }, "high/1024-x-1536/gpt-image-1.5": { - "input_cost_per_image": 0.20, + "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", "supported_endpoints": [ @@ -17202,7 +17202,7 @@ "supports_pdf_input": true }, "high/1536-x-1024/gpt-image-1.5": { - "input_cost_per_image": 0.20, + "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", "supported_endpoints": [ @@ -17356,7 +17356,7 @@ "supports_pdf_input": true }, "high/1024-x-1536/gpt-image-1.5-2025-12-16": { - "input_cost_per_image": 0.20, + "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", "supported_endpoints": [ @@ -17367,7 +17367,7 @@ "supports_pdf_input": true }, "high/1536-x-1024/gpt-image-1.5-2025-12-16": { - "input_cost_per_image": 0.20, + "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", "supported_endpoints": [ @@ -17704,7 +17704,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", - "output_cost_per_token": 1.68e-04, + "output_cost_per_token": 0.000168, "supported_endpoints": [ "/v1/batch", "/v1/responses" @@ -17735,7 +17735,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", - "output_cost_per_token": 1.68e-04, + "output_cost_per_token": 0.000168, "supported_endpoints": [ "/v1/batch", "/v1/responses" @@ -17767,7 +17767,7 @@ "max_output_tokens": 272000, "max_tokens": 272000, "mode": "responses", - "output_cost_per_token": 1.2e-04, + "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, "supported_endpoints": [ "/v1/batch", @@ -17800,7 +17800,7 @@ "max_output_tokens": 272000, "max_tokens": 272000, "mode": "responses", - "output_cost_per_token": 1.2e-04, + "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, "supported_endpoints": [ "/v1/batch", @@ -18503,7 +18503,7 @@ "lemonade/Qwen3-Coder-30B-A3B-Instruct-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 262144, + "max_tokens": 32768, "max_input_tokens": 262144, "max_output_tokens": 32768, "mode": "chat", @@ -18515,7 +18515,7 @@ "lemonade/gpt-oss-20b-mxfp4-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 131072, + "max_tokens": 32768, "max_input_tokens": 131072, "max_output_tokens": 32768, "mode": "chat", @@ -18527,7 +18527,7 @@ "lemonade/gpt-oss-120b-mxfp-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 131072, + "max_tokens": 32768, "max_input_tokens": 131072, "max_output_tokens": 32768, "mode": "chat", @@ -18539,7 +18539,7 @@ "lemonade/Gemma-3-4b-it-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 128000, + "max_tokens": 8192, "max_input_tokens": 128000, "max_output_tokens": 8192, "mode": "chat", @@ -18551,7 +18551,7 @@ "lemonade/Qwen3-4B-Instruct-2507-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 262144, + "max_tokens": 32768, "max_input_tokens": 262144, "max_output_tokens": 32768, "mode": "chat", @@ -18688,11 +18688,11 @@ "groq/moonshotai/kimi-k2-instruct-0905": { "input_cost_per_token": 1e-06, "output_cost_per_token": 3e-06, - "cache_read_input_token_cost": 0.5e-06, + "cache_read_input_token_cost": 5e-07, "litellm_provider": "groq", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 278528, + "max_tokens": 16384, "mode": "chat", "supports_function_calling": true, "supports_response_schema": true, @@ -19353,7 +19353,7 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 131072, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -19366,7 +19366,7 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 16384, "max_output_tokens": 8192, - "max_tokens": 16384, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -19702,7 +19702,7 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 128000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.6e-05, "supports_function_calling": true, @@ -19713,7 +19713,7 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 128000, + "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 9.9e-07, "supports_function_calling": true, @@ -19724,7 +19724,7 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 128000, + "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 2.2e-07, "supports_function_calling": true, @@ -19735,7 +19735,7 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 128000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.5e-07, "supports_function_calling": true, @@ -19747,7 +19747,7 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 128000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -19758,7 +19758,7 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 128000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "supports_function_calling": true, @@ -19769,7 +19769,7 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 128000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, "supports_function_calling": true, @@ -19851,7 +19851,7 @@ "litellm_provider": "meta_llama", "max_input_tokens": 128000, "max_output_tokens": 4028, - "max_tokens": 128000, + "max_tokens": 4028, "mode": "chat", "source": "https://llama.developer.meta.com/docs/models", "supported_modalities": [ @@ -19867,7 +19867,7 @@ "litellm_provider": "meta_llama", "max_input_tokens": 128000, "max_output_tokens": 4028, - "max_tokens": 128000, + "max_tokens": 4028, "mode": "chat", "source": "https://llama.developer.meta.com/docs/models", "supported_modalities": [ @@ -19883,7 +19883,7 @@ "litellm_provider": "meta_llama", "max_input_tokens": 1000000, "max_output_tokens": 4028, - "max_tokens": 128000, + "max_tokens": 4028, "mode": "chat", "source": "https://llama.developer.meta.com/docs/models", "supported_modalities": [ @@ -19900,7 +19900,7 @@ "litellm_provider": "meta_llama", "max_input_tokens": 10000000, "max_output_tokens": 4028, - "max_tokens": 128000, + "max_tokens": 4028, "mode": "chat", "source": "https://llama.developer.meta.com/docs/models", "supported_modalities": [ @@ -19932,7 +19932,7 @@ ] }, "minimax/speech-02-turbo": { - "input_cost_per_character": 0.00006, + "input_cost_per_character": 6e-05, "litellm_provider": "minimax", "mode": "audio_speech", "supported_endpoints": [ @@ -19948,7 +19948,7 @@ ] }, "minimax/speech-2.6-turbo": { - "input_cost_per_character": 0.00006, + "input_cost_per_character": 6e-05, "litellm_provider": "minimax", "mode": "audio_speech", "supported_endpoints": [ @@ -20278,8 +20278,8 @@ }, "mistral/mistral-ocr-latest": { "litellm_provider": "mistral", - "ocr_cost_per_page": 1e-3, - "annotation_cost_per_page": 3e-3, + "ocr_cost_per_page": 0.001, + "annotation_cost_per_page": 0.003, "mode": "ocr", "supported_endpoints": [ "/v1/ocr" @@ -20288,8 +20288,8 @@ }, "mistral/mistral-ocr-2505-completion": { "litellm_provider": "mistral", - "ocr_cost_per_page": 1e-3, - "annotation_cost_per_page": 3e-3, + "ocr_cost_per_page": 0.001, + "annotation_cost_per_page": 0.003, "mode": "ocr", "supported_endpoints": [ "/v1/ocr" @@ -20349,14 +20349,14 @@ "mode": "embedding" }, "mistral/codestral-embed": { - "input_cost_per_token": 0.15e-06, + "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, "max_tokens": 8192, "mode": "embedding" }, "mistral/codestral-embed-2505": { - "input_cost_per_token": 0.15e-06, + "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, "max_tokens": 8192, @@ -20757,28 +20757,28 @@ "supports_vision": true }, "moonshot/kimi-k2-thinking": { - "cache_read_input_token_cost": 1.5e-7, - "input_cost_per_token": 6e-7, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 6e-07, "litellm_provider": "moonshot", "max_input_tokens": 262144, "max_output_tokens": 262144, "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 2.5e-6, + "output_cost_per_token": 2.5e-06, "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", "supports_function_calling": true, "supports_tool_choice": true, "supports_web_search": true }, "moonshot/kimi-k2-thinking-turbo": { - "cache_read_input_token_cost": 1.5e-7, - "input_cost_per_token": 1.15e-6, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.15e-06, "litellm_provider": "moonshot", "max_input_tokens": 262144, "max_output_tokens": 262144, "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 8e-6, + "output_cost_per_token": 8e-06, "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", "supports_function_calling": true, "supports_tool_choice": true, @@ -21650,7 +21650,7 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 128000, + "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.068e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -21662,7 +21662,7 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 128000, + "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -21674,7 +21674,7 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 128000, + "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -21686,7 +21686,7 @@ "litellm_provider": "oci", "max_input_tokens": 512000, "max_output_tokens": 4000, - "max_tokens": 512000, + "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -21698,7 +21698,7 @@ "litellm_provider": "oci", "max_input_tokens": 192000, "max_output_tokens": 4000, - "max_tokens": 192000, + "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -21770,7 +21770,7 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 128000, + "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", @@ -21782,7 +21782,7 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 256000, + "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", @@ -21794,7 +21794,7 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 128000, + "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", @@ -21806,7 +21806,7 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 32768, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": false @@ -21844,7 +21844,7 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 32768, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -21864,12 +21864,12 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 32768, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true }, - "ollama/deepseek-v3.1:671b-cloud" : { + "ollama/deepseek-v3.1:671b-cloud": { "input_cost_per_token": 0.0, "litellm_provider": "ollama", "max_input_tokens": 163840, @@ -21879,7 +21879,7 @@ "output_cost_per_token": 0.0, "supports_function_calling": true }, - "ollama/gpt-oss:120b-cloud" : { + "ollama/gpt-oss:120b-cloud": { "input_cost_per_token": 0.0, "litellm_provider": "ollama", "max_input_tokens": 131072, @@ -21889,7 +21889,7 @@ "output_cost_per_token": 0.0, "supports_function_calling": true }, - "ollama/gpt-oss:20b-cloud" : { + "ollama/gpt-oss:20b-cloud": { "input_cost_per_token": 0.0, "litellm_provider": "ollama", "max_input_tokens": 131072, @@ -21904,7 +21904,7 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 32768, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -21968,7 +21968,7 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 32768, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -22026,7 +22026,7 @@ "litellm_provider": "ollama", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 65536, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -22084,7 +22084,7 @@ "litellm_provider": "openai", "max_input_tokens": 32768, "max_output_tokens": 0, - "max_tokens": 32768, + "max_tokens": 0, "mode": "moderation", "output_cost_per_token": 0.0 }, @@ -22093,7 +22093,7 @@ "litellm_provider": "openai", "max_input_tokens": 32768, "max_output_tokens": 0, - "max_tokens": 32768, + "max_tokens": 0, "mode": "moderation", "output_cost_per_token": 0.0 }, @@ -22102,7 +22102,7 @@ "litellm_provider": "openai", "max_input_tokens": 32768, "max_output_tokens": 0, - "max_tokens": 32768, + "max_tokens": 0, "mode": "moderation", "output_cost_per_token": 0.0 }, @@ -22156,7 +22156,7 @@ "input_cost_per_token": 1.102e-05, "litellm_provider": "openrouter", "max_output_tokens": 8191, - "max_tokens": 100000, + "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3.268e-05, "supports_tool_choice": true @@ -22296,7 +22296,7 @@ "input_cost_per_token": 1.63e-06, "litellm_provider": "openrouter", "max_output_tokens": 8191, - "max_tokens": 100000, + "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 5.51e-06, "supports_tool_choice": true @@ -22491,7 +22491,7 @@ "litellm_provider": "openrouter", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 8192, + "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 8e-07, "supports_assistant_prefill": true, @@ -22506,7 +22506,7 @@ "litellm_provider": "openrouter", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 8192, + "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 4e-07, "supports_assistant_prefill": true, @@ -22521,7 +22521,7 @@ "litellm_provider": "openrouter", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 8192, + "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 4e-07, "supports_assistant_prefill": true, @@ -22535,7 +22535,7 @@ "litellm_provider": "openrouter", "max_input_tokens": 66000, "max_output_tokens": 4096, - "max_tokens": 8192, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-07, "supports_prompt_caching": true, @@ -22693,51 +22693,51 @@ "supports_web_search": true }, "openrouter/google/gemini-3-flash-preview": { - "cache_read_input_token_cost": 5e-08, - "input_cost_per_audio_token": 1e-06, - "input_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "max_audio_length_hours": 8.4, - "max_audio_per_prompt": 1, - "max_images_per_prompt": 3000, - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_pdf_size_mb": 30, - "max_tokens": 65535, - "max_video_length": 1, - "max_videos_per_prompt": 10, - "mode": "chat", - "output_cost_per_reasoning_token": 3e-06, - "output_cost_per_token": 3e-06, - "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 800000 + "cache_read_input_token_cost": 5e-08, + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 5e-07, + "litellm_provider": "openrouter", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 3e-06, + "output_cost_per_token": 3e-06, + "rpm": 2000, + "source": "https://ai.google.dev/pricing/gemini-3", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 800000 }, "openrouter/google/gemini-pro-1.5": { "input_cost_per_image": 0.00265, @@ -22868,13 +22868,13 @@ "supports_tool_choice": true }, "openrouter/minimax/minimax-m2": { - "input_cost_per_token": 2.55e-7, + "input_cost_per_token": 2.55e-07, "litellm_provider": "openrouter", "max_input_tokens": 204800, "max_output_tokens": 204800, - "max_tokens": 32768, + "max_tokens": 204800, "mode": "chat", - "output_cost_per_token": 1.02e-6, + "output_cost_per_token": 1.02e-06, "supports_function_calling": true, "supports_prompt_caching": false, "supports_reasoning": true, @@ -22900,7 +22900,7 @@ "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 262144, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -23285,7 +23285,7 @@ "litellm_provider": "openrouter", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 400000, + "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "supports_function_calling": true, @@ -23301,7 +23301,7 @@ "litellm_provider": "openrouter", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 128000, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, "supports_function_calling": true, @@ -23315,9 +23315,9 @@ "litellm_provider": "openrouter", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 400000, + "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.68e-04, + "output_cost_per_token": 0.000168, "supports_function_calling": true, "supports_prompt_caching": false, "supports_reasoning": true, @@ -23474,20 +23474,20 @@ "litellm_provider": "openrouter", "max_input_tokens": 8192, "max_output_tokens": 2048, - "max_tokens": 8192, + "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 6.3e-07, "supports_tool_choice": true, "supports_vision": true }, "openrouter/qwen/qwen3-coder": { - "input_cost_per_token": 2.2e-7, + "input_cost_per_token": 2.2e-07, "litellm_provider": "openrouter", "max_input_tokens": 262100, "max_output_tokens": 262100, "max_tokens": 262100, "mode": "chat", - "output_cost_per_token": 9.5e-7, + "output_cost_per_token": 9.5e-07, "source": "https://openrouter.ai/qwen/qwen3-coder", "supports_tool_choice": true, "supports_function_calling": true @@ -23530,7 +23530,7 @@ "litellm_provider": "openrouter", "max_input_tokens": 2000000, "max_output_tokens": 30000, - "max_tokens": 2000000, + "max_tokens": 30000, "mode": "chat", "output_cost_per_token": 0, "source": "https://openrouter.ai/x-ai/grok-4-fast:free", @@ -23540,26 +23540,26 @@ "supports_web_search": false }, "openrouter/z-ai/glm-4.6": { - "input_cost_per_token": 4.0e-7, + "input_cost_per_token": 4e-07, "litellm_provider": "openrouter", "max_input_tokens": 202800, "max_output_tokens": 131000, - "max_tokens": 202800, + "max_tokens": 131000, "mode": "chat", - "output_cost_per_token": 1.75e-6, + "output_cost_per_token": 1.75e-06, "source": "https://openrouter.ai/z-ai/glm-4.6", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true }, "openrouter/z-ai/glm-4.6:exacto": { - "input_cost_per_token": 4.5e-7, + "input_cost_per_token": 4.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 202800, "max_output_tokens": 131000, - "max_tokens": 202800, + "max_tokens": 131000, "mode": "chat", - "output_cost_per_token": 1.9e-6, + "output_cost_per_token": 1.9e-06, "source": "https://openrouter.ai/z-ai/glm-4.6:exacto", "supports_function_calling": true, "supports_reasoning": true, @@ -24107,7 +24107,7 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 8192, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -24119,7 +24119,7 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 8192, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -24131,7 +24131,7 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 8192, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -24143,7 +24143,7 @@ "litellm_provider": "publicai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 16384, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -24155,7 +24155,7 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 8192, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -24167,7 +24167,7 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 32768, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -24179,7 +24179,7 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 32768, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -24191,7 +24191,7 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 32768, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -24204,7 +24204,7 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 32768, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -24217,7 +24217,7 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262000, "max_output_tokens": 65536, - "max_tokens": 262144, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.8e-06, "supports_function_calling": true, @@ -24229,7 +24229,7 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 262144, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.8e-07, "supports_function_calling": true, @@ -24241,9 +24241,9 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 262144, + "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 6.0e-07, + "output_cost_per_token": 6e-07, "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true @@ -24253,9 +24253,9 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 131072, + "max_tokens": 16384, "mode": "chat", - "output_cost_per_token": 6.0e-07, + "output_cost_per_token": 6e-07, "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true @@ -24756,12 +24756,11 @@ "supports_reasoning": true, "source": "https://cloud.sambanova.ai/plans/pricing" }, - "snowflake/claude-3-5-sonnet": { "litellm_provider": "snowflake", "max_input_tokens": 18000, "max_output_tokens": 8192, - "max_tokens": 18000, + "max_tokens": 8192, "mode": "chat", "supports_computer_use": true }, @@ -24769,7 +24768,7 @@ "litellm_provider": "snowflake", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 32768, + "max_tokens": 8192, "mode": "chat", "supports_reasoning": true }, @@ -24777,293 +24776,339 @@ "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/jamba-1.5-large": { "litellm_provider": "snowflake", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 256000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/jamba-1.5-mini": { "litellm_provider": "snowflake", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 256000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/jamba-instruct": { "litellm_provider": "snowflake", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 256000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/llama2-70b-chat": { "litellm_provider": "snowflake", "max_input_tokens": 4096, "max_output_tokens": 8192, - "max_tokens": 4096, + "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3-70b": { "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3-8b": { "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.1-405b": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 128000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.1-70b": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 128000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.1-8b": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 128000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.2-1b": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 128000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.2-3b": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 128000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.3-70b": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 128000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/mistral-7b": { "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 32000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/mistral-large": { "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 32000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/mistral-large2": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 128000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/mixtral-8x7b": { "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 32000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/reka-core": { "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 32000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/reka-flash": { "litellm_provider": "snowflake", "max_input_tokens": 100000, "max_output_tokens": 8192, - "max_tokens": 100000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/snowflake-arctic": { "litellm_provider": "snowflake", "max_input_tokens": 4096, "max_output_tokens": 8192, - "max_tokens": 4096, + "max_tokens": 8192, "mode": "chat" }, "snowflake/snowflake-llama-3.1-405b": { "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8000, + "max_tokens": 8192, "mode": "chat" }, "snowflake/snowflake-llama-3.3-70b": { "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8000, + "max_tokens": 8192, "mode": "chat" }, "stability/sd3": { "litellm_provider": "stability", "mode": "image_generation", "output_cost_per_image": 0.065, - "supported_endpoints": ["/v1/images/generations"] + "supported_endpoints": [ + "/v1/images/generations" + ] }, "stability/sd3-large": { "litellm_provider": "stability", "mode": "image_generation", "output_cost_per_image": 0.065, - "supported_endpoints": ["/v1/images/generations"] + "supported_endpoints": [ + "/v1/images/generations" + ] }, "stability/sd3-large-turbo": { "litellm_provider": "stability", "mode": "image_generation", "output_cost_per_image": 0.04, - "supported_endpoints": ["/v1/images/generations"] + "supported_endpoints": [ + "/v1/images/generations" + ] }, "stability/sd3-medium": { "litellm_provider": "stability", "mode": "image_generation", "output_cost_per_image": 0.035, - "supported_endpoints": ["/v1/images/generations"] + "supported_endpoints": [ + "/v1/images/generations" + ] }, "stability/sd3.5-large": { "litellm_provider": "stability", "mode": "image_generation", "output_cost_per_image": 0.065, - "supported_endpoints": ["/v1/images/generations"] + "supported_endpoints": [ + "/v1/images/generations" + ] }, "stability/sd3.5-large-turbo": { "litellm_provider": "stability", "mode": "image_generation", "output_cost_per_image": 0.04, - "supported_endpoints": ["/v1/images/generations"] + "supported_endpoints": [ + "/v1/images/generations" + ] }, "stability/sd3.5-medium": { "litellm_provider": "stability", "mode": "image_generation", "output_cost_per_image": 0.035, - "supported_endpoints": ["/v1/images/generations"] + "supported_endpoints": [ + "/v1/images/generations" + ] }, "stability/stable-image-ultra": { "litellm_provider": "stability", "mode": "image_generation", "output_cost_per_image": 0.08, - "supported_endpoints": ["/v1/images/generations"] + "supported_endpoints": [ + "/v1/images/generations" + ] }, "stability/inpaint": { "litellm_provider": "stability", "mode": "image_edit", "output_cost_per_image": 0.005, - "supported_endpoints": ["/v1/images/edits"] + "supported_endpoints": [ + "/v1/images/edits" + ] }, "stability/outpaint": { "litellm_provider": "stability", "mode": "image_edit", "output_cost_per_image": 0.004, - "supported_endpoints": ["/v1/images/edits"] + "supported_endpoints": [ + "/v1/images/edits" + ] }, "stability/erase": { "litellm_provider": "stability", "mode": "image_edit", "output_cost_per_image": 0.005, - "supported_endpoints": ["/v1/images/edits"] + "supported_endpoints": [ + "/v1/images/edits" + ] }, "stability/search-and-replace": { "litellm_provider": "stability", "mode": "image_edit", "output_cost_per_image": 0.005, - "supported_endpoints": ["/v1/images/edits"] + "supported_endpoints": [ + "/v1/images/edits" + ] }, "stability/search-and-recolor": { "litellm_provider": "stability", "mode": "image_edit", "output_cost_per_image": 0.005, - "supported_endpoints": ["/v1/images/edits"] + "supported_endpoints": [ + "/v1/images/edits" + ] }, "stability/remove-background": { "litellm_provider": "stability", "mode": "image_edit", "output_cost_per_image": 0.005, - "supported_endpoints": ["/v1/images/edits"] + "supported_endpoints": [ + "/v1/images/edits" + ] }, "stability/replace-background-and-relight": { "litellm_provider": "stability", "mode": "image_edit", "output_cost_per_image": 0.008, - "supported_endpoints": ["/v1/images/edits"] + "supported_endpoints": [ + "/v1/images/edits" + ] }, "stability/sketch": { "litellm_provider": "stability", "mode": "image_edit", "output_cost_per_image": 0.005, - "supported_endpoints": ["/v1/images/edits"] + "supported_endpoints": [ + "/v1/images/edits" + ] }, "stability/structure": { "litellm_provider": "stability", "mode": "image_edit", "output_cost_per_image": 0.005, - "supported_endpoints": ["/v1/images/edits"] + "supported_endpoints": [ + "/v1/images/edits" + ] }, "stability/style": { "litellm_provider": "stability", "mode": "image_edit", "output_cost_per_image": 0.005, - "supported_endpoints": ["/v1/images/edits"] + "supported_endpoints": [ + "/v1/images/edits" + ] }, "stability/style-transfer": { "litellm_provider": "stability", "mode": "image_edit", "output_cost_per_image": 0.008, - "supported_endpoints": ["/v1/images/edits"] + "supported_endpoints": [ + "/v1/images/edits" + ] }, "stability/fast": { "litellm_provider": "stability", "mode": "image_edit", "output_cost_per_image": 0.002, - "supported_endpoints": ["/v1/images/edits"] + "supported_endpoints": [ + "/v1/images/edits" + ] }, "stability/conservative": { "litellm_provider": "stability", "mode": "image_edit", "output_cost_per_image": 0.04, - "supported_endpoints": ["/v1/images/edits"] + "supported_endpoints": [ + "/v1/images/edits" + ] }, "stability/creative": { "litellm_provider": "stability", "mode": "image_edit", "output_cost_per_image": 0.06, - "supported_endpoints": ["/v1/images/edits"] + "supported_endpoints": [ + "/v1/images/edits" + ] }, "stability/stable-image-core": { "litellm_provider": "stability", "mode": "image_generation", "output_cost_per_image": 0.03, - "supported_endpoints": ["/v1/images/generations"] + "supported_endpoints": [ + "/v1/images/generations" + ] }, "stability.sd3-5-large-v1:0": { "litellm_provider": "bedrock", @@ -25090,13 +25135,13 @@ "litellm_provider": "bedrock", "max_input_tokens": 77, "mode": "image_edit", - "output_cost_per_image": 0.40 + "output_cost_per_image": 0.4 }, "stability.stable-creative-upscale-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 77, "mode": "image_edit", - "output_cost_per_image": 0.60 + "output_cost_per_image": 0.6 }, "stability.stable-fast-upscale-v1:0": { "litellm_provider": "bedrock", @@ -25204,12 +25249,12 @@ "output_cost_per_pixel": 0.0 }, "linkup/search": { - "input_cost_per_query": 5.87e-03, + "input_cost_per_query": 0.00587, "litellm_provider": "linkup", "mode": "search" }, "linkup/search-deep": { - "input_cost_per_query": 58.67e-03, + "input_cost_per_query": 0.05867, "litellm_provider": "linkup", "mode": "search" }, @@ -25388,7 +25433,7 @@ "litellm_provider": "openai", "max_input_tokens": 32768, "max_output_tokens": 0, - "max_tokens": 32768, + "max_tokens": 0, "mode": "moderation", "output_cost_per_token": 0.0 }, @@ -25397,7 +25442,7 @@ "litellm_provider": "openai", "max_input_tokens": 32768, "max_output_tokens": 0, - "max_tokens": 32768, + "max_tokens": 0, "mode": "moderation", "output_cost_per_token": 0.0 }, @@ -25406,7 +25451,7 @@ "litellm_provider": "openai", "max_input_tokens": 32768, "max_output_tokens": 0, - "max_tokens": 32768, + "max_tokens": 0, "mode": "moderation", "output_cost_per_token": 0.0 }, @@ -25842,7 +25887,7 @@ "supports_tool_choice": true }, "together_ai/zai-org/GLM-4.6": { - "input_cost_per_token": 0.6e-06, + "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 200000, "max_output_tokens": 200000, @@ -25925,7 +25970,7 @@ "source": "https://aws.amazon.com/polly/pricing/" }, "aws_polly/long-form": { - "input_cost_per_character": 1e-04, + "input_cost_per_character": 0.0001, "litellm_provider": "aws_polly", "mode": "audio_speech", "supported_endpoints": [ @@ -26357,7 +26402,7 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 128000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.6e-05, "supports_function_calling": true, @@ -26368,7 +26413,7 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 128000, + "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 9.9e-07, "supports_function_calling": true, @@ -26379,7 +26424,7 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 128000, + "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 2.2e-07, "supports_function_calling": true, @@ -26390,7 +26435,7 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 128000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.5e-07, "supports_function_calling": true, @@ -26402,7 +26447,7 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 128000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -26413,7 +26458,7 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 128000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "supports_function_calling": true, @@ -26424,7 +26469,7 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 128000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, "supports_function_calling": true, @@ -26489,7 +26534,7 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 128000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -26542,7 +26587,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 40960, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.4e-07 }, @@ -26551,7 +26596,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 40960, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07 }, @@ -26560,7 +26605,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 40960, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-07 }, @@ -26569,7 +26614,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 40960, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-07 }, @@ -26578,7 +26623,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 262144, "max_output_tokens": 66536, - "max_tokens": 262144, + "max_tokens": 66536, "mode": "chat", "output_cost_per_token": 1.6e-06 }, @@ -26587,7 +26632,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 300000, "max_output_tokens": 8192, - "max_tokens": 300000, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-07 }, @@ -26596,7 +26641,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 128000, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.4e-07 }, @@ -26605,7 +26650,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 300000, "max_output_tokens": 8192, - "max_tokens": 300000, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.2e-06 }, @@ -26625,7 +26670,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 200000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06 }, @@ -26636,7 +26681,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 200000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05 }, @@ -26647,7 +26692,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 200000, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06 }, @@ -26658,7 +26703,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 200000, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05 }, @@ -26669,7 +26714,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 200000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05 }, @@ -26680,7 +26725,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 200000, + "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05 }, @@ -26691,7 +26736,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 200000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05 }, @@ -26700,7 +26745,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 256000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-05 }, @@ -26709,7 +26754,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 128000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07 }, @@ -26718,7 +26763,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 128000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-05 }, @@ -26736,7 +26781,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 128000, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.19e-06 }, @@ -26754,7 +26799,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 128000, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9e-07 }, @@ -26763,7 +26808,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 1048576, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07 }, @@ -26772,7 +26817,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 1048576, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-07 }, @@ -26781,7 +26826,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1000000, "max_output_tokens": 65536, - "max_tokens": 1000000, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.5e-06 }, @@ -26790,7 +26835,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 1048576, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05 }, @@ -26835,7 +26880,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "max_output_tokens": 16384, - "max_tokens": 32000, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -26862,7 +26907,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 128000, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07 }, @@ -26871,7 +26916,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131000, "max_output_tokens": 131072, - "max_tokens": 131000, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-08 }, @@ -26880,7 +26925,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 128000, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.6e-07 }, @@ -26889,7 +26934,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 128000, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-07 }, @@ -26898,7 +26943,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 128000, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-07 }, @@ -26907,7 +26952,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 128000, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07 }, @@ -26916,7 +26961,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 128000, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07 }, @@ -26925,7 +26970,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 131072, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07 }, @@ -26934,7 +26979,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 131072, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-07 }, @@ -26943,7 +26988,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 256000, + "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 9e-07 }, @@ -26970,7 +27015,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 128000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06 }, @@ -26979,7 +27024,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 128000, + "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-06 }, @@ -26988,7 +27033,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 128000, + "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 4e-08 }, @@ -26997,7 +27042,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 128000, + "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1e-07 }, @@ -27015,7 +27060,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "max_output_tokens": 4000, - "max_tokens": 32000, + "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 6e-06 }, @@ -27033,7 +27078,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "max_output_tokens": 4000, - "max_tokens": 32000, + "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 3e-07 }, @@ -27042,7 +27087,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 65536, "max_output_tokens": 2048, - "max_tokens": 65536, + "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.2e-06 }, @@ -27051,7 +27096,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 128000, + "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.5e-07 }, @@ -27060,7 +27105,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 128000, + "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 6e-06 }, @@ -27069,7 +27114,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 131072, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.2e-06 }, @@ -27078,7 +27123,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 16384, - "max_tokens": 32768, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06 }, @@ -27087,7 +27132,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 16384, - "max_tokens": 32768, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.9e-06 }, @@ -27096,7 +27141,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 16385, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06 }, @@ -27105,7 +27150,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 8192, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06 }, @@ -27114,7 +27159,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 128000, + "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05 }, @@ -27125,7 +27170,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 1047576, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06 }, @@ -27136,7 +27181,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 1047576, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06 }, @@ -27147,7 +27192,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 1047576, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07 }, @@ -27158,7 +27203,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 128000, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05 }, @@ -27169,7 +27214,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 128000, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07 }, @@ -27180,7 +27225,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 200000, + "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05 }, @@ -27191,7 +27236,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 200000, + "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06 }, @@ -27202,7 +27247,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 200000, + "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06 }, @@ -27213,7 +27258,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 200000, + "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06 }, @@ -27249,7 +27294,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 127000, "max_output_tokens": 8000, - "max_tokens": 127000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -27258,7 +27303,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8000, - "max_tokens": 200000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.5e-05 }, @@ -27267,7 +27312,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 127000, "max_output_tokens": 8000, - "max_tokens": 127000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 5e-06 }, @@ -27276,7 +27321,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 127000, "max_output_tokens": 8000, - "max_tokens": 127000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 8e-06 }, @@ -27285,7 +27330,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 128000, + "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-05 }, @@ -27294,7 +27339,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 128000, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.5e-05 }, @@ -27303,7 +27348,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 4000, - "max_tokens": 131072, + "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1e-05 }, @@ -27375,7 +27420,7 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 96000, - "max_tokens": 128000, + "max_tokens": 96000, "mode": "chat", "output_cost_per_token": 1.1e-06 }, @@ -27394,7 +27439,7 @@ "supports_tool_choice": true }, "vertex_ai/chirp": { - "input_cost_per_character": 30e-06, + "input_cost_per_character": 3e-05, "litellm_provider": "vertex_ai", "mode": "audio_speech", "source": "https://cloud.google.com/text-to-speech/pricing", @@ -27938,7 +27983,7 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 163840, "max_output_tokens": 32768, - "max_tokens": 163840, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 5.4e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", @@ -27957,7 +28002,7 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 163840, "max_output_tokens": 32768, - "max_tokens": 163840, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.68e-06, "output_cost_per_token_batches": 8.4e-07, @@ -28042,10 +28087,10 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 65536, + "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, - "output_cost_per_image_token": 1.2e-04, + "output_cost_per_image_token": 0.00012, "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image" @@ -28154,7 +28199,7 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 128000, + "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.6e-05, "source": "https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama-3.2-90b-vision-instruct-maas", @@ -28167,7 +28212,7 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 128000, + "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama-3.2-90b-vision-instruct-maas", @@ -28180,7 +28225,7 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 128000, + "max_tokens": 2048, "metadata": { "notes": "VertexAI states that The Llama 3.1 API service for llama-3.1-70b-instruct-maas and llama-3.1-8b-instruct-maas are in public preview and at no cost." }, @@ -28196,7 +28241,7 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 128000, + "max_tokens": 2048, "metadata": { "notes": "VertexAI states that The Llama 3.2 API service is at no cost during public preview, and will be priced as per dollar-per-1M-tokens at GA." }, @@ -28494,7 +28539,7 @@ "vertex_ai/mistral-ocr-2505": { "litellm_provider": "vertex_ai", "mode": "ocr", - "ocr_cost_per_page": 5e-4, + "ocr_cost_per_page": 0.0005, "supported_endpoints": [ "/v1/ocr" ], @@ -28505,7 +28550,7 @@ "mode": "ocr", "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, - "ocr_cost_per_page": 3e-04, + "ocr_cost_per_page": 0.0003, "source": "https://cloud.google.com/vertex-ai/pricing" }, "vertex_ai/openai/gpt-oss-120b-maas": { @@ -28993,13 +29038,13 @@ "mode": "chat" }, "watsonx/ibm/granite-3-8b-instruct": { - "input_cost_per_token": 0.2e-06, + "input_cost_per_token": 2e-07, "litellm_provider": "watsonx", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 8192, + "max_tokens": 1024, "mode": "chat", - "output_cost_per_token": 0.2e-06, + "output_cost_per_token": 2e-07, "supports_audio_input": false, "supports_audio_output": false, "supports_function_calling": true, @@ -29015,9 +29060,9 @@ "litellm_provider": "watsonx", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 131072, + "max_tokens": 16384, "mode": "chat", - "output_cost_per_token": 10e-06, + "output_cost_per_token": 1e-05, "supports_audio_input": false, "supports_audio_output": false, "supports_function_calling": true, @@ -29056,8 +29101,8 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.6e-06, - "output_cost_per_token": 0.6e-06, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 6e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": false, @@ -29068,8 +29113,8 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.6e-06, - "output_cost_per_token": 0.6e-06, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 6e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": false, @@ -29080,8 +29125,8 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.6e-06, - "output_cost_per_token": 0.6e-06, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 6e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": false, @@ -29092,8 +29137,8 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.2e-06, - "output_cost_per_token": 0.2e-06, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": true, @@ -29104,8 +29149,8 @@ "max_tokens": 20480, "max_input_tokens": 20480, "max_output_tokens": 20480, - "input_cost_per_token": 0.06e-06, - "output_cost_per_token": 0.25e-06, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 2.5e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": true, @@ -29116,8 +29161,8 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.1e-06, - "output_cost_per_token": 0.1e-06, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": false, @@ -29128,8 +29173,8 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.2e-06, - "output_cost_per_token": 0.2e-06, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 2e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": false, @@ -29140,8 +29185,8 @@ "max_tokens": 512, "max_input_tokens": 512, "max_output_tokens": 512, - "input_cost_per_token": 0.38e-06, - "output_cost_per_token": 0.38e-06, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 3.8e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": false, @@ -29152,8 +29197,8 @@ "max_tokens": 512, "max_input_tokens": 512, "max_output_tokens": 512, - "input_cost_per_token": 0.38e-06, - "output_cost_per_token": 0.38e-06, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 3.8e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": false, @@ -29164,8 +29209,8 @@ "max_tokens": 512, "max_input_tokens": 512, "max_output_tokens": 512, - "input_cost_per_token": 0.38e-06, - "output_cost_per_token": 0.38e-06, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 3.8e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": false, @@ -29176,8 +29221,8 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.1e-06, - "output_cost_per_token": 0.1e-06, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": false, @@ -29188,8 +29233,8 @@ "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, - "input_cost_per_token": 0.35e-06, - "output_cost_per_token": 0.35e-06, + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 3.5e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": true, @@ -29200,8 +29245,8 @@ "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, - "input_cost_per_token": 0.1e-06, - "output_cost_per_token": 0.1e-06, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": true, @@ -29212,8 +29257,8 @@ "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, - "input_cost_per_token": 0.15e-06, - "output_cost_per_token": 0.15e-06, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 1.5e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": true, @@ -29236,8 +29281,8 @@ "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, - "input_cost_per_token": 0.71e-06, - "output_cost_per_token": 0.71e-06, + "input_cost_per_token": 7.1e-07, + "output_cost_per_token": 7.1e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": true, @@ -29248,7 +29293,7 @@ "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, - "input_cost_per_token": 0.35e-06, + "input_cost_per_token": 3.5e-07, "output_cost_per_token": 1.4e-06, "litellm_provider": "watsonx", "mode": "chat", @@ -29260,8 +29305,8 @@ "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, - "input_cost_per_token": 0.35e-06, - "output_cost_per_token": 0.35e-06, + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 3.5e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": false, @@ -29273,7 +29318,7 @@ "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, - "output_cost_per_token": 10e-06, + "output_cost_per_token": 1e-05, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": true, @@ -29284,8 +29329,8 @@ "max_tokens": 32000, "max_input_tokens": 32000, "max_output_tokens": 32000, - "input_cost_per_token": 0.1e-06, - "output_cost_per_token": 0.3e-06, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": true, @@ -29296,8 +29341,8 @@ "max_tokens": 32000, "max_input_tokens": 32000, "max_output_tokens": 32000, - "input_cost_per_token": 0.1e-06, - "output_cost_per_token": 0.3e-06, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 3e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": true, @@ -29308,8 +29353,8 @@ "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, - "input_cost_per_token": 0.35e-06, - "output_cost_per_token": 0.35e-06, + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 3.5e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": false, @@ -29320,8 +29365,8 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, - "input_cost_per_token": 0.15e-06, - "output_cost_per_token": 0.6e-06, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, "litellm_provider": "watsonx", "mode": "chat", "supports_function_calling": false, @@ -29611,15 +29656,15 @@ }, "xai/grok-4-fast-reasoning": { "litellm_provider": "xai", - "max_input_tokens": 2e6, - "max_output_tokens": 2e6, - "max_tokens": 2e6, + "max_input_tokens": 2000000.0, + "max_output_tokens": 2000000.0, + "max_tokens": 2000000.0, "mode": "chat", - "input_cost_per_token": 0.2e-06, - "input_cost_per_token_above_128k_tokens": 0.4e-06, - "output_cost_per_token": 0.5e-06, + "input_cost_per_token": 2e-07, + "input_cost_per_token_above_128k_tokens": 4e-07, + "output_cost_per_token": 5e-07, "output_cost_per_token_above_128k_tokens": 1e-06, - "cache_read_input_token_cost": 0.05e-06, + "cache_read_input_token_cost": 5e-08, "source": "https://docs.x.ai/docs/models", "supports_function_calling": true, "supports_tool_choice": true, @@ -29627,14 +29672,14 @@ }, "xai/grok-4-fast-non-reasoning": { "litellm_provider": "xai", - "max_input_tokens": 2e6, - "max_output_tokens": 2e6, - "cache_read_input_token_cost": 0.05e-06, - "max_tokens": 2e6, + "max_input_tokens": 2000000.0, + "max_output_tokens": 2000000.0, + "cache_read_input_token_cost": 5e-08, + "max_tokens": 2000000.0, "mode": "chat", - "input_cost_per_token": 0.2e-06, - "input_cost_per_token_above_128k_tokens": 0.4e-06, - "output_cost_per_token": 0.5e-06, + "input_cost_per_token": 2e-07, + "input_cost_per_token_above_128k_tokens": 4e-07, + "output_cost_per_token": 5e-07, "output_cost_per_token_above_128k_tokens": 1e-06, "source": "https://docs.x.ai/docs/models", "supports_function_calling": true, @@ -29650,7 +29695,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.5e-05, - "output_cost_per_token_above_128k_tokens": 30e-06, + "output_cost_per_token_above_128k_tokens": 3e-05, "source": "https://docs.x.ai/docs/models", "supports_function_calling": true, "supports_tool_choice": true, @@ -29665,22 +29710,22 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.5e-05, - "output_cost_per_token_above_128k_tokens": 30e-06, + "output_cost_per_token_above_128k_tokens": 3e-05, "source": "https://docs.x.ai/docs/models", "supports_function_calling": true, "supports_tool_choice": true, "supports_web_search": true }, "xai/grok-4-1-fast": { - "cache_read_input_token_cost": 0.05e-06, - "input_cost_per_token": 0.2e-06, - "input_cost_per_token_above_128k_tokens": 0.4e-06, + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 2e-07, + "input_cost_per_token_above_128k_tokens": 4e-07, "litellm_provider": "xai", - "max_input_tokens": 2e6, - "max_output_tokens": 2e6, - "max_tokens": 2e6, + "max_input_tokens": 2000000.0, + "max_output_tokens": 2000000.0, + "max_tokens": 2000000.0, "mode": "chat", - "output_cost_per_token": 0.5e-06, + "output_cost_per_token": 5e-07, "output_cost_per_token_above_128k_tokens": 1e-06, "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", "supports_audio_input": true, @@ -29692,15 +29737,15 @@ "supports_web_search": true }, "xai/grok-4-1-fast-reasoning": { - "cache_read_input_token_cost": 0.05e-06, - "input_cost_per_token": 0.2e-06, - "input_cost_per_token_above_128k_tokens": 0.4e-06, + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 2e-07, + "input_cost_per_token_above_128k_tokens": 4e-07, "litellm_provider": "xai", - "max_input_tokens": 2e6, - "max_output_tokens": 2e6, - "max_tokens": 2e6, + "max_input_tokens": 2000000.0, + "max_output_tokens": 2000000.0, + "max_tokens": 2000000.0, "mode": "chat", - "output_cost_per_token": 0.5e-06, + "output_cost_per_token": 5e-07, "output_cost_per_token_above_128k_tokens": 1e-06, "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", "supports_audio_input": true, @@ -29712,15 +29757,15 @@ "supports_web_search": true }, "xai/grok-4-1-fast-reasoning-latest": { - "cache_read_input_token_cost": 0.05e-06, - "input_cost_per_token": 0.2e-06, - "input_cost_per_token_above_128k_tokens": 0.4e-06, + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 2e-07, + "input_cost_per_token_above_128k_tokens": 4e-07, "litellm_provider": "xai", - "max_input_tokens": 2e6, - "max_output_tokens": 2e6, - "max_tokens": 2e6, + "max_input_tokens": 2000000.0, + "max_output_tokens": 2000000.0, + "max_tokens": 2000000.0, "mode": "chat", - "output_cost_per_token": 0.5e-06, + "output_cost_per_token": 5e-07, "output_cost_per_token_above_128k_tokens": 1e-06, "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", "supports_audio_input": true, @@ -29732,15 +29777,15 @@ "supports_web_search": true }, "xai/grok-4-1-fast-non-reasoning": { - "cache_read_input_token_cost": 0.05e-06, - "input_cost_per_token": 0.2e-06, - "input_cost_per_token_above_128k_tokens": 0.4e-06, + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 2e-07, + "input_cost_per_token_above_128k_tokens": 4e-07, "litellm_provider": "xai", - "max_input_tokens": 2e6, - "max_output_tokens": 2e6, - "max_tokens": 2e6, + "max_input_tokens": 2000000.0, + "max_output_tokens": 2000000.0, + "max_tokens": 2000000.0, "mode": "chat", - "output_cost_per_token": 0.5e-06, + "output_cost_per_token": 5e-07, "output_cost_per_token_above_128k_tokens": 1e-06, "source": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", "supports_audio_input": true, @@ -29751,15 +29796,15 @@ "supports_web_search": true }, "xai/grok-4-1-fast-non-reasoning-latest": { - "cache_read_input_token_cost": 0.05e-06, - "input_cost_per_token": 0.2e-06, - "input_cost_per_token_above_128k_tokens": 0.4e-06, + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 2e-07, + "input_cost_per_token_above_128k_tokens": 4e-07, "litellm_provider": "xai", - "max_input_tokens": 2e6, - "max_output_tokens": 2e6, - "max_tokens": 2e6, + "max_input_tokens": 2000000.0, + "max_output_tokens": 2000000.0, + "max_tokens": 2000000.0, "mode": "chat", - "output_cost_per_token": 0.5e-06, + "output_cost_per_token": 5e-07, "output_cost_per_token_above_128k_tokens": 1e-06, "source": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", "supports_audio_input": true, @@ -29942,7 +29987,7 @@ "source": "https://docs.z.ai/guides/overview/pricing" }, "vertex_ai/search_api": { - "input_cost_per_query": 1.5e-03, + "input_cost_per_query": 0.0015, "litellm_provider": "vertex_ai", "mode": "vector_store" }, @@ -29954,7 +29999,7 @@ "openai/sora-2": { "litellm_provider": "openai", "mode": "video_generation", - "output_cost_per_video_per_second": 0.10, + "output_cost_per_video_per_second": 0.1, "source": "https://platform.openai.com/docs/api-reference/videos", "supported_modalities": [ "text", @@ -29971,7 +30016,7 @@ "openai/sora-2-pro": { "litellm_provider": "openai", "mode": "video_generation", - "output_cost_per_video_per_second": 0.30, + "output_cost_per_video_per_second": 0.3, "source": "https://platform.openai.com/docs/api-reference/videos", "supported_modalities": [ "text", @@ -29988,7 +30033,7 @@ "azure/sora-2": { "litellm_provider": "azure", "mode": "video_generation", - "output_cost_per_video_per_second": 0.10, + "output_cost_per_video_per_second": 0.1, "source": "https://azure.microsoft.com/en-us/products/ai-services/video-generation", "supported_modalities": [ "text" @@ -30004,7 +30049,7 @@ "azure/sora-2-pro": { "litellm_provider": "azure", "mode": "video_generation", - "output_cost_per_video_per_second": 0.30, + "output_cost_per_video_per_second": 0.3, "source": "https://azure.microsoft.com/en-us/products/ai-services/video-generation", "supported_modalities": [ "text" @@ -30020,7 +30065,7 @@ "azure/sora-2-pro-high-res": { "litellm_provider": "azure", "mode": "video_generation", - "output_cost_per_video_per_second": 0.50, + "output_cost_per_video_per_second": 0.5, "source": "https://azure.microsoft.com/en-us/products/ai-services/video-generation", "supported_modalities": [ "text" @@ -32367,4 +32412,3 @@ "mode": "embedding" } } - diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index cd76c438ded..bfa162e019b 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -749,6 +749,57 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): raise AssertionError(error_message) +def test_max_tokens_consistency(): + """ + Test that max_tokens == max_output_tokens for all models. + + According to the spec in model_prices_and_context_window.json: + - max_tokens is a LEGACY parameter + - It should be set to max_output_tokens if the provider specifies it + + This test ensures consistency across all model definitions. + """ + import json + from pathlib import Path + + # Load the model configuration + config_path = Path(__file__).parent.parent.parent / "model_prices_and_context_window.json" + with open(config_path, 'r') as f: + models = json.load(f) + + inconsistencies = [] + + for model_name, config in models.items(): + # Skip the sample_spec + if model_name == "sample_spec": + continue + + # Check if both max_tokens and max_output_tokens exist + if isinstance(config, dict): + max_tokens = config.get('max_tokens') + max_output_tokens = config.get('max_output_tokens') + + # Only validate if both exist + if max_tokens is not None and max_output_tokens is not None: + if max_tokens != max_output_tokens: + inconsistencies.append({ + 'model': model_name, + 'max_tokens': max_tokens, + 'max_output_tokens': max_output_tokens + }) + + if inconsistencies: + error_msg = f"\n\n❌ Found {len(inconsistencies)} models with max_tokens != max_output_tokens:\n\n" + for item in inconsistencies[:10]: # Show first 10 + error_msg += f" {item['model']}: max_tokens={item['max_tokens']}, max_output_tokens={item['max_output_tokens']}\n" + + if len(inconsistencies) > 10: + error_msg += f"\n ... and {len(inconsistencies) - 10} more\n" + + error_msg += "\nTo fix these inconsistencies, run: poetry run python fix_max_tokens_inconsistencies.py" + raise AssertionError(error_msg) + + def test_get_model_info_gemini(): """ Tests if ALL gemini models have 'tpm' and 'rpm' in the model info From 43dd0e6ef51c41bfdce6cc9453be9f3b09ce962c Mon Sep 17 00:00:00 2001 From: Shivam Rawat <161387515+shivamrawat1@users.noreply.github.com> Date: Fri, 9 Jan 2026 11:13:38 -0800 Subject: [PATCH 48/56] remove model before casting it in the transformation (#18810) --- .../audio_transcription/transformation.py | 48 +++++++++++- ...sonx_audio_transcription_transformation.py | 73 +++++++++++++++++++ 2 files changed, 120 insertions(+), 1 deletion(-) diff --git a/litellm/llms/watsonx/audio_transcription/transformation.py b/litellm/llms/watsonx/audio_transcription/transformation.py index 186d858321a..c7e6a77b96f 100644 --- a/litellm/llms/watsonx/audio_transcription/transformation.py +++ b/litellm/llms/watsonx/audio_transcription/transformation.py @@ -7,13 +7,14 @@ WatsonX follows the OpenAI spec for audio transcription. from typing import Any, Dict, List, Optional import litellm +from httpx import Response from litellm.litellm_core_utils.audio_utils.utils import process_audio_file from litellm.types.llms.openai import ( AllMessageValues, OpenAIAudioTranscriptionOptionalParams, ) from litellm.types.llms.watsonx import WatsonXAudioTranscriptionRequestBody -from litellm.types.utils import FileTypes +from litellm.types.utils import FileTypes, TranscriptionResponse from ...base_llm.audio_transcription.transformation import ( AudioTranscriptionRequestData, @@ -156,3 +157,48 @@ class IBMWatsonXAudioTranscriptionConfig( url = f"{url}?version={api_version}" return url + + def transform_audio_transcription_response( + self, + raw_response: Response, + ) -> TranscriptionResponse: + """ + Transform the audio transcription response from WatsonX. + + WatsonX may include a 'model' field in the response, which needs to be + removed before creating the TranscriptionResponse object. + """ + try: + raw_response_json = raw_response.json() + except Exception as e: + raise ValueError( + f"Error transforming response to json: {str(e)}\nResponse: {raw_response.text}" + ) + + # Extract only valid fields for TranscriptionResponse.__init__() + # TranscriptionResponse only accepts 'text' and 'usage' in __init__() + text = raw_response_json.get("text") + usage = raw_response_json.get("usage") + + # Create response with only valid fields + response_kwargs = {} + if text is not None: + response_kwargs["text"] = text + if usage is not None: + response_kwargs["usage"] = usage + + if not response_kwargs: + raise ValueError( + "Invalid response format. Received response does not match the expected format. Got: ", + raw_response_json, + ) + + response = TranscriptionResponse(**response_kwargs) + + # Add other fields using dictionary-style assignment (like duration, task, etc.) + # Skip fields that TranscriptionResponse doesn't accept in __init__() + for key, value in raw_response_json.items(): + if key not in ["text", "usage", "model"]: # text/usage already set, model should be excluded + response[key] = value + + return response diff --git a/tests/test_litellm/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py b/tests/test_litellm/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py index e36a494998b..fd5f8f3eff8 100644 --- a/tests/test_litellm/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py +++ b/tests/test_litellm/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py @@ -14,6 +14,10 @@ import pytest sys.path.insert(0, os.path.abspath("../../../../..")) import litellm +from litellm.llms.watsonx.audio_transcription.transformation import ( + IBMWatsonXAudioTranscriptionConfig, +) +from litellm.types.utils import TranscriptionResponse class TestWatsonXAudioTranscription: @@ -189,3 +193,72 @@ class TestWatsonXAudioTranscription: # Verify file is sent separately files = captured_request.get("files", {}) assert "file" in files + + def test_transform_audio_transcription_response_removes_model_field(self): + """ + Test that transform_audio_transcription_response removes the 'model' field + from WatsonX response before creating TranscriptionResponse. + + This test ensures that when WatsonX returns a response with a 'model' field, + it is removed before creating the TranscriptionResponse object, since + TranscriptionResponse doesn't accept a 'model' parameter. + """ + handler = IBMWatsonXAudioTranscriptionConfig() + + # Mock response with 'model' field (as WatsonX may return) + mock_response = MagicMock() + mock_response.json.return_value = { + "text": "Hello, this is a test transcription.", + "model": "whisper-large-v3-turbo", # This field should be removed + "duration": 5.5, + } + mock_response.text = '{"text": "Hello, this is a test transcription.", "model": "whisper-large-v3-turbo", "duration": 5.5}' + + # This should not raise a TypeError - model field should be removed + result = handler.transform_audio_transcription_response(mock_response) + + # Verify the result is a TranscriptionResponse + assert isinstance(result, TranscriptionResponse) + + # Verify the text is correct + assert result.text == "Hello, this is a test transcription." + + # Verify duration is set via dictionary assignment + assert result["duration"] == 5.5 + + # Verify the model field is NOT in the serialized result + # Check via model_dump() or dict() to ensure it's not in the output + try: + result_dict = result.model_dump() + except AttributeError: + # Fallback for pydantic v1 + result_dict = result.dict() + + # The 'model' field should not be in the result + assert "model" not in result_dict, "Model field should be removed from response" + + def test_transform_audio_transcription_response_without_model_field(self): + """ + Test that transform_audio_transcription_response works correctly + when WatsonX response doesn't include a 'model' field. + """ + handler = IBMWatsonXAudioTranscriptionConfig() + + # Mock response without 'model' field + mock_response = MagicMock() + mock_response.json.return_value = { + "text": "Hello, this is a test transcription.", + "duration": 5.5, + } + mock_response.text = '{"text": "Hello, this is a test transcription.", "duration": 5.5}' + + result = handler.transform_audio_transcription_response(mock_response) + + # Verify the result is a TranscriptionResponse + assert isinstance(result, TranscriptionResponse) + + # Verify the text is correct + assert result.text == "Hello, this is a test transcription." + + # Verify duration is set via dictionary assignment + assert result["duration"] == 5.5 From 691505cdae1abbad3f296ca44fcd8a644624d8d3 Mon Sep 17 00:00:00 2001 From: Shivam Rawat <161387515+shivamrawat1@users.noreply.github.com> Date: Fri, 9 Jan 2026 11:14:40 -0800 Subject: [PATCH 49/56] added fix for org level budget enforcement (#18813) --- litellm/proxy/auth/auth_checks.py | 108 ++++++++++++++++++------------ 1 file changed, 67 insertions(+), 41 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index de4973ecc69..26778ece60e 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -147,6 +147,7 @@ async def common_checks( # 3.1. If organization is in budget await _organization_max_budget_check( valid_token=valid_token, + team_object=team_object, prisma_client=prisma_client, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, @@ -2310,61 +2311,86 @@ async def _team_max_budget_check( async def _organization_max_budget_check( valid_token: Optional[UserAPIKeyAuth], + team_object: Optional[LiteLLM_TeamTable], prisma_client: Optional[PrismaClient], user_api_key_cache: DualCache, proxy_logging_obj: ProxyLogging, ): """ Check if the organization is over its max budget. + + This function checks the organization budget using: + 1. First, tries to use valid_token.org_id (if key has organization_id set) + 2. Falls back to team_object.organization_id (if key doesn't have org_id but team does) + + This ensures organization budget checks work even when keys don't have organization_id + set directly, as long as their team belongs to an organization. Raises: BudgetExceededError if the organization is over its max budget. Triggers a budget alert if the organization is over its max budget. """ - # Only check if token has organization info and organization_max_budget is set - if ( - valid_token is None - or valid_token.org_id is None - or valid_token.organization_max_budget is None - or valid_token.organization_max_budget <= 0 - ): + if valid_token is None or prisma_client is None: return - # Get organization object to check current spend - if prisma_client is not None: - org_table = await get_org_object( - org_id=valid_token.org_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, + # Determine organization_id: first try from token, then fallback to team + org_id: Optional[str] = None + if valid_token.org_id is not None: + org_id = valid_token.org_id + elif team_object is not None and team_object.organization_id is not None: + org_id = team_object.organization_id + + # If no organization_id found, skip the check + if org_id is None: + return + + # Get organization object with budget table to check current spend and max budget + try: + org_table = await prisma_client.db.litellm_organizationtable.find_unique( + where={"organization_id": org_id}, + include={"litellm_budget_table": True}, + ) + except Exception: + # If organization lookup fails, skip the check + return + + if org_table is None: + return + + # Get max_budget from organization's budget table + org_max_budget: Optional[float] = None + if org_table.litellm_budget_table is not None: + org_max_budget = org_table.litellm_budget_table.max_budget + + # Only check if organization has a valid max_budget set + if org_max_budget is None or org_max_budget <= 0: + return + + # Check if organization spend exceeds max budget + if org_table.spend >= org_max_budget: + # Trigger budget alert + call_info = CallInfo( + token=valid_token.token, + spend=org_table.spend, + max_budget=org_max_budget, + user_id=valid_token.user_id, + team_id=valid_token.team_id, + team_alias=valid_token.team_alias, + organization_id=org_id, + event_group=Litellm_EntityType.ORGANIZATION, + ) + asyncio.create_task( + proxy_logging_obj.budget_alerts( + type="organization_budget", + user_info=call_info, + ) ) - if ( - org_table is not None - and org_table.spend >= valid_token.organization_max_budget - ): - # Trigger budget alert - call_info = CallInfo( - token=valid_token.token, - spend=org_table.spend, - max_budget=valid_token.organization_max_budget, - user_id=valid_token.user_id, - team_id=valid_token.team_id, - team_alias=valid_token.team_alias, - organization_id=valid_token.org_id, - event_group=Litellm_EntityType.ORGANIZATION, - ) - asyncio.create_task( - proxy_logging_obj.budget_alerts( - type="organization_budget", - user_info=call_info, - ) - ) - - raise litellm.BudgetExceededError( - current_cost=org_table.spend, - max_budget=valid_token.organization_max_budget, - message=f"Budget has been exceeded! Organization={valid_token.org_id} Current cost: {org_table.spend}, Max budget: {valid_token.organization_max_budget}", - ) + raise litellm.BudgetExceededError( + current_cost=org_table.spend, + max_budget=org_max_budget, + message=f"Budget has been exceeded! Organization={org_id} Current cost: {org_table.spend}, Max budget: {org_max_budget}", + ) async def _tag_max_budget_check( @@ -2601,4 +2627,4 @@ def _can_object_call_vector_stores( code=status.HTTP_401_UNAUTHORIZED, ) - return True + return True \ No newline at end of file From 0575bd2d1cebb6b829b7103376924bdeb8c7ff61 Mon Sep 17 00:00:00 2001 From: Robin Date: Sat, 10 Jan 2026 03:18:06 +0800 Subject: [PATCH 50/56] feat: update prices json for novita provider (#18540) * feat: add novita models * feat: ci * feat: add novita support josn --- model_prices_and_context_window.json | 1094 ++++++++++++++++++++++++++ 1 file changed, 1094 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 72e8128d791..2f34d2a7d3d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -32236,6 +32236,1100 @@ "litellm_provider": "fireworks_ai", "mode": "chat" }, + "novita/deepseek/deepseek-v3.2": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.69e-03, + "output_cost_per_token": 4e-03, + "max_input_tokens": 163840, + "max_output_tokens": 65536, + "max_tokens": 65536, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "cache_read_input_token_cost": 1.345e-03, + "input_cost_per_token_cache_hit": 1.345e-03, + "supports_reasoning": true + }, + "novita/minimax/minimax-m2.1": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 3e-03, + "output_cost_per_token": 1.2e-02, + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "cache_read_input_token_cost": 3e-04, + "input_cost_per_token_cache_hit": 3e-04, + "supports_reasoning": true + }, + "novita/zai-org/glm-4.7": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 6e-03, + "output_cost_per_token": 2.2e-02, + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "cache_read_input_token_cost": 1.1e-03, + "input_cost_per_token_cache_hit": 1.1e-03, + "supports_reasoning": true + }, + "novita/xiaomimimo/mimo-v2-flash": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 1e-03, + "output_cost_per_token": 3e-03, + "max_input_tokens": 262144, + "max_output_tokens": 32000, + "max_tokens": 32000, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "cache_read_input_token_cost": 2e-04, + "input_cost_per_token_cache_hit": 2e-04, + "supports_reasoning": true + }, + "novita/zai-org/autoglm-phone-9b-multilingual": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 3.5e-04, + "output_cost_per_token": 1.38e-03, + "max_input_tokens": 65536, + "max_output_tokens": 65536, + "max_tokens": 65536, + "supports_vision": true, + "supports_system_messages": true + }, + "novita/moonshotai/kimi-k2-thinking": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 4.8e-03, + "output_cost_per_token": 2e-02, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "novita/minimax/minimax-m2": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.4e-03, + "output_cost_per_token": 9.6e-03, + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "cache_read_input_token_cost": 2.4e-04, + "input_cost_per_token_cache_hit": 2.4e-04, + "supports_reasoning": true + }, + "novita/paddlepaddle/paddleocr-vl": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 1.6e-04, + "output_cost_per_token": 1.6e-04, + "max_input_tokens": 16384, + "max_output_tokens": 16384, + "max_tokens": 16384, + "supports_vision": true, + "supports_system_messages": true + }, + "novita/deepseek/deepseek-v3.2-exp": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.16e-03, + "output_cost_per_token": 3.28e-03, + "max_input_tokens": 163840, + "max_output_tokens": 65536, + "max_tokens": 65536, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "novita/qwen/qwen3-vl-235b-a22b-thinking": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 7.84e-03, + "output_cost_per_token": 3.16e-02, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "supports_vision": true, + "supports_system_messages": true, + "supports_reasoning": true + }, + "novita/zai-org/glm-4.6v": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 3e-03, + "output_cost_per_token": 9e-03, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_response_schema": true, + "cache_read_input_token_cost": 5.5e-04, + "input_cost_per_token_cache_hit": 5.5e-04, + "supports_reasoning": true + }, + "novita/zai-org/glm-4.6": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 4.4e-03, + "output_cost_per_token": 1.76e-02, + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "cache_read_input_token_cost": 8.8e-04, + "input_cost_per_token_cache_hit": 8.8e-04, + "supports_reasoning": true + }, + "novita/qwen/qwen3-next-80b-a3b-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 1.2e-03, + "output_cost_per_token": 1.2e-02, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/qwen/qwen3-next-80b-a3b-thinking": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 1.2e-03, + "output_cost_per_token": 1.2e-02, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "novita/deepseek/deepseek-ocr": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.4e-04, + "output_cost_per_token": 2.4e-04, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "max_tokens": 8192, + "supports_tool_choice": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/deepseek/deepseek-v3.1-terminus": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.16e-03, + "output_cost_per_token": 8e-03, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "cache_read_input_token_cost": 1.08e-03, + "input_cost_per_token_cache_hit": 1.08e-03, + "supports_reasoning": true + }, + "novita/qwen/qwen3-vl-235b-a22b-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.4e-03, + "output_cost_per_token": 1.2e-02, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/qwen/qwen3-max": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 1.688e-02, + "output_cost_per_token": 6.76e-02, + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/skywork/r1v4-lite": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2e-03, + "output_cost_per_token": 6e-03, + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "supports_tool_choice": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/deepseek/deepseek-v3.1": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.16e-03, + "output_cost_per_token": 8e-03, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "cache_read_input_token_cost": 1.08e-03, + "input_cost_per_token_cache_hit": 1.08e-03, + "supports_reasoning": true + }, + "novita/moonshotai/kimi-k2-0905": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 4.8e-03, + "output_cost_per_token": 2e-02, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/qwen/qwen3-coder-480b-a35b-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.4e-03, + "output_cost_per_token": 1.04e-02, + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/qwen/qwen3-coder-30b-a3b-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 7e-04, + "output_cost_per_token": 2.7e-03, + "max_input_tokens": 160000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/openai/gpt-oss-120b": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 4e-04, + "output_cost_per_token": 2e-03, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "novita/moonshotai/kimi-k2-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 4.56e-03, + "output_cost_per_token": 1.84e-02, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/deepseek/deepseek-v3-0324": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.16e-03, + "output_cost_per_token": 8.96e-03, + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "cache_read_input_token_cost": 1.08e-03, + "input_cost_per_token_cache_hit": 1.08e-03 + }, + "novita/zai-org/glm-4.5": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 4.8e-03, + "output_cost_per_token": 1.76e-02, + "max_input_tokens": 131072, + "max_output_tokens": 98304, + "max_tokens": 98304, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "cache_read_input_token_cost": 8.8e-04, + "input_cost_per_token_cache_hit": 8.8e-04, + "supports_reasoning": true + }, + "novita/qwen/qwen3-235b-a22b-thinking-2507": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.4e-03, + "output_cost_per_token": 2.4e-02, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_reasoning": true + }, + "novita/meta-llama/llama-3.1-8b-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2e-04, + "output_cost_per_token": 5e-04, + "max_input_tokens": 16384, + "max_output_tokens": 16384, + "max_tokens": 16384, + "supports_system_messages": true + }, + "novita/google/gemma-3-12b-it": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 4e-04, + "output_cost_per_token": 8e-04, + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "supports_tool_choice": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/zai-org/glm-4.5v": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 4.8e-03, + "output_cost_per_token": 1.44e-02, + "max_input_tokens": 65536, + "max_output_tokens": 16384, + "max_tokens": 16384, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_response_schema": true, + "cache_read_input_token_cost": 8.8e-04, + "input_cost_per_token_cache_hit": 8.8e-04, + "supports_reasoning": true + }, + "novita/openai/gpt-oss-20b": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 3.2e-04, + "output_cost_per_token": 1.2e-03, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "supports_tool_choice": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "novita/qwen/qwen3-235b-a22b-instruct-2507": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 7.2e-04, + "output_cost_per_token": 4.64e-03, + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/deepseek/deepseek-r1-distill-qwen-14b": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 1.2e-03, + "output_cost_per_token": 1.2e-03, + "max_input_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "novita/meta-llama/llama-3.3-70b-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 1.08e-03, + "output_cost_per_token": 3.2e-03, + "max_input_tokens": 131072, + "max_output_tokens": 120000, + "max_tokens": 120000, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true + }, + "novita/qwen/qwen-2.5-72b-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 3.04e-03, + "output_cost_per_token": 3.2e-03, + "max_input_tokens": 32000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/mistralai/mistral-nemo": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 3.2e-04, + "output_cost_per_token": 1.36e-03, + "max_input_tokens": 60288, + "max_output_tokens": 16000, + "max_tokens": 16000, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/minimaxai/minimax-m1-80k": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 4.4e-03, + "output_cost_per_token": 1.76e-02, + "max_input_tokens": 1000000, + "max_output_tokens": 40000, + "max_tokens": 40000, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_reasoning": true + }, + "novita/deepseek/deepseek-r1-0528": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 5.6e-03, + "output_cost_per_token": 2e-02, + "max_input_tokens": 163840, + "max_output_tokens": 32768, + "max_tokens": 32768, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "cache_read_input_token_cost": 2.8e-03, + "input_cost_per_token_cache_hit": 2.8e-03, + "supports_reasoning": true + }, + "novita/deepseek/deepseek-r1-distill-qwen-32b": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.4e-03, + "output_cost_per_token": 2.4e-03, + "max_input_tokens": 64000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "novita/meta-llama/llama-3-8b-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 3.2e-04, + "output_cost_per_token": 3.2e-04, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "max_tokens": 8192, + "supports_system_messages": true + }, + "novita/microsoft/wizardlm-2-8x22b": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 4.96e-03, + "output_cost_per_token": 4.96e-03, + "max_input_tokens": 65535, + "max_output_tokens": 8000, + "max_tokens": 8000, + "supports_system_messages": true + }, + "novita/deepseek/deepseek-r1-0528-qwen3-8b": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 4.8e-04, + "output_cost_per_token": 7.2e-04, + "max_input_tokens": 128000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "supports_system_messages": true, + "supports_reasoning": true + }, + "novita/deepseek/deepseek-r1-distill-llama-70b": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 6.4e-03, + "output_cost_per_token": 6.4e-03, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "max_tokens": 8192, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "novita/meta-llama/llama-3-70b-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 5.1e-03, + "output_cost_per_token": 7.4e-03, + "max_input_tokens": 8192, + "max_output_tokens": 8000, + "max_tokens": 8000, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/qwen/qwen3-235b-a22b-fp8": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 1.6e-03, + "output_cost_per_token": 6.4e-03, + "max_input_tokens": 40960, + "max_output_tokens": 20000, + "max_tokens": 20000, + "supports_system_messages": true, + "supports_reasoning": true + }, + "novita/meta-llama/llama-4-maverick-17b-128e-instruct-fp8": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 1.6e-03, + "output_cost_per_token": 7.2e-03, + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "supports_vision": true, + "supports_system_messages": true + }, + "novita/meta-llama/llama-4-scout-17b-16e-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 8e-04, + "output_cost_per_token": 4e-03, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "supports_vision": true, + "supports_system_messages": true + }, + "novita/nousresearch/hermes-2-pro-llama-3-8b": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 1.4e-03, + "output_cost_per_token": 1.4e-03, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "max_tokens": 8192, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/qwen/qwen2.5-vl-72b-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 6.4e-03, + "output_cost_per_token": 6.4e-03, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "supports_vision": true, + "supports_system_messages": true + }, + "novita/sao10k/l3-70b-euryale-v2.1": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 1.48e-02, + "output_cost_per_token": 1.48e-02, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "max_tokens": 8192, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true + }, + "novita/baidu/ernie-4.5-21B-a3b-thinking": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 5.6e-04, + "output_cost_per_token": 2.24e-03, + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "supports_system_messages": true, + "supports_reasoning": true + }, + "novita/sao10k/l3-8b-lunaris": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 5e-04, + "output_cost_per_token": 5e-04, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "max_tokens": 8192, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/baichuan/baichuan-m2-32b": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 5.6e-04, + "output_cost_per_token": 5.6e-04, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "novita/thudm/glm-4.1v-9b-thinking": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.8e-04, + "output_cost_per_token": 1.104e-03, + "max_input_tokens": 65536, + "max_output_tokens": 8000, + "max_tokens": 8000, + "supports_vision": true, + "supports_system_messages": true, + "supports_reasoning": true + }, + "novita/baidu/ernie-4.5-vl-424b-a47b": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 3.36e-03, + "output_cost_per_token": 1e-02, + "max_input_tokens": 123000, + "max_output_tokens": 16000, + "max_tokens": 16000, + "supports_vision": true, + "supports_system_messages": true, + "supports_reasoning": true + }, + "novita/baidu/ernie-4.5-300b-a47b-paddle": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.24e-03, + "output_cost_per_token": 8.8e-03, + "max_input_tokens": 123000, + "max_output_tokens": 12000, + "max_tokens": 12000, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/deepseek/deepseek-prover-v2-671b": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 5.6e-03, + "output_cost_per_token": 2e-02, + "max_input_tokens": 160000, + "max_output_tokens": 160000, + "max_tokens": 160000, + "supports_system_messages": true + }, + "novita/qwen/qwen3-32b-fp8": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 8e-04, + "output_cost_per_token": 3.6e-03, + "max_input_tokens": 40960, + "max_output_tokens": 20000, + "max_tokens": 20000, + "supports_system_messages": true, + "supports_reasoning": true + }, + "novita/qwen/qwen3-30b-a3b-fp8": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 7.2e-04, + "output_cost_per_token": 3.6e-03, + "max_input_tokens": 40960, + "max_output_tokens": 20000, + "max_tokens": 20000, + "supports_system_messages": true, + "supports_reasoning": true + }, + "novita/google/gemma-3-27b-it": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 9.52e-04, + "output_cost_per_token": 1.6e-03, + "max_input_tokens": 98304, + "max_output_tokens": 16384, + "max_tokens": 16384, + "supports_vision": true, + "supports_system_messages": true + }, + "novita/deepseek/deepseek-v3-turbo": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 3.2e-03, + "output_cost_per_token": 1.04e-02, + "max_input_tokens": 64000, + "max_output_tokens": 16000, + "max_tokens": 16000, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true + }, + "novita/deepseek/deepseek-r1-turbo": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 5.6e-03, + "output_cost_per_token": 2e-02, + "max_input_tokens": 64000, + "max_output_tokens": 16000, + "max_tokens": 16000, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_reasoning": true + }, + "novita/Sao10K/L3-8B-Stheno-v3.2": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 5e-04, + "output_cost_per_token": 5e-04, + "max_input_tokens": 8192, + "max_output_tokens": 32000, + "max_tokens": 32000, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true + }, + "novita/gryphe/mythomax-l2-13b": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 7.2e-04, + "output_cost_per_token": 7.2e-04, + "max_input_tokens": 4096, + "max_output_tokens": 3200, + "max_tokens": 3200, + "supports_system_messages": true + }, + "novita/baidu/ernie-4.5-vl-28b-a3b-thinking": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 3.9e-03, + "output_cost_per_token": 3.9e-03, + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "novita/qwen/qwen3-vl-8b-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 6.4e-04, + "output_cost_per_token": 4e-03, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/zai-org/glm-4.5-air": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 1.04e-03, + "output_cost_per_token": 6.8e-03, + "max_input_tokens": 131072, + "max_output_tokens": 98304, + "max_tokens": 98304, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_reasoning": true + }, + "novita/qwen/qwen3-vl-30b-a3b-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 1.6e-03, + "output_cost_per_token": 5.6e-03, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/qwen/qwen3-vl-30b-a3b-thinking": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 1.6e-03, + "output_cost_per_token": 8e-03, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/qwen/qwen-mt-plus": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2e-03, + "output_cost_per_token": 6e-03, + "max_input_tokens": 16384, + "max_output_tokens": 8192, + "max_tokens": 8192, + "supports_system_messages": true + }, + "novita/baidu/ernie-4.5-vl-28b-a3b": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 1.12e-03, + "output_cost_per_token": 4.48e-03, + "max_input_tokens": 30000, + "max_output_tokens": 8000, + "max_tokens": 8000, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_reasoning": true + }, + "novita/baidu/ernie-4.5-21B-a3b": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 5.6e-04, + "output_cost_per_token": 2.24e-03, + "max_input_tokens": 120000, + "max_output_tokens": 8000, + "max_tokens": 8000, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true + }, + "novita/qwen/qwen3-8b-fp8": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.8e-04, + "output_cost_per_token": 1.104e-03, + "max_input_tokens": 128000, + "max_output_tokens": 20000, + "max_tokens": 20000, + "supports_system_messages": true, + "supports_reasoning": true + }, + "novita/qwen/qwen3-4b-fp8": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.4e-04, + "output_cost_per_token": 2.4e-04, + "max_input_tokens": 128000, + "max_output_tokens": 20000, + "max_tokens": 20000, + "supports_system_messages": true, + "supports_reasoning": true + }, + "novita/qwen/qwen2.5-7b-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 5.6e-04, + "output_cost_per_token": 5.6e-04, + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true + }, + "novita/meta-llama/llama-3.2-3b-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.4e-04, + "output_cost_per_token": 4e-04, + "max_input_tokens": 32768, + "max_output_tokens": 32000, + "max_tokens": 32000, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true + }, + "novita/sao10k/l31-70b-euryale-v2.2": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 1.48e-02, + "output_cost_per_token": 1.48e-02, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "max_tokens": 8192, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true + }, + "novita/qwen/qwen3-embedding-0.6b": { + "litellm_provider": "novita", + "mode": "embedding", + "input_cost_per_token": 5.6e-04, + "output_cost_per_token": 0, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768 + }, + "novita/qwen/qwen3-embedding-8b": { + "litellm_provider": "novita", + "mode": "embedding", + "input_cost_per_token": 5.6e-04, + "output_cost_per_token": 0, + "max_input_tokens": 32768, + "max_output_tokens": 4096, + "max_tokens": 4096 + }, + "novita/baai/bge-m3": { + "litellm_provider": "novita", + "mode": "embedding", + "input_cost_per_token": 1e-04, + "output_cost_per_token": 1e-04, + "max_input_tokens": 8192, + "max_output_tokens": 96000, + "max_tokens": 96000 + }, + "novita/qwen/qwen3-reranker-8b": { + "litellm_provider": "novita", + "mode": "rerank", + "input_cost_per_token": 4e-04, + "output_cost_per_token": 4e-04, + "max_input_tokens": 32768, + "max_output_tokens": 4096, + "max_tokens": 4096 + }, + "novita/baai/bge-reranker-v2-m3": { + "litellm_provider": "novita", + "mode": "rerank", + "input_cost_per_token": 1e-04, + "output_cost_per_token": 1e-04, + "max_input_tokens": 8000, + "max_output_tokens": 8000, + "max_tokens": 8000 + }, "llamagate/llama-3.1-8b": { "max_tokens": 8192, "max_input_tokens": 131072, From 777ae4f530303522d69b82c2c0ee7392c1b76642 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Sat, 10 Jan 2026 00:57:11 +0530 Subject: [PATCH 51/56] Fix :test_count_tokens_caching --- tests/test_litellm/test_utils_custom.py | 31 ++++++++++++++----------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/test_litellm/test_utils_custom.py b/tests/test_litellm/test_utils_custom.py index 292da4132b9..3e924e9c719 100644 --- a/tests/test_litellm/test_utils_custom.py +++ b/tests/test_litellm/test_utils_custom.py @@ -1,4 +1,5 @@ import pytest +import sys from unittest.mock import MagicMock, patch, AsyncMock from litellm.proxy.utils import count_tokens_with_anthropic_api, _anthropic_async_clients @@ -14,29 +15,31 @@ async def test_count_tokens_caching(): messages = [{"role": "user", "content": "hello"}] model = "claude-3-opus-20240229" - # Mock anthropic - with patch("anthropic.AsyncAnthropic") as mock_cls: - mock_client = MagicMock() - mock_cls.return_value = mock_client - - # Mock response - mock_response = MagicMock() - mock_response.input_tokens = 10 - - # Setup async return for count_tokens - mock_client.beta.messages.count_tokens = AsyncMock(return_value=mock_response) - + # Create a mock anthropic module + mock_anthropic = MagicMock() + mock_client = MagicMock() + mock_anthropic.AsyncAnthropic.return_value = mock_client + + # Mock response + mock_response = MagicMock() + mock_response.input_tokens = 10 + + # Setup async return for count_tokens + mock_client.beta.messages.count_tokens = AsyncMock(return_value=mock_response) + + # Patch sys.modules to ensure our mock is used when anthropic is imported + with patch.dict(sys.modules, {"anthropic": mock_anthropic}): # First call with patch.dict("os.environ", {"ANTHROPIC_API_KEY": api_key}): await count_tokens_with_anthropic_api(model, messages) assert api_key in _anthropic_async_clients assert _anthropic_async_clients[api_key] == mock_client - mock_cls.assert_called_once() # Should be called once + mock_anthropic.AsyncAnthropic.assert_called_once() # Should be called once # Second call with patch.dict("os.environ", {"ANTHROPIC_API_KEY": api_key}): await count_tokens_with_anthropic_api(model, messages) # Should still be called once (cached) - mock_cls.assert_called_once() + mock_anthropic.AsyncAnthropic.assert_called_once() From aba7dcea9ca13fdc18c867bcddf343d5f0291ab5 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Sat, 10 Jan 2026 01:03:50 +0530 Subject: [PATCH 52/56] Fix : litellm import error --- litellm/llms/bedrock/count_tokens/handler.py | 17 ++++++++--------- .../llm_passthrough_endpoints.py | 7 +++++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/litellm/llms/bedrock/count_tokens/handler.py b/litellm/llms/bedrock/count_tokens/handler.py index 60ace7f3369..e8366165b65 100644 --- a/litellm/llms/bedrock/count_tokens/handler.py +++ b/litellm/llms/bedrock/count_tokens/handler.py @@ -6,10 +6,9 @@ Simplified handler leveraging existing LiteLLM Bedrock infrastructure. from typing import Any, Dict -from fastapi import HTTPException - import litellm from litellm._logging import verbose_logger +from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig from litellm.llms.custom_httpx.http_handler import get_async_httpx_client @@ -97,9 +96,9 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): if response.status_code != 200: error_text = response.text verbose_logger.error(f"AWS Bedrock error: {error_text}") - raise HTTPException( - status_code=400, - detail={"error": f"AWS Bedrock error: {error_text}"}, + raise BedrockError( + status_code=response.status_code, + message=f"AWS Bedrock error: {error_text}", ) bedrock_response = response.json() @@ -115,12 +114,12 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig): return final_response - except HTTPException: - # Re-raise HTTP exceptions as-is + except BedrockError: + # Re-raise Bedrock exceptions as-is raise except Exception as e: verbose_logger.error(f"Error in CountTokens handler: {str(e)}") - raise HTTPException( + raise BedrockError( status_code=500, - detail={"error": f"CountTokens processing error: {str(e)}"}, + message=f"CountTokens processing error: {str(e)}", ) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 84550092d2e..d9798dae690 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -776,6 +776,7 @@ async def handle_bedrock_count_tokens( - /v1/messages/count_tokens - /v1/messages/count-tokens """ + from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler from litellm.proxy.proxy_server import llm_router @@ -822,6 +823,12 @@ async def handle_bedrock_count_tokens( return result + except BedrockError as e: + # Convert BedrockError to HTTPException for FastAPI + verbose_proxy_logger.error(f"BedrockError in handle_bedrock_count_tokens: {str(e)}") + raise HTTPException( + status_code=e.status_code, detail={"error": e.message} + ) except HTTPException: # Re-raise HTTP exceptions as-is raise From 8a683d9a6a8ae30b9d5d64de308df669ff1c7f11 Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Sat, 10 Jan 2026 01:09:00 +0530 Subject: [PATCH 53/56] Add fix for bedrock_cache, metadata and max_model_budget (#18872) --- litellm/proxy/auth/auth_utils.py | 70 +- .../hooks/parallel_request_limiter_v3.py | 84 ++- .../proxy/auth/test_auth_utils.py | 131 ++++ .../hooks/test_parallel_request_limiter_v3.py | 629 ++++++++++++------ 4 files changed, 648 insertions(+), 266 deletions(-) create mode 100644 tests/test_litellm/proxy/auth/test_auth_utils.py diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 7a71af1da5c..797540deaa4 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -426,38 +426,65 @@ def get_key_model_rpm_limit( user_api_key_dict: UserAPIKeyAuth, ) -> Optional[Dict[str, int]]: """ - Get the model rpm limit for a given api key - - check key metadata - - check key model max budget - - check team metadata + Get the model rpm limit for a given api key. + + Priority order (returns first found): + 1. Key metadata (model_rpm_limit) + 2. Key model_max_budget (rpm_limit per model) + 3. Team metadata (model_rpm_limit) """ + # 1. Check key metadata first (takes priority) if user_api_key_dict.metadata: - if "model_rpm_limit" in user_api_key_dict.metadata: - return user_api_key_dict.metadata["model_rpm_limit"] - elif user_api_key_dict.model_max_budget: + result = user_api_key_dict.metadata.get("model_rpm_limit") + if result: + return result + + # 2. Check model_max_budget + if user_api_key_dict.model_max_budget: model_rpm_limit: Dict[str, Any] = {} for model, budget in user_api_key_dict.model_max_budget.items(): - if "rpm_limit" in budget and budget["rpm_limit"] is not None: + if isinstance(budget, dict) and budget.get("rpm_limit") is not None: model_rpm_limit[model] = budget["rpm_limit"] - return model_rpm_limit - elif user_api_key_dict.team_metadata: - if "model_rpm_limit" in user_api_key_dict.team_metadata: - return user_api_key_dict.team_metadata["model_rpm_limit"] + if model_rpm_limit: + return model_rpm_limit + + # 3. Fallback to team metadata + if user_api_key_dict.team_metadata: + return user_api_key_dict.team_metadata.get("model_rpm_limit") + return None def get_key_model_tpm_limit( user_api_key_dict: UserAPIKeyAuth, ) -> Optional[Dict[str, int]]: + """ + Get the model tpm limit for a given api key. + + Priority order (returns first found): + 1. Key metadata (model_tpm_limit) + 2. Key model_max_budget (tpm_limit per model) + 3. Team metadata (model_tpm_limit) + """ + # 1. Check key metadata first (takes priority) if user_api_key_dict.metadata: - if "model_tpm_limit" in user_api_key_dict.metadata: - return user_api_key_dict.metadata["model_tpm_limit"] - elif user_api_key_dict.model_max_budget: - if "tpm_limit" in user_api_key_dict.model_max_budget: - return user_api_key_dict.model_max_budget["tpm_limit"] - elif user_api_key_dict.team_metadata: - if "model_tpm_limit" in user_api_key_dict.team_metadata: - return user_api_key_dict.team_metadata["model_tpm_limit"] + result = user_api_key_dict.metadata.get("model_tpm_limit") + if result: + return result + + # 2. Check model_max_budget (iterate per-model like RPM does) + if user_api_key_dict.model_max_budget: + model_tpm_limit: Dict[str, Any] = {} + for model, budget in user_api_key_dict.model_max_budget.items(): + if isinstance(budget, dict) and budget.get("tpm_limit") is not None: + model_tpm_limit[model] = budget["tpm_limit"] + if model_tpm_limit: + return model_tpm_limit + + # 3. Fallback to team metadata + if user_api_key_dict.team_metadata: + return user_api_key_dict.team_metadata.get("model_tpm_limit") + return None @@ -469,7 +496,8 @@ def get_model_rate_limit_from_metadata( if getattr(user_api_key_dict, metadata_accessor_key): return getattr(user_api_key_dict, metadata_accessor_key).get(rate_limit_key) return None - + + def get_team_model_rpm_limit( user_api_key_dict: UserAPIKeyAuth, ) -> Optional[Dict[str, int]]: diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index c416527990e..4d17cca22ad 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -167,7 +167,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): self.token_increment_script = None self.window_size = int(os.getenv("LITELLM_RATE_LIMIT_WINDOW_SIZE", 60)) - + # Batch rate limiter (lazy loaded) self._batch_rate_limiter: Optional[Any] = None @@ -1013,7 +1013,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) # Fail safe: enforce limits if we can't check return True - + def get_rate_limiter_for_call_type(self, call_type: str) -> Optional[Any]: """Get the rate limiter for the call type.""" if call_type == "acreate_batch": @@ -1095,9 +1095,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): now = self._get_current_time().timestamp() reset_time = now + self.window_size - reset_time_formatted = datetime.fromtimestamp( - reset_time - ).strftime("%Y-%m-%d %H:%M:%S UTC") + reset_time_formatted = datetime.fromtimestamp(reset_time).strftime( + "%Y-%m-%d %H:%M:%S UTC" + ) remaining_display = max(0, status["limit_remaining"]) rate_limit_type = status["rate_limit_type"] @@ -1137,7 +1137,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # Check if the call type has a specific rate limiter # eg. for Batch APIs we need to use the batch rate limiter to read the input file and count the tokens and requests ######################################################### - call_type_specific_rate_limiter = self.get_rate_limiter_for_call_type(call_type=call_type) + call_type_specific_rate_limiter = self.get_rate_limiter_for_call_type( + call_type=call_type + ) if call_type_specific_rate_limiter: return await call_type_specific_rate_limiter.async_pre_call_hook( user_api_key_dict=user_api_key_dict, @@ -1233,26 +1235,58 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): return pipeline_operations - def _get_total_tokens_from_usage(self, usage: Any | None, rate_limit_type: Literal["output", "input", "total"]) -> int: - # Get total tokens from response + def _get_total_tokens_from_usage( + self, usage: Any | None, rate_limit_type: Literal["output", "input", "total"] + ) -> int: + """ + Get total tokens from response usage for rate limiting. + + For 'input' and 'total' rate limit types, cached tokens are excluded + because providers like AWS Bedrock don't count cached tokens toward + rate limits. This aligns LiteLLM's TPM calculation with provider behavior. + """ total_tokens = 0 - # spot fix for /responses api + cached_tokens = 0 + if usage: if isinstance(usage, Usage): if rate_limit_type == "output": - total_tokens = usage.completion_tokens + total_tokens = usage.completion_tokens or 0 elif rate_limit_type == "input": - total_tokens = usage.prompt_tokens + total_tokens = usage.prompt_tokens or 0 elif rate_limit_type == "total": - total_tokens = usage.total_tokens + total_tokens = usage.total_tokens or 0 + + # Get cached tokens to exclude from input/total + if rate_limit_type in ("input", "total"): + if ( + hasattr(usage, "prompt_tokens_details") + and usage.prompt_tokens_details is not None + ): + cached_tokens = ( + getattr(usage.prompt_tokens_details, "cached_tokens", 0) + or 0 + ) + elif isinstance(usage, dict): - # Responses API usage comes as a dict in ResponsesAPIResponse + # Responses API usage comes as a dict if rate_limit_type == "output": - total_tokens = usage.get("completion_tokens", 0) + total_tokens = usage.get("completion_tokens", 0) or 0 elif rate_limit_type == "input": - total_tokens = usage.get("prompt_tokens", 0) + total_tokens = usage.get("prompt_tokens", 0) or 0 elif rate_limit_type == "total": - total_tokens = usage.get("total_tokens", 0) + total_tokens = usage.get("total_tokens", 0) or 0 + + # Get cached tokens from dict + if rate_limit_type in ("input", "total"): + prompt_details = usage.get("prompt_tokens_details") or {} + if isinstance(prompt_details, dict): + cached_tokens = prompt_details.get("cached_tokens", 0) or 0 + + # Subtract cached tokens for input/total (providers don't count them) + if cached_tokens > 0: + total_tokens = max(0, total_tokens - cached_tokens) + return total_tokens async def _execute_token_increment_script( @@ -1336,6 +1370,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): def get_rate_limit_type(self) -> Literal["output", "input", "total"]: from litellm.proxy.proxy_server import general_settings + specified_rate_limit_type = general_settings.get( "token_rate_limit_type", "total" ) @@ -1381,9 +1416,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): user_api_key_organization_id = standard_logging_metadata.get( "user_api_key_org_id" ) - user_api_key_end_user_id = kwargs.get("user") or standard_logging_metadata.get( - "user_api_key_end_user_id" - ) + user_api_key_end_user_id = kwargs.get( + "user" + ) or standard_logging_metadata.get("user_api_key_end_user_id") model_group = get_model_group_from_litellm_kwargs(kwargs) # Get total tokens from response @@ -1393,7 +1428,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): response_obj, BaseLiteLLMOpenAIResponseObject ): _usage = getattr(response_obj, "usage", None) - total_tokens = self._get_total_tokens_from_usage(usage=_usage, rate_limit_type=rate_limit_type) + total_tokens = self._get_total_tokens_from_usage( + usage=_usage, rate_limit_type=rate_limit_type + ) # Create pipeline operations for TPM increments pipeline_operations: List[RedisPipelineIncrementOperation] = [] @@ -1518,9 +1555,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): from litellm.types.caching import RedisPipelineIncrementOperation try: - litellm_parent_otel_span: Union[ - Span, None - ] = _get_parent_otel_span_from_kwargs(kwargs) + litellm_parent_otel_span: Union[Span, None] = ( + _get_parent_otel_span_from_kwargs(kwargs) + ) # Get metadata from standard_logging_object - this correctly handles both # 'metadata' and 'litellm_metadata' fields from litellm_params standard_logging_object = kwargs.get("standard_logging_object") or {} @@ -1555,7 +1592,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): f"Error in rate limit failure event: {str(e)}" ) - async def async_post_call_success_hook( self, data: dict, user_api_key_dict: UserAPIKeyAuth, response ): diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py new file mode 100644 index 00000000000..b1bef63933e --- /dev/null +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -0,0 +1,131 @@ +""" +Unit tests for auth_utils functions related to rate limiting. +""" + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.auth_utils import ( + get_key_model_rpm_limit, + get_key_model_tpm_limit, +) + + +class TestGetKeyModelRpmLimit: + """Tests for get_key_model_rpm_limit function.""" + + def test_returns_key_metadata_when_present(self): + """Key metadata takes priority over team metadata.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-123", + metadata={"model_rpm_limit": {"gpt-4": 100}}, + team_metadata={"model_rpm_limit": {"gpt-4": 50}}, + ) + result = get_key_model_rpm_limit(user_api_key_dict) + assert result == {"gpt-4": 100} + + def test_falls_back_to_team_metadata_when_key_has_other_metadata(self): + """Should fall back to team metadata when key metadata exists but has no model_rpm_limit.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-123", + metadata={ + "some_other_key": "value" + }, # Has metadata, but not model_rpm_limit + team_metadata={"model_rpm_limit": {"gpt-4": 50}}, + ) + result = get_key_model_rpm_limit(user_api_key_dict) + assert result == {"gpt-4": 50} + + def test_extracts_from_model_max_budget(self): + """Should extract rpm_limit from model_max_budget when metadata is empty.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-123", + model_max_budget={ + "gpt-4": {"rpm_limit": 100, "tpm_limit": 1000}, + "gpt-3.5-turbo": {"rpm_limit": 200}, + }, + ) + result = get_key_model_rpm_limit(user_api_key_dict) + assert result == {"gpt-4": 100, "gpt-3.5-turbo": 200} + + def test_skips_models_without_rpm_limit(self): + """Should skip models that don't have rpm_limit in model_max_budget.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-123", + model_max_budget={ + "gpt-4": {"rpm_limit": 100}, + "gpt-3.5-turbo": {"tpm_limit": 1000}, # No rpm_limit + }, + ) + result = get_key_model_rpm_limit(user_api_key_dict) + assert result == {"gpt-4": 100} + + def test_returns_none_when_no_limits_configured(self): + """Should return None when no rate limits are configured.""" + user_api_key_dict = UserAPIKeyAuth(api_key="sk-123") + result = get_key_model_rpm_limit(user_api_key_dict) + assert result is None + + +class TestGetKeyModelTpmLimit: + """Tests for get_key_model_tpm_limit function.""" + + def test_returns_key_metadata_when_present(self): + """Key metadata takes priority over team metadata.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-123", + metadata={"model_tpm_limit": {"gpt-4": 10000}}, + team_metadata={"model_tpm_limit": {"gpt-4": 5000}}, + ) + result = get_key_model_tpm_limit(user_api_key_dict) + assert result == {"gpt-4": 10000} + + def test_falls_back_to_team_metadata_when_key_has_other_metadata(self): + """Should fall back to team metadata when key metadata exists but has no model_tpm_limit.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-123", + metadata={ + "some_other_key": "value" + }, # Has metadata, but not model_tpm_limit + team_metadata={"model_tpm_limit": {"gpt-4": 5000}}, + ) + result = get_key_model_tpm_limit(user_api_key_dict) + assert result == {"gpt-4": 5000} + + def test_extracts_from_model_max_budget(self): + """Should extract tpm_limit from model_max_budget when metadata is empty.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-123", + model_max_budget={ + "gpt-4": {"tpm_limit": 10000, "rpm_limit": 100}, + "gpt-3.5-turbo": {"tpm_limit": 20000}, + }, + ) + result = get_key_model_tpm_limit(user_api_key_dict) + assert result == {"gpt-4": 10000, "gpt-3.5-turbo": 20000} + + def test_skips_models_without_tpm_limit(self): + """Should skip models that don't have tpm_limit in model_max_budget.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-123", + model_max_budget={ + "gpt-4": {"tpm_limit": 10000}, + "gpt-3.5-turbo": {"rpm_limit": 100}, # No tpm_limit + }, + ) + result = get_key_model_tpm_limit(user_api_key_dict) + assert result == {"gpt-4": 10000} + + def test_returns_none_when_no_limits_configured(self): + """Should return None when no rate limits are configured.""" + user_api_key_dict = UserAPIKeyAuth(api_key="sk-123") + result = get_key_model_tpm_limit(user_api_key_dict) + assert result is None + + def test_model_max_budget_priority_over_team(self): + """model_max_budget should take priority over team_metadata.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-123", + model_max_budget={"gpt-4": {"tpm_limit": 10000}}, + team_metadata={"model_tpm_limit": {"gpt-4": 5000}}, + ) + result = get_key_model_tpm_limit(user_api_key_dict) + assert result == {"gpt-4": 10000} diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index b76957dbf39..134fc84965f 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -247,7 +247,9 @@ async def test_rate_limiter_script_return_values_v3(monkeypatch, time_controller ) @pytest.mark.flaky(reruns=3) @pytest.mark.asyncio -async def test_normal_router_call_tpm_v3(monkeypatch, rate_limit_object, time_controller): +async def test_normal_router_call_tpm_v3( + monkeypatch, rate_limit_object, time_controller +): """ Test normal router call with parallel request limiter v3 for TPM rate limiting """ @@ -394,8 +396,10 @@ async def test_normal_router_call_tpm_v3(monkeypatch, rate_limit_object, time_co # Manually increment the token counter to simulate token usage from previous call # This simulates what would happen after a successful call - await local_cache.async_increment_cache(key=counter_key, value=15, ttl=2) # Use up most of our 10 token limit - + await local_cache.async_increment_cache( + key=counter_key, value=15, ttl=2 + ) # Use up most of our 10 token limit + # Make another request to test rate limiting - this should fail as we've consumed tokens with pytest.raises(HTTPException) as exc_info: await parallel_request_handler.async_pre_call_hook( @@ -535,7 +539,9 @@ async def test_async_log_failure_event_v3(): ) # Mock kwargs with user_api_key via standard_logging_object - mock_kwargs = {"standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}} + mock_kwargs = { + "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}} + } # Capture pipeline operations captured_ops = [] @@ -785,7 +791,7 @@ async def test_tpm_api_key_rate_limits_v3(): tpm_limit_per_model=tpms, models=[], ) - + user_api_key_dict.metadata["model_tpm_limit"] = tpms user_api_key_dict.metadata["model_rpm_limit"] = rpms @@ -804,32 +810,45 @@ async def test_tpm_api_key_rate_limits_v3(): # Return Error response to ensure HTTPException return { "overall_code": "OVER_LIMIT", - "statuses": [{'code': 'OK', 'current_limit': 2, 'limit_remaining': 1, 'rate_limit_type': 'requests', 'descriptor_key': 'model_per_key'}, - {'code': 'OVER_LIMIT', 'current_limit': 2, 'limit_remaining': -18, 'rate_limit_type': 'tokens', 'descriptor_key': 'model_per_key'}] + "statuses": [ + { + "code": "OK", + "current_limit": 2, + "limit_remaining": 1, + "rate_limit_type": "requests", + "descriptor_key": "model_per_key", + }, + { + "code": "OVER_LIMIT", + "current_limit": 2, + "limit_remaining": -18, + "rate_limit_type": "tokens", + "descriptor_key": "model_per_key", + }, + ], } - + parallel_request_handler.should_rate_limit = mock_should_rate_limit - + # Test the pre-call hook error = None try: - await parallel_request_handler.async_pre_call_hook( + await parallel_request_handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=local_cache, data={"model": model}, call_type="", ) except HTTPException as e: - error=e + error = e assert e.status_code == 429 assert "rate_limit_type" in e.headers assert e.headers.get("rate_limit_type") == "tokens" assert "retry-after" in e.headers - - + assert error is not None, "An Exception must be thrown" assert captured_descriptors is not None, "Rate limit descriptors should be captured" - + model_per_key_descriptor = None for descriptor in captured_descriptors: if descriptor["key"] == "model_per_key": @@ -837,9 +856,15 @@ async def test_tpm_api_key_rate_limits_v3(): break assert model_per_key_descriptor is not None, "Api-Key descriptor should be present" - assert model_per_key_descriptor["value"] == f"{_api_key_hash}:{model}", "Api-Key value should combine api_key and model" - assert model_per_key_descriptor["rate_limit"]["requests_per_unit"] == rpm_limit, "Api-Key RPM limit should be set" - assert model_per_key_descriptor["rate_limit"]["tokens_per_unit"] == tpm_limit, "Api-Key TPM limit should be set" + assert ( + model_per_key_descriptor["value"] == f"{_api_key_hash}:{model}" + ), "Api-Key value should combine api_key and model" + assert ( + model_per_key_descriptor["rate_limit"]["requests_per_unit"] == rpm_limit + ), "Api-Key RPM limit should be set" + assert ( + model_per_key_descriptor["rate_limit"]["tokens_per_unit"] == tpm_limit + ), "Api-Key TPM limit should be set" @pytest.mark.asyncio @@ -861,7 +886,7 @@ async def test_rpm_api_key_rate_limits_v3(): tpm_limit_per_model=tpms, models=[], ) - + user_api_key_dict.metadata["model_tpm_limit"] = tpms user_api_key_dict.metadata["model_rpm_limit"] = rpms @@ -880,31 +905,45 @@ async def test_rpm_api_key_rate_limits_v3(): # Return Error response to ensure HTTPException return { "overall_code": "OVER_LIMIT", - "statuses": [{'code': 'OVER_LIMIT', 'current_limit': 2, 'limit_remaining': -2, 'rate_limit_type': 'requests', 'descriptor_key': 'model_per_key'}, - {'code': 'OK', 'current_limit': 2, 'limit_remaining': 2, 'rate_limit_type': 'tokens', 'descriptor_key': 'model_per_key'}] + "statuses": [ + { + "code": "OVER_LIMIT", + "current_limit": 2, + "limit_remaining": -2, + "rate_limit_type": "requests", + "descriptor_key": "model_per_key", + }, + { + "code": "OK", + "current_limit": 2, + "limit_remaining": 2, + "rate_limit_type": "tokens", + "descriptor_key": "model_per_key", + }, + ], } - + parallel_request_handler.should_rate_limit = mock_should_rate_limit - + # Test the pre-call hook error = None try: - await parallel_request_handler.async_pre_call_hook( + await parallel_request_handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=local_cache, data={"model": model}, call_type="", ) except HTTPException as e: - error=e + error = e assert e.status_code == 429 assert "rate_limit_type" in e.headers assert e.headers.get("rate_limit_type") == "requests" assert "retry-after" in e.headers - + assert error is not None, "An Exception must be thrown" assert captured_descriptors is not None, "Rate limit descriptors should be captured" - + model_per_key_descriptor = None for descriptor in captured_descriptors: if descriptor["key"] == "model_per_key": @@ -912,9 +951,16 @@ async def test_rpm_api_key_rate_limits_v3(): break assert model_per_key_descriptor is not None, "Api-Key descriptor should be present" - assert model_per_key_descriptor["value"] == f"{_api_key_hash}:{model}", "Api-Key value should combine api_key and model" - assert model_per_key_descriptor["rate_limit"]["requests_per_unit"] == rpm_limit, "Api-Key RPM limit should be set" - assert model_per_key_descriptor["rate_limit"]["tokens_per_unit"] == tpm_limit, "Api-Key TPM limit should be set" + assert ( + model_per_key_descriptor["value"] == f"{_api_key_hash}:{model}" + ), "Api-Key value should combine api_key and model" + assert ( + model_per_key_descriptor["rate_limit"]["requests_per_unit"] == rpm_limit + ), "Api-Key RPM limit should be set" + assert ( + model_per_key_descriptor["rate_limit"]["tokens_per_unit"] == tpm_limit + ), "Api-Key TPM limit should be set" + @pytest.mark.asyncio async def test_team_member_rate_limits_v3(): @@ -925,7 +971,7 @@ async def test_team_member_rate_limits_v3(): _api_key = hash_token(_api_key) _team_id = "team_123" _user_id = "user_456" - + user_api_key_dict = UserAPIKeyAuth( api_key=_api_key, team_id=_team_id, @@ -933,7 +979,7 @@ async def test_team_member_rate_limits_v3(): team_member_rpm_limit=10, team_member_tpm_limit=1000, ) - + local_cache = DualCache() parallel_request_handler = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(local_cache) @@ -947,15 +993,12 @@ async def test_team_member_rate_limits_v3(): nonlocal captured_descriptors captured_descriptors = descriptors # Return OK response to avoid HTTPException - return { - "overall_code": "OK", - "statuses": [] - } + return {"overall_code": "OK", "statuses": []} parallel_request_handler.should_rate_limit = mock_should_rate_limit # Test the pre-call hook - + await parallel_request_handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=local_cache, @@ -965,24 +1008,32 @@ async def test_team_member_rate_limits_v3(): # Verify team member descriptor was created assert captured_descriptors is not None, "Rate limit descriptors should be captured" - + team_member_descriptor = None for descriptor in captured_descriptors: if descriptor["key"] == "team_member": team_member_descriptor = descriptor break - - assert team_member_descriptor is not None, "Team member descriptor should be present" - assert team_member_descriptor["value"] == f"{_team_id}:{_user_id}", "Team member value should combine team_id and user_id" - assert team_member_descriptor["rate_limit"]["requests_per_unit"] == 10, "Team member RPM limit should be set" - assert team_member_descriptor["rate_limit"]["tokens_per_unit"] == 1000, "Team member TPM limit should be set" + + assert ( + team_member_descriptor is not None + ), "Team member descriptor should be present" + assert ( + team_member_descriptor["value"] == f"{_team_id}:{_user_id}" + ), "Team member value should combine team_id and user_id" + assert ( + team_member_descriptor["rate_limit"]["requests_per_unit"] == 10 + ), "Team member RPM limit should be set" + assert ( + team_member_descriptor["rate_limit"]["tokens_per_unit"] == 1000 + ), "Team member TPM limit should be set" @pytest.mark.asyncio async def test_dynamic_rate_limiting_v3(): """ Test that dynamic rate limiting only enforces limits when model has failures. - + When rpm_limit_type is set to "dynamic": - If model has no failures, rate limits should NOT be enforced (allow exceeding) - If model has failures above threshold, rate limits SHOULD be enforced @@ -990,75 +1041,75 @@ async def test_dynamic_rate_limiting_v3(): _api_key = "sk-12345" _api_key_hash = hash_token(_api_key) model = "gpt-3.5-turbo" - + # Set a low RPM limit to make testing easier user_api_key_dict = UserAPIKeyAuth( api_key=_api_key_hash, rpm_limit=2, metadata={"rpm_limit_type": "dynamic"}, ) - + local_cache = DualCache() parallel_request_handler = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(local_cache) ) - + # Mock should_rate_limit to track if limits are enforced captured_descriptors = [] - + async def mock_should_rate_limit(descriptors, **kwargs): captured_descriptors.clear() captured_descriptors.extend(descriptors) return {"overall_code": "OK", "statuses": []} - + parallel_request_handler.should_rate_limit = mock_should_rate_limit - + # Test 1: No failures - rate limits should NOT be enforced (rpm_limit should be None) async def mock_check_no_failures(*args, **kwargs): return False - + parallel_request_handler._check_model_has_recent_failures = mock_check_no_failures - + await parallel_request_handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=local_cache, data={"model": model}, call_type="", ) - + # Find the API key descriptor api_key_descriptor = None for descriptor in captured_descriptors: if descriptor["key"] == "api_key": api_key_descriptor = descriptor break - + assert api_key_descriptor is not None, "API key descriptor should be present" assert ( api_key_descriptor["rate_limit"]["requests_per_unit"] is None ), "RPM limit should be None when dynamic mode and no failures" - + # Test 2: With failures - rate limits SHOULD be enforced (rpm_limit should be set) async def mock_check_with_failures(*args, **kwargs): return True - + parallel_request_handler._check_model_has_recent_failures = mock_check_with_failures captured_descriptors.clear() - + await parallel_request_handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=local_cache, data={"model": model}, call_type="", ) - + # Find the API key descriptor again api_key_descriptor = None for descriptor in captured_descriptors: if descriptor["key"] == "api_key": api_key_descriptor = descriptor break - + assert api_key_descriptor is not None, "API key descriptor should be present" assert ( api_key_descriptor["rate_limit"]["requests_per_unit"] == 2 @@ -1069,17 +1120,17 @@ async def test_dynamic_rate_limiting_v3(): async def test_async_increment_tokens_with_ttl_preservation(): """ Test TTL preservation functionality for token increment operations. - + This test verifies that: 1. Keys are created with proper TTL on first increment 2. TTL is preserved on subsequent increments (not reset) 3. Both TTL and non-TTL operations work correctly in the same call - + Environment variables required: - REDIS_HOST: Redis server hostname - REDIS_PORT: Redis server port - REDIS_PASSWORD: Redis password (optional) - + Test scenario: 1. First call: Create keys with TTL=60s and TTL=None 2. Wait 2 seconds @@ -1094,38 +1145,40 @@ async def test_async_increment_tokens_with_ttl_preservation(): # Skip test if Redis environment variables are not set redis_host = os.getenv("REDIS_HOST") - redis_port = os.getenv("REDIS_PORT") + redis_port = os.getenv("REDIS_PORT") redis_password = os.getenv("REDIS_PASSWORD") - + if not redis_host or not redis_port: pytest.skip("Redis environment variables (REDIS_HOST, REDIS_PORT) not set") - + # Setup Redis cache redis_cache = RedisCache( host=redis_host, port=int(redis_port), password=redis_password, ) - + local_cache = DualCache(redis_cache=redis_cache) parallel_request_handler = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(local_cache) ) - + # Verify Redis connection is working try: await redis_cache.ping() except Exception as e: pytest.skip(f"Redis connection failed: {str(e)}") - + # Verify the TTL preservation script is registered if parallel_request_handler.token_increment_script is None: - pytest.skip("Token increment script not available - Redis Lua scripting may not be supported") - + pytest.skip( + "Token increment script not available - Redis Lua scripting may not be supported" + ) + # Test keys - use hash tags to ensure they map to same Redis cluster slot test_key_with_ttl = "{test_ttl}:with_ttl" test_key_without_ttl = "{test_ttl}:without_ttl" - + try: # Clean up any existing test keys try: @@ -1134,88 +1187,108 @@ async def test_async_increment_tokens_with_ttl_preservation(): except Exception: # Keys might not exist, ignore cleanup errors pass - + # First increment: Create operations with mixed TTL scenarios pipeline_operations_first = [ RedisPipelineIncrementOperation( - key=test_key_with_ttl, - increment_value=10.0, - ttl=60 + key=test_key_with_ttl, increment_value=10.0, ttl=60 ), RedisPipelineIncrementOperation( - key=test_key_without_ttl, - increment_value=5.0, - ttl=None # No TTL - ) + key=test_key_without_ttl, increment_value=5.0, ttl=None # No TTL + ), ] - + # Execute first increment await parallel_request_handler.async_increment_tokens_with_ttl_preservation( pipeline_operations=pipeline_operations_first ) - + # Small delay to ensure Redis has processed the commands await asyncio.sleep(0.1) - + # Verify keys exist and check initial TTL ttl_after_first = await redis_cache.async_get_ttl(test_key_with_ttl) - value_after_first_with_ttl = await redis_cache.async_get_cache(test_key_with_ttl) - value_after_first_without_ttl = await redis_cache.async_get_cache(test_key_without_ttl) - - assert value_after_first_with_ttl == 10.0, f"First increment should set value to 10.0, got {value_after_first_with_ttl}" - assert value_after_first_without_ttl == 5.0, "First increment should set value to 5.0" - assert ttl_after_first is not None and ttl_after_first > 0, "Key with TTL should have positive TTL after first increment" + value_after_first_with_ttl = await redis_cache.async_get_cache( + test_key_with_ttl + ) + value_after_first_without_ttl = await redis_cache.async_get_cache( + test_key_without_ttl + ) + + assert ( + value_after_first_with_ttl == 10.0 + ), f"First increment should set value to 10.0, got {value_after_first_with_ttl}" + assert ( + value_after_first_without_ttl == 5.0 + ), "First increment should set value to 5.0" + assert ( + ttl_after_first is not None and ttl_after_first > 0 + ), "Key with TTL should have positive TTL after first increment" assert ttl_after_first <= 60, "TTL should not exceed the set value" - + # Check TTL for key without TTL (should be None, meaning no expiry) ttl_no_ttl_key = await redis_cache.async_get_ttl(test_key_without_ttl) - assert ttl_no_ttl_key is None, "Key without TTL should have no expiry (None from async_get_ttl)" - + assert ( + ttl_no_ttl_key is None + ), "Key without TTL should have no expiry (None from async_get_ttl)" + # Wait a moment to ensure TTL decreases await asyncio.sleep(2) - + # Second increment: Same operations to test TTL preservation pipeline_operations_second = [ RedisPipelineIncrementOperation( - key=test_key_with_ttl, - increment_value=15.0, - ttl=60 # Same TTL value + key=test_key_with_ttl, increment_value=15.0, ttl=60 # Same TTL value ), RedisPipelineIncrementOperation( - key=test_key_without_ttl, - increment_value=7.0, - ttl=None # No TTL - ) + key=test_key_without_ttl, increment_value=7.0, ttl=None # No TTL + ), ] - + # Execute second increment await parallel_request_handler.async_increment_tokens_with_ttl_preservation( pipeline_operations=pipeline_operations_second ) - + # Small delay to ensure Redis has processed the commands await asyncio.sleep(0.1) - + # Verify TTL preservation and value updates ttl_after_second = await redis_cache.async_get_ttl(test_key_with_ttl) - value_after_second_with_ttl = await redis_cache.async_get_cache(test_key_with_ttl) - value_after_second_without_ttl = await redis_cache.async_get_cache(test_key_without_ttl) - - assert value_after_second_with_ttl == 25.0, "Second increment should update value to 25.0" - assert value_after_second_without_ttl == 12.0, "Second increment should update value to 12.0" - + value_after_second_with_ttl = await redis_cache.async_get_cache( + test_key_with_ttl + ) + value_after_second_without_ttl = await redis_cache.async_get_cache( + test_key_without_ttl + ) + + assert ( + value_after_second_with_ttl == 25.0 + ), "Second increment should update value to 25.0" + assert ( + value_after_second_without_ttl == 12.0 + ), "Second increment should update value to 12.0" + # Critical test: TTL should be preserved (not reset to 60) assert ttl_after_second is not None, "TTL should still exist" - assert ttl_after_second < ttl_after_first, "TTL should have decreased (not been reset)" + assert ( + ttl_after_second < ttl_after_first + ), "TTL should have decreased (not been reset)" assert ttl_after_second > 0, "TTL should still be positive" - + # TTL should not be close to the original 60 seconds (proving it wasn't reset) - assert ttl_after_second < 59, "TTL should be significantly less than original, proving preservation" - + assert ( + ttl_after_second < 59 + ), "TTL should be significantly less than original, proving preservation" + # Key without TTL should still have no expiry - ttl_no_ttl_key_after_second = await redis_cache.async_get_ttl(test_key_without_ttl) - assert ttl_no_ttl_key_after_second is None, "Key without TTL should still have no expiry" - + ttl_no_ttl_key_after_second = await redis_cache.async_get_ttl( + test_key_without_ttl + ) + assert ( + ttl_no_ttl_key_after_second is None + ), "Key without TTL should still have no expiry" + finally: # Clean up test keys try: @@ -1224,7 +1297,7 @@ async def test_async_increment_tokens_with_ttl_preservation(): except Exception: # Ignore cleanup errors pass - + # Properly close Redis connections to prevent warnings try: await redis_cache.disconnect() @@ -1239,115 +1312,125 @@ async def test_async_increment_tokens_fallback_behavior(): Test fallback behavior when Lua script is not available. """ from litellm.types.caching import RedisPipelineIncrementOperation - + local_cache = DualCache() parallel_request_handler = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(local_cache) ) - + # Mock the token_increment_script to None to simulate unavailable script parallel_request_handler.token_increment_script = None - + # Mock the fallback method fallback_called = False - original_method = parallel_request_handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline - + original_method = ( + parallel_request_handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline + ) + async def mock_fallback(*args, **kwargs): nonlocal fallback_called fallback_called = True return await original_method(*args, **kwargs) - - parallel_request_handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = mock_fallback - + + parallel_request_handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = ( + mock_fallback + ) + # Test operations pipeline_operations = [ RedisPipelineIncrementOperation( - key="test_fallback_key", - increment_value=10.0, - ttl=60 + key="test_fallback_key", increment_value=10.0, ttl=60 ) ] - + # Execute increment await parallel_request_handler.async_increment_tokens_with_ttl_preservation( pipeline_operations=pipeline_operations ) - + # Verify fallback was called - assert fallback_called, "Fallback method should be called when Lua script is not available" + assert ( + fallback_called + ), "Fallback method should be called when Lua script is not available" # Redis Cluster Compatibility Tests def test_group_keys_by_hash_tag_regular_redis(): """ Test that keys are correctly grouped for regular Redis (non-cluster). - + For regular Redis, all keys should be grouped together under a single group. """ local_cache = DualCache() handler = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(local_cache) ) - + # Test keys with different hash tags test_keys = [ "{api_key:sk-123}:window", - "{api_key:sk-123}:requests", + "{api_key:sk-123}:requests", "{api_key:sk-123}:tokens", "{user:user-456}:window", "{user:user-456}:requests", "{team:team-789}:window", "{team:team-789}:tokens", - "no_hash_tag_key" + "no_hash_tag_key", ] - + # Group the keys (should be single group for regular Redis) groups = handler._group_keys_by_hash_tag(test_keys) - + # Verify all keys are in single group for regular Redis assert len(groups) == 1, f"Expected 1 group for regular Redis, got {len(groups)}" assert "all_keys" in groups, "Expected 'all_keys' group for regular Redis" - assert set(groups["all_keys"]) == set(test_keys), "All keys should be in single group" + assert set(groups["all_keys"]) == set( + test_keys + ), "All keys should be in single group" def test_group_keys_by_hash_tag_redis_cluster(): """ Test that keys are correctly grouped by Redis cluster slots when using Redis cluster. - + This ensures that keys are grouped by their slot number for cluster compatibility. """ from unittest.mock import patch - + local_cache = DualCache() handler = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(local_cache) ) - + # Mock _is_redis_cluster to return True - with patch.object(handler, '_is_redis_cluster', return_value=True): + with patch.object(handler, "_is_redis_cluster", return_value=True): # Test keys with different hash tags test_keys = [ "{api_key:sk-123}:window", - "{api_key:sk-123}:requests", + "{api_key:sk-123}:requests", "{user:user-456}:window", "{user:user-456}:requests", ] - + # Group the keys (should be grouped by slot for Redis cluster) groups = handler._group_keys_by_hash_tag(test_keys) - + # Verify keys are grouped by slot assert len(groups) >= 1, "Should have at least 1 slot group" - + # All group keys should start with "slot_" for group_key in groups.keys(): - assert group_key.startswith("slot_"), f"Group key {group_key} should start with 'slot_'" - + assert group_key.startswith( + "slot_" + ), f"Group key {group_key} should start with 'slot_'" + # Verify all original keys are present across groups all_grouped_keys = [] for group_keys in groups.values(): all_grouped_keys.extend(group_keys) - assert set(all_grouped_keys) == set(test_keys), "All keys should be present in groups" + assert set(all_grouped_keys) == set( + test_keys + ), "All keys should be present in groups" def test_keyslot_for_redis_cluster(): @@ -1358,16 +1441,16 @@ def test_keyslot_for_redis_cluster(): handler = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(local_cache) ) - + # Test basic key slot1 = handler.keyslot_for_redis_cluster("user:1000") assert 0 <= slot1 < 16384, "Slot should be in valid range" - + # Test key with hash tag slot2 = handler.keyslot_for_redis_cluster("foo{bar}baz") slot3 = handler.keyslot_for_redis_cluster("{bar}") assert slot2 == slot3, "Keys with same hash tag should have same slot" - + # Test keys with same hash tag should have same slot slot4 = handler.keyslot_for_redis_cluster("{api_key:sk-123}:requests") slot5 = handler.keyslot_for_redis_cluster("{api_key:sk-123}:window") @@ -1379,67 +1462,70 @@ async def test_execute_redis_batch_rate_limiter_script_cluster_compatibility(): """ Test that the Redis batch rate limiter script execution handles cluster compatibility by grouping keys and falling back gracefully on errors. - + This simulates the Redis cluster error scenario and verifies fallback behavior. """ from unittest.mock import AsyncMock, patch - + local_cache = DualCache() handler = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(local_cache) ) - + # Mock _is_redis_cluster to return True for this test - with patch.object(handler, '_is_redis_cluster', return_value=True): + with patch.object(handler, "_is_redis_cluster", return_value=True): # Mock script that simulates Redis cluster slot conflict mock_script = AsyncMock() mock_script.side_effect = [ - Exception("EVALSHA - all keys must map to the same key slot"), # First group fails - [1234, 1, 1234, 2] # Second group succeeds + Exception( + "EVALSHA - all keys must map to the same key slot" + ), # First group fails + [1234, 1, 1234, 2], # Second group succeeds ] handler.batch_rate_limiter_script = mock_script - + # Mock in-memory fallback (returns 2 values for 2 keys: window_start, counter) handler.in_memory_cache_sliding_window = AsyncMock(return_value=[1234, 1]) - + # Test keys from different hash tags (would fail in cluster without grouping) test_keys = [ "{api_key:sk-123}:window", "{api_key:sk-123}:requests", - "{user:user-456}:window", - "{user:user-456}:requests" + "{user:user-456}:window", + "{user:user-456}:requests", ] - + # Execute the method results = await handler._execute_redis_batch_rate_limiter_script( - keys_to_fetch=test_keys, - now_int=1234 + keys_to_fetch=test_keys, now_int=1234 ) - + # Verify results: 2 from fallback + 4 from successful script = 6 total assert len(results) == 6, f"Expected 6 results, got {len(results)}" - + # Verify script was called twice (once per slot group) assert mock_script.call_count == 2 - + # Verify fallback was called for the failed group handler.in_memory_cache_sliding_window.assert_called_once() - + # Verify the calls were made with grouped keys call_args_list = mock_script.call_args_list - + # Both calls should have keys, but we can't predict exact grouping without knowing slots # Just verify that keys were grouped and calls were made assert len(call_args_list) == 2, "Should have made 2 script calls" - + # Verify all keys were processed all_processed_keys = [] for call_args in call_args_list: - all_processed_keys.extend(call_args[1]['keys']) - + all_processed_keys.extend(call_args[1]["keys"]) + # Should have processed all keys (some might be duplicated due to fallback) unique_processed_keys = set(all_processed_keys) - assert len(unique_processed_keys) >= 2, "Should have processed at least some keys" + assert ( + len(unique_processed_keys) >= 2 + ), "Should have processed at least some keys" @pytest.mark.asyncio @@ -1485,23 +1571,23 @@ async def test_multiple_rate_limits_per_descriptor(): "current_limit": 2, "limit_remaining": 1, "rate_limit_type": "requests", - "descriptor_key": "api_key" + "descriptor_key": "api_key", }, { "code": "OK", "current_limit": 10, "limit_remaining": 8, "rate_limit_type": "tokens", - "descriptor_key": "api_key" + "descriptor_key": "api_key", }, { "code": "OVER_LIMIT", "current_limit": 1, "limit_remaining": -1, "rate_limit_type": "max_parallel_requests", - "descriptor_key": "api_key" - } - ] + "descriptor_key": "api_key", + }, + ], } parallel_request_handler.should_rate_limit = mock_should_rate_limit @@ -1560,9 +1646,9 @@ async def test_missing_descriptor_fallback(): "current_limit": 2, "limit_remaining": -1, "rate_limit_type": "requests", - "descriptor_key": "nonexistent_key" # This won't match any descriptor + "descriptor_key": "nonexistent_key", # This won't match any descriptor } - ] + ], } parallel_request_handler.should_rate_limit = mock_should_rate_limit @@ -1597,14 +1683,17 @@ async def test_get_rate_limit_type_default_is_total(monkeypatch): # Mock general_settings to return empty dict (no token_rate_limit_type set) import litellm.proxy.proxy_server as proxy_server - original_settings = getattr(proxy_server, 'general_settings', {}) - monkeypatch.setattr(proxy_server, 'general_settings', {}) + + original_settings = getattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "general_settings", {}) try: result = parallel_request_handler.get_rate_limit_type() - assert result == "total", f"Default rate limit type should be 'total', got '{result}'" + assert ( + result == "total" + ), f"Default rate limit type should be 'total', got '{result}'" finally: - monkeypatch.setattr(proxy_server, 'general_settings', original_settings) + monkeypatch.setattr(proxy_server, "general_settings", original_settings) @pytest.mark.asyncio @@ -1619,14 +1708,19 @@ async def test_get_rate_limit_type_invalid_falls_back_to_total(monkeypatch): # Mock general_settings to return an invalid token_rate_limit_type import litellm.proxy.proxy_server as proxy_server - original_settings = getattr(proxy_server, 'general_settings', {}) - monkeypatch.setattr(proxy_server, 'general_settings', {'token_rate_limit_type': 'invalid_type'}) + + original_settings = getattr(proxy_server, "general_settings", {}) + monkeypatch.setattr( + proxy_server, "general_settings", {"token_rate_limit_type": "invalid_type"} + ) try: result = parallel_request_handler.get_rate_limit_type() - assert result == "total", f"Invalid rate limit type should fall back to 'total', got '{result}'" + assert ( + result == "total" + ), f"Invalid rate limit type should fall back to 'total', got '{result}'" finally: - monkeypatch.setattr(proxy_server, 'general_settings', original_settings) + monkeypatch.setattr(proxy_server, "general_settings", original_settings) @pytest.mark.parametrize( @@ -1638,7 +1732,9 @@ async def test_get_rate_limit_type_invalid_falls_back_to_total(monkeypatch): ], ) @pytest.mark.asyncio -async def test_async_log_success_event_with_dict_usage(monkeypatch, token_rate_limit_type, expected_field): +async def test_async_log_success_event_with_dict_usage( + monkeypatch, token_rate_limit_type, expected_field +): """ Test that async_log_success_event correctly handles usage as a dict (Responses API format). @@ -1664,13 +1760,13 @@ async def test_async_log_success_event_with_dict_usage(monkeypatch, token_rate_l # Create a mock response object with usage as a dict (Responses API format) from litellm.types.utils import BaseLiteLLMOpenAIResponseObject - + # Use spec to make isinstance checks work correctly with MagicMock mock_response = MagicMock(spec=BaseLiteLLMOpenAIResponseObject) mock_response.usage = { "prompt_tokens": 25, "completion_tokens": 35, - "total_tokens": 60 + "total_tokens": 60, } # Create mock kwargs for the success event @@ -1760,7 +1856,10 @@ async def test_async_log_success_event_with_dict_usage_missing_fields(monkeypatc # total_tokens is missing } from litellm.types.utils import BaseLiteLLMOpenAIResponseObject - mock_response.__class__ = type('MockResponse', (BaseLiteLLMOpenAIResponseObject,), {}) + + mock_response.__class__ = type( + "MockResponse", (BaseLiteLLMOpenAIResponseObject,), {} + ) # Create mock kwargs for the success event mock_kwargs = { @@ -1805,7 +1904,9 @@ async def test_async_log_success_event_with_dict_usage_missing_fields(monkeypatc assert tpm_operation is not None, "Should have a TPM increment operation" # Should default to 0 when field is missing - assert tpm_operation["increment_value"] == 0, "Should default to 0 when completion_tokens is missing" + assert ( + tpm_operation["increment_value"] == 0 + ), "Should default to 0 when completion_tokens is missing" @pytest.mark.asyncio @@ -1813,68 +1914,154 @@ async def test_execute_token_increment_script_cluster_compatibility(): """ Test that token increment script execution handles Redis cluster compatibility by grouping operations by slot. - + This ensures token increments work correctly in cluster environments. """ from typing import List from unittest.mock import AsyncMock, patch from litellm.types.caching import RedisPipelineIncrementOperation - + local_cache = DualCache() handler = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(local_cache) ) - + # Mock _is_redis_cluster to return True for this test - with patch.object(handler, '_is_redis_cluster', return_value=True): + with patch.object(handler, "_is_redis_cluster", return_value=True): # Mock script mock_script = AsyncMock() handler.token_increment_script = mock_script - + # Create pipeline operations with different hash tags pipeline_operations: List[RedisPipelineIncrementOperation] = [ + {"key": "{api_key:sk-123}:tokens", "increment_value": 100, "ttl": 60}, { - "key": "{api_key:sk-123}:tokens", - "increment_value": 100, - "ttl": 60 - }, - { - "key": "{api_key:sk-123}:max_parallel_requests", + "key": "{api_key:sk-123}:max_parallel_requests", "increment_value": -1, - "ttl": 60 + "ttl": 60, }, - { - "key": "{user:user-456}:tokens", - "increment_value": 50, - "ttl": 60 - } + {"key": "{user:user-456}:tokens", "increment_value": 50, "ttl": 60}, ] - + # Execute the method await handler._execute_token_increment_script(pipeline_operations) - + # Verify script was called (at least once, possibly more depending on slot grouping) assert mock_script.call_count >= 1, "Script should be called at least once" - + call_args_list = mock_script.call_args_list - + # Verify all operations were processed all_processed_keys = [] for call_args in call_args_list: - all_processed_keys.extend(call_args[1]['keys']) - + all_processed_keys.extend(call_args[1]["keys"]) + # Should have processed all 3 keys expected_keys = { "{api_key:sk-123}:tokens", "{api_key:sk-123}:max_parallel_requests", - "{user:user-456}:tokens" + "{user:user-456}:tokens", } - assert set(all_processed_keys) == expected_keys, "All operation keys should be processed" - + assert ( + set(all_processed_keys) == expected_keys + ), "All operation keys should be processed" + # Verify args structure is correct for each call for call_args in call_args_list: - keys = call_args[1]['keys'] - args = call_args[1]['args'] + keys = call_args[1]["keys"] + args = call_args[1]["args"] # Each key should have 2 args (increment_value, ttl) - assert len(args) == len(keys) * 2, f"Each key should have 2 args, got {len(args)} args for {len(keys)} keys" + assert ( + len(args) == len(keys) * 2 + ), f"Each key should have 2 args, got {len(args)} args for {len(keys)} keys" + + +class TestGetTotalTokensFromUsageCacheExclusion: + """ + Tests for _get_total_tokens_from_usage cache token exclusion. + + Issue: AWS Bedrock and similar providers exclude cache tokens from TPM calculation, + but LiteLLM was including them, causing up to 10x difference in rate limiting. + """ + + @pytest.fixture + def handler(self): + """Create a handler instance for testing.""" + local_cache = DualCache() + return _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache), + ) + + def test_excludes_cached_tokens_from_total(self, handler): + """Cached tokens should be excluded from total token count.""" + from litellm.types.utils import PromptTokensDetailsWrapper + + usage = Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=800), + ) + + # Total should be 1500 - 800 = 700 + result = handler._get_total_tokens_from_usage(usage, "total") + assert result == 700, f"Expected 700 (1500 - 800 cached), got {result}" + + def test_excludes_cached_tokens_from_input(self, handler): + """Cached tokens should be excluded from input token count.""" + from litellm.types.utils import PromptTokensDetailsWrapper + + usage = Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=800), + ) + + # Input should be 1000 - 800 = 200 + result = handler._get_total_tokens_from_usage(usage, "input") + assert result == 200, f"Expected 200 (1000 - 800 cached), got {result}" + + def test_does_not_exclude_cached_tokens_from_output(self, handler): + """Cached tokens should NOT affect output token count.""" + from litellm.types.utils import PromptTokensDetailsWrapper + + usage = Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=800), + ) + + # Output tokens should be unchanged + result = handler._get_total_tokens_from_usage(usage, "output") + assert result == 500, f"Expected 500 (no change for output), got {result}" + + def test_handles_no_cached_tokens(self, handler): + """Should work correctly when no cached tokens present.""" + usage = Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500, + ) + + result = handler._get_total_tokens_from_usage(usage, "total") + assert result == 1500, f"Expected 1500 (no cache), got {result}" + + def test_handles_dict_usage_with_cached_tokens(self, handler): + """Should handle dict usage format (Responses API) with cached tokens.""" + usage = { + "prompt_tokens": 1000, + "completion_tokens": 500, + "total_tokens": 1500, + "prompt_tokens_details": {"cached_tokens": 600}, + } + + result = handler._get_total_tokens_from_usage(usage, "total") + assert result == 900, f"Expected 900 (1500 - 600 cached), got {result}" + + def test_handles_none_usage(self, handler): + """Should handle None usage gracefully.""" + result = handler._get_total_tokens_from_usage(None, "total") + assert result == 0, f"Expected 0 for None usage, got {result}" From c29d042df44212b7c5c5ac016b3c06ed56633e9a Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 9 Jan 2026 12:51:00 -0800 Subject: [PATCH 54/56] Case insensitive email login --- litellm/proxy/auth/login_utils.py | 2 +- .../proxy/auth/test_login_utils.py | 77 +++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index 8cc33ce6cdd..5be44f479b8 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -188,7 +188,7 @@ async def authenticate_user( # noqa: PLR0915 _user_row = cast( Optional[LiteLLM_UserTable], await prisma_client.db.litellm_usertable.find_first( - where={"user_email": {"equals": username}} + where={"user_email": {"equals": username, "mode": "insensitive"}} ), ) diff --git a/tests/test_litellm/proxy/auth/test_login_utils.py b/tests/test_litellm/proxy/auth/test_login_utils.py index c04b8114939..e7b27908c14 100644 --- a/tests/test_litellm/proxy/auth/test_login_utils.py +++ b/tests/test_litellm/proxy/auth/test_login_utils.py @@ -248,6 +248,83 @@ async def test_authenticate_user_wrong_password(): assert "Invalid credentials" in exc_info.value.message +@pytest.mark.asyncio +async def test_authenticate_user_email_case_insensitive_login(): + """Test that email lookup is case-insensitive during login""" + master_key = "sk-1234" + stored_email = "testemail@test.com" + login_email_mixed_case = "testEmail@test.com" + correct_password = "correct-password" + hashed_password = hash_token(token=correct_password) + + # `LiteLLM_UserTable` does not define a `password` field, but `authenticate_user()` + # expects `user_row.password` to exist (invite-link login). Use a simple object. + mock_user = MagicMock() + mock_user.user_id = "test-user-123" + mock_user.user_email = stored_email + mock_user.password = hashed_password + mock_user.user_role = LitellmUserRoles.INTERNAL_USER + + def mock_find_first(**kwargs): + where = kwargs.get("where", {}) + user_email = where.get("user_email", {}) + if user_email.get("mode") != "insensitive": + return None + if str(user_email.get("equals", "")).lower() == stored_email.lower(): + return mock_user + return None + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock( + side_effect=mock_find_first + ) + + with patch.dict( + os.environ, + { + "DATABASE_URL": "postgresql://test:test@localhost/test", + "UI_USERNAME": "admin", + "UI_PASSWORD": "admin-password", + }, + ): + with patch( + "litellm.proxy.auth.login_utils.expire_previous_ui_session_tokens", + new_callable=AsyncMock, + return_value=None, + ): + with patch( + "litellm.proxy.auth.login_utils.generate_key_helper_fn", + new_callable=AsyncMock, + ) as mock_generate_key: + mock_generate_key.side_effect = [ + {"token": "token-1"}, + {"token": "token-2"}, + ] + + result_mixed = await authenticate_user( + username=login_email_mixed_case, + password=correct_password, + master_key=master_key, + prisma_client=mock_prisma_client, + ) + result_lower = await authenticate_user( + username=stored_email, + password=correct_password, + master_key=master_key, + prisma_client=mock_prisma_client, + ) + + assert result_mixed.user_id == result_lower.user_id == "test-user-123" + assert result_mixed.user_email == result_lower.user_email == stored_email + + calls = mock_prisma_client.db.litellm_usertable.find_first.await_args_list + assert len(calls) == 2 + for call, expected_username in zip(calls, [login_email_mixed_case, stored_email]): + where = call.kwargs["where"] + assert where["user_email"]["equals"] == expected_username + assert where["user_email"]["mode"] == "insensitive" + + @pytest.mark.asyncio async def test_authenticate_user_database_required_for_admin(): """Test that database is required for admin login""" From dfb298792c1380acec4dde5bc40d999da236abeb Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 9 Jan 2026 16:49:22 -0800 Subject: [PATCH 55/56] New endpoint for router fields + react query --- .../router_settings_endpoints.py | 60 +++ .../test_router_settings_endpoints.py | 71 ++++ .../hooks/router/useRouterFields.test.ts | 387 ++++++++++++++++++ .../hooks/router/useRouterFields.ts | 69 ++++ 4 files changed, 587 insertions(+) create mode 100644 tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.ts diff --git a/litellm/proxy/management_endpoints/router_settings_endpoints.py b/litellm/proxy/management_endpoints/router_settings_endpoints.py index 167160c72d1..4d4c41a3dc0 100644 --- a/litellm/proxy/management_endpoints/router_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/router_settings_endpoints.py @@ -4,6 +4,7 @@ ROUTER SETTINGS MANAGEMENT Endpoints for accessing router configuration and metadata GET /router/settings - Get router configuration including available routing strategies +GET /router/fields - Get router settings field definitions without values (for UI rendering) """ import inspect @@ -37,6 +38,15 @@ class RouterSettingsResponse(BaseModel): ) +class RouterFieldsResponse(BaseModel): + fields: List[RouterSettingsField] = Field( + description="List of all configurable router settings with metadata (without field values)" + ) + routing_strategy_descriptions: Dict[str, str] = Field( + description="Descriptions for each routing strategy option" + ) + + def _get_routing_strategies_from_router_class() -> List[str]: """ Dynamically extract routing strategies from the Router class __init__ method. @@ -120,3 +130,53 @@ async def get_router_settings( ) raise + +@router.get( + "/router/fields", + tags=["Router Settings"], + dependencies=[Depends(user_api_key_auth)], + response_model=RouterFieldsResponse, +) +async def get_router_fields( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Get router settings field definitions without values. + + Returns only the field metadata (type, description, default, options) without + populating field_value. This is useful for UI components that need to know + what fields to render, but will get the actual values from a different endpoint. + + Returns: + - fields: List of all configurable router settings with their metadata (type, description, default, options) + The routing_strategy field includes available options extracted from the Router class + Note: field_value will be None for all fields + - routing_strategy_descriptions: Descriptions for each routing strategy option + """ + try: + # Get available routing strategies dynamically from Router class + available_routing_strategies = _get_routing_strategies_from_router_class() + + # Get router settings fields from types file + router_fields = [field.model_copy(deep=True) for field in ROUTER_SETTINGS_FIELDS] + + # Populate routing_strategy field with available options + for field in router_fields: + if field.field_name == "routing_strategy": + field.options = available_routing_strategies + break + + # Ensure field_value is None for all fields (don't populate values) + for field in router_fields: + field.field_value = None + + return RouterFieldsResponse( + fields=router_fields, + routing_strategy_descriptions=ROUTING_STRATEGY_DESCRIPTIONS, + ) + except Exception as e: + verbose_proxy_logger.error( + f"Error fetching router fields: {str(e)}" + ) + raise + diff --git a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py new file mode 100644 index 00000000000..1f5473e75d4 --- /dev/null +++ b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py @@ -0,0 +1,71 @@ +""" +Tests for router settings management endpoints. + +Tests the GET endpoints for router settings and router fields. +""" +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +sys.path.insert( + 0, os.path.abspath("../../../..") +) + +from litellm.proxy.proxy_server import app + +client = TestClient(app) + + +class TestRouterSettingsEndpoints: + """Test suite for router settings endpoints""" + + @pytest.mark.asyncio + async def test_get_router_fields_success(self): + """ + Test GET /router/fields endpoint successfully returns field definitions without values. + """ + # Make request to router fields endpoint + response = client.get( + "/router/fields", + headers={"Authorization": "Bearer sk-1234"} + ) + + # Verify response + assert response.status_code == 200 + + response_data = response.json() + + # Verify response structure + assert "fields" in response_data + assert "routing_strategy_descriptions" in response_data + + # Verify fields is a list + assert isinstance(response_data["fields"], list) + assert len(response_data["fields"]) > 0 + + # Verify each field has required properties and field_value is None + for field in response_data["fields"]: + assert "field_name" in field + assert "field_type" in field + assert "field_description" in field + assert "field_default" in field + assert "ui_field_name" in field + assert "field_value" in field + assert field["field_value"] is None # Ensure field_value is None + + # Verify routing_strategy_descriptions is a dict + assert isinstance(response_data["routing_strategy_descriptions"], dict) + assert len(response_data["routing_strategy_descriptions"]) > 0 + + # Verify routing_strategy field has options populated + routing_strategy_field = next( + (f for f in response_data["fields"] if f["field_name"] == "routing_strategy"), + None + ) + assert routing_strategy_field is not None + assert "options" in routing_strategy_field + assert isinstance(routing_strategy_field["options"], list) + assert len(routing_strategy_field["options"]) > 0 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.test.ts new file mode 100644 index 00000000000..1e57b482915 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.test.ts @@ -0,0 +1,387 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { renderHook, waitFor } from "@testing-library/react"; +import React, { ReactNode } from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { useRouterFields, RouterFieldsResponse, RouterSettingsField } from "./useRouterFields"; + +// Mock the networking module +vi.mock("@/components/networking", () => ({ + proxyBaseUrl: null, +})); + +// Mock useAuthorized hook +const mockUseAuthorized = vi.fn(); +vi.mock("../useAuthorized", () => ({ + default: () => mockUseAuthorized(), +})); + +// Mock global fetch +const mockFetch = vi.fn(); +global.fetch = mockFetch; + +// Mock console methods to avoid noise in tests +vi.spyOn(console, "log").mockImplementation(() => {}); +vi.spyOn(console, "error").mockImplementation(() => {}); + +// Mock data +const mockRouterFieldsResponse: RouterFieldsResponse = { + fields: [ + { + field_name: "routing_strategy", + field_type: "String", + field_description: "Routing strategy to use for load balancing across deployments", + field_default: "simple-shuffle", + options: ["simple-shuffle", "least-busy", "latency-based-routing"], + ui_field_name: "Routing Strategy", + link: null, + }, + { + field_name: "num_retries", + field_type: "Integer", + field_description: "Number of retries for failed requests", + field_default: 0, + options: null, + ui_field_name: "Number of Retries", + link: null, + }, + ], + routing_strategy_descriptions: { + "simple-shuffle": "Randomly picks a deployment from the list. Simple and fast.", + "least-busy": "Routes to the deployment with the lowest number of ongoing requests.", + "latency-based-routing": "Routes to the deployment with the lowest latency over a sliding window.", + }, +}; + +describe("useRouterFields", () => { + let queryClient: QueryClient; + + beforeEach(() => { + queryClient = new QueryClient({ + defaultOptions: { + queries: { + retry: false, + }, + }, + }); + + // Reset all mocks + vi.clearAllMocks(); + + // Set default mock for useAuthorized (enabled state) + mockUseAuthorized.mockReturnValue({ + accessToken: "test-access-token", + userRole: "Admin", + userId: "test-user-id", + token: "test-token", + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }); + }); + + const wrapper = ({ children }: { children: ReactNode }) => + React.createElement(QueryClientProvider, { client: queryClient }, children); + + it("should render", () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => mockRouterFieldsResponse, + }); + + const { result } = renderHook(() => useRouterFields(), { wrapper }); + + expect(result.current).toBeDefined(); + }); + + it("should return router fields data when query is successful", async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => mockRouterFieldsResponse, + }); + + const { result } = renderHook(() => useRouterFields(), { wrapper }); + + // Initially loading + expect(result.current.isLoading).toBe(true); + expect(result.current.data).toBeUndefined(); + + // Wait for success + await waitFor(() => { + expect(result.current.isLoading).toBe(false); + expect(result.current.isSuccess).toBe(true); + }); + + expect(result.current.data).toEqual(mockRouterFieldsResponse); + expect(result.current.error).toBeNull(); + expect(mockFetch).toHaveBeenCalledTimes(1); + expect(mockFetch).toHaveBeenCalledWith("/router/fields", { + method: "GET", + headers: { + Authorization: "Bearer test-access-token", + "Content-Type": "application/json", + }, + }); + }); + + it("should handle error when fetch fails", async () => { + const errorMessage = "Failed to fetch router fields"; + const errorResponse = { error: errorMessage }; + + mockFetch.mockResolvedValueOnce({ + ok: false, + json: async () => errorResponse, + }); + + const { result } = renderHook(() => useRouterFields(), { wrapper }); + + // Initially loading + expect(result.current.isLoading).toBe(true); + + // Wait for error + await waitFor(() => { + expect(result.current.isLoading).toBe(false); + expect(result.current.isError).toBe(true); + }); + + expect(result.current.error).toBeDefined(); + expect(result.current.data).toBeUndefined(); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + + it("should not execute query when accessToken is missing", () => { + mockUseAuthorized.mockReturnValue({ + accessToken: null, + userRole: "Admin", + userId: "test-user-id", + token: null, + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }); + + const { result } = renderHook(() => useRouterFields(), { wrapper }); + + // Query should not execute + expect(result.current.isLoading).toBe(false); + expect(result.current.data).toBeUndefined(); + expect(result.current.isFetched).toBe(false); + + // API should not be called + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it("should not execute query when userId is missing", () => { + mockUseAuthorized.mockReturnValue({ + accessToken: "test-access-token", + userRole: "Admin", + userId: null, + token: "test-token", + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }); + + const { result } = renderHook(() => useRouterFields(), { wrapper }); + + // Query should not execute + expect(result.current.isLoading).toBe(false); + expect(result.current.data).toBeUndefined(); + expect(result.current.isFetched).toBe(false); + + // API should not be called + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it("should not execute query when userRole is missing", () => { + mockUseAuthorized.mockReturnValue({ + accessToken: "test-access-token", + userRole: null, + userId: "test-user-id", + token: "test-token", + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }); + + const { result } = renderHook(() => useRouterFields(), { wrapper }); + + // Query should not execute + expect(result.current.isLoading).toBe(false); + expect(result.current.data).toBeUndefined(); + expect(result.current.isFetched).toBe(false); + + // API should not be called + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it("should handle network error", async () => { + const networkError = new Error("Network error"); + mockFetch.mockRejectedValueOnce(networkError); + + const { result } = renderHook(() => useRouterFields(), { wrapper }); + + // Wait for error + await waitFor(() => { + expect(result.current.isError).toBe(true); + }); + + expect(result.current.error).toBeDefined(); + expect(result.current.data).toBeUndefined(); + }); + + it("should use relative URL when proxyBaseUrl is null", async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => mockRouterFieldsResponse, + }); + + const { result } = renderHook(() => useRouterFields(), { wrapper }); + + await waitFor(() => { + expect(result.current.isSuccess).toBe(true); + }); + + // When proxyBaseUrl is null, should use relative URL + expect(mockFetch).toHaveBeenCalledWith("/router/fields", { + method: "GET", + headers: { + Authorization: "Bearer test-access-token", + "Content-Type": "application/json", + }, + }); + }); + + it("should handle error response with different error formats", async () => { + const errorFormats = [ + { error: { message: "Error message" } }, + { message: "Error message" }, + { detail: "Error detail" }, + { error: "Error string" }, + { unknown: "format" }, + ]; + + for (const errorFormat of errorFormats) { + vi.clearAllMocks(); + mockFetch.mockResolvedValueOnce({ + ok: false, + json: async () => errorFormat, + }); + + const { result } = renderHook(() => useRouterFields(), { wrapper }); + + await waitFor(() => { + expect(result.current.isError).toBe(true); + }); + + expect(result.current.error).toBeDefined(); + } + }); + + it("should return empty fields array when API returns empty fields", async () => { + const emptyResponse: RouterFieldsResponse = { + fields: [], + routing_strategy_descriptions: {}, + }; + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => emptyResponse, + }); + + const { result } = renderHook(() => useRouterFields(), { wrapper }); + + await waitFor(() => { + expect(result.current.isSuccess).toBe(true); + }); + + expect(result.current.data?.fields).toEqual([]); + expect(result.current.data?.routing_strategy_descriptions).toEqual({}); + }); + + it("should have correct query configuration", async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => mockRouterFieldsResponse, + }); + + const { result } = renderHook(() => useRouterFields(), { wrapper }); + + await waitFor(() => { + expect(result.current.isSuccess).toBe(true); + }); + + // Verify the query was called + expect(mockFetch).toHaveBeenCalledTimes(1); + + // The hook should have the expected properties from useQuery + expect(result.current).toHaveProperty("data"); + expect(result.current).toHaveProperty("isLoading"); + expect(result.current).toHaveProperty("isError"); + expect(result.current).toHaveProperty("isSuccess"); + expect(result.current).toHaveProperty("error"); + }); + + it("should handle fields with null options", async () => { + const responseWithNullOptions: RouterFieldsResponse = { + fields: [ + { + field_name: "timeout", + field_type: "Float", + field_description: "Timeout for requests in seconds", + field_default: null, + options: null, + ui_field_name: "Timeout", + link: null, + }, + ], + routing_strategy_descriptions: {}, + }; + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => responseWithNullOptions, + }); + + const { result } = renderHook(() => useRouterFields(), { wrapper }); + + await waitFor(() => { + expect(result.current.isSuccess).toBe(true); + }); + + expect(result.current.data?.fields[0].options).toBeNull(); + }); + + it("should handle fields with link property", async () => { + const responseWithLink: RouterFieldsResponse = { + fields: [ + { + field_name: "enable_tag_filtering", + field_type: "Boolean", + field_description: "Enable tag-based routing", + field_default: false, + options: null, + ui_field_name: "Enable Tag Filtering", + link: "https://docs.litellm.ai/docs/proxy/tag_routing", + }, + ], + routing_strategy_descriptions: {}, + }; + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => responseWithLink, + }); + + const { result } = renderHook(() => useRouterFields(), { wrapper }); + + await waitFor(() => { + expect(result.current.isSuccess).toBe(true); + }); + + expect(result.current.data?.fields[0].link).toBe("https://docs.litellm.ai/docs/proxy/tag_routing"); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.ts new file mode 100644 index 00000000000..589508c5ddb --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.ts @@ -0,0 +1,69 @@ +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { useQuery, UseQueryResult } from "@tanstack/react-query"; +import { createQueryKeys } from "../common/queryKeysFactory"; +import { proxyBaseUrl } from "@/components/networking"; + +export interface RouterSettingsField { + field_name: string; + field_type: string; + field_description: string; + field_default: any; + options: string[] | null; + ui_field_name: string; + link: string | null; +} + +export interface RouterFieldsResponse { + fields: RouterSettingsField[]; + routing_strategy_descriptions: Record; +} + +const routerFieldsKeys = createQueryKeys("routerFields"); + +const deriveErrorMessage = (errorData: any): string => { + return ( + (errorData?.error && (errorData.error.message || errorData.error)) || + errorData?.message || + errorData?.detail || + errorData?.error || + JSON.stringify(errorData) + ); +}; + +const getRouterFields = async (accessToken: string): Promise => { + try { + const url = proxyBaseUrl ? `${proxyBaseUrl}/router/fields` : `/router/fields`; + + console.log("Fetching router fields from:", url); + + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.json(); + const errorMessage = deriveErrorMessage(errorData); + throw new Error(errorMessage); + } + + const data: RouterFieldsResponse = await response.json(); + console.log("Fetched router fields:", data); + return data; + } catch (error) { + console.error("Failed to fetch router fields:", error); + throw error; + } +}; + +export const useRouterFields = (): UseQueryResult => { + const { accessToken, userId, userRole } = useAuthorized(); + return useQuery({ + queryKey: routerFieldsKeys.detail("fields"), + queryFn: async () => await getRouterFields(accessToken!), + enabled: Boolean(accessToken && userId && userRole), + }); +}; From f72394cac14a39e61d0bc926f3d78d131ca7d6cc Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 9 Jan 2026 16:50:08 -0800 Subject: [PATCH 56/56] fixing build --- .../src/app/(dashboard)/hooks/router/useRouterFields.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.test.ts index 1e57b482915..fe4680cedeb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.test.ts @@ -2,7 +2,7 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { renderHook, waitFor } from "@testing-library/react"; import React, { ReactNode } from "react"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { useRouterFields, RouterFieldsResponse, RouterSettingsField } from "./useRouterFields"; +import { RouterFieldsResponse, useRouterFields } from "./useRouterFields"; // Mock the networking module vi.mock("@/components/networking", () => ({