From fa08f157d8255f5f96299002c45aaceb7c47a589 Mon Sep 17 00:00:00 2001 From: Akiva Kraines Date: Mon, 5 Jan 2026 00:14:14 +0200 Subject: [PATCH 001/121] fix: Improve error messages and validation for wildcard routing with multiple credentials - Enhanced error messages to show which deployment/credential was used when routing fails - Added debug logging for pattern-matched deployments to track which deployment was selected - Added validation warning at startup when multiple wildcard patterns exist for same provider with different credentials - Helps diagnose intermittent authentication failures caused by non-deterministic wildcard routing Addresses issue where multiple wildcard deployments (e.g., openai/*) with different credentials cause intermittent 403 errors due to random credential selection. --- litellm/router.py | 56 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 6821ab9e6c6..a040b3bfeaf 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4458,9 +4458,21 @@ class Router: if hasattr(original_exception, "message"): # add the available fallbacks to the exception - original_exception.message += ". Received Model Group={}\nAvailable Model Group Fallbacks={}".format( # type: ignore - model_group, - fallback_model_group, + deployment_info = "" + if kwargs is not None: + metadata = kwargs.get('metadata', {}) + if metadata and 'deployment' in metadata: + deployment_info = f"\nUsed Deployment: {metadata['deployment']}" + if 'model_info' in metadata: + model_info = metadata['model_info'] + if isinstance(model_info, dict): + deployment_info += f"\nDeployment ID: {model_info.get('id', 'unknown')}" + + original_exception.message += ( # type: ignore + f". Received Model Group={model_group}" + f"\nAvailable Model Group Fallbacks={fallback_model_group}" + f"{deployment_info}" + f"\n\n💡 Tip: If using wildcard patterns (e.g., 'openai/*'), ensure all matching deployments have credentials with access to this model." ) if len(fallback_failure_exception_str) > 0: original_exception.message += ( # type: ignore @@ -5713,6 +5725,37 @@ class Router: return True return False + def _validate_wildcard_deployments(self): + """Warn if multiple wildcard patterns exist for same provider with different credentials""" + provider_wildcards: Dict[str, List[Tuple[str, str]]] = {} # provider -> [(credential, deployment_id)] + + for deployment in self.model_list: + model_name = deployment.get('model_name', '') + if '*' in model_name: + # Extract provider from pattern (e.g., "openai/*" -> "openai") + provider = model_name.split('/')[0] if '/' in model_name else model_name.split('*')[0] + + litellm_params = deployment.get('litellm_params', {}) + # Get credential identifier - use litellm_credential_name or first 10 chars of api_key + credential = litellm_params.get('litellm_credential_name') or \ + (litellm_params.get('api_key', '')[:10] if litellm_params.get('api_key') else 'none') + deployment_id = deployment.get('model_info', {}).get('id', 'unknown') + + if provider not in provider_wildcards: + provider_wildcards[provider] = [] + provider_wildcards[provider].append((credential, deployment_id)) + + for provider, cred_deployment_pairs in provider_wildcards.items(): + unique_credentials = set(cred for cred, _ in cred_deployment_pairs) + if len(unique_credentials) > 1: + deployment_ids = [dep_id for _, dep_id in cred_deployment_pairs] + verbose_router_logger.warning( + f"⚠️ Multiple wildcard deployments found for '{provider}/*' with different credentials ({len(unique_credentials)} credentials). " + f"This may cause non-deterministic authentication failures. " + f"Deployments: {len(cred_deployment_pairs)} ({deployment_ids[:3]}{'...' if len(deployment_ids) > 3 else ''}). " + f"Consider: 1) Using concrete model names, 2) Using one credential per provider, or 3) Using tag-based routing." + ) + def set_model_list(self, model_list: list): original_model_list = copy.deepcopy(model_list) self.model_list = [] @@ -5762,6 +5805,9 @@ class Router: # Note: model_name_to_deployment_indices is already built incrementally # by _create_deployment -> _add_model_to_list_and_index_map + + # Validate wildcard deployments after all models are loaded + self._validate_wildcard_deployments() def _add_deployment(self, deployment: Deployment) -> Deployment: import os @@ -7629,6 +7675,10 @@ class Router: ) if pattern_deployments: + verbose_router_logger.debug( + f"Pattern match for model='{model}': Found {len(pattern_deployments)} deployments. " + f"Deployment IDs: {[d.get('model_info', {}).get('id', 'unknown') for d in pattern_deployments]}" + ) return model, pattern_deployments if ( From 229d2801413e1df54ac63a741f52144c32a01291 Mon Sep 17 00:00:00 2001 From: Chesars Date: Mon, 5 Jan 2026 15:36:32 -0300 Subject: [PATCH 002/121] feat(ui): add custom proxy base URL support to Playground Add ability to configure a custom proxy base URL in the Playground UI, enabling control plane/data plane architecture where: - Control Plane: Has UI but LLM APIs disabled (DISABLE_LLM_API_ENDPOINTS=true) - Data Plane: Has LLM APIs but UI disabled Changes: - Add "Custom Proxy Base URL" input field in Playground settings - Persist custom URL in sessionStorage for user convenience - Modify getProxyBaseUrl() to check sessionStorage first - All API calls now route to custom URL when configured This allows users to run the UI on control plane and point API calls to a separate data plane endpoint. --- .../src/components/networking.tsx | 6 +++++ .../components/playground/chat_ui/ChatUI.tsx | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index f0464c61f88..ed43c8aa455 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -92,6 +92,12 @@ const updateServerRootPath = (receivedServerRootPath: string) => { }; export const getProxyBaseUrl = (): string => { + // Check for custom proxy base URL from sessionStorage first + const customProxyBaseUrl = sessionStorage.getItem("customProxyBaseUrl"); + if (customProxyBaseUrl && customProxyBaseUrl.trim() !== "") { + return customProxyBaseUrl; + } + if (proxyBaseUrl) { return proxyBaseUrl; } diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx index a963aa706be..145560e9922 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx @@ -118,6 +118,9 @@ const ChatUI: React.FC = ({ return disabledPersonalKeyCreation ? "custom" : "session"; }); const [apiKey, setApiKey] = useState(() => sessionStorage.getItem("apiKey") || ""); + const [customProxyBaseUrl, setCustomProxyBaseUrl] = useState( + () => sessionStorage.getItem("customProxyBaseUrl") || "" + ); const [inputMessage, setInputMessage] = useState(""); const [chatHistory, setChatHistory] = useState(() => { try { @@ -1112,6 +1115,26 @@ const ChatUI: React.FC = ({ )} +
+ + Custom Proxy Base URL + + { + setCustomProxyBaseUrl(value); + sessionStorage.setItem("customProxyBaseUrl", value); + }} + value={customProxyBaseUrl} + icon={ApiOutlined} + /> + {customProxyBaseUrl && ( + + API calls will be sent to: {customProxyBaseUrl} + + )} +
+
Endpoint Type From 31430156c7e9654352ab4a133a2dce36dfd66c63 Mon Sep 17 00:00:00 2001 From: Akiva Kraines Date: Mon, 5 Jan 2026 22:26:46 +0200 Subject: [PATCH 003/121] refactor: Remove incomplete wildcard validation per maintainer feedback Per maintainer feedback, removed the wildcard validation logic as it doesn't cover all auth mechanisms (Google, AWS, etc.). Keeping only the core improvements: - Enhanced error messages showing deployment/credential used - Debug logging for pattern-matched deployments The validation logic needs more work to handle all provider auth mechanisms properly. --- litellm/router.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index a040b3bfeaf..b709dc764c5 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5725,37 +5725,6 @@ class Router: return True return False - def _validate_wildcard_deployments(self): - """Warn if multiple wildcard patterns exist for same provider with different credentials""" - provider_wildcards: Dict[str, List[Tuple[str, str]]] = {} # provider -> [(credential, deployment_id)] - - for deployment in self.model_list: - model_name = deployment.get('model_name', '') - if '*' in model_name: - # Extract provider from pattern (e.g., "openai/*" -> "openai") - provider = model_name.split('/')[0] if '/' in model_name else model_name.split('*')[0] - - litellm_params = deployment.get('litellm_params', {}) - # Get credential identifier - use litellm_credential_name or first 10 chars of api_key - credential = litellm_params.get('litellm_credential_name') or \ - (litellm_params.get('api_key', '')[:10] if litellm_params.get('api_key') else 'none') - deployment_id = deployment.get('model_info', {}).get('id', 'unknown') - - if provider not in provider_wildcards: - provider_wildcards[provider] = [] - provider_wildcards[provider].append((credential, deployment_id)) - - for provider, cred_deployment_pairs in provider_wildcards.items(): - unique_credentials = set(cred for cred, _ in cred_deployment_pairs) - if len(unique_credentials) > 1: - deployment_ids = [dep_id for _, dep_id in cred_deployment_pairs] - verbose_router_logger.warning( - f"⚠️ Multiple wildcard deployments found for '{provider}/*' with different credentials ({len(unique_credentials)} credentials). " - f"This may cause non-deterministic authentication failures. " - f"Deployments: {len(cred_deployment_pairs)} ({deployment_ids[:3]}{'...' if len(deployment_ids) > 3 else ''}). " - f"Consider: 1) Using concrete model names, 2) Using one credential per provider, or 3) Using tag-based routing." - ) - def set_model_list(self, model_list: list): original_model_list = copy.deepcopy(model_list) self.model_list = [] @@ -5805,9 +5774,6 @@ class Router: # Note: model_name_to_deployment_indices is already built incrementally # by _create_deployment -> _add_model_to_list_and_index_map - - # Validate wildcard deployments after all models are loaded - self._validate_wildcard_deployments() def _add_deployment(self, deployment: Deployment) -> Deployment: import os From 9e6714fe1b9344b06fe819c486d9869e588480ce Mon Sep 17 00:00:00 2001 From: Wiley Kestner Date: Tue, 6 Jan 2026 16:04:02 +0100 Subject: [PATCH 004/121] Tally total_tokens in response if missing (#18468) (#18445) Calculate `total_tokens` in usage data in Response manually if: - `total_tokens` is missing - `total_tokens` can be calculated from input and output tokens Run the test for this feature with: `poetry run pytest tests/test_litellm/responses/test_responses_utils.py -k "test_transform_response_api_usage_calculates_total_from_input_and_output_tokens_if_available" -v` --- litellm/responses/utils.py | 17 ++++++++++++----- .../responses/test_responses_utils.py | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index a92b5d25a37..7667d1bad84 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -443,11 +443,18 @@ class ResponseAPILoggingUtils: completion_tokens=0, total_tokens=0, ) - response_api_usage: ResponseAPIUsage = ( - ResponseAPIUsage(**usage_input) - if isinstance(usage_input, dict) - else usage_input - ) + response_api_usage: ResponseAPIUsage + if isinstance(usage_input, dict): + total_tokens = usage_input.get("total_tokens") + if total_tokens is None: + input_tokens = usage_input.get("input_tokens") + output_tokens = usage_input.get("output_tokens") + if input_tokens is not None and output_tokens is not None: + total_tokens = input_tokens + output_tokens + usage_input["total_tokens"] = total_tokens + response_api_usage = ResponseAPIUsage(**usage_input) + else: + response_api_usage = usage_input prompt_tokens: int = response_api_usage.input_tokens or 0 completion_tokens: int = response_api_usage.output_tokens or 0 prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index 96ac2e2c345..09628cd4a76 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -203,3 +203,22 @@ class TestResponseAPILoggingUtils: assert result.prompt_tokens == 0 assert result.completion_tokens == 20 assert result.total_tokens == 20 + + def test_transform_response_api_usage_calculates_total_from_input_and_output_tokens_if_available(self): + """Test transformation calculates total_tokens when it's None and input / output tokens are present""" + # Setup + usage = { + "input_tokens": 15, + "output_tokens": 25, + "total_tokens": None, + } + + # Execute + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + usage + ) + + # Assert + assert result.prompt_tokens == 15 + assert result.completion_tokens == 25 + assert result.total_tokens == 40 # 15 + 25 From cd38e1c9deaf7b9c216bec3e8469572de63520ef Mon Sep 17 00:00:00 2001 From: Chesars Date: Tue, 6 Jan 2026 12:17:22 -0300 Subject: [PATCH 005/121] Scope custom proxy base to Playground --- .../src/components/networking.tsx | 6 --- .../components/playground/chat_ui/ChatUI.tsx | 41 ++++++++++++++++--- .../playground/compareUI/CompareUI.tsx | 8 +++- .../playground/llm_calls/a2a_send_message.tsx | 6 ++- .../llm_calls/anthropic_messages.tsx | 3 +- .../playground/llm_calls/audio_speech.tsx | 3 +- .../llm_calls/audio_transcriptions.tsx | 3 +- .../playground/llm_calls/chat_completion.tsx | 3 +- .../playground/llm_calls/embeddings_api.tsx | 3 +- .../playground/llm_calls/fetch_agents.tsx | 7 +++- .../playground/llm_calls/image_edits.tsx | 3 +- .../playground/llm_calls/image_generation.tsx | 3 +- .../playground/llm_calls/responses_api.tsx | 3 +- 13 files changed, 68 insertions(+), 24 deletions(-) diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index ed43c8aa455..f0464c61f88 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -92,12 +92,6 @@ const updateServerRootPath = (receivedServerRootPath: string) => { }; export const getProxyBaseUrl = (): string => { - // Check for custom proxy base URL from sessionStorage first - const customProxyBaseUrl = sessionStorage.getItem("customProxyBaseUrl"); - if (customProxyBaseUrl && customProxyBaseUrl.trim() !== "") { - return customProxyBaseUrl; - } - if (proxyBaseUrl) { return proxyBaseUrl; } diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx index 145560e9922..9fda837c37f 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx @@ -365,7 +365,7 @@ const ChatUI: React.FC = ({ const loadAgents = async () => { try { - const agents = await fetchAvailableAgents(userApiKey); + const agents = await fetchAvailableAgents(userApiKey, customProxyBaseUrl || undefined); setAgentInfo(agents); // Clear selection if current agent not in list if (selectedAgent && !agents.some((a) => a.agent_name === selectedAgent)) { @@ -377,7 +377,7 @@ const ChatUI: React.FC = ({ }; loadAgents(); - }, [accessToken, apiKeySource, apiKey, endpointType]); + }, [accessToken, apiKeySource, apiKey, endpointType, customProxyBaseUrl, selectedAgent]); useEffect(() => { // Scroll to the bottom of the chat whenever chatHistory updates @@ -862,6 +862,7 @@ const ChatUI: React.FC = ({ useAdvancedParams ? temperature : undefined, useAdvancedParams ? maxTokens : undefined, updateTotalLatency, + customProxyBaseUrl || undefined, ); } else if (endpointType === EndpointType.IMAGE) { // For image generation @@ -872,6 +873,7 @@ const ChatUI: React.FC = ({ effectiveApiKey, selectedTags, signal, + customProxyBaseUrl || undefined, ); } else if (endpointType === EndpointType.SPEECH) { // For audio speech @@ -883,6 +885,9 @@ const ChatUI: React.FC = ({ effectiveApiKey, selectedTags, signal, + undefined, // responseFormat + undefined, // speed + customProxyBaseUrl || undefined, ); } else if (endpointType === EndpointType.IMAGE_EDITS) { // For image edits @@ -895,6 +900,7 @@ const ChatUI: React.FC = ({ effectiveApiKey, selectedTags, signal, + customProxyBaseUrl || undefined, ); } } else if (endpointType === EndpointType.RESPONSES) { @@ -933,6 +939,7 @@ const ChatUI: React.FC = ({ handleMCPEvent, // Pass MCP event handler codeInterpreter.enabled, // Enable Code Interpreter tool codeInterpreter.setResult, // Handle code interpreter output + customProxyBaseUrl || undefined, ); } else if (endpointType === EndpointType.ANTHROPIC_MESSAGES) { const apiChatHistory = [ @@ -956,6 +963,7 @@ const ChatUI: React.FC = ({ selectedVectorStores.length > 0 ? selectedVectorStores : undefined, selectedGuardrails.length > 0 ? selectedGuardrails : undefined, selectedMCPTools, // Pass the selected tools array + customProxyBaseUrl || undefined, ); } else if (endpointType === EndpointType.EMBEDDINGS) { await makeOpenAIEmbeddingsRequest( @@ -964,6 +972,7 @@ const ChatUI: React.FC = ({ selectedModel, effectiveApiKey, selectedTags, + customProxyBaseUrl || undefined, ); } else if (endpointType === EndpointType.TRANSCRIPTION) { // For audio transcriptions @@ -975,6 +984,11 @@ const ChatUI: React.FC = ({ effectiveApiKey, selectedTags, signal, + undefined, // language + undefined, // prompt + undefined, // responseFormat + undefined, // temperature + customProxyBaseUrl || undefined, ); } } @@ -991,6 +1005,7 @@ const ChatUI: React.FC = ({ updateTimingData, updateTotalLatency, updateA2AMetadata, + customProxyBaseUrl || undefined, ); } } catch (error) { @@ -1116,9 +1131,25 @@ const ChatUI: React.FC = ({
- - Custom Proxy Base URL - +
+ + Custom Proxy Base URL + + {customProxyBaseUrl && ( + + )} +
{ diff --git a/ui/litellm-dashboard/src/components/playground/compareUI/CompareUI.tsx b/ui/litellm-dashboard/src/components/playground/compareUI/CompareUI.tsx index ff35188bbb5..92bcb21c4b5 100644 --- a/ui/litellm-dashboard/src/components/playground/compareUI/CompareUI.tsx +++ b/ui/litellm-dashboard/src/components/playground/compareUI/CompareUI.tsx @@ -106,6 +106,9 @@ export default function CompareUI({ accessToken, disabledPersonalKeyCreation }: ); const [customApiKey, setCustomApiKey] = useState(""); const [debouncedCustomApiKey, setDebouncedCustomApiKey] = useState(""); + const [customProxyBaseUrl] = useState( + () => sessionStorage.getItem("customProxyBaseUrl") || "" + ); useEffect(() => { const timer = setTimeout(() => { setDebouncedCustomApiKey(customApiKey); @@ -171,7 +174,7 @@ export default function CompareUI({ accessToken, disabledPersonalKeyCreation }: } setIsLoadingAgents(true); try { - const agents = await fetchAvailableAgents(effectiveApiKey); + const agents = await fetchAvailableAgents(effectiveApiKey, customProxyBaseUrl || undefined); if (!active) return; setAgentOptions(agents); } catch (error) { @@ -598,6 +601,8 @@ export default function CompareUI({ accessToken, disabledPersonalKeyCreation }: undefined, (time) => updateTimingDataForComparison(prepared.id, time), (latency) => updateTotalLatencyForComparison(prepared.id, latency), + undefined, // onA2AMetadata + customProxyBaseUrl || undefined, ) : makeOpenAIChatCompletionRequest( prepared.apiChatHistory, @@ -618,6 +623,7 @@ export default function CompareUI({ accessToken, disabledPersonalKeyCreation }: useAdvancedParams ? prepared.temperature : undefined, useAdvancedParams ? prepared.maxTokens : undefined, (latency) => updateTotalLatencyForComparison(prepared.id, latency), + customProxyBaseUrl || undefined, ); requestPromise diff --git a/ui/litellm-dashboard/src/components/playground/llm_calls/a2a_send_message.tsx b/ui/litellm-dashboard/src/components/playground/llm_calls/a2a_send_message.tsx index 94206937f02..0654db32920 100644 --- a/ui/litellm-dashboard/src/components/playground/llm_calls/a2a_send_message.tsx +++ b/ui/litellm-dashboard/src/components/playground/llm_calls/a2a_send_message.tsx @@ -113,8 +113,9 @@ export const makeA2ASendMessageRequest = async ( onTimingData?: (timeToFirstToken: number) => void, onTotalLatency?: (totalLatency: number) => void, onA2AMetadata?: (metadata: A2ATaskMetadata) => void, + customBaseUrl?: string, ): Promise => { - const proxyBaseUrl = getProxyBaseUrl(); + const proxyBaseUrl = customBaseUrl || getProxyBaseUrl(); const url = proxyBaseUrl ? `${proxyBaseUrl}/a2a/${agentId}/message/send` : `/a2a/${agentId}/message/send`; @@ -242,8 +243,9 @@ export const makeA2AStreamMessageRequest = async ( onTimingData?: (timeToFirstToken: number) => void, onTotalLatency?: (totalLatency: number) => void, onA2AMetadata?: (metadata: A2ATaskMetadata) => void, + customBaseUrl?: string, ): Promise => { - const proxyBaseUrl = getProxyBaseUrl(); + const proxyBaseUrl = customBaseUrl || getProxyBaseUrl(); const url = proxyBaseUrl ? `${proxyBaseUrl}/a2a/${agentId}` : `/a2a/${agentId}`; diff --git a/ui/litellm-dashboard/src/components/playground/llm_calls/anthropic_messages.tsx b/ui/litellm-dashboard/src/components/playground/llm_calls/anthropic_messages.tsx index 3f8c90424c6..304fb5124ff 100644 --- a/ui/litellm-dashboard/src/components/playground/llm_calls/anthropic_messages.tsx +++ b/ui/litellm-dashboard/src/components/playground/llm_calls/anthropic_messages.tsx @@ -18,6 +18,7 @@ export async function makeAnthropicMessagesRequest( vector_store_ids?: string[], guardrails?: string[], selectedMCPTools?: string[], + customBaseUrl?: string, ) { if (!accessToken) { throw new Error("Virtual Key is required"); @@ -28,7 +29,7 @@ export async function makeAnthropicMessagesRequest( console.log = function () {}; } - const proxyBaseUrl = getProxyBaseUrl(); + const proxyBaseUrl = customBaseUrl || getProxyBaseUrl(); // Prepare headers with tags and trace ID const headers: Record = {}; diff --git a/ui/litellm-dashboard/src/components/playground/llm_calls/audio_speech.tsx b/ui/litellm-dashboard/src/components/playground/llm_calls/audio_speech.tsx index de2bb761012..c5d4ae4d686 100644 --- a/ui/litellm-dashboard/src/components/playground/llm_calls/audio_speech.tsx +++ b/ui/litellm-dashboard/src/components/playground/llm_calls/audio_speech.tsx @@ -13,6 +13,7 @@ export async function makeOpenAIAudioSpeechRequest( signal?: AbortSignal, responseFormat?: string, speed?: number, + customBaseUrl?: string, ) { // base url should be the current base_url const isLocal = process.env.NODE_ENV === "development"; @@ -20,7 +21,7 @@ export async function makeOpenAIAudioSpeechRequest( console.log = function () {}; } console.log("isLocal:", isLocal); - const proxyBaseUrl = getProxyBaseUrl(); + const proxyBaseUrl = customBaseUrl || getProxyBaseUrl(); const client = new openai.OpenAI({ apiKey: accessToken, baseURL: proxyBaseUrl, diff --git a/ui/litellm-dashboard/src/components/playground/llm_calls/audio_transcriptions.tsx b/ui/litellm-dashboard/src/components/playground/llm_calls/audio_transcriptions.tsx index 61460951ec4..cdc512ba2f7 100644 --- a/ui/litellm-dashboard/src/components/playground/llm_calls/audio_transcriptions.tsx +++ b/ui/litellm-dashboard/src/components/playground/llm_calls/audio_transcriptions.tsx @@ -13,6 +13,7 @@ export async function makeOpenAIAudioTranscriptionRequest( prompt?: string, responseFormat?: string, temperature?: number, + customBaseUrl?: string, ) { // base url should be the current base_url const isLocal = process.env.NODE_ENV === "development"; @@ -20,7 +21,7 @@ export async function makeOpenAIAudioTranscriptionRequest( console.log = function () {}; } console.log("isLocal:", isLocal); - const proxyBaseUrl = getProxyBaseUrl(); + const proxyBaseUrl = customBaseUrl || getProxyBaseUrl(); const client = new openai.OpenAI({ apiKey: accessToken, diff --git a/ui/litellm-dashboard/src/components/playground/llm_calls/chat_completion.tsx b/ui/litellm-dashboard/src/components/playground/llm_calls/chat_completion.tsx index 799f050fc8c..4b9054dbf88 100644 --- a/ui/litellm-dashboard/src/components/playground/llm_calls/chat_completion.tsx +++ b/ui/litellm-dashboard/src/components/playground/llm_calls/chat_completion.tsx @@ -23,6 +23,7 @@ export async function makeOpenAIChatCompletionRequest( temperature?: number, max_tokens?: number, onTotalLatency?: (latency: number) => void, + customBaseUrl?: string, ) { // base url should be the current base_url const isLocal = process.env.NODE_ENV === "development"; @@ -30,7 +31,7 @@ export async function makeOpenAIChatCompletionRequest( console.log = function () {}; } console.log("isLocal:", isLocal); - const proxyBaseUrl = getProxyBaseUrl(); + const proxyBaseUrl = customBaseUrl || getProxyBaseUrl(); // Prepare headers with tags and trace ID const headers: Record = {}; if (tags && tags.length > 0) { diff --git a/ui/litellm-dashboard/src/components/playground/llm_calls/embeddings_api.tsx b/ui/litellm-dashboard/src/components/playground/llm_calls/embeddings_api.tsx index d0939c00437..84192a1d866 100644 --- a/ui/litellm-dashboard/src/components/playground/llm_calls/embeddings_api.tsx +++ b/ui/litellm-dashboard/src/components/playground/llm_calls/embeddings_api.tsx @@ -7,6 +7,7 @@ export async function makeOpenAIEmbeddingsRequest( selectedModel: string, accessToken: string, tags?: string[], + customBaseUrl?: string, ) { if (!accessToken) { throw new Error("Virtual Key is required"); @@ -18,7 +19,7 @@ export async function makeOpenAIEmbeddingsRequest( console.log = function () {}; } - const proxyBaseUrl = getProxyBaseUrl(); + const proxyBaseUrl = customBaseUrl || getProxyBaseUrl(); // Prepare headers with tags and trace ID const headers: Record = {}; if (tags && tags.length > 0) { diff --git a/ui/litellm-dashboard/src/components/playground/llm_calls/fetch_agents.tsx b/ui/litellm-dashboard/src/components/playground/llm_calls/fetch_agents.tsx index 054e2216b63..889b012c5f7 100644 --- a/ui/litellm-dashboard/src/components/playground/llm_calls/fetch_agents.tsx +++ b/ui/litellm-dashboard/src/components/playground/llm_calls/fetch_agents.tsx @@ -16,9 +16,12 @@ export interface Agent { /** * Fetches available A2A agents from /v1/agents endpoint. */ -export const fetchAvailableAgents = async (accessToken: string): Promise => { +export const fetchAvailableAgents = async ( + accessToken: string, + customBaseUrl?: string, +): Promise => { try { - const proxyBaseUrl = getProxyBaseUrl(); + const proxyBaseUrl = customBaseUrl || getProxyBaseUrl(); const url = proxyBaseUrl ? `${proxyBaseUrl}/v1/agents` : `/v1/agents`; const response = await fetch(url, { diff --git a/ui/litellm-dashboard/src/components/playground/llm_calls/image_edits.tsx b/ui/litellm-dashboard/src/components/playground/llm_calls/image_edits.tsx index 504798ae8b2..233f4201b17 100644 --- a/ui/litellm-dashboard/src/components/playground/llm_calls/image_edits.tsx +++ b/ui/litellm-dashboard/src/components/playground/llm_calls/image_edits.tsx @@ -10,6 +10,7 @@ export async function makeOpenAIImageEditsRequest( accessToken: string, tags?: string[], signal?: AbortSignal, + customBaseUrl?: string, ) { // base url should be the current base_url const isLocal = process.env.NODE_ENV === "development"; @@ -17,7 +18,7 @@ export async function makeOpenAIImageEditsRequest( console.log = function () {}; } console.log("isLocal:", isLocal); - const proxyBaseUrl = getProxyBaseUrl(); + const proxyBaseUrl = customBaseUrl || getProxyBaseUrl(); const client = new openai.OpenAI({ apiKey: accessToken, diff --git a/ui/litellm-dashboard/src/components/playground/llm_calls/image_generation.tsx b/ui/litellm-dashboard/src/components/playground/llm_calls/image_generation.tsx index 4eb8ea1b551..102b26c6d0d 100644 --- a/ui/litellm-dashboard/src/components/playground/llm_calls/image_generation.tsx +++ b/ui/litellm-dashboard/src/components/playground/llm_calls/image_generation.tsx @@ -9,6 +9,7 @@ export async function makeOpenAIImageGenerationRequest( accessToken: string, tags?: string[], signal?: AbortSignal, + customBaseUrl?: string, ) { // base url should be the current base_url const isLocal = process.env.NODE_ENV === "development"; @@ -16,7 +17,7 @@ export async function makeOpenAIImageGenerationRequest( console.log = function () {}; } console.log("isLocal:", isLocal); - const proxyBaseUrl = getProxyBaseUrl(); + const proxyBaseUrl = customBaseUrl || getProxyBaseUrl(); const client = new openai.OpenAI({ apiKey: accessToken, baseURL: proxyBaseUrl, diff --git a/ui/litellm-dashboard/src/components/playground/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/playground/llm_calls/responses_api.tsx index 8488f012e72..1b255d86065 100644 --- a/ui/litellm-dashboard/src/components/playground/llm_calls/responses_api.tsx +++ b/ui/litellm-dashboard/src/components/playground/llm_calls/responses_api.tsx @@ -32,6 +32,7 @@ export async function makeOpenAIResponsesRequest( onMCPEvent?: (event: MCPEvent) => void, codeInterpreterEnabled?: boolean, onCodeInterpreterResult?: (result: CodeInterpreterResult) => void, + customBaseUrl?: string, ) { if (!accessToken) { throw new Error("Virtual Key is required"); @@ -47,7 +48,7 @@ export async function makeOpenAIResponsesRequest( console.log = function () {}; } - const proxyBaseUrl = getProxyBaseUrl(); + const proxyBaseUrl = customBaseUrl || getProxyBaseUrl(); // Prepare headers with tags and trace ID const headers: Record = {}; if (tags && tags.length > 0) { From 9f68081f6d044faa018284fea99532c3b9374c6d Mon Sep 17 00:00:00 2001 From: minijeong-log Date: Wed, 7 Jan 2026 03:16:24 +0900 Subject: [PATCH 006/121] feat: Add built-in migration lock to prevent concurrent Prisma migrate deploy (#14440) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: prisma migrate deploy with lock Author: Mini Jeong * fix: use redis cache from proxy server Author: Mini Jeong * fix: add type checks and fix unit tests for migration lock - Add DATABASE_URL validation in _create_baseline_migration() and _resolve_all_migrations() - Fix MyPy type errors by adding None checks before using database_url in subprocess calls - Add _resolve_all_migrations mock to failing unit tests to prevent filesystem errors - Apply Black formatting to modified files Fixes: - MyPy type errors: database_url could be None when passed to subprocess - Unit test failures: _resolve_all_migrations tried to create directories in read-only /test path * fix: resolve MyPy type error in vertex_ai vertex_llm_base Fix MyPy type checking error where vertex_api_version parameter type was incompatible with function signature expectation. * fix: Return 403 exception when calling GET responses api * fix: added new step into rotate master key function for processing credentials table * Add redisvl in requirements.txt * fix: fixed the issue of handling root paths when processing Discovery protected resource metadata and authorization server metadata URLs. * fix: added additional grant type into oauth_authorization_server response for fixing mcp auth register bad request issue * fix: added RFC RECOMMENDED property(scopes_supported) to protected resource and authorization server metadata * fix: removed initialize the tool name to MCP server name mapping(oauth2) on startup for avoiding 401 error * fix: upgraded mcp sdk depency version for fixing ClosedResourceError * Use already configured opentelemetry providers Users that instrument using opentelemetry-instrument can now setup exporters as per their environment. * Handle all protocols for all telemetry * Add more tests * feat(mcp): parallelize tool fetching from multiple MCP servers (#18627) * feat(mcp): parallelize tool fetching from multiple MCP servers Replace sequential tool fetching with asyncio.gather() to reduce client timeouts when using multiple MCP servers. Changes: - mcp_server_manager.py: list_tools() now fetches tools in parallel - server.py: _get_tools_from_mcp_servers() now fetches tools in parallel Real-world impact (7 MCP servers example): - Sequential: ~4.5+ seconds (exceeds typical 5-second client timeouts) - Parallel: ~1.2 seconds (max of all servers) Fixes #18626 * fix: copy oauth2_headers to avoid shared dict mutation in parallel tasks * feat: add display_name, model_vendor, and model_version metadata * added the option of adding langsmith tenant id in the env (#18623) * fix(router): Validate routing_strategy at startup to fail fast with helpful error. (#18624) Invalid routing_strategy values (e.g., "simple" instead of "simple-shuffle") previously failed silently, causing confusing "No deployments available" errors downstream. This change adds upfront validation in routing_strategy_init() to: - Check if the provided strategy matches valid string values or RoutingStrategy enum - Raise a clear ValueError listing valid options if invalid - Fail fast at startup instead of at request time Fixes behavior reported in #11330 where users had to debug cryptic errors. Valid strategies: simple-shuffle, least-busy, usage-based-routing, latency-based-routing, cost-based-routing, usage-based-routing-v2 Co-authored-by: Flibbert E. Gibbitz * Add libsndfile to database Docker image for audio processing (#18612) The litellm-database Docker image was missing the libsndfile system library, which is required by the soundfile Python package for audio file processing. This caused failures when using audio transcription endpoints that attempt to calculate audio duration. This adds libsndfile to the runtime dependencies in Dockerfile.database, consistent with Dockerfile.alpine which already includes this library. * Fix: Map Gemini cached_tokens to Langfuse cache_read_input_tokens (#18614) * Fix: Map Gemini cached_tokens to Langfuse cache_read_input_tokens Fixes #18520 ## Problem Langfuse integration was not capturing cached tokens from Gemini models. Gemini returns cached tokens in `usage.prompt_tokens_details.cached_tokens`, but Langfuse only read from top-level `usage.cache_read_input_tokens` (which only Anthropic populates). ## Solution Updated langfuse.py to check both locations: 1. First check top-level cache_read_input_tokens (for Anthropic) 2. Then check prompt_tokens_details.cached_tokens (for Gemini, OpenAI, others) This ensures all providers' cached tokens are properly reported to Langfuse. ## Changes - Modified litellm/integrations/langfuse/langfuse.py (lines 742-761) - Added 3 unit tests in tests/test_litellm/integrations/langfuse/test_gemini_cached_tokens.py - All existing Langfuse tests still pass (11/11) ## Testing - test_cached_tokens_extraction: Verifies Gemini cached_tokens extraction - test_cached_tokens_not_present: Backward compatibility (no cached_tokens) - test_cached_tokens_is_zero: Edge case when cached_tokens = 0 * Refactor: Extract cache token logic into helper function Address review feedback from @officer47p - Created _extract_cache_read_input_tokens() helper function - Reduces code bloat in _log_langfuse_v2 method - Improves testability and reusability - All tests still passing (11/11) * Adding Role Mappings * Fixing Edit SSO Settings Modal * feat: add user_mcp_management_mode for view_all visibility * Fixing tests * fix: missing mcp_allow_all_ui.png * docs: add user_mcp_management_mode * Align responses API streaming hooks with chat pipeline * Clarify responses API streaming context * Address review comments * feat: Add GigaChat provider support (#18564) * feat: Add GigaChat provider support Add native support for GigaChat API (Sber AI, Russia's leading LLM). Supported features: - Chat completions (sync/async) - Streaming (sync/async) - Function calling / Tools - Structured output via JSON schema (emulated through function calls) - Image input (base64 and URL) - Embeddings Closes #18515 * fix: resolve mypy type errors in GigaChat handler - Fix _prepare_file_data return type (use 3-tuple for cleaner type flow) - Add type annotations for lists in _process_content_parts methods - Add type annotations in _collapse_user_messages - Use ChatCompletionToolCallChunk for proper tool_use typing - Add type: ignore[override] for astreaming async generator * refactor(gigachat): migrate to BaseConfig pattern * fix: remove unused imports * fix: resolve mypy type errors * fix: mypy type errors * refactor: address review feedback for GigaChat provider - Remove singleton pattern, reuse litellm HTTPHandler - Move constants/errors to transformation files, delete common_utils.py - Add models to model_prices_and_context_window.json - Fix ssl_verify not passed to HTTP client for embeddings * docs: update GigaChat documentation with ssl_verify requirement * Revert "Add redisvl in requirements.txt" * Put reasoning summary behind feat flag * fix: model eol * fix: anthropic claude-3-opus-20240229 EOL * Revert "fix: model eol" This reverts commit 5aa1665d79d75e0842ec44a8dc23d2e15fdfadd8. * Fix: ImportError: qualifire package is required for QualifireGuardrail. Install it with: pip install qualifire * fix: test_secret_manager_failure_does_not_block_email * fix: test_update_ui_settings_allowlisted_value * fix: test_aaamodel_prices_and_context_window_json_is_valid * fix: test_all_models_have_display_name * fix: async def test_bedrock_apply_guardrail_blocked() * fix: test_databricks_embeddings[True] * fix:test_anthropic_beta_header * fix:test_api_error_handling * fix:mypy mcp management * Revert "feat(model_cost): add display_name, model_vendor, and model_version metadata to model entries" * [Feat] New API Endpoint - Responses API (v1/responses/compact) (#18697) * init transform_compact_response_api_request * init acompact_responses * init async_compact_response_api_handler in llm http handler * init transform_compact_response_api_request for openai * init acompact_responses * fix acompact_responses * add OAI Compact API * docs responses API Compact * code qa checks * test_openai_compact_responses_api * fix mypy linting * fix: remove display name * Add the LITELLM_REASONING_AUTO_SUMMARY in doc * fix model map * [UI] - Feat add request provider form on UI (#18704) * add request provider form * fix link to github * add button * fix link * fix(streaming): normalize status code extraction to prevent 4xx errors from triggering mid-stream fallback (#18698) 在流式处理错误时，添加状态码标准化逻辑，确保 4xx 客户端错误直接抛出而不是被包装成 MidStreamFallbackError。 - 新增 _normalize_status_code 函数用于从异常对象提取状态码 - 优先从异常的 status_code 属性获取，其次从 response.status_code 获取 - 当映射异常或原始异常的状态码在 400-499 范围内时，直接抛出映射异常 - 添加单元测试验证 Vertex AI 400 错误正确抛出为 BadRequestError - 确保流式处理中的客户端错误能够正确传播，而不会触发回退机制 --------- Co-authored-by: Eric84626 Co-authored-by: Eric84626 <97266539+Eric84626@users.noreply.github.com> Co-authored-by: Sameer Kankute Co-authored-by: mangabits <1457532+mangabits@users.noreply.github.com> Co-authored-by: Costa Tsaousis Co-authored-by: Nik Co-authored-by: Shivam Rawat <161387515+shivamrawat1@users.noreply.github.com> Co-authored-by: FlibbertyGibbitz Co-authored-by: Flibbert E. Gibbitz Co-authored-by: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Co-authored-by: yuneng-jiang Co-authored-by: Yuta Saito Co-authored-by: LingXuanYin <3546599908@qq.com> Co-authored-by: YutaSaito <36355491+uc4w6c@users.noreply.github.com> Co-authored-by: 0717376 <103773680+0717376@users.noreply.github.com> Co-authored-by: Ishaan Jaff Co-authored-by: Kris Xia Co-authored-by: Krish Dholakia --- docker/Dockerfile.database | 2 +- docs/my-website/docs/mcp_control.md | 17 +- docs/my-website/docs/providers/gigachat.md | 283 +++++++++++ docs/my-website/docs/proxy/config_settings.md | 7 +- docs/my-website/docs/reasoning_content.md | 65 +++ docs/my-website/docs/response_api_compact.md | 104 ++++ docs/my-website/img/mcp_allow_all_ui.png | Bin 0 -> 137849 bytes docs/my-website/sidebars.js | 9 +- .../litellm_proxy_extras/utils.py | 133 ++++- litellm/__init__.py | 4 + litellm/_lazy_imports_registry.py | 4 + .../transformation.py | 23 +- litellm/constants.py | 1 + litellm/integrations/callback_configs.json | 6 + litellm/integrations/langfuse/langfuse.py | 40 +- litellm/integrations/langsmith.py | 19 +- litellm/integrations/opentelemetry.py | 256 +++++++--- .../litellm_core_utils/streaming_handler.py | 54 +- .../llms/base_llm/responses/transformation.py | 27 + litellm/llms/custom_httpx/llm_http_handler.py | 214 +++++++- litellm/llms/gigachat/__init__.py | 23 + litellm/llms/gigachat/authenticator.py | 241 +++++++++ litellm/llms/gigachat/chat/__init__.py | 12 + litellm/llms/gigachat/chat/streaming.py | 134 +++++ litellm/llms/gigachat/chat/transformation.py | 473 ++++++++++++++++++ litellm/llms/gigachat/embedding/__init__.py | 7 + .../llms/gigachat/embedding/transformation.py | 212 ++++++++ litellm/llms/gigachat/file_handler.py | 211 ++++++++ .../llms/openai/responses/transformation.py | 66 +++ litellm/llms/vertex_ai/vertex_llm_base.py | 7 +- litellm/main.py | 65 +++ ...odel_prices_and_context_window_backup.json | 63 +++ .../mcp_server/discoverable_endpoints.py | 44 +- .../mcp_server/mcp_server_manager.py | 141 ++++-- .../proxy/_experimental/mcp_server/server.py | 24 +- litellm/proxy/_types.py | 7 + litellm/proxy/common_request_processing.py | 2 + litellm/proxy/db/prisma_client.py | 6 +- .../guardrails/guardrail_hooks/lasso/lasso.py | 2 +- .../mcp_management_endpoints.py | 58 ++- .../proxy/response_api_endpoints/endpoints.py | 82 +++ litellm/proxy/route_llm_request.py | 2 + litellm/responses/main.py | 202 ++++++++ litellm/responses/streaming_iterator.py | 217 +++++++- litellm/router.py | 22 + litellm/types/integrations/langsmith.py | 2 + litellm/types/utils.py | 2 + litellm/utils.py | 4 + model_prices_and_context_window.json | 63 +++ provider_endpoints_support.json | 4 +- requirements.txt | 2 +- .../test_bedrock_apply_guardrail.py | 34 +- .../test_litellm_proxy_extras_utils.py | 337 ++++++++++--- .../test_openai_responses_api.py | 46 ++ .../test_responses_hooks.py | 165 ++++++ .../test_anthropic_completion.py | 2 +- tests/llm_translation/test_databricks.py | 6 + tests/llm_translation/test_gigachat.py | 349 +++++++++++++ tests/local_testing/test_completion.py | 16 +- tests/local_testing/test_streaming.py | 6 +- .../test_langsmith_unit_test.py | 111 +++- .../test_opentelemetry_unit_tests.py | 25 - .../test_router_helper_utils.py | 67 +++ ...responses_transformation_transformation.py | 94 +++- .../langfuse/test_gemini_cached_tokens.py | 90 ++++ .../integrations/test_opentelemetry.py | 103 +++- .../test_streaming_handler.py | 23 + .../test_function_call_args_serialization.py | 355 +++++++++++++ ...test_vertex_and_google_ai_studio_gemini.py | 35 ++ .../mcp_server/test_discoverable_endpoints.py | 57 ++- .../mcp_server/test_mcp_server_manager.py | 19 + .../guardrail_hooks/test_qualifire.py | 135 ++--- .../hooks/test_key_management_event_hooks.py | 10 + .../test_mcp_management_endpoints.py | 79 +++ .../test_proxy_setting_endpoints.py | 48 +- .../(dashboard)/hooks/sso/useSSOSettings.ts | 20 +- .../ModelsAndEndpointsView.tsx | 25 + .../Modals/BaseSSOSettingsForm.tsx | 18 +- .../Modals/DeleteSSOSettingsModal.test.tsx | 46 +- .../Modals/DeleteSSOSettingsModal.tsx | 103 ++-- .../SSOSettings/RoleMappings.test.tsx | 92 ++++ .../SSOSettings/RoleMappings.tsx | 74 +++ .../AdminSettings/SSOSettings/SSOSettings.tsx | 92 ++-- .../AdminSettings/SSOSettings/constants.ts | 7 + .../AdminSettings/SSOSettings/utils.test.ts | 12 + .../AdminSettings/SSOSettings/utils.ts | 21 +- .../src/hooks/useMcpOAuthFlow.tsx | 2 +- 87 files changed, 5826 insertions(+), 566 deletions(-) create mode 100644 docs/my-website/docs/providers/gigachat.md create mode 100644 docs/my-website/docs/response_api_compact.md create mode 100644 docs/my-website/img/mcp_allow_all_ui.png create mode 100644 litellm/llms/gigachat/__init__.py create mode 100644 litellm/llms/gigachat/authenticator.py create mode 100644 litellm/llms/gigachat/chat/__init__.py create mode 100644 litellm/llms/gigachat/chat/streaming.py create mode 100644 litellm/llms/gigachat/chat/transformation.py create mode 100644 litellm/llms/gigachat/embedding/__init__.py create mode 100644 litellm/llms/gigachat/embedding/transformation.py create mode 100644 litellm/llms/gigachat/file_handler.py create mode 100644 tests/llm_responses_api_testing/test_responses_hooks.py create mode 100644 tests/llm_translation/test_gigachat.py create mode 100644 tests/test_litellm/integrations/langfuse/test_gemini_cached_tokens.py create mode 100644 tests/test_litellm/llms/vertex_ai/gemini/test_function_call_args_serialization.py create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RoleMappings.test.tsx create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RoleMappings.tsx diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 0e804cbfd12..9a4e9a315ea 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -48,7 +48,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root # Install runtime dependencies -RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip +RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile WORKDIR /app # Copy the current directory contents into the container at /app diff --git a/docs/my-website/docs/mcp_control.md b/docs/my-website/docs/mcp_control.md index f6ab4a73087..a7d66a6b7fc 100644 --- a/docs/my-website/docs/mcp_control.md +++ b/docs/my-website/docs/mcp_control.md @@ -108,7 +108,7 @@ Some MCP servers are meant to be shared broadly—think internal knowledge bases 3. Toggle **Allow All LiteLLM Keys** on. MCP server configuration in Admin UI @@ -634,3 +634,18 @@ Control which tools different teams can access from the same MCP server. For exa This video shows how to set allowed tools for a Key, Team, or Organization. + + +## Dashboard View Modes + +Proxy admins can also control what non-admins see inside the MCP dashboard via `general_settings.user_mcp_management_mode`: + +- `restricted` *(default)* – users only see servers that their team explicitly has access to. +- `view_all` – every dashboard user can see the full MCP server list. + +```yaml title="Config example" +general_settings: + user_mcp_management_mode: view_all +``` + +This is useful when you want discoverability for MCP offerings without granting additional execution privileges. diff --git a/docs/my-website/docs/providers/gigachat.md b/docs/my-website/docs/providers/gigachat.md new file mode 100644 index 00000000000..13eec298c25 --- /dev/null +++ b/docs/my-website/docs/providers/gigachat.md @@ -0,0 +1,283 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# GigaChat +https://developers.sber.ru/docs/ru/gigachat/api/overview + +GigaChat is Sber AI's large language model, Russia's leading LLM provider. + +:::tip + +**We support ALL GigaChat models, just set `model=gigachat/` as a prefix when sending litellm requests** + +::: + +:::warning + +GigaChat API uses self-signed SSL certificates. You must pass `ssl_verify=False` in your requests. + +::: + +## Supported Features + +| Feature | Supported | +|---------|-----------| +| Chat Completion | Yes | +| Streaming | Yes | +| Async | Yes | +| Function Calling / Tools | Yes | +| Structured Output (JSON Schema) | Yes (via function call emulation) | +| Image Input | Yes (base64 and URL) - GigaChat-2-Max, GigaChat-2-Pro only | +| Embeddings | Yes | + +## API Key + +GigaChat uses OAuth authentication. Set your credentials as environment variables: + +```python +import os + +# Required: Set credentials (base64-encoded client_id:client_secret) +os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here" + +# Optional: Set scope (default is GIGACHAT_API_PERS for personal use) +os.environ['GIGACHAT_SCOPE'] = "GIGACHAT_API_PERS" # or GIGACHAT_API_B2B for business +``` + +Get your credentials at: https://developers.sber.ru/studio/ + +## Sample Usage + +```python +from litellm import completion +import os + +os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here" + +response = completion( + model="gigachat/GigaChat-2-Max", + messages=[ + {"role": "user", "content": "Hello from LiteLLM!"} + ], + ssl_verify=False, # Required for GigaChat +) +print(response) +``` + +## Sample Usage - Streaming + +```python +from litellm import completion +import os + +os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here" + +response = completion( + model="gigachat/GigaChat-2-Max", + messages=[ + {"role": "user", "content": "Hello from LiteLLM!"} + ], + stream=True, + ssl_verify=False, # Required for GigaChat +) + +for chunk in response: + print(chunk) +``` + +## Sample Usage - Function Calling + +```python +from litellm import completion +import os + +os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here" + +tools = [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a city", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name"} + }, + "required": ["city"] + } + } +}] + +response = completion( + model="gigachat/GigaChat-2-Max", + messages=[{"role": "user", "content": "What's the weather in Moscow?"}], + tools=tools, + ssl_verify=False, # Required for GigaChat +) +print(response) +``` + +## Sample Usage - Structured Output + +GigaChat supports structured output via JSON schema (emulated through function calling): + +```python +from litellm import completion +import os + +os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here" + +response = completion( + model="gigachat/GigaChat-2-Max", + messages=[{"role": "user", "content": "Extract info: John is 30 years old"}], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "person", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"} + } + } + } + }, + ssl_verify=False, # Required for GigaChat +) +print(response) # Returns JSON: {"name": "John", "age": 30} +``` + +## Sample Usage - Image Input + +GigaChat supports image input via base64 or URL (GigaChat-2-Max and GigaChat-2-Pro only): + +```python +from litellm import completion +import os + +os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here" + +response = completion( + model="gigachat/GigaChat-2-Max", # Vision requires GigaChat-2-Max or GigaChat-2-Pro + messages=[{ + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}} + ] + }], + ssl_verify=False, # Required for GigaChat +) +print(response) +``` + +## Sample Usage - Embeddings + +```python +from litellm import embedding +import os + +os.environ['GIGACHAT_CREDENTIALS'] = "your-credentials-here" + +response = embedding( + model="gigachat/Embeddings", + input=["Hello world", "How are you?"], + ssl_verify=False, # Required for GigaChat +) +print(response) +``` + +## Usage with LiteLLM Proxy + +### 1. Set GigaChat Models on config.yaml + +```yaml +model_list: + - model_name: gigachat + litellm_params: + model: gigachat/GigaChat-2-Max + api_key: "os.environ/GIGACHAT_CREDENTIALS" + ssl_verify: false + - model_name: gigachat-lite + litellm_params: + model: gigachat/GigaChat-2-Lite + api_key: "os.environ/GIGACHAT_CREDENTIALS" + ssl_verify: false + - model_name: gigachat-embeddings + litellm_params: + model: gigachat/Embeddings + api_key: "os.environ/GIGACHAT_CREDENTIALS" + ssl_verify: false +``` + +### 2. Start Proxy + +```bash +litellm --config config.yaml +``` + +### 3. Test it + + + + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data '{ + "model": "gigachat", + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] +}' +``` + + + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="gigachat", + messages=[{"role": "user", "content": "Hello!"}] +) +print(response) +``` + + + +## Supported Models + +### Chat Models + +| Model Name | Context Window | Vision | Description | +|------------|----------------|--------|-------------| +| gigachat/GigaChat-2-Lite | 128K | No | Fast, lightweight model | +| gigachat/GigaChat-2-Pro | 128K | Yes | Professional model with vision | +| gigachat/GigaChat-2-Max | 128K | Yes | Maximum capability model | + +### Embedding Models + +| Model Name | Max Input | Dimensions | Description | +|------------|-----------|------------|-------------| +| gigachat/Embeddings | 512 | 1024 | Standard embeddings | +| gigachat/Embeddings-2 | 512 | 1024 | Updated embeddings | +| gigachat/EmbeddingsGigaR | 4096 | 2560 | High-dimensional embeddings | + +:::note +Available models may vary depending on your API access level (personal or business). +::: + +## Limitations + +- Only one function call per request (GigaChat API limitation) +- Maximum 1 image per message, 10 images total per conversation +- GigaChat API uses self-signed SSL certificates - `ssl_verify=False` is required diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 87064d442ae..f4359a86ba9 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -111,6 +111,7 @@ general_settings: master_key: string maximum_spend_logs_retention_period: 30d # The maximum time to retain spend logs before deletion. maximum_spend_logs_retention_interval: 1d # interval in which the spend log cleanup task should run in. + user_mcp_management_mode: restricted # or "view_all" # Database Settings database_url: string @@ -230,6 +231,7 @@ router_settings: | image_generation_model | str | The default model to use for image generation - ignores model set in request | | store_model_in_db | boolean | If true, enables storing model + credential information in the DB. | | supported_db_objects | List[str] | Fine-grained control over which object types to load from the database when `store_model_in_db` is True. Available types: `"models"`, `"mcp"`, `"guardrails"`, `"vector_stores"`, `"pass_through_endpoints"`, `"prompts"`, `"model_cost_map"`. If not set, all object types are loaded (default behavior). Example: `supported_db_objects: ["mcp"]` to only load MCP servers from DB. | +| user_mcp_management_mode | string | Controls what non-admins can see on the MCP dashboard. `restricted` (default) only lists MCP servers that the user’s teams are explicitly allowed to access. `view_all` lets every user see the full MCP server list. Tool list/call always respects per-key permissions, so users still cannot run MCP calls without access. | | store_prompts_in_spend_logs | boolean | If true, allows prompts and responses to be stored in the spend logs table. | | max_request_size_mb | int | The maximum size for requests in MB. Requests above this size will be rejected. | | max_response_size_mb | int | The maximum size for responses in MB. LLM Responses above this size will not be sent. | @@ -669,6 +671,7 @@ router_settings: | LANGSMITH_DEFAULT_RUN_NAME | Default name for Langsmith run | LANGSMITH_PROJECT | Project name for Langsmith integration | LANGSMITH_SAMPLING_RATE | Sampling rate for Langsmith logging +| LANGSMITH_TENANT_ID | Tenant ID for Langsmith multi-tenant deployments | LANGTRACE_API_KEY | API key for Langtrace service | LASSO_API_BASE | Base URL for Lasso API | LASSO_API_KEY | API key for Lasso service @@ -707,6 +710,7 @@ router_settings: | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development) | LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers | LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60 +| LITELLM_REASONING_AUTO_SUMMARY | If set to "true", automatically enables detailed reasoning summaries for reasoning models (e.g., o1, o3-mini, deepseek-reasoner). When enabled, adds `summary: "detailed"` to reasoning effort configurations. Default is "false" | LITELLM_SALT_KEY | Salt key for encryption in LiteLLM | LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections. | LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM @@ -774,6 +778,7 @@ router_settings: | OTEL_EXPORTER_OTLP_HEADERS | Headers for OpenTelemetry requests | OTEL_SERVICE_NAME | Service name identifier for OpenTelemetry | OTEL_TRACER_NAME | Tracer name for OpenTelemetry tracing +| OTEL_LOGS_EXPORTER | Exporter type for OpenTelemetry logs (e.g., console) | PAGERDUTY_API_KEY | API key for PagerDuty Alerting | PANW_PRISMA_AIRS_API_KEY | API key for PANW Prisma AIRS service | PANW_PRISMA_AIRS_API_BASE | Base URL for PANW Prisma AIRS service @@ -888,4 +893,4 @@ router_settings: | DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL | Time-to-live in seconds for health check lock in shared health check mode. Default is 60 (1 minute) | ZSCALER_AI_GUARD_API_KEY | API key for Zscaler AI Guard service | ZSCALER_AI_GUARD_POLICY_ID | Policy ID for Zscaler AI Guard guardrails -| ZSCALER_AI_GUARD_URL | Base URL for Zscaler AI Guard API. Default is https://api.us1.zseclipse.net/v1/detection/execute-policy \ No newline at end of file +| ZSCALER_AI_GUARD_URL | Base URL for Zscaler AI Guard API. Default is https://api.us1.zseclipse.net/v1/detection/execute-policy diff --git a/docs/my-website/docs/reasoning_content.md b/docs/my-website/docs/reasoning_content.md index fca3df638c7..04c6d7ee6cc 100644 --- a/docs/my-website/docs/reasoning_content.md +++ b/docs/my-website/docs/reasoning_content.md @@ -591,3 +591,68 @@ Expected Response + +## OpenAI Responses API - Auto-Summary Control + +When using OpenAI Responses API models (like `gpt-5`) via `/chat/completions` with `reasoning_effort`, you can control whether `summary="detailed"` is automatically added to the reasoning parameter. + +### Enabling Auto-Summary + +You can enable automatic `summary="detailed"` in two ways: + + + + +```python +import litellm + +# Enable auto-summary globally +litellm.reasoning_auto_summary = True + +response = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=[{"role": "user", "content": "What is the capital of France?"}], + reasoning_effort="low", # Will automatically add summary="detailed" +) +``` + + + + + +```bash +# Set environment variable +export LITELLM_REASONING_AUTO_SUMMARY=true + +# Or in your .env file +LITELLM_REASONING_AUTO_SUMMARY=true +``` + + + + + +```yaml +litellm_settings: + reasoning_auto_summary: true # Enable auto-summary for all requests + +model_list: + - model_name: gpt-5-mini + litellm_params: + model: openai/responses/gpt-5-mini +``` + + + + +### Manual Control (Recommended) + +For fine-grained control, pass `reasoning_effort` as a dictionary: + +```python +response = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=[{"role": "user", "content": "What is the capital of France?"}], + reasoning_effort={"effort": "low", "summary": "detailed"}, # Explicit control +) +``` diff --git a/docs/my-website/docs/response_api_compact.md b/docs/my-website/docs/response_api_compact.md new file mode 100644 index 00000000000..f5caa32ea33 --- /dev/null +++ b/docs/my-website/docs/response_api_compact.md @@ -0,0 +1,104 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# /responses/compact + +Compress conversation history using OpenAI's `/responses/compact` endpoint. + +| Feature | Supported | +|---------|-----------| +| Supported LiteLLM Versions | 1.72.0+ | +| Supported Providers | `openai` | + +## Usage + +### LiteLLM Python SDK + +```python showLineNumbers title="Compact Response" +import litellm + +response = litellm.compact_responses( + model="openai/gpt-4o", + input=[{"role": "user", "content": "Hello, how are you?"}], + instructions="Be helpful", + previous_response_id="resp_abc123" # optional +) + +print(response.id) +print(response.object) # "response.compaction" +print(response.output) +``` + +### LiteLLM Proxy + + + + +```bash showLineNumbers title="Compact Request" +curl http://localhost:4000/v1/responses/compact \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "openai/gpt-4o", + "input": [{"role": "user", "content": "Hello"}], + "instructions": "Be helpful" + }' +``` + + + + +```python showLineNumbers title="Compact with OpenAI SDK" +import httpx + +response = httpx.post( + "http://localhost:4000/v1/responses/compact", + headers={"Authorization": "Bearer sk-1234"}, + json={ + "model": "openai/gpt-4o", + "input": [{"role": "user", "content": "Hello"}], + "instructions": "Be helpful" + } +) + +print(response.json()) +``` + + + + +## Request Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `model` | string | Yes | Model to use for compaction | +| `input` | string or array | Yes | Input messages to compact | +| `instructions` | string | No | System instructions | +| `previous_response_id` | string | No | ID of previous response to continue from | + +## Response Format + +```json +{ + "id": "resp_abc123", + "object": "response.compaction", + "created_at": 1734366691, + "output": [ + { + "type": "message", + "role": "assistant", + "content": [...] + }, + { + "type": "compaction", + "encrypted_content": "..." + } + ], + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "total_tokens": 150 + } +} +``` + diff --git a/docs/my-website/img/mcp_allow_all_ui.png b/docs/my-website/img/mcp_allow_all_ui.png new file mode 100644 index 0000000000000000000000000000000000000000..f074deb801eb4ce4c9ad2a48d80f66007a8cb6d2 GIT binary patch literal 137849 zcmeEubyVC>vM=rgg1fs*aCdhP4#C~s-3d-`cX#(-L4$j+!3pm0CcAs@{@#7Nd-(UA zb7#JDrl-5BrK-BVpQ`TfL@3HjAj09mfq;M@N=b?;gMdIsf`EWW!$1LZIuQBIKtMqC zEkr~Vr9?!C6dmnMEv!vIK&ZaD#0$xQDWQc-6_O;0!Uti{N3TF&u#`dCvP8k?M1THr z`hDip^aV33`sw4xnSMfKdTnwsp;cEgMP&MwxqLOWAH};aIqC&scMh$DVNQgp@Xima z-^Xl0(AQaJuUG?4#w>fE5MT-Jkq5%d)9I?Iq$0OE+){i*EXt3WFBX5z%&F1rZ)Q#| z=wQ|`-uYeO&u1~F;oKl8YZ`|_N)j8Hj2i{Ig}*@2i*?erE^UMNwPED2vql|4T#!5D zs4HI$A`SNGWawM(NPCDptq574(;moRAAuJN{V@6<Sf~R;gh{ZZlEd*Phcj;jlqex=D851$xIBqq&$hb=SMD7w zCy!})-m8v!{rdh&@Y;4Z_tx^xvkms|Sg;uF$<5g;(;F^^*_GB6GB0tfxl3IpWs}u|EdignG613Y4B)Z8HkXIh?ErYTgBMX#KhLg z+|K!a>X90l0c$U*k@7LeQY2t42&z@|Z z{xL1!1R4L7FfubRG5#wuXA9GRL-wcS?___Q*WbJ2{nHq?qJ_JOwWg?r4UpBqq46_w zu`==gt)Ks==$|9~JE@wJiKB>}4Up2A|DVG82k~DE|0m(!`qcVopIl6wf9>)wC4Zs( zGY4*E6DK=smp?pIv$b&MXW?c1SK0rQO6#9w{LG&|{e$SQx&Mu%lO|~%MV9Ga^(gBA_yWSDx~TT zdXfd5HZO`9D)F(-ueTu1itr2fH8`+50XBasS9|&W>N5 zI`18SZho50JGtWzu^kP9KIGs!;L-NZ@sJ6MF|WRFo_YMht$Qt~;Q$;odszPi7?&SEpwTC8h8S34-ISHevglEQ) znJV*MZBRvPKzyf`#S3PB%+2SVwB=+>ecxK~|D*W0|Eu`_P8k2I`2WEK{jcKxhjsb? zx5c*y{YZv#f4BKIm*e+R_4@i0%c+*sOZ_)tw{-%e%8df@?I9GX}oT4;n6)gVZs5+fl5gG7N2!v|)Ul z%n|RixkQMl{#&gkIqKE zzSOwk3gCt;@4^|Gau6u}ldpO5S`oux>x!B6=KY zBBma+C*aB6!RK&eFea=JdE|DX0~qQ6h;3=5=kc0h7g?j$jAC*J1CcF^_s!M{A{RB- z_V;dt3S7IeG)X|K5}K9%?(kbl4IXm@^}F;)5DdT%T7b*T&@Oy{Kn)AAEw1GKd`yUM8Ka0Xq!47}w=jM_IV`Uj;>UfTfN5hL<^lZC5#|ga75izvw=En=R zCcaNWw$?}toP(#ah212TXc*9j{9bQbx)(9^()5ae%6ymgzE+50VFV>$bcqKKKI|M^ z@-edgY3a>JSiiIFA)72s^8Xg^zeA?>lPFB>!Ua6MW{jVHz()Zm6_H5E_TiqezzY9F{KMRp%@$fM#1+7O?4XC8d_t2VKBjp0JL_W&;Dj7hJ0^_+W0Sm}B5 z(0>k)O{mxHV>m_(d!!Q%~K`PrC9k7u!x zOq0*>*&JU0RopGmjFnNV=qrX0P#4jtdJEHk?$mj?+w@af_-^z^Ly&m_{9Yd?_0QwwBatG?qdMU&RP&T2dcp=1h z{%~hLBQe*>HN~^4LmZ$9V-PsEsP9+M{cx*&v+lN8A;Z`us_VDza`YXyMuTI`-iJ=D z!5R*^wWqh>D8mk=`&fyuyW+Qx*uO?#0g^CyJhzFH**6riZ#u}ZocnwnFwc+2cI83? z7>`D-XDi1#z=2)y9GmV1USpQx*7WF^v{5(EJWCQQVeM-Gauh_I2ZpUYPM!z@lLklb zj;xMVZ!D9$A`8S>rPBsaXw?+gTmArG^8;<$iB39J#uSH5xK%<(mBAVw|w+ z?RRR>9|&f(c6#0ja>>LWfm5|`9d5K<=*$xKivc*_ zW&5;OlZ3#REwh%CULXD(R^@;%u>q@H1alDd{~X2HI(jDad|cC$R7J%8)HVBqpKdrNC!gLr z6!EY`8jaENe7jwYXvZYg0zP%q?`=R@%_!Yjx7YRaS}WJF+|d?g4piof9E;SJPV%{! z)b9M*+g*=9fifB)|8vndKDRu|YLyasC8ZOo&Z;va8BC&73`@#sz)GGAhX_`I8*+T@ncLJv~Sz!tNycNTOIiL45r1VhfmYS27N<$Hj3d;HQm`7R6L_+U2-V}emn2nPn2^6w?88M)A$b#U7HOL?OgZsrWA)8b6 z{dlS=*G*)1Hk^1FBO}O=!yx2G1~c^i9^`brPUp>@kACf*SzTa!CBH&6d)9FWBa6p@ z=;pZ%YUlo$j7KO*ymX?X3}cN`#!lbWGYf-~4Fy{W7r?EsxG?NdD+oH@?vha&%VpT_ z-ed)k*L9s-^H7@a6Q-5`8pHE>2fYP5T(x2`Suwdf6a2-r8?40-}dsNJeNNZZ%Q$JND? za?+)?tSNq91Oz?$80Zh|s1c}4#Sj`&4mEbWZFUIlUG29~Qqj)_C(|z8V`z16qtitC z9A>Wbuv;%xW>+tM+K%ci3}3L}so?XvD-#0uEW4WIVch1PRR(pRU1~k7@>f}DG;C_; z^J@10WJ`twilFZ6y%ql`t&{2Zbb8l5YgIE^c-GF^(SEv+kq987)`747HN*u!e~a%m zzqm=!`Z`IT7#NCllo$7Wbc{O}OwD+7W46ks5r;vY)^9d=1ibqh5&wR`)Gu$&@~rS1 ztdMS-9a60@MKmg7D6UVFU9I(?ycviSDEonOX}B`@%k0IAor1`s^!L^;)VZ^+^8?n; zx3kV0sbAVT1|$@eGd5Q87|@W!QHMXMr<1S2l0U%IDo7c??u31(&r#D{C{xo_tI}a0 z+s*NP6^j<<|C#6Rdw*!=GM}tbid)^IZZhRs5xmz%@nv~#Ej?vJf7D~$FH%UO%9GwD zSqxsFaHYwza`CGkAbsH|$M01Q@pw@+E}1qxGt5Mn4EZA{RXAldVU25KeoJoU=c)8Qrfj{Bw^( z!mdP?OIw?oqOao*JgUcU`%1TVVT9IqlS7S>}Z?!{<&} zv>{mV++UeL%2eJ;mz^+z6O%oFj&na*?YK!@bZZ?)qJo!k}k z-pgdp;RZyr-6nc+!cw)(!U^QjWi+C~djcQ+A`p@msFC>Qgon(pL z#x7^m!t05&VT)&<64|ZiM4L=jtNpLIJT8M9x~%s;o4uGkgcjy6_4T2nNG3w)X}179 zKnF)FlYEKs;+v^76#LB%LUVHFNk~+sTfgUsv1D08aYagUPp}?*fog*`2`PRs#YaIY z1klwR+)ZVL#ps@hhf#@7trWM^u(En-FA1-&gTcY|MFfaYF5G*GA~_XpjPz<=8@^K< z3X^zl<|bENZKSM)zpI!+@huKt&<9K)&|fsr`igr(QVB>g^_{=cXmy5vZrVp=v-z&0 z)@UyzcFq_2A+<~nM14^47Kr#8H8BaF$%s#kS^)tp$W##pO_Cq(G=E^?1MynHj;nmi zx9hc1|Kmp`(;MI02TaR(TB~^^0Wyf&W>#nK<`NnPhSHi2_R}8>4O8c`HGpxqoc_&a z3VVU|e2;>L)5gRXZ%l^I)$J~aRB%^O{pDWcDOK51Qko&@K$o5<90aYs!?g!LUaxd3 z2y}JEI#<5N?VIjmI|s*61dA)q;*=g3@FtylbYFQlm}50q-(YjRUtars&@Dul-p%c( z+Q)W9TcLjwtEH%Pp~jdu;O3LSPMzNiEMYJJ4JegQy=j{gKG$PRw`faA;R^~$(RpvM zL)%fDK(VDN4@ZA?u#jfJLP1B!}6|sA}P5Hu{?|(C7= zAa@wsDA2Sl$>+rIWKQ9GUiY&+ja517Qg~C^e7?`AKK))XrE+JgP%_5n9MHb^IOk*o zLt8bLJm#gR6la??7;`Dg=HU1ndT_Rgv0{G$$1}y41U7R5V$1TnX>k)lTkDfu-*&@; zqsJLp{5}jG_jS7Cu&3YFt2c8FU@W;JvC{akK6SwNHnB1qdy*5>d%%5&j6u0I{@cZ~ z6KD`+cwI~njskCWk5?!HKECrSICPBScxBc(<;K!x>595LaU%nwcdSC2)oNqQ+$tZ>92o@(uhDD!hwzFl= zGAV#+QhoKlI}%#wz1w%3Z4(CG``BUEa^CpaeCI%JCR&D71D>W%1}QbgC(`-lPZJ~9 z9>fApWBsz(VJ|Bsi9ErKC9X9(28-^hI5rDyQ_jJ(nc6Op9MBQ!s^Cs}0Y{HW0?!9J zJe~FKu(N(VuN@m9&wBm*Vq9bTvfb1>#&XXq?J<_mg_wB3wY zzONGDnLsH9qxRlf>he`;DKK0X>EbY^*%$Wn_2=xhF|t z<+9B7)09eTZ&h^#uvduOgY`C6A+6}N0V!(Vuh5YuBRb#7Z=o}wZWWchLv8y{G!)AA zwB@1j@!@vte0huXR$Zj|^f)X2Wan#?|BS1VJ7bQsl<)wK_}V|yjH^?*XcFM569xFG#l=X<%r&_6~esj34#+t=)19>w)Q-j}qxQEr{>AK)jrFZqs)8dzM z(_2AYlbV*igN@61SDRYUqXP6$;ftoJ^CXH}KcRHzN|#*^ErUXhMhGEwA%Qnh4vk)A zc>C+)ai;MSEoX~J4#kH`-FSwqqnWNmegz)F1!h{uaqZO`=G*by?Nz&;0_aAEs=IZ! zxON_i{s(mL15me_TpKDib{k)rvw5MLK4`)+pYzU&nhQ{DkdbuOu<8oI0BE$M+XT6c zmU4~-@*ljPo^a{S56#tVjppV(MZO`=7wPV}Z zL{;2j@sP5nr%umiqn_Tt?I#7I>Ow~Sfo30&IVwDEr}HlQ6hW|e8;LSdm*=Vqi@sxb zpTTIN9E~@g#SE#VV)38!s(f5@lO&Xm3N;CYsAB?l;tCN@Hs9)O!Ob(;oB+ygjucc3 z1PNZL@+%3c_e~zVJ}TStQyD#)nr9l}tg)_{JR0&YQ+!-%1!lmzX?5#b#h8C1q3^R& zRVsFgJ^kg3#B6sVgQiKo&E8~|%3dc=7_UTIV0NCrbY%<>saIn#>g8>$wCno3J{aEo zoDvxC>eQ`f3oHNJl+Zmq!te8)ZoOJoiC2ct%Q?k&AS1{lJNJOc{kO`^17=uLJ+pSR z_I%YPvu^%<0i1)gh;9R+%(YnAj9;475DQl|C07n(zO*A z7n1Yrl_a5h*Jl$mo`htdbBKU(1Ca@p6r%OT^bO3R&E(5VDE*Na zws-68w+cDsgmJs>2X$ikhi02gy#%*xCE5JbUs)Q>vao8^)@AWFQ~WU7g^XJD!rCpq z9=o(JC;4yVbk!{vwp9AnNi?_ShEsQux=0I&1^CM@1n9KkRjaNCuW$fZcpaEZYbR;H zfruF+hADXr8W_u)Ce*=;z!6 zy~v6onUsooEB(cDv(9nbu>`FuJ#=0N8m_d~DXuEp9%@@M8*+yziW}9CdU7y%Ok1@I zRl15T4Eu*MjhpVp%{4lP4EVCA-Sx~A2z6?4IE)cHb?_8-y-RYD=X-zYHehf&J|DuM zEknK1jn$-GI@$gDD_WPHx;&Ug2ndA;+Q%upq$E=0DW5U7VP8e)FDK~ATLGQi7|j`< zzW=6>Pd_9ZXc)}()kQ-~5Z~gWD2Wy$X}*~LI#p)Fb9bc6NbjM=JI-4=tS-ATr&nBb zr(vnoBQf`hfc^b9idIWC5Mz@NA z9G_e$M1s6xxg7L(TRZ3T1)(PE4b6^vYz(alo>e^927}px;rgdqKLwY=qSUlPe{v}Qj88T@RFq(@?d*>? zhvhhtfzzj^*4JqfmUC$UFvqEEF0mx5fKy0p1dNYI>;Kp^T95mvUtT zJvJXUE|RCpe1?#=Z&YbZG67pFH7p7pFb$+ocTeytxP4iZ7G;lGGd*i^a6vB+Z|BV%sC zdtnhGwwXp$n2!{cd6z1O-CJ*7BOP|6p{F4yCds)+h(TvCddfl4vH z?z4Dmxf{RHUGg%{*mH!HE%5SO(JoEl@#_iSA@*fAsnzsZhQe0(_ltS=)7Ir5xWEFPPwr4!pZJMgTE};?pNZAi zkP#*`V9V{x5TR1_vuD~UIyQjumeDrfg6@|yFPC2-hePwXH=t+Dr=4G`cQU(EBD^HJ zA3tvft=e`i8)D1rnnOzzJ=dT-;h@?#eCh6MYmrJG_IWv~u)F?3rHNqDacUYSrEKD` zQ7iZ&zM*oDZ$EIecod-YEze#u(#&&)D*HwhCxK())&dx zz^lSBy35?-x?hcIgL(>adI)DB(~7Afy?TpU;~ERN-T@Con*?_tQrbAz@k{aY6QmH< z^IK=TJTX!Mkuod#;{YvF+7k#9p}BuDKCNS-!L?E}!2NX6woxfQv}BPE}1>zhItrMHSi&^5C4d`vAHhaiPO< z1f22%+2|mmYYAOmNjD*yjwqi~#_7)YmCP#9NVBvqKmS+rjsuI#CRL8T;1<868k0ikz^h#OWcW_T=kJq>~DVaR;U$AqV1xg31KnQtb#Hdf%1Wz<1a=Jx=kD)EEM~E*k z(yskc&`O(4^7Xr3pg%08tgF#z{2oV9c{H-%$l__bsRAdQXc#);#Sv<=<$Da%MT7fE znO#{?)nw3=oYY&+o|J&KHxrvvy(((6dikvxq~&;!$Y5B0y(zgci@?&Std$~g{t_4m zAMdsNO@^ttdUwD^q>Apj8N~`1C0h58+GoXnP7*Th@rr1^xfq6?pC*X2{YO6!rp56Nvfsa?5(DFn?N4vDQ~Swv(P6+9hvvT>vNLp*(ib# zG*^~Cph~u_+-USMlSZXoRFPeCf1zxV?BuS$`LshdBvPStxuLjxwZRO&oRMqw<73b( zcvv|$;}1mcSPigl#O(5$=%cy3Cd9)*le&wsqs2gFLi$HZO=%;v!CpLy z_?tvnx#lgOeRVW1Inmnj*2Q0Cg`#EB?L7z_`wlGxFTykt4hW6IkBHi(h{x(f&p%qH zmaUB)H8?jx%K8Q~RX%GPCFCoi=+j6?mt>b_?pn-OyaX3ey-FsDVa>VRx#ZwI>d5)} zHX9QgX~0!P?#CQQE~X7~=9s?Kqg%~YdPLpq=XUGPF)K}XbDHJtG|%jcto>S;$br4j zu2HKgiVZQnG~hBY-yS5?d~-4XDSGaE6BA=O+o`YBpBA}0cDW+#PS#87QtawH zQsW!3nysVL%@4O!?MbO#MG#zpi=e;k;`aXWMYqrP+kqwE zN_%)~BMtWFTy_iU%f%lW7OpxvF8PQy zo>pPHsx`*(trx6KB6^Fa)OXKOS508E)mui`2-Qke$|IcL?0T-caX1`838t?j z?;^sx07Ypr&6z$3UhC@^1hg8;a%s(r8|T3c2eD#*Ql` zf_hKu6REeQimFskvR1ilMgR=<8<~aXg(G5{?a**tp{P7&;yiBZRm9rmoHd)7xu-JTpQO9qvYnq3W66avu20s~b-iqI-BNbjDe%B;zogQ6 z4K?Wd7VF7I^kF(wPR@MuyglJ5AjtR@+|uQ$B5h^@3X6HWK-DNWzh= zl=yA%LJu*=meZevJR;7KdCjoJ3@vu0lpKY>=EzZ?Arh{sOk}r+R_VO7FIT0ypKDYq ze(rvqoB>x;_t=9outXGf zsU3%f$hin(*%l8P{8XKTt|9OoKW(E_Ygo=2Z}B^5=6 znMRj^+PRqs5>nPQ~|RLne0c^JL%$8 zWU_)=2FPd?wU2|W1QGr!A(3ozb?1ICR4Hqx2PP4xCnaRxdEirG^jI9PA%Hu;wq_cR z-EW!FB9$j-ZU?qPOHq=AN(1djS0d+7&xkl!&kU~G6=6;YYj>eB+7|HJg_o4qRS^uc zB74JkU>o|>R+0{#8r4{!kkrVN-kuSXDpf`V?gszAsCZ*t% zHuB2R3S+VSm5?kXZFDl+QMS1?6PrJk1ZxI+yay$>;zbi7x6b$YLB(G8j2sK=bg9WN zJ^{<458aj`qv>p0O8{-doK#QRC~4S$r`oD7AAJb{r7_De{|5^kG!Hn1xB*K)7%b&B zxxsS9TQ~@0$!OS>UcFJ1<$Slq^h;cGW*JY}-jq>t+)jJ)AQklZ_DuftX5F#Pl%>JL zoE#l}SJEIA$l?`#`?Z>)f9uePvQMAUdh~|fQu`o&@Pms(F%SBL?ET8_Ixi>i35BGN zP(nIGI?Jj;6KsF^(Vj#5e0DhR*VJmB_p|S#5A{lic5bQ7Msr4>ORlzZIeoS7jof^(F{1)-#|0Gb*S<}M z=k4h}UO^W{Qdk1k;{GC|+sTT;FbUgmL@#8(}XuEubve};mH|j_P%vYA4*4z!PUE<#Uc><263q< z)-PjaV_Bxifv%Q^1iKr?l#RQ+KgtpeMSweOWu(`**Jc%&xE`Kch%8Pp$~;Dot2Kc6 z*j4Yz_uGrvV+S^>ka%>~$wODnE9{F=hgsrnw?*}^-i8a{bh4Pxj}lx%k|-Gl3z8{m z$R%6;dIg4@d6vitE0r_(O#CD)Pt4&2VW8R&81**4Pod-ruVLMjsd{=-SdEBn_DA!#Tl&SGYAixc$=CN|Eg`_`|n4qjEBM~D| zr&P$Ci`-3b06*5m))@=!6E&DR)O#1Mw9^#0F-k<+YKF^h zJHyO#dTqWSlghwgTdVK9Mv~F)FsXMr+iwCo%9c}3u|O>~wUEVOXDG;J?Qht^fbU$n z3EWiG(;4+}-VzMdf9N8qL`x>}Qy{nZZBQ+6fm>m`bS zdmF9qMY+a8`9jsVZ*-tCjpQxc*w{>5CHpG#YD$OjAK`H?44{#)(dPh2c%O98sXO~U z05R7GtBQpEz(Y5cnKWeTOeJ6JaV~yH#AO{^D~*7g>?BdMs-ElV)|QdCt~U&+l+ zqMWUIIUnzOLklu5RZR8f0`ADuIgNy6$pBANTwjrC&)n$i`LCygh@~OU1B)_J!ytGl z*RH%^`^A$O)O0HO&Xc4d&nwtarrd76^)BDIPhFpKh{oXZ{HTb`{Y15AC4;y9WRgEX zjcxCK)5L*J(cL<97WQ;=TD{V2QQhF|*(@>Lg4B&|i)w{V+OkQok^Av+C=ik${!ZQ2 z=3_YLzPNH?1lNALo{_=k$SFa#`w#v*mW-i|jW8iSs!XO@h&~YAnSRJ2np;4tJgL=xF@iJ`fD{qLTHkvsGc^U2@e~xo@QBG z!Zay1a|9gvX(}C|5EW{n1dAn#UeM#8QiVFd_l`MMH6r1;odud~yFPsPZ_OCqTZb@7 z`U!i>NI8> zY6GDjDdjku)3w$$+2ROdN@!r1i&v@J`cOj6wvHrpcI(C^V-E9@9^S;#OleYp!!#Zs z$afU9jXFhUlsHi2<>_!5eZObR>_AML!dO&w(`Wsa>YhmoV%BV$sw?>Bw~B)5jElG1 z2{fC+S&35QWM&7|$gDo*LJ<2M+BxB zFdOfK0hTj4ZHkXp3ksaLspl~aLHX4vrm?7X4ulRducBNuF+#jJ{C+w5_~jksaJm%R zNd}r%!wv@rB1jvN)+&Rew$f}i0q~n_UrfW=Mo~u{M=$3m2Sc?@P`smxZ{{bq9WiWN zgES|q6MYH67@=!rTh^_IXOV_iHpLuc@-DvthR=0()SBh1j>{FxfP_j81kvhT8MuDC}@^O`v#NK!k%*uHxGBB*uY&Xctr$aF&!~{A-)dGEY ze5BcCI_@eHm|VUa#Afa00WE^eUtl$y;<@f?b`;F?8g<*Qm`@8sz2yY=!^TMQlM@*b)py03m1K>cn`RD&ZEKBW&p{}#-;F$> z`q284V%6~9%WDEpNT@Z4qd#3I>a?6w$BGdLGzx;875{R@VJMC>W7^0A&&<_l4vL+K?enI zlmiE7(7?CEnGSMT%|58r2=xnANQDI=Tzp~L!l)Mfb(n%Nfx`vFftu*yX%+g{RV`y%p$nzu4hZ5O8`ne2hKj% z3mI1ONwa)0ExiX&eRbcYBs$dRiw64{7N1z(VWz{{v(st*Ups^rNNYI?M|5S8XtbEq zuC?dCe@&uYL5r>n)uWU**$?*0_A#o)3JpQzR+Aq~mED3o9^5L+nzQ@qBVxZ}g!n zObFWj+)vt_=IXsXNH6YZI$ZMuVq*}hNSIAw{4cUbFs7;9_l$dcSzrta7v9U+Jm1(Y zh$3^XRF9jpz>hLo4C;=^9lDs8QczAu_^=_M&q*UKuA+@`Ulv=L2UPdUn+AtY?<8k0 zn+DHV5;D&PR7y5yU*wD~Ln6)0FGcHvTY}fUZXwYv*6SmHI|N;cANXMsr^_cAgc4Cw zB%gxw1Z(1>qtEFKMFH1fi0xX$K5906w^H-J=JuawG)Xs*kXUV zcGS0B1n5b~KF*LrL$^tZch7y<=C5IGdpWKd&Ke-a(5eK(Ec>311h=9j;>81g)FxV| zDwfoZ#T1f{1pXadEP%~c)Nga(_H)afl#{4Y7f_E%wqfl!WOm|^LB=CG|I84Qola@|Qf!pjSlN?Exgy^mw|z4Dsw ztLp6bO^D;>2q!|!+@8;>`g<`Qj+#is)zlEu8y|zbAO{Zyh)AH8^Wr7e(;(sp>Tv?B zJ{!Y+D1u*$`*sI5W|aF874lZ*hZ;=M6I;33-0`?Zwc2@SkA98uwCi(ez-3jclwFYD z1Iy67OqBk@S@TyIc>MyZ(1dD<&0tWE6RTOlGyS2#CukF|jVDJQhEa*gD0^85Ch*(l zCK-CJ;iEW8&olpgJex|DmS9jfD)X3p&J{<_VMz$MNa~$1?!>HKlvC7{^>axqk~MhV zAXITA=kgFE!xqio!iofuThW(@NCEZ5!pkV-Rjs%~L`+E;G9hMiN*kWAI-GF?3yVL+ znt1$^`z0WR9YWgbx45?x zsUi*|#^R`Aum}_ab_K>%MzL|*H{RPzDG!FD$P7zT^Ptvq`W2T&DZ6v{XP*>hn+y7M zGXGmhYVt@ls+}0351+F8LMH~F(x?ExVnC+yJ*s8-8=|kjO)vItPmG|(X~c83M)bg6 z#0m;5cgkr1@Z>h^Q{A(P(_jKzEjq1or}_x{HHGp*eGwCtWc6{a1n^}1FwdGoGNyoc zBn<+G;ft_Q5pkCBLcba}Ti~`e{`tqW>+K?{4Wep0;#O#1+wBR56O+a}hL!G*J=#Cm z@)_A8jbyW#&$*m@N5alOlaAPpeQT6V{pyXGXWP2K0SM=JNRXvqaVVZi-CAciUXWRd zirj1y9bc;9=fOAp8h!79@d*&;?H0_%JU8c&s!7=Mx=+#dP|ymFM1F7DL@8{9vR0qE zXwFXsJ%t}PLWUjPBR2agf`29J%dykR0Z+yUX%=qqz#xNh6pjjUab=H4qm(|U98GerhDfB@IiZB=9I|LUJij*OgUzzQ6=EN zV*LHT0CXu*>Xituo%{Zr-O$0kL#W3ZJfke9eEoeB`2qv4r7dMWNgnkaKD3putsbRj ztUOy|*AkvQSlVEACm-uLkGCWJkOyl;*djMk)~Nt2X>h;Q zZT#dhFX$;v&Mac7?N*|@Vc>IsS|d+2x$A&%jm@#33Eynu4nZz~&zGvO z3>GRZn*bB*S8tCKvdtTfMLIQ!p68K=fv(!*!D6B_+?6bCYw~EXoaNAj^@^(9QKyYc!HXYH?0Ch@^@M*_;no|C&*dqay^%}$Hi{$#w` z2Q>xB=iNLQpYlMeAMCohDpX8>Pej6dGUS>MKv1+ilLhpItL<%pz*4<~O>c+k zMZcD53xI`d)9)74H|ZRcu}${O#c_<*DzHb_-jCbG`qU$h_)@RY?;u0A7C~Ox;K7PE zLhjYm#flpvm;!O!u7}Umj$+l53%d}H)iN9yih`DsqZoBn8daQ`>LL3uZCERS5ilPa zu?$bHD4=w?q7FRzyV}*>{1Xn%df;=P-BxL?x5l!x(Qa50ZN@#HLFFyV?cnQMNsiwy zRWNu#pf(axQR>C&HrTi#r(RNc;>{;6Xi#-d@}|Z~5RB zL!0_TH9QAKW;aUo*B*u!5g~?=`1FV9d`$2-aPoyluitnc9j=x|xuT&YE7_S>N4U6h zYUO4c5pG8pJo~_Njg28VV04VWjdD5Rr^jp3%Kv^COUl zL4XpG!*I^7Ef&ej;b>9q$s{BOVaHONf3zlxigRUVe_KZNa>@)k31P%)O@T)o)GBfZyo<}q`df2k0YDi@c%akcvwiFdcEiIFE)`y!rSZT~6^+^mS_;yC=BofMn&>8E?$=2wF}`O+H-m zI-l67IUi--&^1#^uH!l;R~p#5OW+PamsicCb5VnB&GWr+ejsMH9LdMdmY5|tvqaOB z3EV|QMU)KZL90n81^EW-*{1G9>L1CCzt`8wbXnJHe&JitMof^exevI|Tv_bk$bvJOacJd^GF+;}%1XMv|KC^4*!&Sz+#Jq)`@nj5~F`;uPOP zSY%mCZfnssPH>+uqu7>I#2#A2Zc` z@4ApO`v}~!&6_uqmv3wX=G`N<#nwkBZxh+dxsOdDtNRm_Px?)2y#(iy!uAriyBjJO z*Pck$L$$3n+gHG>7+Zd1r80S~7cQd8eCcsw_sG}k=PS@oP;Mj@NDA=XzIpax#9*8m zo7d1sSDix;HW&^7?*lgh9h4;bye=Ac!e%-&c;id^v-+}?PanpvCUJdW9Vr8{*T^E6iZ zX_LjeY*oT2#%)eI#AWli3r}p26Enu^eygbk#p{xbA)ibN9k+~Q3+c5LIp5<$J$iYW z#nhBD+T8xLFn!|H3+55$65vjU(oLYtr~ z0@YjJqE$Y~*mnD41j+{!r7(+OFZ$!LXOD9sb%g+JBU1Z4t3ml3KD!9byh*po;oXbR z;@XoH3_YKy^zA0U{ho9N1YF+pAKww9Ss{CS;ihIN&fX_W&?#QY;&Y~r4ESU!p3Y3d z)3PxJ9DgBYXH5*#$8#+hjk84U{|CDpk1d=vuNxb8dA07xcOw-Ddp zvk|gup15Qn+IqCH^y)0B&<>hW85`TIh>UC*IvH)CU|E}D`VO{=i*#^5^Fv& z4FhIFug!d$)g#G-46iSP*;vqVV58&#n=%$x9oR2P*gVa{VH9$g?5PUT*l!ZS!W+;2Q+K93tD$ypr!bdwYfOJi|pG>)+} zrE909`rH}FL%qw6wvY!wX#8ss<))mpqjmf$A&Q}$it52M0%@1BHpNk*(o&l)i1 z($f6wfHXoZPxnfssM@c_u+7bXDcJ{dm(4wJ_=fDYAhX4d?WcGka4*Pz+Khb>^s5`4 zuyiIo72T@3qYIi*_QjqW!Ah{DLkU6eX&p6bfM*{4>4hCxO9^uQk={zU34F< z7Aqz7l9-%}P9suw*GC3k2?pARAX??rtF??z|?*zMe z<$DHD$;bc2*J%4v?C;c9nB$95fah21&(5$RBe`bR%6*#vJhI?1Z&&g#q!=R8rbf#0mRUI}!skb0t?$rYDo={4 zBGqzaqU6+hNsg~WR* z#`w=Mhy0@>42)Y1WjX!&6e|s9U0J#jqU?Yyny&enMCek9#8@q9RqQkqb;&hd(VIeE zVa(VQ)_n7>%ab}W$552g!;ND9)%EUC5CGoAzAl~+b3J>v|Kddf7OwLM{U(9f@+R`; ztX5S=FH7zU=?aL?fPWL?_f%cOP|#{0!$?8@@hR}_QMl62>pZgsSOIYPmEqNZ=;AbG&90BYS<)90j)Va=3(V&@8g_I z^8{J8fT5sb0H?F=UGOGJ!IjjFiGanmy25_T2MO2RnpW@5qoeICRD;Ov$N6Jc+E`Oh z)!-4knqi&M`Q_!oP@r9ZX!Et4FI^85%R&Q{k$ccZY6pR2#LI zCjde7r?cAcy3tmKaHUS1BhreszC$Rh#yC+WkT| zY=qCGR2*wUyt$H7{pP8SXQB!Ep_MJ|54*NE1FLn!o9CnQZA&$VWu>Q@0iu}V zI-MMvsf9+)- zkPzE0Q0RdG?fR`}Z6LL5p#oHMO7%`p6VpPqpBt)?r7PsInQ0r|dPA#HYxg2j;D;=` zB=THKeX@AN{98308DnsaN8mev3Rl?a!;qv0MkA4$WMy)9NuN=3kF*nw2UB&i0O@rI)2$-5x-%xw)hr2UP7Rd+SryJM z_rm*ZuKxHmv%ffA_BWfyYuxT)mZ62T$e!WXeB9A;=3CloO!jao@vBnxl90R zBMNF9-xo)siN0aweO>QlX)SgmH6H>;%&vVQoh~2T3b&35KL~i_^moU-4Q@0#E_4zy zIz%v5&93MAUyWiubWu?8XHzS9H$ac|2{KnxLs$EDT)4|_M40?ty~n0KC^1+cYI^{w zCC#;Oy+~-M>k(1DLXgA^C}}vs%bJ;&Qy9}OX7)X2;KqQQ={|3%usBIQW#9dIO|YZC ze@siZ=RMGmBOiJ{WOGedSe6gH|IIJ@Fi|zv9AA^d*=}2`#QskXv3o}**^TNJdVmlu z|LhoB&6gmra|i~m|64oHGm(2N*z-ROYsF0Lk4;{9N%>xUtHLDlM0&E~vZ$}SPBv&S zLcR0oXqyBvQl6l4%yYNn*?kKQul;bB>BZ7fy>g|pGU8MZxOT>@aN2}gSNqnD3|}H( zPEZ;~Bv(UT>)q|OAk(<|(Usk!C^5ZvqI|}wLJIq&8f5ZM)j4RwYzequb*`&hge1-* zHNNn!A1dLljy(d<3OQwyim+s#+0gywD|n-XA|p^80BXPraAzEWg<51qP6i*N1|r`a z4vIvDtfzG3 z`s5SN9@n*;(cPZ1^IObdG<#h>Z?@}=eNHl>$tD2aAWt92+2zf=*h%Y|_RH!WJ?&7F zEVr33-pdyG)K`~rt+gD|&PnDdp$#6U9RDG>8d&)`rt+kVj?1?1Sst~&4|JuHTF{xa zergu792ff-U@F9=0oC*M(XYBYQ`@7rgWVLg#N+0x`)dZQw)==ro=O87F((SsHOIzD z9fDHzxcwFG!uqoA(!LMK}T^edyZ+!dP{*#Luyv|Kzy ze3km16imYH1Ij*&4c3*^k}iHy9H%IIiL?ms<#{LX#%|{BGF-+6*9kWE7-lX|RWWwZ z=gy2n^(U6;i&v)K-O}_Zqc-OoAlj?L*bEqBRZ4td&bz^!qEi*~e%J7K3wfWF`sKk- zA8J0Rk~`G<1oxM+4?mjL+gg3pgL5p7i>vs6%q(~%+^j3C8xwuh_Z`&|VSQ=5EbW23 z(NDlpyCKIx4T#6(B)9KErr>~L>FM6kWGxsd<6>$a6hbfld0tMMqQPg;x%`KvHQEN) zVB!4viX~Nl?8l{V)G4<_lpVu-&9h5&bqk1G>7hi%GO)1BLJZ{nE_MHBeiCP4Es63H zcl*QiiG(veM~e@J3ri{+($~@JVMss19A~r=ZVQ7=!HE`Hz>Gw@Ens^k(-wn5^LIO9 zA2wXQuHzq&&~ly{0E!{0X3c zqgRC-e@CpR)#|V+qe7mJ*fII9yww7T)+y;GN82j;TAt7>v-kLh(+vOvbs68I+y16f&}ZM#?7{+lR2muymcN`W?kpY z-0m_sdvWHGJi{acl$>~u)bLO+_~indbj{#ZCO$>a)iP@j-cC&)L9AF`Wz|l6O}fgA zhj)wt9$id-7BYz8kVj)$lV@ahDv$@1VeVQU9dFm0vk*|Rg( zY?blyxeol8J%|?+@H6OMNZ0I=Gj4WAvhxb1f>m|eLS`$qbpUXf)`%=lr~TR7#|W3Y z<7&O_XU&$7tovNr=+C{K{q%|2IU|i_&v%}%QzS3sQFCRE))yMj`4u{Z7W$Qs2E#yQ zKpRIO{@}fzj3ipZMz8R6`r+pd0kX!vyKP!n4{Ijf_m2DgVK=g`ViMdE$(_3sB_+Pl z)xw)>-PxdK3k0#}>;tEwMcIrNpQA_+U!R=9P`zqp*QQ#8RhiD?EMSg;wr^%JqH3UV zDAD5~D_n!Zl}bKqP0BZSHW)u@`8anMFJ6z3rd_v)ahyam=HR>jkh!e$8}?|Vx*&cs=o)g(dgdrb)2Fvl1kS*s`^z<+aNQWHhZB_*T~49>V;) zG_|&*8Y$vy$hqK6OKSa^BfM9e@0U+jTd%;6UcG~t)~zn9&N{P}+lm#aGGCD;n}L?c z0`_{}s(X^u#JbDIBx<))Jyhh(_6Z#C-f(<5T=lK1Yu^0&IiSyYqYbg!qjpz-$Ab2B zr?Otp7=HTso@{-&;G#!q=PGact6)IkAaucXc6MIuB<}mSzN5}51CehwpV;6ZZu{ET z^>YL)Yl`-wAclq-PUDiOZ;7R`jPuQ1!GKlo zMO_YL9WqdEdbMN@$fA?$bD2Ew&{39WD(-q;P>_W!hHvC`vEHiX`if{4vVBExBPi7E zpG0O)gOD0*nP4W#3;TzzOHTGP-;cp1dkOZra3ZlW>Az}A1 z7uA}=i7<%}81Z%xOX1&3Y8?(!Lh@3%ks1%0_lQY;_w}V|Gc+RPn}}S&@;=#*;wCBK zZMy#Wtqf2A-e=35w~#cIr{ydzXHReK2jSJ{^c%W?$Lqn(&P%G!7m$(%A%kP{7d04oAM3JyLNF*n! zwSLvsG@xYTMbT*hq;8Da;&f{LIoD?LYP1;(7XoX$;WnshK=d$FY1OzE4~B11aLoX^ zDif8U%~&q|Tf?Mpku%5UNO8-=ZtnV2-CBi5j|rc%;pmr0x{cV_r#l-6p3yc1IzoJW zlk%ZijW#$QD)Bg^NqRr9wF9t7$D1M1WZUVy{zKwn-_C85<|1#7hC*$sbR525rw|J` z1uCVo82(&s|7BnQCMdNJ~cuzo6oqya? zg6gpx2hPPJZ1X~hH8l?T>(`v1txjAv;wG8P-nKrpA>=jQJP%&l_L9tg3FeF#@WiR) z;X8(-dPC0FXnDU;gN6pNB{Wbzd!y|TkzQOU;#Que@`tlNLO9yiZXh89M+xH3&8gL# zS3&J2U>a?0-}9;ned-W@NwzMJoNYPvyT#@E&24hs@*Ro8Wn|0IJyEhNn`3eqd0+ib zsK?lwcn9(^hp!${ZRT(iR2fbD*LY?YywF!^qkm)b;Tk5cIrqLDx{8)b&C}b*GYB9#qE3 z)w^>1o@T>vt8-G-ZLIFYJMXsi8@35{GSuRw(7t~DJcMOB&Vz`wjXN1`hHF#`CmXPt zo-_7ph?b0xw;NfwSb~`OR^9~vlCFYI-ufN|4QiD~AAO9^Y894sPZ(x?Hc4@p>>IH^ zVkUf8!u0*}=;QprvnMSUSU>FaPA&0zIcYdbVL=5gYB?g?ZKD%xqybO~61fK-%n7hQ z00_d8AEpp9)SvgYy+;MUX97TbxcFy1rYY((GQ?3trhYx&kS~pX(X>u)k5CYvu|+Qw zcKJS=6qF9LHymWiwchxgH{fKEuw^=u!xyDPR^%ab9sASL3wnVv{@TdqAmf>%f; zcVzs7J-G7$MhJ1hwuM0hiC@c3(G&NWL#iVoTfA}kD-TJU8vc@r zlVl~XxrhnqVv9P>Tvs8{^&IC?3+uSF&rxG5F<f1~Bs>i``pJ7F4(`fO(^K9@p|vs$i`B1U38NQd>_T5v8Z`?J&% zR3-|DaqVG^F>O_TB)P1mfae=lXGX}JNTKrRUftQ@r#9P7R;JsvIt1;n3b@&|xXF~WL=0J1|A6MB;5GanG^Ls$kpo~#-kZzd? z=|v-6pp0fC?W;_;5l+F%=c(*pcE2a8*VgDC`2RpL6~GY7RH#t4?ZJ1^pGsJSbo;4~ zKl!Y!6)-UK@QPcF6(Tmeg5(P^&DBei*Z7Hd#zkEbKR!RV&%n&f_TYLE{T@3PyC3Hz z8uL#tJw}2HB(0mve%F07#|t+zf&}P zVJ9GKQT{##+in+O$gSU5*N9sq=Dc0AaKA)@PvaxM%VpJV1n-N-!%ppV?|J7q)gL&Y z)=PxXs`a8>dJ`L2A|zyfcebG8T@lh1S%!QoY1D9fs&>g6{Ue-*G#8ZCiP|PjWCPzA zUKDVU>3)xu&jomv$EvYET)kdnfDe=uGHkd%y%aIHub?6-iD>mr%=BT#!T7M)N*)R`^8dbGb4j+zF6Cq z-(>@@`o@T1Rmd3?D;0ue1*tuw_}-mrAnLT;0cBbIQK(BZS6j|nV7|FzxmuZEeB}?9 zolgv|kY9bB{^N}!_=buj5OY-{17m|C@a!oW4hCuc;VyxFQcK7u9uJgz0d*Cx9gaoO zkETqS6LW4U`Ee|fK8qq5bmqNZ!i+iYlC3H%;FG*y^e3{<) zBA)EIo|O3Xnth}HI2Sg4QXAP^K1c8ipM#%^?v4&zk6|>BH`&YDwVe*h=To1MI?F@i zg8P@DK?^Rx*_0=rE}L@$q~obcf4QH^W5m$1@hwxd2>D$TfM*Ufwa{$v7-@GG8oXJn zu-=p>4_GnqSxOlgfWM%Ms}B|c`JMsaExs|4QKqXUb#UHpr(;+rl%tQtQALnzy@|@& zQcvp`N+GczQ}qF-n`{InWC=w1h{uYY?zHQwvACbfbUKZ` z9fg=id{nVB`kC4)^SQA_m&0G!_sA=;>GaoS{l;|zD>L2zZf1+A?96(ns(hFOrEbC9^ zx;w_5L`E8{6-6NBT@A)8`Rgmp>Djo-h1U1k<8L;SFiymtH12TfxDMr;H$UHIey2oH z{U{=GSI6At6U|VRpLa_M<&;RXV~s%*wA73ojs3ganCP>%*7vhySf}gbR9pg2)SJAI zmJMplO76^_y$`Eqwh+b-+fIqzbQ&ks$y6tirrM{_%LluGn*$g~V>H>AAhW^JV7ET+ z)A;kQ-<(s<3;SG@`vS``s>%B1`jtx@(^*g}L-uNQPV->g&-`h}g@_e<%+tQgluGdJf@TJ2OX#TN`g?O&a z;!oRCaMO}IW?Y!AVf*`DfL^vMnWM$ABl{1-rb#rytyymq?`r#cN=rwzmT6bch8$xmSF&z!tS^krKCP1y5W^;*t*nUMep>a3 z=OeyuZynpqhqv;v{U+nWZW-)0AL)58Ji$R_;f5+~hw;sW{graV5N+9ItW{sN zZ`bQN2#L4qqtsy#LsY%9vUu>w;E(i~&H|&}%X~==j`ccN>RGSg0xArHew7jygbJ6~ zo#MO8F1<(A3>NREtjZd0FDaU80bvzh(N7mi?=3QL$T*L@)M9B*d(^r=T@;~nldsxF zc&A*oNze5wv{RnfVY?X0B{{Vz!4`x|wytjUZNN!4Q$0+Au;78&{fF#3&9$y>6$9?^ z{fmTyQ@^)m-;F9)c~NwRImYVme26ejc=qW}kY>3utPxdYpQ2;;Du1R^V;a%Qm&LMnnqF zo~z3xJf2cgqn4o=1l$Yi1|Mx=97r0=sD1OuQ|5>Rd;2@vrVOjdNr<=&Rbg?O$1j?6 zbIK)TRUp>&2e$H0t(Eo==j0>?c&luCu|%2!%DK`z?=NzvIwPKYGkZaW^~eMEf{`kC z)_hlDKv&3ey|sD>t^Ktx?P*-FAmfT?cf?PY zieOc(FY}b4SO*0!cCO-I`8WFO;b2=gkmOcrXyX&fW^;Z33wix|G~1>TWc%}QNLPM# zC}|UYSs;s7TYULvMv%QT4)X?z9}!nm?($i@!zjQbe5M(MsqnPR!?@ zyOEfcA0}u8s@s$c*w3k|Wbw{vWEa%@<_LN}?7lpqtWYo1R4w~W*hO($II9Mm4!6@P zhHc0}T-(?<$eX1W{A4=6s)0EXk8O!B78;~b-S-p(9lPhuY`Sut{JetG%JWA)j~L7R z=_15cB#8Y=&)3NXOsUuj`yLMwZ5&ICY5u-d&KG`lgH|Z?B1qUst*=>hxqd19s&VC$ zEc;_~=h$e1<2P&DG2Ox7#RSLQEf;yo5ex!^X9QHxG(>;fZJ;TfWT~3zJ3rXNQzyuN z_~N!vn7MgHt)BZZ*%6#kzp4x_$C0QxO(y*w!!%s`GQwepqDk6z8KywBk*P z4l6B{bY4-?<7`II&N>1b=F6!Hnhq@QyqTCaStVT&t~0mWN6R&uMKRQ3ft^99 zhq`9Zk;vmF;W8B$ihPt2L0(<$M>s^ZBbORuJ=NXtakr8W zY%My&QtU>17Ihb?&5-jvW4ij-BvdD12Az9=88!OH4553D-kTJQpI(XXfhx5=0@o6p zvmBI^+i@8i*Omztq-0GROYWUH;b zcaid&I?UBrabQ!*{k3ibocGuDsIKFyiSa0JHtw5cZ{S&JiE~o~IyUw}uZyyBhJ z?tUt}v2u4g%pERkI9~qM60;NFWzU{MeziYLqCWc7U}D6g*04+`_as@o0$=o5$)W|S zRqO-rtvHxIoTrnM?IwppOf6%yejvu$3C=u7vV!K;V>7DQ)~e3do~r1pWP0zm1#se) zfG^lz{aL(7E-|kn5AX?5S!7%(_i(kf<^RBCw+TXo%*zep$fEk{GFDRg3$^5Pw`8^- z(!KbY<>0dPMlppp2w^krNS9JVcYV0>V?olg<t+kG6sSK4b0X(eSWU{a;h9YPf z=M3~ExNyuuiP3k}36^+$maQTs*(_bs7OvDkTD$;rKKdF(!b^uS2XYxPaRbN)zpx0E zY`qZ*O~v47Gt=Io*i4{K!!t=UL?v8xW7DhD>wbPeoJJAOX*TYR`ffyJUVXhuiek+2 z*ROTq5 ziOaVx)Zdy<^@H1SzeHKT-~qHOkr^F-ONC}8WA_`3iziubZ0$b`As+~UZ76YRi2A|^ zG+ClVOqE&m8FeyYdU?Ut`Ct01aM*&87>fPBsbgC#hV605T!cNMwbRd){F!tXKqS+y zD3-YhGFi6iDtkMSf=}18u3!P<2^B%>lm7eLEVuc;P;3+L4j5f>`-s_2!7&TrQ>L%k z*-cdIOd3aYD-E7GG3%8ml82Am4mYIJ%DVSqdC#Ej31yJ&O|A&YRUzH6v z>A*9Kxd}h>njG2MB2&H}!&?D&n6yl<(O7oP3Lmb-ySy?rCp$mRIJpp94SyYO?echkALK>@lc{buJ0y+qW?f6}5?r=hcfJOl9Stw*W6; znTN_O3km`YV=$JjkX1w8i}9RQ1dthzNvlj%EmX{7aoA$zc>IkF6tT$o`pM{1ibm$l?PZ{5o|Nv6M|WGv`*Ih zdvhp5D}14g-a2yXIU>#3*DEm2!LCH{>K-+^pkr14lhoF#wd^h<(+ga0DD1lAu6vuq zr*%4S+a+safn~8@JKdCxe@rHPQM@G~X^)Hg<6J(xoHdW=B32gxhc{tKpHoYm-Gr#H z_`it#X;PM{u^3ro^KL0@gF`n9ytIG_!x93^MksQQ>3tW>aOrjgJ-_*w7J%2etK)x` zL%79vRssyBa-!@qiKpSOgUK|XKNoplffc*zcOAs)xv-*&kQ~Oqn-Jt$jEDg4QO-@^ z(b-vB0Q?Wy9qTky_=n8WH(<6$^b?ivuP5L55{KApyBSkdu*!gTL{di)c9!6SbQpD; zAK1^%^}dT*wLHB++u??%_+aiuM+?kyY5HWj6^3wZLKvq>ZXML{9g1&UW&hkfvb_Z$ z#|kEf&bWKbzYph7!QG;25`BUPz-o}DrA=?TtIPAfucER*OS-e|e(9B~f$M?N^!sB* zu^cc@AH&x!lw%D|Fxj!}pp%7HJNGkfABwLP9$6!me@;9U{c3S-2UH4Bg|{Pwp;ntP zzCQZEw|mri{sDx-ciAhYO4qE=F;pNiz)OA_S6_==uL``{vC0?_?+v~H84-JP{7Mca zNZ&fEsR{t#N* z*p^B%?>k^^7GSZaPVCNQZVf2Gaktf2OZJ0qE|-yuk8p4K?kA~_R-XxY*J3rqoF;<` zI`%sDcI*p+0rNfv(VLdg39B-UWMlh^*;X(8I;9p7?}v9$o)yi_9E(N93v?^D8FezV zg2v6G83H^``tCL_9WRQt#@WsX{KdS3eucOy3^K|X6WW7Xs|j!jEeVdVtX1!bUgD^xlG}sWj?E8OF|Jx+dC2Yjy$8zd)HcO zVuu*wP2X7a0=m`j83I2gJl@n8tQBvU-^gsD) zowY7=X|fnlv=6art%Ej;LIh0!Pn`5M_=@C0%z3+A@Rpo5erf1X4#6V*ZIr(f$kW<- z*YBDHd-sfN*ND2fb2pUc(VTh|;4Mh*AlJM&Yp5K>)kIlsettDB?~t|mCXN`joGSBG zvu%70^9S!w6Xy-jg5KLA4l=5PYTNoNJWFuWyu@KM(u3Xvk9gK+zuT|Iv(Iq0Vg;kJ zf-BNT1s@KoPMV@ZekGg>X8I;AMC0z1poe&@C`z-ApRl6-h2}{fv{`v5Ne~GWd@rFm^(bDYOIpV@8Vj~ zMRo@4N3!NjU;re?+-U7OWK^5*+DdwiR(nhfz{&pf_y7btmD{y=WP-*F8q(?4R+0Hs z;Xv42>F^CwccvZ;xtK6UI?Rk8E7h!p14o&JnTKX2UNhQKsLiHQz@P5Hf~zet-S( znQ*l+H#c7wuH6yO0(pZ~$eEO03X(d?+VuPMF|DXiOacJKxKEK%-4L~xZcaV)X0|Zo zXwTnq@(Iq7jiw@hdS#E0;!PFA(+eMbK zB~r59bD zHe;saw&4<9TLgMcn;9Z=7a}Q_<$c9_5syyNl7k6E5lTD~M?VgHRXA|PLy)7h2qWvW zuNqBjlb_1@whF>cgcAh&#Fy>ps1}o_nRdx(H?nP-5 z2fDBL6>yqv$>h$Y9=#9d>=m%J$Gu&myV^qG=!z|}CDWv7wE5*Cb5$YTg#0F&KnzhJ{2Xy!H>csPk`arYJp}?+iBx#110v zb9Wto^6xl);j8Vg)@mPA7s$Uu+TMB8?2Zj=RvIg)6*FXe^BD1J#9any3E#k-;3P8> zf{d;VIHcZUCt_7K0k#TPSXr~UG7_MR0~mJUrcUIL(1Cdcbn_y)PFL@sYEy!N=ZIAR zkm?SmdpRV=Z}4`~z4hW1_Sn>1zMfTDteW*Bh0i*iUyt)?T-It1!|N#k3>>+OZJW;vS|m4D~LeTMNB&Z8@K{>0BqV zy=vNSHv9zDe)wF8-h8L2Nh`}$JzTHYcr0eD9OLN{eORQ1WZfJN!Ixq#%6+|PlI(n; zBmphd1;);LfI@F`^+RgAF1j1zZEC+izY@%}*-Z7BoWL97n9H-G#Fvx9{XFPdmAs67 zP}XaWe5N{?Z{&{dx#$9w;Gfi7MsA2F_@sladV)5Iv(n0>VF|5UIeQ-))Lis?9{ZF4FD|xu zvHz(3^FB2dQwzk=qTB28d=pW1_y!7lX)tL9O=Yq4TAnA)ePw~aA)97L@tGDrU`s!Y zM1a@51ZmrA$00oQ^yi_ek>r^QPOe9E6Qd>+klgd-Zi{t@9f%w{3y~=Q2sNId3i<{d zoaBn<|FWG9!#3uHmXTFpk1i}u$0;5vTH%@e$8Vh6ca&# zUvI~hi4O0!l&6hryG(SyYpq5oZAri`W4XoTOVwyyG2@p`#)(zW>krpsD>{7av!^I0g#!9q>=dP*`xG22r^xe6^Q;5F z*YRb1L*l_W&@on7c3@%ZB*h?XoTcnSWFWJJ^?N)aQvNfS=uF(c$a)i4OO>7mY>m z?Y>4Xg9IQ)!T6!wXLIc4uPmx%lz2n#FdgIWOlhyo=FSy7MpQul!8NjT#bBGB+k-Qm zMDj`AXlT%kVeJuXUzC1c7Tlv26w585+kLZPSxJeFe9F}D`gZ*1MZJWfS5V8~^&*5% zzIFBtbaoJI(JFmpXq>lw1z<>;cnt;_C)*cZ1O%CMBo2Bx;OtRCZ8O~W4hyN&5Sd&E zi5tI9c33lIjDtg{pKvkvK67&QRNiHTD*V=Dlv6DARmQBHN=G5;$PL9RV_|_@zV3I9UcGmR6*0OL2pxrlkv)YWMi{*BC;~xWnjxpFT#KL^9H5}| zteRk*8UxO&;^u^`vYG&~+Wfdi%NW(-=}S9Gv&*o+GygnVM93ouzr#X@jGj${ME>#U zbcE)4jGy4h?cprI7+5faXiU&5CLEsO18cAtG>tS{??}vDwYV(VfPtUEb4~lfQ@X|5yx?5t#hxjzM4+^Q^Rkt9 znZcmdqP?!8{Abvr!))9wmG(^i%V`tmCR{=Bv|x>5#H&6m{Ec8&GVeI6>Q$HS(0S_) z)3J#WLH$*LAHi%=oMlOh0`GXbNI!MYQLlV%8hywmCreZU&6mx+gNzX}3`H3R1JHDd zJIBFml$I!J<;&+Cuh_lxyIf8oBw^!HIS=o+c-1P8O^8DS072@Y1g?y}%O1v=UeQqR znqQ2c;q6ND1RAN`hT^>pMr#a#GP+$Idi?+!U$Vi{@*bt0j{fipjTsA z!vmDr%pw9Tm|;^SKe4l19=qjfBZY0@$(|Drk2t?HB`9_MzI6lUa@a;`DdYy|+VkpQ zPN55XEG1~ATr)30`(}gPb(p(nJSpDhu|3f4Yq$P%P>I$cdf`q8@`*h7dL_o%r?S@WJONf!3=^B`Oe|xs1AYmLliw z;dddo0|lqh*6D98vsBQU>tCB46b9qr=M;WN$i%H3!y^Jw9H@@|pjp{(BYbdZg0`qq z8XTrS5jR`eg|9*MUogNB9F|%+r)Rn8pe8f0Kg@c|SZ}R?CW3W*!0)J%D5h=2JsnH+ zdlxOm(1r1UEUJbwY~v%Q7D`z6&`Q{j!87|FI}7VzD`pR&F&5N9n2sBd$bLiXl=T_S zWMMOw@>bIUyj@m`RxSMx)cN<9um&Oq^~ajD_rO7Z{Z0?7QYlk?9pL_VKLm_9mv z_qMu2{u{P%#U?Q?YHx=_!pf0OC#sxDf?`=K*TN3(-*?})WB#o@za{stFU9ru!Zu!Z z`@N+79SaZ)n-0i%YN~c69RS~>)17X1Jm^gnXE^G*{`|IFQ@AE*o~;XG?zUN#f?JG_aa19N5`a(?1@R)^FV5PLh&x zn5|*HfPhj}jlMeT(SIBL|GXV0 z%7^TKeENT+|L^nspEDmY)Y0IgbjjGLBlo|R>0d?iPpAI=!0H`JLUC=Q*-ZbpdH#=7 ziAYe7tHJEyZ-w~xXZbJ3h92BU8=!K}6ju1VGo=`qr2fbF|K~=38t_MAG>jO&4PMXW zznJ@feNpO7sFeFq__M#iIU`D|ZTqcvnEv;=_tKa7kj zJpI!&{^i&cl$f62mi~=2QO;)^%22Y2(=$_k1M^>>E{zhC42ANOf4R}022Aw?C3#c? zjWzl&<^KCR?|vIUN5-G6_@4*FD~*!8oRN(x$v+?c)SJg^m9PSBRuSrFam3H+hs7X8s$e55qk>{scPzdMwNW_IKhzxgLGLL3$o@*XwUV z8a0(=^nbm@AH*c}9tAz)b-3gEfA{e3p{CM~{YUuv=LIkfj}!&Z=9JmfvVSY^2V|av z|F#1E;Vl&l3Y+`5gvw(7R^UWzJ9Pg%m_G}YR~ZG&?~UD5DgXTWKh2RreN_BEiueaH z{ePi|R<`q(9)DvgJQ%0ld2IgEtb_C6+7F6rrMBY@tG*vqb<1n2KCMJfCVRgI70+nT zr4q4mIl3oA{guiT*M(w5pRES)M;gAFCSvQBnhiHTi#3%k5TE8(R|ZW-axvrnlP>&O zvBJQHp;9NXpdew#d8=^ zfQ9ix-)!?crw(p~?keMY+SbI4+!m-E=P12Gl!#4rmR$z;SCa`NBmT+YKHvybNSgbJ zhhk!?b6#+8+%Ji8zpO^mjk7%9|lZd2S-0@U%ZT zVgKYv7AJ*bMNr<5e~g5nH3Z5~^NWp_I)sY)mJLyyH3pM#AmH$h>t5cJa#MSP;NU_+(KNa{zX-A@=yx$!bW>)$}G zBk1qx_W}=JBCTj>=!#n9{7< zxx+RZW_I*Y)02AtaMU=7`%GtNWSQJQEp*&bxGs!p?OVWD^3DZvytDaZ$e&Odb?o=j z$yM58Y}LJMl=T!}anfd9|NEFDcCi+zq_Jk%o_9l*Z2o-O8u@R8XV^!hCZZGeOvfR@ zM(<0)u1jC4?daA8KAdXg?8$vvogx26@r0ccXLLPhAf?23JqRi3XU)HV5eYdhf!s>i zYbV0KY-+JzSN~$^J{%K%il%{idWOy$w3FBXul1gSW~geGC~kjQYCsEnOYwn%VGeHt zoyLmX;pzhwFxwjbvezQ&9nPP97dFp~n-j$cdHsnbMsezg$ z;cp_pnZ|xMv~G849q}S*pA8HGx&x%o+PCFWtw8 zjiq3}=6pbw-qpTKj2;&1wAw9mdUnRl%{}pu$i|j3$Uh*@vz{+?v5KbunX$mNSuW^hF1oM)QK+X?dlu&i2Ce# zcg$92kvNQg1Hy2A=h7|BMVoR;BeUHHBw@Dn5Spfk_^%Qy)h`R!Y50cnu$(~ z_gps|D+Tj26< z@2!`FUK#_Xk4Be2c*6yNeGo_2E^co8pb_CzQI{^&@k4N5W}<+GfzKd9NVmlWY(wj_ zCf=%;n351X_fC3bQ03;kK71+LU&qEB#f_h1ZOyNd?5TIYOCo%B^HTbZ2MrnW&7TB;wfVciRv6aCRc4DL_CZyf992_7x!4^X?Y=w(OylI+I)$*-5X#Bo}9 z#=8LuOcAF6;(r!jZmCxza3*?4;L2y7MdgXk3ow_W;NV+mbwqCnE*kzZ>!x}WJzP(i zJ=4rXZLL}vBRtqunZ>ccD6!C!v@h3DuC^$E(7%5YF#9?|;7qdS!O&<)Scd+0YB%~@ zrS5Z)d2Ya4NlArPh8Ap`~DQ0qD`3<#ur` z?oc%@=O`F0->b!n%>p4r^@lNH`F7yzum8Awu=(B}V@hCsvusYE=e5jD#&CPJ8yc6H z#!YI&OhL((N1A{tiD;}YMb|*;x3Dm1h2TSd57#C3S9-Z~46)I-sX6Xo6QRn6B#SJk z%P)^>T7nEx^dH-6nVez7j3}*m8;v>9-rBFP&Gip!73(IF8XNLbJ$rEtYHtV}sHhw0 zxw!()(jp|{Y2I_HMP=enh{_l|_6xmfG%xZxnD16kX?0q79&+a_awlkwyjpWCi1%s4 z{G20hKGo{sfpC${UHIG_lIdJV)^4mI`0J$1bh`ye?@?jjuqwg(YZ6dwF)$j8rwB}F zQF4#yhu%VovRh&wn-K#W8~Bf`#DN>kp_ zg8avJezw4#HEUbqQ)vA#4Xsqa1xa-yT{>*p<$~a;yB-wpF(eX)`ddJ8mk!C|nL4!+ zeBV8QEm1=!;Ep$ime-}T(Jb@xgzpn$91|P-CT`mO%8h)_z^VZFqNAGsdS(=##I*yQ9wgC6o3`4H)*zh%%r)0Cd0-~|hxpHLJ0hwL#S*fNS6D8r8M=(R zu4Xu)F9mLAMQsfuYqv{YebhtmA?`KBC(6n@zS~DMUs<}k_UvRaUy)wr!73%G8R&ln zxrYNGnV@_50kt!e7+0cYESk*LOa^dv*(w*&W zIatHgec!*(yG(1L(?7P#&{g7zO1Wl9!}P`Z!K|b8bpmaJ4>(}g>m|_dB-wf{(8ot- z#ywQfWs$>km-C}`p_<9l{@AJ!DyENj3O9+6X3G4 z_7!bv4UY`rN|U_Rb22_ERe>jckEbSE9Pd(dFM~4FWBE2i_Vfm5&!Gzfb7>Z{{Jyfe zK)O`vpaLpQR5~FP=^ac$M?^(IK&5vD>4DH& z2#ECF6CebX4rx*nN(k)ue(T+Puk}99`u^}E97pENoOc=H8s|0cdJd1joy}Tb_D|QC zGc2zqk42?MXKn?7)NCcMxAvuKcjSq*UvA?ls4Bu{$v1k$ybX!Ex!(*_|3sFo{swy5 zCVw070EIz|n2IxvfOhI%LTv><>Ilmm5BvyC*AOv2mIoR#VP(*1DhdmMn>ahvB>k|` zS*>j9I903dS;d8qmd;tuuskHlAh5&~u^Y1#n60lO9ZU5JSIM=^sHGAk6nnqH+5)UX zx;NCH|H`|$J(NT&cXBjzhB9jDm(ggUPd(AE>)OMh8b#R>O289hr53S9x>XFky6UD5 zA5e$Pt0Q$5O1&A!kP~z{ zp%x=~j5!lQ$qIQ>xuDMpKjs_MKst8S7`yuXQ`IbT`ZVw0l@@lcKqh&RG34WZm+|&W_U(-e zH{$OIvX8Q`U10jA*CVkpqij7ZAA1a_V*F61xcg8}Zgy=8PogZG+ujj8kloI-;ihH_ zs~==l){O9NS3!21nhr`^7dFB-#_y9tO`^Ko=L3y%CyT$KP5SQ%9VSaBAfNxd-D9}l zt&EMzAWCO&1TJp%+!@f|SLjYRBENQ4b_ah`eas(po?rD~I7AR6%LfXJ21l!A4y7F= z%6S>PzC;4#UvgQ>-(-p@7w9V8{~Yb8c>yrt@m~`KhPR6srV)^az#X%Hz(bbCLFYRf{vJUjLLXCjpXtcLn<(O5pcBmnJN3*EGcASkL56QY}0zxy*e%`^Hyf z*Y*w``V=tEc{i>ISki*DA_9#E{Dg%y*;KUP?v8z_1IT|o8g(4W7`eDZ=kSOEj-D>N zF9$jC2Sr203k^@&En(~`j5V0#3dCYht_wQRfASMPYa>uZzL&bv*f+e|1HwNy6oh!X zxXkp9v|h&N1VcnKB*O++PB>Zqn$@W(2Bja*`C&n12P|maqr;$K4yZB^uTqUT-?-uK zVa+m#UKz3%w6L5ehn6Cc(5BR^;QlSM8d~5fyd)Biow+}BBS*$9rp^fNp>aGdm4oJh z{BksI6-{i*Ud4q|BvFr0`3e532;rCzmS(|)sS=}+gPz=Q&?^2c0;b6;H=m=RD{^Oo zc)TEDq+2aDi9JR}7{p53)vBwdKE(LT*St5C}sqW0?yI4`kDc@rR4F5p1Qqt>}I#)GJ^kI24r;_N(^~c-rFQKyt zYd+Jc?`=crYgzgYj;|s`_;>c!jrg-xfJt|qacE+=C;>We?Sm76shy;4#FHhOHe;Bb zrg`t(ha>6+XsFwr^q!2SX&Q}+GnJ*F7P8&4(;QBB5c1k9ns!cnIak)0e!S|V&8$^u zEW}}P+3xY<2GmQp0nVXv0J=eSk_T<>vfKEyXs9-&h@x4PHEB4kN_jD6*9v$6wFDGu zx{wL>()AVC0a)wOAkhEU%r=dR>ZMUc)%9VtU}WVyiX4>_Xj{v?{Y(-Ol8uwccO>X3 zPY}++#a{plIXT2LfR7%EsSxTYjfWSyEW?cAEa9yUv^Q8Qj;&-;Xic|Tx2PH1Um=Xw z2}M~6_U_MdXoUy<#3MnPjVqC6ze8ymN&nd%emrh-9%*Tz%N_=t7d~Y7dEcOq4a7SB zlM=7R;4Bd*@C^rk*_WcA({sZJPAKfCen}{`p!)osapt?NZ(W<6Z5wg5z6z8N0 zw~&jSG8(92vXIU>>1<}qQLU75qt(F@b<+cN2o;|cMrSIu!-_t$$E$Xy3mk8_ypi?K zPVUO`b;)#BLh#)GAd#WaWW4lt*nwYqMoJH&F`ko@8#hnv#o3jrzo_Lsw1PYeT?mG^yyv#6WT~5N80pmE`?cv*gSUl zRUs-wNx;Wkt|JCdQ~@kO!?$QX5P7)s9YH@=7Nd?A$ZwoZ?rd_j6c8U4uHydtDDzv% zOXq5-g`b#xR?7Z2TZP=Cs%t_~Z#j-Drb1xbHk*e6%GVy)1dO~?o6^*q#lalUry7kl zldc3yKq~jE?1NJ55v3X%l?i)QB%Y6^IIpL(`e!ev*83(M*73SPD#_p2o8NlPG(C0o zG#mPmt+CTnNxGSSK*?n>OC1CA&l(SF zLY2h+UGA?T1wlK`)ikTgfEGV4w;Wv`t12E!XnEWXDYe`{lvqxH{UE4{fm+IQe6Uq4 zxC&FLt?toWdp!1Y3&F-Jb-4v32>XW8THpeYEm1XmTGc(`#E+6aJa?YjU>=&6*eFZz z9uO#ki6XgSGySp)fM$D$P})D0U>=G79(ZK#J7sEK$M-3o!r;gyecm`)sH$x1qP7Db zeE7AsZ^L~Sba~_h(Pt?5RR?HN99sB-?xQP7UD=jCr;Q2K3*m8dHTDxj83gR0 z$9FMJ&4n{y>Ug0kvr7{=7cq2(cKUY3WyMvuOf$)XOm0;RI75#2t3k4&W@EPqA@KR) z+VZd`QrFdd+FKQJ>KV#7ZMQrN>@IrmX)1o$>7d_qcq43g%XKU_?~xcpz1$1MbgMSj zQ&KQt|87O=_7kl3+n+~kNPlV%hM}QG<}5o`4&<$bW%VF;(SGKwg@MYw=m8z)Ci7bx z3Zqr&s+m>&q&rr^5psrB0+(^NRMf)i`&OXDzZD;WEbS2=qeUb?`T9K9UeUA}a?<5cr z)mKtrHmaK-wB^P1Ok&Dl@c|RfU*O2fQ~xxSE)nJ~T6CNt?O3!%OCG7?^;Nx{44*<6 zU~g;_d6Lh;93O6+{b*tnTz}gG3{frTwFOTZ`a6v5{0PTMWq}ajuh8_aKG94ipR;9! zW@Gj&=ThJ2@Eq7?7X~w+fXA0Bs97lW6 z58TpL%-;z=w9meket*^0(k57LqrE)J*siq&2NvE_^ji)}3lg`>k_NEt3~cCSvWqR{ z^-Ma#X{)-d-7(}rSS~GbnOU`?&j%#RdWY-s$A4J#Z=9QDfWpxmftg>HK5quL+>{JK z77Lhl^_zP9=q>}nCcSeoI&xR=;|IlLS*Ohx9%enakYB2^H>OL3-pw8Vek>F?yDK;t z*hx`IpUk+O?P|7^wXinuxnKY<+(|0MJd*}il}^HM#FyPJX!!)5fTDU?IR!B#QM)Rn z`y=`8dlSjxrm~oLeT!3*0ZW%HrYW_F*?U9Lf(h00%o*k0sWp@=v6-2^lLv4jA+ww? zwl`PPtfumtwP~{%jl0+X5-yOmw-EHO)Vyp{LLaFx`oRIduHH8NT%HIwzV7ETnH#!T zMHaS)%{sLlyQVH?Btwaj#KNKT=+!o#BV&Jm|K)1^VHRuKtaMu5VaeT9kZ!8x&@ENw z7A$e@?-zHW$4?^PU|XV+B{vzPI)NvA9}n~q4YxL`^qnFgidAQO9|-{~)-7o(tobff zl-(u$h62moG}#EZ)=+99FkYimglTH4k4@mK07BS{m!UzLLgZIN9jEJ-ls_8%}>fcys1U%Nwi|0b4GwFcyl zI>$TE2tYr+5SYT9X;GtrA8*BcMSf#^#pd4H#_b4-G&t~X-l#ql3Sn0+9QZW+qJ^Et zY+tC~a8pOXlZG+L9!~aX-UNQ7hsU5XWirm2+H9QvHl-ow8@MA3cRL##yE8=?-gBE9 z4SxKy2pFCiw8PcU)S;)mhOCK(XwA)hxx#*ETVhd_S5q{<6DAGUdfTcvXjRBPL@yZr zL?B{c)1Daj+=hN=7DYiHpHDj)dZt|J7_z3#9Hv>XvaeO-Wy{W#al4eHH-Mxm)y~^D zf{s^|vY1qMBa@$>iOEEVcNdcpLY6iNc3u5 z)Ig7ce!iyz#^{@~{PdQ}}ue$jIJd{G_q=F1OM4ddq8t@k`9l{Qho ze`(tzgoC~O-`NH)WEE6i-h84ZS=v9+{UJcn&8u5DeC41g>!9~NE1oS;l^8T>rRH@g zG4?%)@!g{*@{}r?!kOpD>~XN3IP$IMVGq~l53?RBzp8h2K(SP|>YWLdvB@)66+RYP zc4k;+WpkY8RPcOk%%gCgp_LYV{5WUhZaz!!>?Pw_nag0s=`Ag4avJ!xVQ2FrZ0H6` zsU_3Rr69fddAC7*&l>-2cbqqusBN?78_J994OTrq!kHKmgPd#Tw&7ppa$q%|(nrrv z)p8Q_R?qfI1hE)xglDmfBKI6@ZP(cnHH15?4CkCW-o{nCldL>UjEr<< ze1LM(_q0o%G*#~sK%Z#_&;IrA1mviQFKrdw3YND6wAf&CPrQxC5Wf9tPhfqlcDpq+- zmIW}di_SZ3XNCzI@l_g3nZ=cvp6@NpCZ~KgM<^PSSx*S6wJ9afnO~28{|t~L1)cMp ztF>(9IjO55EE6zUZm_d(W|t65YQ^_*1oRG!uw|@ZceMNQ>*rj-b9eD`O-~8UL=%yn z#|al_hRPXBn!y|EeBs*-6VMH*LbxMDdHjqNUUs$YA^EulxT;2``!yKg1u21Mg2b;q zB{x=NXV1zDIVa@fBn&#2A!_ zzcP(;z6_Upsc+SQbb`2H$^gHw+$>L|{=MiXRKtMuBx{`LhA^g%q-C;dFFxi4;eZpU z7r%R+vlFdHM7+q-CpIQtQT00T_Ik9Eqqi?~@ofrdo$uB5wXupgL=I~CT+#{P^W4tE zgIQsNLq%vDLCa%0=2d#B>7ERLNzlnjax?E!ep?drFn+1d?e4%ZF06F$m;@;bS>4mv zf2R9EKs?(3RiEKkZ26fmezG~RQhCp+iOX_A4}3>WV!)gWj2l(-gqum;!x*B)U*K#Y zsjcK~3mCAmIU5Oeg6m^kDYfcQ&~zXI_#s0t|V+Oy^AHCb(*gaOT0r7dnG zwjK|c=+^vD2vD%PwV|8^Mz;NURchPQgssP}&_eg%@^jXOGrtc%MumLv@j5Zlt7k@6 z)VlCo=?Nh=KlFxUudY2-vE73Es7W8@7$OyZQ>gxL`}mMsuQ56yXGXVTuE%P8vv)bb zHXkdU^z3r_X7L60wy$b`d>QH`usV7>7VmGC`n{12CwonZ!TTVD}l2UbpzZOd_xu2Ii7AT*0sgv%*LW&$_tV7A0((rf|yB)su}(Gs?@j2R_kEjs~<`+j1C8 zcn^+#(|xNxRWRJuBV;_d{P9)vIWU|{VCr_&_xTh_>+CP}$b5hkmuYoPO~>gkvtD(K z?E}rl9#BE`hQNV~&bYN+yDD(x5v`?#y=j);;Vu8<4gXoGS@r#O>F&M=V1$yy11NU^ zto6=xjfZ;(Ff48=9AG}Fe7Wv~j7MXlW0f)JGe(#dzKN9@=YtfGGEJHBe=D9CbP!{@ z{qu`UmNRYYWvyS-g6#eElW){uT|FZAtQ{6q7i+s0&8RM(pmuCy@zJ73vvvGKL)M^! z3x`K`lpeuXHZnHgJj*2ye(==?p8CK4A=tagF@0f0d?SM)J6OuX?9NNeamN(GaUp_7 zrv6noog1bW*Ct2nMABaWdbYvNvJzrFFdL;jTld~fb3mWQk~TgMKZ{4qpTUOjyVchu zzPeid@IV5sV2K*+d5RkYi}F8qL=$yHpc~zJ~`a22;bk+3e9E?CdR1joHC1jEZWR zyv0FmUtYcbLmW=!^VTK=pal8;G%Ea)PE93HK(_bXX^&M`=s4;OokGnW*-9wIRmcQJ zS^wn|URyK%Y&AnplZ%vbGd;J~D zc2opj{FeApo)5^Z_J$PljeOwu0%T(jLsO-!y|RHzx*!O4N20H9<4U~;vLZ3!c*ZWQ zc(TIusXGDD;JSMIrUJumIowdrZ^*@!)*zwSvaT7^Op3_FQc_*D>dnd#LvMwJ4 z4j$9DY$p1oISgzk&MK|;BSkHH>{XM{2HL(;p(ldpFK7BifcSoDGe_z>{9@fU$S z{~%lM+~=S-CF1sUL(NbiES{A3dl>gv>U4&Jb%B3z>@U~2QM~)q9QyEmJ|TDX2k}V z--A~RKj16rQv3T5CF$9B@dr?l`$vlbq$qb!d)=*l7>5!vVld&XK+&U*76s7yG{yJQ z(;9VCx6oD%eyOggZUoCj-yeKjP2c^GzvlBb8Jd1=rbLo+rt&EkzUX1~(t*z#r8*>t zL+KpgjYW7Tg8AuAZOGphXU6*Bd(_nD8(wh+qZXFS~k~ zHgF7xy^{U!ANmbLp`K8~ba;&~$l7r2or1xaK;q`cMNwApB2I&irJwORsadULR#c%i zcB=jin*00}B0zF_UK0L;zL#vdUiURA`_$aOuibct27}ydrV0FU+JGV>$+6q(q z+0thbLQ*bAy?p{ceZx03TS4#78(+LsY?9wBKjR}(aD|Md?y zp~nGiM6KVOhV2rIW`+nFM%M0aU#P28q%PDWE20%fIKd{!g|?3qEJ=N?P?|{Of;EbNo5)& z-u!u|3oyIC{IYu<{hIjcbby&0NpQ#UbalHZ6li<={9coSRG=rzJx25-^b=5}jy2x+ zh?4Qe@A*<5Ah8#pW}HK-kS$w2`GR*2D^x&mJ4>-J#u$4N6R)$8Sw%yLmS+0Qv&%0OyaZ8a zJ}{>o82BPxUezYFG}(q{_`IrjdbVp3Y@l90#kfPfC`t7XRrN|R4!7Wx(XXYpN03Zo zZJ80mN!1)ek8SGb`B?sa!M_yW@Lc4^ zs}Huo6sW>I4xnmaL7&9@YFPC{i&7Bw_O69SfB|J+DLc{WJheJFws*IH9Enq8S|jnS z*L_JsXq{V_aY|^v_z(NTe?1ggy)S^Jb-MnoohZvLgbHQl2cd-X33DBq&_DBNYDy|M z`-1~N*PS~jsb_!M9*>^m)@V1l(d3$K)gaKJk&|Y(rQG8(P?BA3s8|IQmXu*q(DuTI zTg6fqEKuKP(;^-j3QZ9Hbf|=)b>}?~-5Ap+g?<)YpY5Pfl(*r3I@|;)}Md zF@>%Ucb<23cPra}`pVd4IptOJ$RKk)gZ78oo_`MzcWRfNXak>(MScaj{kf@o2?d%( zpuQCGkPX?k&^jR@kA~N$(MWq{=cyde#Y?vDF5zfr!iCNJ3rPLvTZ*jZT%YWeW=5^( zd&>y(fu#b>V*h|1`I(2yg#kxTl=YUgZwv6g#haSR@pvg%j_?R6N}C9@yQI8b&L9N*nNMz{KLxQj}}o~ zYkXYiFHxxB>D3m?dkXCai*|~BGD+p-eTsgN|F_J&*89IN<0}%6X>>D+8n!*9CO&CZ ztWcA}kV*qGC6OkZ^|41s8We%Z17%rNDVCck$+V5IMBuRHX~IlJq9TQl={ z+u_XkS5pqRng}O1q^RB|9-v)nc9zjs$!93l6btDS6b{*6V?aNj#g%Wb_1o06IZL8v z`kk%i%6F8q0PQc70NO{#66xDT#^E-W7jD#Bv8y#JxVb<6|BChhw_kBfXtXqExFW&m z-@O2gk)D?hf}D2kw(0?N8{uDoF9sK*3`h6h{5p^&nG^8?ge3X>P4B35)CrS;nV;{t z74P8Elov?^7!YS@hiYNmUKuf6by*P(IvI#_ZYmk**9B)sCKuRL!U{#LF$Nw&%2yauy#)St zP5IZjqAe`V{67;T>Y1MREzHM_c^9L+`nWI^mh`!5eqJdbvIRau+6Ge(^fB#LM5?|| z38T%>RyNc@u58N&QhBFQ<%*EvOlNa>8jXaCi_)aUZNIC$uLMb1wY|-#dy>r>9|^@6 zDS4#bbm6zwj`v7)Vt9x0CJijV(Kzsk^=6;bs2`{ooubW zxS<~=>zM9*QSa=fObphH-Nf;cJIyO^6o=Ik|6iUb?V}RC{l5uq<-& zeZOu2ZO6Rah_#ABm^DlYi%$>Vs0W4#C*GkYW%c8ePNmSSr9^~*Z-cgBo4ni4)D+6W zO66*J$&)@GhIiD$)|s*)i_U?t>Y&t;g)?#Z!KF$A2k{5E8qcY;&pfAXVkR3|Yft~* z#s;}qbiAkzs5{b^Fw0V>7?61O*vRI>N4Hq|@w^J-3>0UH`4B6Q$Ip74{&$7eN+au& z33G~wrqDs*z(#1c(u9g`l>(sBmcG+j7Mf;Jd8=KhhA~3nPK$G_@v+5PVvvVU{Tid( zJ5V+|Up=_`ZnxR@pj!d-%zJ(Q~syNU5)V$Ga?FQ@1DVy6U8|0i#^xDJ$qEcQ(PC-Wj1QY z9tBFLYPR)>D5iIXv$G7N&O6A(SN-`Ze^rrAMh{jqetl?IJv4Z%CKTZ#N9Qm>BFC@@ zhXlHa`r>vxw5IPTDm?Z?Z1!0`HT&3T7^dlXDRrDAjTF~Z>Ifk@f*RGa?t@G zi-l`3&fFuG<(lBisc=b$o^P=FYppzclA=79rP7!y#Qr*$IVJ4tS?))xYNmjCUircl zR#?wa41Mp&3l8Sg*SwFSynmkgGZ00kj+ff{BBYAGrJTywN$ntwo?52WF&2+!S^=sR z#&O%!4}vR6^IX=N!1!DnsP4`D_(i@z8Y;T{fw|)I{2D2Fi@l8mb(km|lKuUuXeBH5 zu&-}b%~*wcjhlSjlY{1j*gqW6%-Iq3Mo&vq&r~#RNnxphzgDOdu&u(bn_-&k4c1`(?Xx5Xmwdj(n(|>4bMkva7IX-0EGxAs`TXJNH>t5fdWfZgIDJ*s@Qd{Tviz&` zIJ>g5?B;{zKytlcsM>5!Qi2B6fR?fSp(0l$tqT@dApNCNh4;eDXPizm^-0`RKgk=u zp6*ppG0C-0gCme3{vB@(OBO5=I8%11UrfC*%|iz^b) zag~A)LZfy(VNzS;ipHa3%4!I*IRPV3vlNxYTX2k!HYGu3koXS9XvL@X!h#7Y(= zEKp$2%9^9+j&t>vi)I3#P93%(Mw`eIQ{ng$8q${2JAi4|S@a`ES@ zMcmWJMN-H*7Q+TKo`+Ju zkyI^6t32mu)su&wJg%;JHDOsn`2DoXBNUD(AX#PPMp{~~p6z|*fOcu3>2U}I0BSz8 zuQ9Rv{A>GIiX`htJwls0a+TJD@;}$f2lR9-FE>Z>NOT8X9{?@B03P{fp$ncM8U7F4 znYx);-oH;O*#BaLfLa_NRj+)1Ae|)~PIoe>xozM5(wHSZ`B3CV zGGb3Q!Msy+>q8N*BgBm3uOmuK>^H>O3oZv#|LLpc=bDZbA(1t>=J1ox>BQ=Ysg zeMbv3PW;c$4O5$+1?=m}eTz(~^O#s{yRxV(S^sy*kj!cD_rR#L3=;eL^xYdDojmV0 zBTtx5x0{XLx%|71zTL>~PIodAAs^D{W|)fu4oGMC89vnf*JeIM!_ws4EZS_98?MQ* z!T`5wRiTEHiGY^Cl9~#~==$LorG}S50wqY>*{D;eySOT&I;`#)b=f^`{JgRCU5X%7 zi136&NDhr?;SGOC1@lS%nos9SYfumTyWR!AIY&>qp>;K|E_xNcF>!R3j{ZzYue0wM zKPhF7ZNCWeZrwXxsi_sZ<7o?(aD-p{^}Z81?m2N582?_FV83lahDtmMV9TAXaDUY~ zMMe6DnA=V~rP-u;Y#Zk0)Mo0{)5@(t31Wpl2S0g(sfQ8EdqK?4iHN<=VS8?(1nCyX z?810riKDFdZO2_jk7n8z)j^;`>ZtM)ir~S-87mrDp3F$XKFWVf$_57!F09>E7r{#f zFBKX-DFREn%{6Mzt#JhXh5vyb-Uh+FW?Q;uUr^$+?cvh{d&N&?&QA+$#>;!=zv4y+ z1Ow`Q*y0jObHMz;k~nVz1R3R&X@H#ZX&X^O`56C;dhhuF7O*8%STtF2Qky253n_TC zg;=xfMMQl9#L^Et72?%b&9RLFV!!FeZ6d>}Q8@~E0^n!`VQ@tRVf~amkkWbzIcC!K-FoDS|hORvhlvSPdw;`{psH57bdOIH9LhAJs(P z4puBvjh$BaLKWaq74aD4;mqzD%~%j(tf^1K1!OisqlC{5X?vw97P%3sL;JMz5D4c( zB4)jQ2TQ&$gSXz-q&F|Oy*`Uvr6|P(Nn>w&8bi1y$g%)IVwfH8D*SEy=w1v}RT~R? zqP(A8NNSD^@0PT#dSV8%^?3XGtcBw>CGT&!_xcVJCAr&nFnGy0w>=k++7V8S@cf7e z1@gXrib`21!*3?K%+nvLqOn9##L2Nu%OFqbT}|H4q`i1^Ym2x4A+vyaB!3kTR>w9%woUr(+wZtfSkh$gyJk+$SpNSi38HgsC0<2LGD4K+5t}d zj|Ie~it(bAz@!c1d|1tWusQ)GFdXlEm;P>4Zjfbbj_lAm&L-3b*X-V0c+^>qZcRcp zBI4mDrqVuNZEu`cb_+|yZDyhVfSF3^lxOr;KdlUhF!dSNtXafr@*bzm?R>XO-|83{ zcBc2nxeVVn&XEbX3MSprKHP&3S_&%r(JYF008t71ZKzIG4J?>l*!WZG+g8B3%4Qh& zc%~@U`9zPCWz{xp34mPu!vHZg+8a*`;6_xXe*qX}?#_!3~O$|tC=oxLyOS;Rg4gp;&#;`fWCh8FV zjahz5yJQB}>`BShljN!;Ve5f~Fsd>(GX5@aI)h6VqyQ%$DT`oQ2t0TQGt869+$B9L z^QtdH*o#l((7c&$BPsi)#0jpqtv8njUGm}UE{inRW(uvt4BnPc5il2`2ECM!{#&am z+XP35S4pDtAXel8c|?&+Mmga}8bSD>y_Ka;!0EDbb3~C5G3fIVWt*8kVB%* zeWlv_?~u%c8R_uRSH#Tza$<$UBH{d1g`}P3PwgYoUe$AIjZ4Rv?Wy9EH|41BYkxeE zf??a~BwIh&WJu-Ot)7A8RDhiZ+bc&?II4$cBUm`t;w$=wF7BXJPOd&J9I7oXHE*at z7z`A{MP}dEgb79*-rS+Ml$tJJ=nwT#`KrN+PX&WjDS6>Y6{_lLgwNZGuNuBGw8)m3 zipb{`93ww&qNaF2M?yzpEdSL+?|=54(TX{@|Fjk$FaGvr$b~IfQK%0?AMW-A4BYS- zoM8jPc;%%Ve$t70$i^n{MDzg^?Ixo~AIx*ArY85=9?gL!M1XYg=KwR@M|U*onXI|1 zZ7JPWg)^n71%H^!_eXma{~xG4y8BdF*_q8ezMj!{c!I+)wXk>1_H2D+9{*%}#w|wloC8DGD~I;^5bdhiYhq@X zFFvkx*qa@QT_lNt5oXJIe<|KKsk|nnb3y0Z;cx_NP5SZ;W#zTkPkVii_-bZkRVkRE zx5hbP&X@FEBbFo~T@3H3&FXmii>*g{`2AD4xiA#0EuDkmJx3O6R3|%b&2>5svdnzR zL)GcWX8VVp6XHT}X7&2l_gLMzOAPOyR-`$!vpeR>r2-Vx7V5rCfuwpgROhq&2nCAy zvt*OcdFT24%F=(g_4(0S-Ikuh4ngL8=)bUk_AW8N(50S#c#xiXfMHwB8>j%Ov)wSe zV2l1$m)@8YabirD%h8q-L0ztKb1p^zZ!*5G3!ODzNShczbUJZUNU?{_C4LKg2OsN;vVs}ExX;zY`%v({?fDU2{lP|H@|EOB7NEOsw+*@YPX`I`8OTMRdd zEqkAQ6I(WZ-haEM?X9I93`HF=aXWWG8#2cG+cAe5-)}yz}bbl)r%9a+(7z= z(IyDk{UlO%LE(KD+N|VKNW2uM+OR~K4d|_x!JJ;{UhvPA5|^A2t{ce1X2#sGV-DU> zfkgM6+MSxM>|8LDrByK&Kg;X={#9o=j{g4MoW?b&2R2C*5K$cxXSFWq`QG^qd+_(~ zsbS>VA>(FZPU3K*%tjlec3khNhC)xae@@80#O%Ajx1je&YT;5x{YNb_(ttnTq|W=! zyp86s&E06mn@N3?@f+`<@x&l#o?t&Pa08U8yxNeX81|O>RZY4^cPHuWj8mBrExl6T z85$&h7%?(S?K-KlbQPf$K^Ei*pd)dg=DowK4SNFcX3q|3ub-=N4f6pQW{+PqGnt)0 z6EWp9oU+RyE)({Z#bQIxFWIObJ%<1j?Q0A(Ge=s}2=O}Fehge3`%mhv{T{_S8xBfd zhxwU)sOHw|-+!{TBVR6+$t-p~+SVIhXY1V^&+8*>n6qi7x$^pYzIXH#KcsrQ+nNZU zVZyyZ|naOwq8{w^)dfD;d+9m<1h|#w|Z!2h?jz6Hu+Lzh)r32Rbw0g7#yyS z)W8p%XoW`)K?zdLOOR z9m{gr2A4Mx<3HW1xda*#WA;zL(E1ej+v4Wj0#I?q)lIwi5Ss`6;DvHR3j07NnT{TK zNI;>Dd7Q>e<64W4v+-A^rW8xvHFCC_g0`lVFinBeInkU$8HI!QchZOZTQP3Y6@_tl zo~FfS?_Xh<65A;}P}At365!Id+G;gcb8RTaHtd z@8RU+Fvz>1%b4n?die%M42@yVt+ITOnW_+*)Q-)0KxFVq`cuwQ*>%!PrhODPxY+Kk z!V^2c?_aL20wD`YvL1Kb4N`vAFRIx75QiDSxz-r14QKku?P$hSJA-^Y;WLGPj@d3dJjO^cAiDKg4WO*- zb^DU8yAqxe%sE`L-}c+t^L3$uKP3 ze*GOfZ%9FkTA>b>iR1Bd5DvRD)j}dL5o*3gr&qL)b$5QM=jVeIentHVS6lbBEyNgk z@H$CiL+)E1M^!Z1))+(ugNA4u&(A{k5XQjWK9_u~#h!9lQK?>hOtM(oL&31c3Q(9X zNYJzKA^pUH^D7stYEpLY#&Jw@j)uMpBzRh<%cV0K5q+5yUQYaLDxb2PWUQ&Q8KW}e z(i5TQenMeXmzlE!E)=_w$=qongzrBvzoTYTLPdeqW@DE z` z;6=sXp{oa}O+UYc-$9UunFc}TE5v!_SvQ&6{<{DZv!d{6e#Pa_qG?TDsgi(~xJfyt z<}WF)K2T<)OcEIwqQoYQ+z0g-Bu#hoVQiNvL2gQ{oc)Gf)bH7QybZi;a*4rGY7MzF zZ6i4*UMd1qEsgz*i{3e>1=BfmP09j7P?8;a6&(%u@tJjFN9oeIbM@G0{E%wxQ1NCS zvW$saBkSlrI=Wf2)n~A`SlkQX!ZiD`y!sC5@vB#h#wQ^y%E#;X66+t*wcJ)>Go00_ zYtAzJVxbc``5g7S8DRR!a3mWJcXL9sH9MqbB2R`jsP_ZbwJEwpj^Yhx!S9W;;HXmB z(A_IsFCLxW-e+MYz!Q`kPWq0$x?$&Z5B-1Kj?Tzu(;RIBq28Vj>aRBc@}TFF#A7JS zZj(6~aL26jv>A0+;4%YtsyrkRK-)S~&sCOrj#Xthg-wFST#T^%;NRg=qoj?s34qY^ z-pb4&K*Rh`ByiybzL5gYKLw5kdktojGsuqBSlfW*uVG%OEqcHnWyUr3bm`uLvh2Cn zKIpDSLNm#(Ylj@ko9q#dap+D65uF7n!nTUX?efZ~bFIGzqrBqId< zm4bTKze;__yKLsbuAV1^{XVON)y1}CvJ?t4XB~LO>{wk!<>8FECxh?wajDDr8J#2^ zS?{R_jc@6tIT~xA00TVIfJnWt*3hvFbRv8qX<@zBk-z#)qWN7}l`D;3;156(2>0_Z z!onwof$|@;_Gfl>rWhHngmL>}R>*U+X7Oc}sDYr{b*;wpeFY49}#UF?)zUmC;cWP4v zrqZb{p)JF~t>z4K-Yv&5q7QK9#Wqc!xo^LyWbSp#sZN)@#Y2yBSrQWs#3ysDQ0#4I zKBwBg06zn$WV#l}s(tmu$vnVimI~%@SeR~U+EAQIY)SXvZ2P9A7Ps6`MfWNid$U8b zPYj|uyGr&}Q1dYvs_y%yCLgCqo_z_QtQ4nO%i#wmQIAe0;`MoMAiL2%4RIzik8~mz zJM*+Y5Ocy30Fd{m*ZZIBIm@y|b+)p`q44xI{h|~<0Js)N0D^`%Z$5zHxg>0s%ihm& z4pp)TKVDv069BCdbXBE-N@ag$h0{WVgUqaNlEtQrtCCH3 ze-HeyQ`_>iUHYX#tvzjY?gB-qY89KcAM}20z-J9OqPmmRC{&dTV+y|fn(UNkQ6kgss&eSsf%TWb-zC-H957{uRo&Hytir0Huzd+@5apGpo8QDBxBX-QW6`s z|HUB)%^WS=!w>Scp*=Lz{!i*FKLRjf2hdjh9pxFe#?_dAt>O#u3JLr(M8!mgisX&% zWO#21@*|>{&agdylitxf^Xubz9=6$3omfuF!w|{kh}I{l?nJgYzdaXB(b-FSHU@uJ z$4H|8Bb5ETVOsI*mXmm*lZcxWwGvMsx6#dQWLDL!1*r4pd$us;P)(YodQcBRiIV;p zi-wunWb49H*{{SIk~e@orxy=R)J&Cff8922z?@HAt2SMee4@AjF)t7JV({}-%xT^J z`(`Gq8yjoqWqHb;E){VWU9Pir8ZQZP<6t;U3d@0tCZd658Zv2~2!djVn&}wZ&ZZP? z1q8!_uCJ^a@Rf-Ruf|-*On!f_F6=+fB-`G=XZ>@nAptXXxI9Jufgqj^93jb5~gKHg6zS|DJzXGMQEris#dH3`gnvXkusH?Yp zTJp#+Q@O4)-lW+XpAlNN&SXSgFC_#g7t{o95v8td(90btK9J472Os)87&qjOss|;p zP|AnyN)%@fGSs7Wfs++f*1lYp)l>>fW^`pOxrlZwq%d2~<`s8TDF=W5?D>d~8q`V0 zAt!Sd2)d-qK+~+~bniJN&I7(lrdu!GJ}(^egy=)g*w#UrHt%e}R2hOucG}f1SAmuy3XHl)tG*@ide3Bn)mCL3XKDVmm$lslk`Ca_p+71tA%jUExW6qfgoK|kP|vCa z%76=iCKwH_y7gbt!{lG@+_1rHfn-f-Ug|AyllWoN{j-Zv|jQmrfe&;4jdZUDCVt_X@J{U=@H#cb#B@5+-;f<<`q2f-Uc1 zaS?-4h6{zq7}c65F`-~m1jcE6w$MZW%)-*Wi=qfum>lit_(^WsaxtXOZ@l#ob*ExI z;0v76?@Q-vO?O;X{H`4}{P=uRpPj2HEH{pLRI4hwcy}T8-9qTk1D8f7%n{|c0`DG{ zo{Sj)HB~TG{vZ^sY=jU;uaNA?jjxbAm$a7T6lA;ZzaFfB_Cq5Q_A5ra1|95%jp>0+KmY-CuBd7Dq?onMM&VpB+Egc1*NY5ND5A+oo-_TDj9ZK1aWD|Trk(B^HKDe=of`Q zTPcTVzC*IYvsvK+Q|I(kosr$evZ}Wuj(`02GAo>-f7!f>ywY=yaFF-CsXIwPWm_@e z=f$E7zxvF>X9Z6(a+scY?L@QmQ^vnNysnwPTj!S8J$AIfK9|Q}ne`Wu+mPASy2iPx zo}=#U(78Z65zixaOlH|mNfK@aKxiMm0#T$09oc|uc)Z?b`xQ9lSBu(lU&zUnYI z*S?d}!4xC{_VanQfz<+-%G<7aKQVA&7keXQ2nh+VZmv=H`K@q$aC7QF(0d}h+OG|1 zG1B@PW8AiLd|wkSnwDi@RPSgF^d-9$+D9FZU1P}9%p@m|-P4Wc3utcoRe1XrTEMIG z%-<53P6jVOEXsoKKFU*0wN)N;%`)L|>fGIIN_2CCKZlw(#1FhHaf0WXqGA|8T+|($ zvRfuEqkuf;bM3B^Qx&AzzvnP*Nb{mF&23a@j%`_z;2&ow&@^_UwAK?5>!5$$*7 z>=&QQtxQM{isTx_3Y>Y2oaA1!br-0f!EIEtu+uSH`=nhgZ?0&@z3LRp-#ZR<*Y$f9 zei(TY8J+8I*|!;PW!dF3jdK8!u{F`TFJN_1o&z1Mv#1xC#K6hTf|XLXlCVJ1V^d zLFo`$2qci8^bS%&2%$<15PE=+a2E6J{p~4^v;UpzI_LXGfR(k@`#$}>pXZifPQE`} zc?YFwdV&(OX7S1;4QAgun&=Y1x5-ooV`M?^RwuMbWww{^`p(f%4cdHA8rW?)9qE)J z_gZSP#cysD;jDKfYL25>YP9AhtM7_GKxYP5(hjyTfg1E43x4sOUC8h=R~im!Hck&K zyg4%@s=b|BLO#&n=J2QQ_!%GbT`Ugq&bJRubE_vfn9nx7ilrh}K?E0Kd(i%#50$uE zyFRYyAdJ&*Je8m3MO&Q8IfP0!Ym`&wJ$9KDvfdT%^j|I2a*-o{5lSXd2>?txmkUgk z&QP8AHh}V*LJah?n;z1QUasWqW}efa>^UOkQZd}KFXA3#dV?7)t65z-CpGd@V!^;W z&#RSY@gut_1=ZHUPGPENWW*vyC-dl%TICG6{u77E@^B{UAlW@uRy(8jrPqTW+jFIOR?iY# zN%X1%H7?oynr2>&{gD#-7S^)MG)wlf?U+A0wOBKiveLzyt#XE|357q4b~b$OHKV%T zGJ5}5md{Lv8v99BYuwIM{ZGc%QqykD}t1+hcq{Ty9;DPsdD+s)N?Hmzm)n zLv%_8OH7NI(rzep+V6D<)pr64K`WP&g9Fn_kTG;nRj&p5ecj33mW6`_^1GS1n^?lQ zaVBKSU~_k5e#{1ZQMBsAwC2+;TP!#H%gL?EF;u2Vv^`j%dT~<}6Xir6@@{>1XPS{Y zM1uKgIZw`+^L%W^4-?9zTjS!3R^tpLMV3{$*wgpW;FsNtmC@(@F4Y^*I-cvDdsm$J zY9UkRwjVgvDP4tDV!}G%n}?DZszK$YKY3i$o-|;8Z(xxx$`p>$b0Z46pSEKxw(Hsd zVb*P{5!GW~NV-1z{5$5YzPm|1W;L~(NwUj+`wcV*f3d5E48Z^?x-@;uz@*kI!5KE1 z-=JO{%}^}o9wI^&vK&e?Z7e#;@}i|nU->#da8J6+N_96SjlM0zcezq_%>2VkZKXr; zRD{8Ncg~{6*r0w2*0Pifa%)r|{JRWwm{dXaBIZr;!Pcgz?yzsg?6VZgD)sfN($d}r zdGevC>2NSHx&6IotXgcF$23A_LKBtyblSSQo+a3M)?*}8e_kIqS$CSDxe#7fI+`n> z!eYF;v2RAMJ^2`Liv}$1>49^~-d=~%F}**XNrJ;2&#b>J&UX&699-2`Ii_z}P2wc9 z^s4Wqc9Rbt1DZS@M?|9p*`Cxd)D7e;wTx|+JvfN#7^ay5E>wn{J>#&9ut;MK%;=K~ z>{FrGL-R=9tQWunu%M5>s+#m;elckY=J}hgHUQ#e@OD>mp5L=O8MT{p^MvT&bKG*% z!$F&`yLF^{nv+CP3A)@5z^44{Jzvja z|HwD8L@#GwwI8FnzdPW7`iYMZDmipHI4FeTzY{9-YIL#H&GJoo!h`Et^-Nk+@xHj| zl28vDU5j1=|gj=aQtRUp#WyY&R?2C8f zW4EKTjTlcnKaIvCv|WSTKSuOdxM-$>MNs{Qso zCuO$+Fk2m58?ogDt8P+m`J-7CLy2i>Gw+t5$&veA-l9&1=Kc12!(uE=-JM&pERW|A zuEv<^g4H`omM}+O#!{J}t>kyM+v7eYg=!&+iW_7P2RD)_ECO`jVg8avzR*rnXqq(S z^SXVPk%S$JlTFguxpjBvT6%Ef36aBnA^OxQDBo&T*zJrYjfcAC5(P4z+zJho5jXqx zI0H6`kwwWscqN7#_lSnnDBoGzaW$Ja<4=g(XcTI{h2lfZfcGIZw;|I6v zgAdy0=i2iQS-Y18AQT5V;yr2X=D-R+SLc?=(g4~>e+X<8|6=nCM-yWR0Y2BJR)pB)bvk`X<+sb1)@bV;PfxZ_qV{!AiD7t5N z{KkR!m!|~^Ff;flfI94_ejvPe$rwcpRR3&y0+w!9H(tfXFFcqW%gN%2w+u(w3lfYFIrBPVO zj^hKQrnKJ7dBFj7rF@|mZx=zX9D^D1zCQTIB`xD)3YKy5OXrkK_$HY*CbCd4nNm@^-MEE;kXKuX27GUpErMpTx~zUgaKzEVc`hk@ zZcv?lUu;fsYGbdcQD&`RqgjoFLYT3aKv=*|mz z=Q>Y*3|t;mU^*4CZCI$^KK6t*J69LI+43gHW;vv*O4L4?8%bX3!Cg+fQ(T$}eW?S> zs9&DHMwoqxfp!sDZqF6g+l@GQZ%v&{btAaAE)5mgVh|}K{zS@swl{=IBR`7L?zFcC zw(?|(igal1`W@}tG=@rQX#mU~q(&aq7`1dcfsIn^#hu3VX;Ke;t6&gIAMHyw7dXA! z!lf8t*~Q zwpfKagT=?O5DpJI#}pG7ORdTRdmZ}g3b^!kKoCXRT^5umKl5oj6jozcFO75LB=gqZ zqg#}ZLa{9m-N%ay@V3N8p%NKa$5wu&_Ly4A_sAYZqq`@lm!$YEJ0eb!uT(TI<4F}) z2obAINnIfy_)t4qDwum$?9sJ{ChcX_^1m6loovX$>B@g zY0t}5hvWVJ^WHO-zLcDdWr}Os@QrR!zvYeA%s}ZS6e@uxV<_A7fy$~B_55z1y<2qL z%8d99E2SCo#XR<*DFAtS5Y4|#&R_#&JTGz#!nrU;8IS|)Gi+32Fas+;he`5kY*GhV zG7?;nADA}Y390b}hB9BwPjg}%I3Haxs7mxOcr%qya~ls~%x)RGJ$T}UsGriDkn;AR zL@&SDkh$VQ-8TJd$S3Z+l-hx_pd70bLp}eNeZ5KeM0@Kye8q0`;F;3k$GicZpw(8g zV>Q~6c1G8WEz9wf3p#&^IJfessd_HMh(0 z+~ylh0?XwD5l`Y0PuYaU^&$hiZC|6al17$Fqa``Kn~hgeCv;$q1T_$e;X<%QwFSHE zBbf#!%yo~qp$PhUbv1$lX-r0LL(J1@TNpR2#yMvO7Z_=q>I`A@ zaj&~33X#D3;t=Zg65W2jtH*uz1hvKP+6qL31tD&%2WJ&801TpYt18zE+a3HUZXoj( z^eSX2PTC}?X+LOB*xu3+kksG_#(Ke*25>EIiS0_qY6$Vcs;DOyG3}Ltq*wi!7u3+A zr|UTY&d*IxL9#zP;C0QFz%eG$INOkv2`HsYug|y1H!1Co(PwZfNg_cwpIGN;>pk?I z7(Y|m4xqaEj<@<9aax-(lkfLBZ)q)pO)22E)N6b zkGysEZ@z0a+G2Nd8RddwKZ5Cd0d%S2t0^k4TVlIF$g}FE@TETbYQ62fH?gy%8w*ln zMk6P%pzKPRWwH@t3EP4pGI5IjQSKu&&s2>hq)bw8j$%J}Y+i2-GPRDkK#zEfkAiRV z50_9nvLI`#y3%FCLWQEQBc}(8mYZ}N;o?l+@>f5g8s@?n5F96F^!BB%nN5hQQ#kFO zU(Xuqwu{IWk|)<>IEVARx-aY$-Xaq`{Z@Vh{44^7^^hx%dpb@+)l#duG?da11y*-$ zbd_hCY*~=F;cZo0o9E4B9QJwVfzK0H(ho12vQNuG=ibmVi}O2;R?qM3JG z=2noi_x-eWLuW^ePN))|x%PLhaccQt*!y%R?}yblWv^@c*2|3jobeSrr26NbxrqTG* zMA)8m+sD%i`tM0<6qL_wj9$D1OVe7i zbqm}cYrjp06JlFT&vphw$kxD=YqFhde>o^@LS?F26z;OK*}=bv1ROhzaZA@G|@w1emszp4uA_h!f9v)P6=H$}a>l$;Mei6Xhw z4~U!FMdQF=nPhM$-76(V)Aih~)A!kOOJ7;|Z!byB!Dl;5+D!>(+;8M`HNFyIc6pvE zKEih2x#q!m{1D#Zt}ZE|{eY1fFQF5$5rS{Ro0+HqsR}rd#FAXK(@-{4M1p8pletr}`t0z)I|W}e*OKd*qxnct_PpDCUP8jb zj5c<})h9UZvD9(1nNAV&-8A&cDkpqUeQmKxWsC21YF<5AGT*Y7u|a}%koe3w2o@ga zZBo!7fC=e}T^(E~vY7cq+RQt7+{c17e20FS57TjX-e-GhC>Pvm_YB4J5fwGVIjonw zsI=^Rn~d9AMHi?<=jN(zcLhEY=u%{=EG;dN8rG%M$1dCdWYS;z34v*_APJ&a7UjCO zI5iCOU`Zw9AWKU}?1lMYXf(GxNWnZmKjPKRMtP#A~NeZ4AyyFMVIn)H5U+tm(1ANJS%#vp7V`%Oh)tWcZrU~lGa zNf94Ubx-qXlsn_P(@+=PUr(3t`azk_{KGJ78W{bUB(FqR5p;s8)LZtJH=6EVDHnh0 zPb$CLqS@Ez^sHX#vDdhS+c!dAWc+%m0wh8vP45K%*L#Om@MpOk;=Sim;r3mU+6vrU z$LWKt@g1^g%rZCojia{RzjUl*Jpt+=(etgt|FOzobF0xUcz1J>B`@w_EHu+;dA)`` z0D$HG=U@K)Q^N^C9na$mB4?R@`|7ZCHXS3B!OD|O-{p}7pgi6ts#EYtWx#uWr5kFi zI_fznG;0}l>)%)Z;Tu1{0A^6%^2)xYarBU3W9N1$dX)K=Ft0bK*nZCP&WoGtr@BjV z;XOx4ILkxZ432Ea;xCS!P%Z$7(nO>J{r)rl08s7(AtUK-7o&kM*X+-|_^xl;r|Z8V z-{c3HiRYUf=dOIQy!879tFR;1yX1cn&wlZp|J})?bws`yInZ_IiyQq1b0r3By4Zh1 zzL|bRzR8-G68@JSh`&gI4Il!x{|))(tt0YHel3ZU|4hEw1T0hQe?z|Mbws|Ym2P+T z>+~Q;vikzq*tq|OeDm`WJ8Zga_vQbB9Tu2kaJlloVTb(=SovF?skiR`Rf+zjSXYjC z@a+DF8u|}1{^xVQm#o0bdrwPxzxfM^t$KC!k-jLBC(&|JManKTcEpH!AEu zDD5A1S2;Kgp~sh;;BS!>a?JajfIY)dl8E;W3lqr9&q#Av_?d!p0^=`x_*#FX2mib8 zDP{=+Qgy^m{8Qxsj{MryZ-x^o$J>9VE(Xdmzt9i+{i6Kg?*IP78-75XXBS=i4gXo3 znE@%S4870(#V`83Oe+VRw~CqG@TqnX`r1a7Wikr-PWUR2!ZHGE3Nl5 zi*Svz+A1;~VDpY2~#~!Bs73Kb?ry=Qu)>vyKmFE4QCHsq$|D%0s z`#($ef3Ef)WaR(ft~Sfd<<<4y!cg<`$zt{;dv>d)61F{@{v(0#Pvs`#68E^%48*xp z4P(a9&kk7r98lc}s`UQ%EP&rxSLq{ud9EhM&sUBD+%878b397mgfr*n8K1Kvw>D`- ziM%x+_5mc9<)ui&x-U<*JGxa?9n)G-NPvQO6&LVP;GJ^eH|63l5){&nEq0$6NOo$o z3di05+qu)g_2$g_yRC^{&~GnE(&zD56vDLHA^pQBDI~$5$S`fF$Vg3o)FHn9l9Epn1T-t(>SFHBMi`sA9?O#m>zYlwW2x#4XRW}{hOmsmt;IGYO)hGtr`iE zC&^3TEYzbpM8!%>rDeEB-R(>z-8=TKmsjf4a9_WF+vy<6;VNSU)vxH074p+eUa=oY zF9P4pK`Aku^y`oN+S{J~?hT^G+VK;R6fn|+p2tYSF?7GG`82h&V(9HLu|UhdCT@Qv z2VfrJq60$xWTtK0?PB*W#dKrz4)4)i<7741MSINQRGx>fBr$)AZ|&qq8!26)-~Cn*U#T^qfQS7$nF<}H{eax*L(D+6J{enDe$1+P9H(jm`k z4^pS*My+M?PMng^j&pZM7X4KWf-8K8gMW zt*Y*?#^m?h>>TNx|00C)UkcL^0fgz|9+?2-(q*ozo5X|8nyZtseziw}&oY+Su|4MY z+1s9rJ;ZPz-H@(GX?|}1UvBNc2scj}nEI&36Eb@GqGXewbG>+ahEL$;*_D_>_2sDj zJXnYrtp_tAY+NI-Se??7zB7Azvok%wvnXVJ7N@$c_t0>1)@VXJ9L~do)B|Vp+Et=^ zm;ImKn(jLuBi;e+my)yUld@Z`;W!YsP=BDD(tu%;VKycjM0}WD*RuglQlWo0@9?)x znZ29kwdXMRe4=ArwBCB3B9Cb}&P^e6Fq^9h)}+$#f&156pI0VZ#4~ir@+vOH)hMI`r%-bvIcLQT3Y*-12$Z#+eisiQ!ieV- zx^B*8MrcgX0{OXda-~0y7juM_p&fi>N|fDE(-QFmv7ll({Um^^M75mGR47AKH2GTK z_5SL)=#4i$bUaBe3;^c#r~7U6=X%W!=H2^%Yu8g0>T?+?9J2eoAU`v#*B~u+znFd0 zZFuzd4xG?k&JSY+lgBCW*6wmO9v4A_LN+5clnlVGBZ&hkv>mLs!shCIL1%^1_PpeC zJyu2$pIlRYQcxp2hhD{m#n;-h;+#n)L|qEdu%P$$=V$K-&n1lD7Ot56z}!7wI>Y9C z1z3cHfTE>=I@HpX)Ac;|Z(!TBcoBJ&Aq7R`Ehxj{c&oz|C~tHYo#@c5;<-wv@kH-)=n`UUy06`OYIq ztguC0ldGQhOsr=3iS4niWy`r##{qj|Guxx`rGC)`(wwnSzS$OFtcJTP=e-s`DW%(c zLqvQO_1J5jK?#Arq<3`}LQ*7f#TFU3Ye56Kv6@4I2HAoknF_7NAoOrOaN-n6jRANP zKY}<+4Qn5PDU0r1q^Ck&gF=Oe ztd$O@oeM5Bcf`>De8Oft*x%;U?0klA;jMl-qvB*G`7Nxww#U4#R-{U6P@`{B zil!L?Xq(!oe>yZ1?)g^Uwf}?p?a8h;Bv=1r-|y= zTBo(pH?Xu6Y1tsWLF4N<4hy&iNaAx;v2o#FME zTLCj{uMN_*qQL)gD}Vm!jb_-vuelZYo-@@3XmYi;k4vv2#Y-|n_U&mi#)nY6ZxZ*x zV+;;JL0OK~y^d27Uxil4KzGK8>5#eWbLn(`G`7&7h|jE~A)wnu+RpeD0ImpfPOJGFShqIAQ;5x5wEx|ys7mxA@Pt-edIm7 z7ycUPq%*%`%%~)a+A`gohjmH{l?p`HOqg)+17m5M7cHSPEgps!oNe}|T4CL~{OJke z_lvj5HPk!az>2Gylw#rtC?Bu5CBA!iR$W;;A!O|+hET{#|A22!T1{NYVK;JO?}1U0xD~S?r}NGm z)_6e&1Va)Ki^;lW$RV6t>w7=PqWBDvLzrc}Glxv={Hj7sZ=qo#>6fz2XM@i}!gQ!t32Jx1~4CF$eA(WqL9b7$1V4bJ`-FD7;5w*~~q| zwsg5bBYF4NJ=|686rj_1-UtCQ7nQUJ-INW4L&?bA)!e((>QchvSa@+(*3)h#Ri*ug z6`)@!x&O8HA-{-2V7rQaT92RkVy$$e2`-c+D?~P+@XW53)Y0J@A!&!&HlwXZB?h#v z&ei0wNDsMk5IQiy8;6-&b|RFzn%o9_m$bN=Yk(YH3)oBC$^9hn-qk_sN904-hx4lxsW&k76Pt zcsnHw<#)!tjgoXrN$r-ba&Qhy#X&zTRL>lTM~-=hR2e}rQZ3$P~8cRE16jwdv;V8FUrFv959Dw&F#%H<~2ofIG(}7Nmi(8WO!qRQ)t!U|Im6W@BrmMD{0X4tnbS#GOqk9Igci+1C3OO=kq8v4Ysw^5oiI$OzN z2|Qq(o_VY07TwHDyn8xU;B&49CvBYV)6Dq!Giv>##BUmAomv zThxn)LZ$D7J+8-2#7z~KM;gevj(^C&dBm(Cm+h&O80gr>EBeGDMZ_OnhYk6_lH{P` z>&Wfr3Hr1G4*9cT{apb~#|Ov|Rgj&diBvM$o=qUQ<-{-A@_PB%X)taNqo)_ON^_4zmXa=TgNZYp7PttF9oh71EH8}?xaSx#M&|`>)b81WkOKqQ{b82@FhEMNbchG$9AABI!2|Sg*L9Cpi6@yq z@ss4K63=0G7(9L)AP$YGXhG0NrOHdgX*-2_A(7yz?1^lXLgxepUh8UvxNn|Dkd!I^ zK;+un@HlB96p(N3Nz=OR?jKV3k~X_p>@Nqe9rH{Xn5rPCv2+7n&0b)a2w>lQ@7O3= znoQp^iNNC{NrG^{rP%vY}!><%U8k(|ywR)2pDt=AMaGgNb^13AE>eyr;RvucITJ`mQB7H5S3Kp_RP{P8@Gq63bEPzIF{F& zsx=lA=88pWXc-MQ2aYsXTN<$_9tFcduSbToLLxq6)qHhWLb1pk*RapTMz(M5+#%XT z#3ty$fLwT9r0&9b_y<-h$SO{0Ux)HCyaL6%8E}@;F8Q@eu+C*b;G6q9r>i4)E3nLL zMIH7q$So$eqN1rh7&q^VBUTZjI#9sJ7PNV+*Q0J)pTyyZVW_pfDdP!YjEt5uTqgOQ zo+y(Xdl%L@xV<@A71zsVz%h1`yDZ+)<$|y|ng>1&_d_hV5!RvO?W{8{Me)sZX z3s>af5Bzpjd+yj9apF#Zj$M>!i}NIPviiAL(+V8AEMv9n?cqGRcftdM*(s=u6;o4? z2DRJ`OSj9d^&9DSOLPM@PQf;XbTN`gNJxnhacs>c+3x8*%QJ?2<2 zXrRH=A1Vb#FE<7P>ezkPqg*>rv^)B1?6&c;q7k=H9TqU#-;TmC%C}o)Q7&BH0qww z7_9Vq3DU9XLa^s3?k2%#yVw*ls$XzEgy2x`$h^4*MYarW&h%DxOWjpd*?a|_Qf1cD z_+`FAH)qTCs&kMnufVYE5UFTEBDL@*5k3FO?ldXiBFiV^mvV#a1LxKcss4jnH*{2M z1JC!tI=Zh$BsZOQL}pz)OaqddMZyhwI(v?s^w%8inR}rNcP`!fwPWM<#u~;B06^AW za9kdhpk`PBJ~kS9>vI;A_IMQ6Enn?>Or_-AWF~_f=Sf7CI0?-=CR=X`PdOt{DD?U9 z?7p*z7-x}`px>A&SX0_*qyem>y2*U&ZB{i(Q$$18uCfA1Pz?&)CDfrn#EK6V&%vR5 zomxxs0u%bv9SdHU(ekH^-j8jGa$PRfge z@1nqPapY^o9*(wR!;(~dp?5LKxg<~7(>PVi^)ja00cAmy>km}w%{&UscY%mH2k#rc zlAYrBv#k!lP{AC(ng5Qc_Xc;7B{}JqH|-t7_42n!_CBS(U#`0P&qv_bTa=`DHR*#I z)${z9JB_(OK8xpm3#S2SK}XbDu5IN*Roj#B-t}Zfz>r+xh17Jcs{&0&XI%XgUj1F7m#SvB=esqzss=dT@m%s87tD5RzS*&gJ$< z$&Kh%Ex+J#@VVyMrMg82wAnDM!?H6e4LAM-Hx@u{)Kx9y;X8G_8S{`ah6)TE`A<+@EY^0OWsEDL_J7vSE|d!PvPPkIxSrqH z_nCi`f|ld6{I~)jtVPO?Hssl2;Xc7cDOo?>xody5HGdx{%ic^7n#a~3L4Kj72lnbL zUI+8fjJoR&18BUi~{`x0teG3iox0f3j!o7oMUC}O!EiT(+c1gpv>jox36kOYR;+w`# z;G)mRK6zxgXce95tzUbn560@n`QpnvUTPkoSC{V0-IUL*6g`TTVUh%Q4(42;C%b^V zRO|d0NmvbPa6xurav|~1ZblBvl?y|KCIbQu^PMA}qBWj&G|1sT>EM|iZaf0zil&V8 z28pdWBORQ%lvz4~V&z3lY_8O56~pE}b*oY|XAgUb8D#WjA2}oO81b|M&2pP~ZKZ=sIe3T{F&M%aO+wMhK%XA%ugENVO&g&In^Gzio-P<``YP=zw zGTQEg@5zq#XC+}!F$GgUz$2w*$6V(!(D@~j#r1?!GCQi(ZLtzWG3U_1m423_k=(~D z5`00T3&H!Jgd$Q*CR)W65U)_S!CR4@ZcrK=0Ow^dhM#=_784SX+uxzaJRQLlThx!9 z<_=aLG-TYbUVg)2UJ61d;-Sb4TH)Lh$t}uz=qUivN##{Ik<>Pg$a_R z$6B;`p-p$(4qOg{2TLf)TyCgF?r2z^5qCsO%ubkpw;q?aMbb(M<=J&2lEAlWZV|B3 zR^wJ>kc9e(nWwAeIHCPXHk>kXvG&Un6^N%TLBV?nWVWx=+pL2UpgNP5v>JO_wB9D! z1hp0KfB^fskcw8GbXlK!QlwR0S_2E*SL(0r51?)182hVJ`=s7lRfuXv_h}+0q}LJd zNI|zm@uFm6y{O;IcDd-d*spBE9}IFYfx zTn`x=L8n}~epfd{F~{)ENM#SU%Vqp+Mt7>K`Uw^lDl1t0S@EJ<-K3f+R`l) zDT6V4yrNEYi8zLNOeI$XGkQ)fh65zQ^ti>UfRlRfAA-H-Z#0@)^|<>3X;CanX}r!> zp5rIOhCU8P@y4vpwdbH#Mp#KFXfsX`+%ItSdj6BGZjV<1%;?4 zAfQwKuz)5i4ys;yj2hi3EE!e2jRb(7B1_rEAUUD7mE)*+mb#AR3486z*zR+dz^O~l zX)u_y@744cN_%%}HUDnc_EmRpvQ+MRdmhV81wZ__%V@>3{u)%g!nR9QDWBvCuUv3W zY+;JJh@6l}BS>U_pu7OGgvWNpwltq5)z582GeKAOSBeY^Y*sHS48|4|2W-fH&L0@7 z9doJF>^|zoOb3cdR8oX_VS9d+JJLk5mMVyN5FIjZKa(xtjr z(_PuR0_P?~RX5{qZCK?j<6YPptwgbQ_w>Z9$GJ_@{kGVHIE z%o?vS#^@^Js59Y0cwZT?m{4a0m)7$zw3w1bFm;T3Q%?=NdytcFw?`RqgqV?$A_P-r zL%i6pvk4>ITjkQS5Y)<#@~7F1iVVBAMt%8n){lyQ?nY$zCP$4(E;e1UQJ6}CJ6N`I zLtiZVdkG7`1CM%}fs><0Z@xcR=_2QRQ_iAT#1K0X1PtR-xnsxZj6Gb>%a4^qcIX6C zi7e*d7`0|)+hVqJf5^>%Udg)oUXx_#=RzzMDdXe`kJB`%^`37Qw}IA&i`?$K{G?%= z?z&azS3q!OM?iSxGrbD4yXJwYUs{weU43Pfa;3=V*kJ@2J0y9g=lxQ#W2>{<@VoMW zdD@{0T4$$JypMg%us>jp7(uEZT_))o4N;V~-r1qO@XhlhR#1yvWE^xHzDfI$GFk}s z_6CAvwUqHx6SxJpUcsa0g=AmQg6Wg_`bbrxMFj41^YI=ZmXl2erd={;oD3vcvF)8v zIOkTogf2dWAX|YUW29PaFKN}B-Fysb`s+t%i%I_0EMa%f6N98u}9SobJ+QSt)N)4WqVjD|+Op^Oge8<`$j&WN1 zMxrUswrIJ(=@H%!%$<1iCTEZ6lFvmpBkqpz?sv-wwRi_9uv73lc-;6yYHj^W_3pU5 zYAQF%KX!PXwc|F?9j4GV(z@ygwEJN~5(sV9NrWq_Vj|9Xpdj0!H7whHTy(+xc<8#c zY2JewRiNyqns$7L(CL1jQYbujRL>ES!-k9n)i0KB?#$uyR%Pm4TMC}SB2{b!SkOD- z{uYhi?@K?boxzTtuC>Y1)sZO|P%d76%(u(J*^e<^1|2)fqsIApK3 zH3n6zW%zTYKF+NsV-!M8=IYjx_hGhI8|bJB`AjEh$ntnr^=WMpvB*onp>B@LUX$0V zNnWydxL7jcHgUUtDA6qW5TgGabiSwpluUMO8FBH8C8Zm0DK|7yfF=~H`{sP=+yqvZ zSh2uUR|zVYnyi(L!p5_G?v}WoI&V*1)&WKp|E?zV2LRZ6ew`0?T{OKwv}lUL;casa9K{iT{oDGu=^9h@-Vejnc0-tcMn=W%Go@x@e?MbbfEK3xU z9YdP2KW&OM{(A_9fDC|$v&|D6KI!GcqxaixapfydxCJ0-H;pfA@yoj~G_~s|NBkRX z_m}Nd-n(!_%FxSOas=~Wx{+m7P9H4UlJUlk;-pH?)*G!8ANso(w);4kiUpD{Y$*M` z7M~lr0zhru&*l9Z4xmc5uj%BdZs#v`BnYVtaDDuR;bQ>)V-UN`srlEozsDy3@f;!R zCd0jxOgYV{Q&(Q_g*g>M*Yf6x(=Ir}>2p8>|_)`0b@|Mtrb zy}w@^<+vjx^slVl)?Ybg{MRE{gesROO@0ku@IUsh?R%gy^1s0HKfd@MfpN+WK)}3n zzV4szed|X6;j5-48S&3R=qJG4EuI>EWdr|bU#_Sf0n6JW=Wc!_0{qwWiYjM~{4MF? z?~Dh4sSlgJe5?O5-`|^*fBpL;FmfI8-Ph8I|G9VAWdLrHew9D9@t?Q74q!zS{qR*N z=x^%z@+u%6-k}>9zBu*!kK*lh55P;ur8}Hu{pyY%z108r zgj461T3lrp?;K#HyFR^F@2&uM#^eI%2+$ls=a+puoS{+NF0Ek$ru#sN6tz71ArNrt7D9CN-)6$)$Yy2 zD|G==eaxS}d-YSAl(QYsk!)m`ma)0=qmKE*q|k~N+xMPDF*L<;O8qEn{&4;&@$2f% z22%d+<-Q(rAS&SciQLfg^Lz=8#mhDi@j#=1In(^TlqSXdV^xlNde~svBL_zGKd8YK zGy~H`8TawA?JVF_z?_41;9|?;3--8MTnK|M)oX*l)`qtHv_sTn{i$axR_g0}(-9wf z{u(>8p010AnD|5ddEkLQ&ff)T6#;+^@ksTHFyhx~;oc;N+E_t{sFid@4*kXE zsDGJT0Ay)!jF)S(yyB7TY591-POGtfafaI;m#_AyTSlfwLhYZO#67?OUS;C@Q-J>0 zk3d@#V12pfQ~nF5`y=3ZzGr0p_v`=d;wZZv`P|seF5Ryy#?j0D3iy*3Prp0>`1enH zMdrwvwFJxwd|je}Yr^gW#^v6JZyCUrR=}ai94v%>J>fZ>MwxhDVd?k94zK@xm6y=4 z4m|u7XJ=|ds+4mVaH%7}sXUijz23r=2K*-VVS-=@GMBzJpl!Faj4!1FtSKqLbfyv{ zboODhgnf@sE`WHI0=RgjxLm7rBQK*E2K4f@q(TnXBMS8L;_$#}p2jmK_J@xrmcRX~ zAF-kVi)f)#3eGlU+_CFQo_gBU&Z%Eyc%yP}E{%&Zt`Y>y{C71hHD4^eQF403g;&pU zIL`{pQrGD__3>FQFe>ocr*{(}heXCmgoGyP?QxL&-*l?|%gu#t2m#uAkcqCk0h4jh z+}kI?2TF64H$Jz&?oKpwa|EnyvLEq`<6CpKHGOqI6_PSYS$oWnmOFOS?&;Ml^)42#p;In=~ev*iB*%gudV_A2O1DQW7$E2Re zXE>Jbg)dMv-k(jfRF zj;+=DO^yr`Fx1YHQ)T8yQtI13YWyG8K^cF37Z5y9_@IO%uTeH|NW=sMMtcC6SzBL$LYvM(q$MOForW64L03!5t6Z`1In1i&eqN`1b)A0q<%7 z72gik)-36K8j#MZn_gYo!P5mRqteBCtrt=ko-pfup6^X~I#Ds^ms$1YBawVzUm6>U|HF z4;u2co1v2kBtjnamlKPn@`w+Xy>cQTb-6efcIcb|*1|V-Nz~whZ`|i1xks*+=4#OA zK0H1pY&&Sxxp9R$k@NBWT% zR|?AEg>N@H*!0KO116z<4^WZw9JQXY3FYalCuu=kj1m`an0{U|EW07>M9EEaGo7qS zskHBv?#LXDr&#y1>~eZboPI?*xyqurd&Rt4Q#<+Q52^gofL^GPnN$1XYk=_sSsE_k z3=vFq5i+(k9a_H{uL`M9XmNj=G*5_iYf+-_=A^;b?;6B+ASZ%J4Ew38_)2?>cMV7R1id85j>A@z-+VrW~!zZwrJ+L^jpoM##RVBqfAinAe_q0_+zI36) zZE27?r>#s~&!9peO1WB*V<1;E+BvhDGH2RXWF%VYG4*rK>aM}#cO*H36gl5Qh_P}f zWNTqc(J~UJKGPBlBA)@y5&NNfuoa(_eLMzeTU)c8+pk`bm?F!d+snQjAfM z2u13s(e%Y$SkT4;AJDFJ(0VgR@TUxa=<|<{#>kCjB1$FHdr!0n{G*r8W8psVezAA^ z8l===#Oo(oTJJipgI||gM*%mCni-6b#nN`nMs0^#Xh3&_XGo-w;6icegNiHV z!$Zbxfa9+1{6>AOI*a0z6fCQK8sK|4yy{p;bK<5slqefcvy7YO@*>#vhgdMyljfZj zCO$Lz9ug*8TQgN|Eu1(UM+O&TO^~jYxR^shH8m@i;X%syOZK%#H)Tv-8dK8%2g$r3 zDDMbmXFRD$U9PN|%$P{RWNqbOdt7bSaVZQJBUKc=HuxetC7|#X_T>9p!#jZ^%Ey{C zO4&1bONwy%Dmc4B($BaktRE}VNLldp`BDz82!a!)G-O4nyiIviFJ;L)PuZyAo9OC< z97RgOPDTxO4XW$A|jr9DX`}-deTXJ)Ebl4|)Ya z#?Ovgw7&Ihea(JYd*7E=xPvA$RQ?kIJ6CW9rJd&#u6S}kPRA_jkx%bS?TT;S^;FJj zXWqWGw91E9WILs~9Wb!p?5WSSbyK+z67h75$=Er-n2BT#NQcM7l+Van==gThGWFp^ zQu``( zj;v6eLv4?{&3ExRKXnJmcYO4(b!Yx-F-M2~?Z>7cX{(8j>j2pX%u(aV0h9@Rnfaxz z`A)CyTeJMrcX=Chl+{d}fm|#qxTd@6XWEnHkOl+Q_zJo0Lf%&P-9B;NWgS(WTq0hu zCd+RxJ4Tpxu6SkpYC-UmVV0UvY_Nyc5_n0h7YDZJyCM;>ZQYfC3>7*!DU7VItt zC*D1kRb({`QzPdH+rU)bFPCXVFYUOlnPhtzS>0FKFNzi1537ezR&`a=<5X6LylCIR zxypJk8d8U(Eu|-dHfw}$C@JBcFSfnRtElnm+J9?)ho{ydq+{~Yj&ZfZAYVw=C^Akj zjNw+qoy*+r7lmFWBh7Q8-OwIpjMzI8k6mkcK zHpA?p<3Xw}4gz^_%)TH~Tc;VJLsx`eSukT*CVQafARgTmfHGQwG@Lwc0LSF{XgwCO zT)9i}qU8+RyYLU^kRGbEzKzKA=_d6iP0KEAE``VLi~I`fc~;5}ilAePdh6WLJ4G(R z2s4Vv;Wsg>Xu90S9sLrsG4kD^Umx__bq<=;j93iaHy?j>5Zflvy?1-N*i6JCPI3oK*`GGqwl4u*|wkRP3H>r zC>~JJ#kxYHr#%_+YKJG*2)npf(gVRXsrQQ-m`(%_<6p!1Q;cj2LuAK&1yig$8{6)X z+`is*2N3f7Az>1T)Z9Jt25TDfgU@X%c?jv`DbZe(@|tg7EvJ9XYB;R)=l4j|pZ##i z{MCnhI>ud?%Bqey|9zuc>t;)x>f!gV zFD6~g$v$i-XWZypbI`~8ojsYuyF*(yTW$-WGCg>HMw zk~NabI`+vjmb+w6_T8N#`)=%%tqe2CGMKT1F~X1;jA8J7x!>>4_xrw=PyPP?z5nre zn0awsuj`!aoaZ^`d7aC+%<9sPro9l``_H@&eh*4cq7jRVx26NJcK>dw?ylTVElALK zI$7@jjiu0Wrn;+d>e$SCZad=6;$TT}i zEN3G&s8CLYE$lw_W27b~qBp)amWMdcg#l+?PpkDg~>CVu^(8F)dnCrC{P z$BLOt3GPy5bBo;m@W_F!;P8@kpYyij+`2iyeby0!|YBhL)+r}YCyH!N=?Rw~S?e>hpXfZrnihV6n;%Yqt0IovPFgQ8YJl6o(Dpd@)|!!f&A z;Unan;4{m};E#C2Nm+cGk^p;l;AVlq)MLFw*svJ>X7~%C_+#5(B#9fM{;42k14)p< zBbu|tN*-J2gota-1tAr@SW<%fE>}2bN6f3R^jWt%DUeE#UAD}ndzkHv8q-K+djusx zwmZbjrho4v`T35?S+`V@v(WMjh0dlVl9s%MVNP%PTV<5@T2!~6*K0oMX0^Z``8eFu zHcm73QnX+{FFvrMgK7~WyXayjiPq0L^Hc)mwB1H`N;E(8Pd_?CPw!S%?pxBNqo|{A z>4oJIe0Ss|)M!1_)tOhakwGf;r6<9BmD#-X--*_S;W?vBnA!XYbv7*|JzL*3mCbYV zXRpv@Z!~>N!8r)A@}a1A+4%-!zP!5dt2oMwwQ*J*#a*^0IQ;FU$j2Tvl~vO2fPex{ z_A`Y4eOR`X$O!$ZJ8tycI41vaKc81RozFUNgGCm&2|e3_pI;$9Y;)iGB*zdC?=mF| zrD)9z7u%x0A~)&ZnB>LYj^O}5oRt$+*5ySQC&(1=!T(H5SeN&(9y}EzG#| z?(rC@BbkZ}vOb?X265cJ&gPGmG zfR@pI+6R?ej&8kV(x9tcb#A!1jVVxlH5HaSv#CE#Jysc1zR(BS5ai!h&x2pGqr*|| zarS{or233Q&jk*!?pJG$KJ8&yYt_m4%4{iL7QFOWX$${`jd0%j{7C`jya4d3Q(r` zG!^K0$k?Yn9nE+Fngyo4kN()oCS(bw-%Zs2Io@x0j6ai3Ym{&Y-1kVS5RTI_2Genl zsuA%GYF-#uD@hc>?TUvp$GV`BU7C4rTAUZ3m+kEKvH@*#b5K@c+?VG9FqZuD&X;uA z3(S@MO8eLgVg+&ef%-A#lXu|Qu1jJr4ea8>P3f;y60x+)#(&S1fF?{C1(5gi8NS@^w>uk-;c-=MZ=&|U7c zU3RhI1=mwU9em<@Yn>dsSy1t@H~KavoB{f@z~*^A#KNHN(DKLm-;qHr?yJ%s~>u^ZjwIYwlOpkLBzQ$?%D>h%Sz$`BdoGu$+FpoE^Va zQt2}*7oKU=^lFEe2Iey|Y-xDmIdaF{*AzHOgo3n;Hg9D%@P#?_XC4D&G)b(G%mU-> zb=H1EA-V&@FDhHG4wGE!nMkcz$;ruwNWsb^()n$5+6G~7A~ILG@%raTM+t2j64vBk zTp&ENPSDmnv@ES=ai4cgcyFtQpp|z2jVtwa#nZiS+|NF?cQgO8+%QbAK)Sv122tWj zR4en|=EqH^m31xQuI_r2Y;(A)J_6w@7wwyAyAlHGseuZNPv@g$kb~6x2&utrqmOAW zHq()XHNIK72{Bp~kyAtzDJxzT+2(s?T>zcrj@Kr>AVC-U>QAa7Wg0gkLR(M2hXkFY z4A#G#)xf9nes^Qc@AI3JPg54Kk^y&kRWQQFN=e08+WhmfS;;O?KjOcQF8QU_-lmmpk;mBi}dtg7;?q_YCkWt+`kCwGJKg18DDpb8@MyG#9IMUn+!mw|ePW zsA!VP!@hpXsdTdzM{{O7OlcA0k=8)C4?3BttyGOwGEEM1+kBFQj;b*UajzcqP8WoT zA+^{9-^qN<=ws!5dn`B93{-FNTxCu1BiaRe;*1yqsGV7sK8vYxlDDlJIK;k54Xn+<>=iFmX^ZR-N9*-FeRL|bQiQ*4wVkSwk&r@5 z99k63pj*$3#Wi3h@>X?iNZAWi$7uHzvK;KCY;W}l%Fm)<#+z`#2?4Km&VqKU(K>`Q za`>Cr^Uo;Gfkmt9)Nn9eFt9YAhSC4+TEOdc@q8kWwh~ln-OgSRPkCjlA=2$41;68lmun^}}Reb)% z?`{0ows4`W>*<~$FTlk4@%g67RUK?@(_>Yc;8(7%_1jw6PUBl&a8=f!gY+S-^_S`5 z+^&RoiL)1Qk(6J3D^NRpWhZqd?RSjELc@p;!gRW#n-3!AD!T-naV+{JAD7Ub1}SngWFINVm;=%oamT3dwx0-`rVzm&+ley-_bIc$$`AwdoL z);kxhEi7|88uPr@t@xbHz$2nHGju8=#Ax3{gAK0Wc1lg#^Y$U%C)P8dOYsZ-=xQTJ#RFUVn5l8&82~gCHc?wpB+-IrOlSKd{*` zHtu4+h;0=K514UnYk8B81Pb>L&{02^hW5TThfZfYo9y5l8ak_f=?>7+i?-e6QUsVz zdFnPO%2G1i%xA4vLXbfTr>Q9qUS!`{z8uig+%(@n$cT(JzGQdsx;A&Kz`9lB-DL8( zzhRKtbtRRYl~ler1>p7Tu|ZGzRktr{bM<*Ovik_K#;R@&>ZwMC-vAqXkku=2^t)J5;S`97Hg2BO>T>ozrL{! zmyOTRg$W}o5%^$0?|R+wknM{Jx`Qjm<}$yumk+XytX-~8xrjn&kAU5-2zXaZguli< zyMSZ|IV`8f;7Bghs8&Mj>Gdqv5=7jx9KN#Q22Ip_Qa9!K(6X1T-Uv zX{lblCj^h1FxzV8K|pe%9P?U0H>_eQ5$#Ma&t;zbqQ8bz;v&`}R<8U7{^xg6Ir0Ur zo_r3?+bdke+qRguo#GZz6cGMaRMJ9A@UgZ=>8JxHPE2b!x8Z82ID~OFxT*X2OgaDpI|ymY)#r954GJuAvJL9BAy2r~hKvgvi25R^&84)6r;fxI1|Yyzn9! zeg8baQ215Wm*OJSRe^9(U&^A>oO--3)DwF{`gt>omM7e+?yW}%DXadyR!AqS*6A^X zn@EEdIg{*doYfq3G9&d^2(Xo^Ez|5kzs>6?RqIa|rJIraDRKi28u&(pR7YzqVL7b^ zk^^(xqj0`AAY4fc!@ooCbHDI50=1kou$)R5x7|u1M2g{6<8Mux8Q$-vXi^?^uTr?;S=}?;>VyqwI{N*mYWegVn1sH45ofB8ad zr1L0tKxmv(@X(>|SD*Yv!#la)bXAXUjc*0B?WMJt==2u!bYv()%DnFl`}L~iPG(MM zgNzwK+L0cy#Apnp82@pR0+DqmzWYt27mv+;~G)-%tM*?0SFXqfz zHL|HO3N-ZjO=yJ0^J+%nN-JY!g?4z~EA(qtMQNU_1R2to8B4OkzYNgE&^yQ_ux83k zvVpg=)TA%6q(w9@@DzkWLtK+nE|B6Br*(mwt3cM$f-Jq;m(s5n1v3rQ7)tSjxF8Nz z6u+*zsCRnXf(c8P)^(M1@*e={YcXQEhGoLOL!I!JHna*SrR{h-p|$9AT~vq)S*AO> zLHh$I^KzV@W^U{wM;~w31jdJTO}6C3;McB0k^yhe7z?;2sa&FGb4wC9(e16$#So$7 zzH{+7v_=E>A)|SKA(Hp=rOzysT3+Z4Om{kk(sxeHQT1f>1YQ)Z(I^uqYoBuLd0hc7 zs2^9-F7V7?)v9C5PJ%ISLdROSPH*5H-qi$|y&LJ6vyGA+5saEw78u5L=j|t97?X&U zq=sG@L)91Cx}H&Ze6R8=-{XCBD35UgKopXJVgwpxpN{4FFa#BiQESvPf`_i^orYLrTW zt5APGq{q8?o)_gaX;^Pi#e{ez9pjidA5EPQK*AtQ1oBghhgc%BFQ;3o_dS~rs_l%k zDKHu>zOMrzFPXP)20r9l4&8FFAr{7uS7twQS$N-G%O8c8!ILlK4Po%yxrI0Llqyr6^_&zouOp#H$=hlk#3W`Ia!p2=D`K- z+Ud^6CT-oGz7@r@eR~WdH7^(4C!xkj5qP@vyHltL9j(tD9cNz~2>l0E(F$QYOP`7_ zw+uzx*I1B<8u$7LH58BsmCLk}=o4KiB(gSM4t=Ex`4kiHN;k}UP4E#Ps%a2&DgPGB z@0p@5AjEhF_iOZj&8mMfF@7T${yfOxf2*A%m$#tILL^Mx!1yveEASKC!i{hC6fh+7gl z<1&nDXqJcx9%uVLm#{hBr_m5Jc*rv1NKtZ@P6+;X*(497&QQ~{VQG`#Z9OHU0VL8e zc#GK>3u`V>L0H|%e)cQBec z-?762{&?gHAE}Se-@3_&RWvGS59&0-;K0%uYhqQcL2DR(8kFoQBml>fQ<#bW>Ju%iA5mm!N^V-1P%AhTvTca-JcQn)&VItYzwD$%B%;A%fg{je{Ks4Q#<+|0eY5Cvd78hI3128pOFWdE5UGHwE_pM5J#>msZ#eo9#fZn^FYx3eopnFQFQO zOKx<$4x<@}b+t>e@bD&cq8b0W{#8OIkdo<1js>cSERQfuVNG5^N5-6zBq%>HqShxa` zZA9;-aCuwGUDd$q{o72Ul;y^t|L{9W3_%Y1)o`l&-KIsPoA<%%sa{`+#f^K;FY)Fx zXbEuagfyD?WP)1u=ltov@RX($ot6IaO8!PJdnrBCkdxW;cTN0a&3+)yafOPh1?AW` z)PWI*n;4!4-X2H8&eu#G^?kLtMGVYj&=Pak`t%gLT${J$Z97T^9cN&wIivCd6HYhk zOVQ7;$RmC(wnIa#;QYf?UlVQ^JAdYQuhU>4vLOvUbMLtYvyQeZdvbp6?iIv4wb$ZP zwpScM*RMD;oyPkzJjn4zD&vGyb{J+iUwsI7dMq$vV7iO+Z#Qia40jgE_WWhoA=n_n z@BOb4b#B{PD)*;^v#c-VO2j?}a!fIYJpJ?M3bj^~jqKYO!`{m0FRxwqT0Yiq5 z1Pt)kvo@VQpV!kjO*w_K0uHR$kddgnUYwDv*{-nwQ-+yV_{ zPHIbbDIvY<vSw5gn_^k2i-HC-|GBKV zFU^Btpd}V(p$o8xLi*En$0tt=W+5>R@g#Q&n=$vbcb_?Vmm8+kU@tG&+rpC9i^b>` zBi@)mH=EtU?O17gU(&eM#|>HX!_SG#gy=|lG2WScwHm7_jxS`Zj#uD%}FRChKrhsMhA8IB4*sKtYY*8I{aTpo!3k(}EpY1qm3! z6&zJ{P4@Gc_UQD==?sMd4f|`W@^xI!so$KM3WIs^n;Gv4OwwQt>X9-9ZSi~Q-1TtE z-|#!2jRWem$9lXgk>b^KB<)Z`&s@7t-a-uLNv*eYlMoU~-)5ab!thpH$_>v%Xq&Lg z)>|n-^H%FBTPs)Ywy5d-Y2-DjOrq&U1ZD{Q|N2c7{hGnf5MSm#cI88+Ic# zNc(Dk*?QkCk(@j6wjxW@Mm%4YyAz&{6g+IBa{l>^|074Q8RibWeG7xK9qsXlcS7^A zjXK+GRPPZ&tl3I7X1o!&_4HVE=ij%(za>X9v`0YUI%_ZgRP_045ctQ#nx(*7nejVc z{l{#Aa20oON;0{scRxSi1g4Wob=BD2Z7@?_uJNwo+n@aPD#IUtU2Dz;&pv*d!#Pf- zW9D&DPQL^e$d%nZT`fQo9{hXe@lWp*X32g*Slcz0qq@dYES4u%*giktd|v=qIy(fP z<1eY`g>i$&`8zIF{A4QPj5qXSxijFoxw!s!uG5;3;}5U`s|Ir2h-w8_V z(Sf3(o+ZABQ_c)Xz53dz)~bHMdn_H4Drs{M3p?CDex0DKIaXm~wFFnTkn8DtFXRk zn;vyC*0jv7qrFvrOPp(P@F!CFd^faZiH$UQKawvx$8Yiz4aB_qlLfE=ABSA`^X7Z6EmBBu6_SiRAaUOD(Zg~^{>0_F9^)o{=XLWzZUhs7WMy&>mPqY zjDB$B`NRCe#M~$ESVKvLbJ1A}sf@%7pYe8P>Nc(_nCmQpJW)ex%KiraG4S96ydK}F z8PWXCH;gPS8ZJH0i)l8Fg?)Vjb$r??dx#DRS(!^_R~x-s{?AFJe|um?tInlQA1YvE z+c4#epE)i(1j!UcmVRTzyDjNQlTXB`3bmX32mwS(gs7w}wsiILxJ&p$CJq7(x+~nr zJz)^w_%8WwYEKM&#f3tt`#xE}qS7*4+STm467y&h2%`+DajD)b5bP?+k^gWL83H?y z6OdxEXt2YRetCNR<&L^@`TDBt`>&>kZ&CLkFsRG2TJw&4kt@cx4f#B%;0GWA zh)loCJ+Eb0eKBYG8MgAxnbU5Uh~Aqda)HUk4tZ_lRhJw})pvQctsQl2y{)lU%x-5@ zBxZkms)*F|ZLk3L?QxE!bM8j!T(t^oez3nb#Hl-+v(l=tR&tj$8w z=~U7_%kHUQp=M_lv-hK*Zl8Xj{C?{G>I6ut1v1~;u4jIRcRNSx-q$=05@jweZ+C#W zz6B#9uQ?N^-AhgfR0gdUcV;OJ?rD+Kl?QsB5p^BtS|hL;nZ6c^Y|urMUYRNLA0b}* ziwsJy1L&Rvb_A2x?nS#bOf;{^wn=)T)XALwScr4e&hjX8>vnSm;)=8>T`Va6+N!9% zySV{BYrr4sWM%BZ#NX!^^ur*BGa7(YmKEPjbBVvrm3xG2iKl%S6qB~3qxcPB-7_^k ztfFZ=F|{$DB3c{I$O}eOR#H7M3OPkTklm>Pr=yfp+4fqSzZ+yP!QA)PHgs+v2gZ#& zo0#Zd-qa%ZBp^njLG(ylAS>l?u9dM$DR;a2=30${n97!e=R}g3mBg~zA-VkwPjr?? z`pY>M+&3)zbrRENjH2Vq$R2>Ti@>DZf_-Q2Hf?uZY=_XF95t z7G{q{A7dWxuxJfvd5nmgyRK%MuIUcP80dUXBjkGhvN!f=<;b#VIp~~)TPd6W07Ngs z0RbwTb%2EONFRn(-d6A=>eGC`ECmaOP*TiO5UfH6tKi_eovvY9yA!)AVn%KEGiCx7 z1$>0dmG<;-dwLojYO~(lApcwN5Ez_Rsc$#;Z9%Hx|+RLlNO`t50iD)1z=PUODrJoZw^14-z28B0$~ulfTG*e z;YidW2F58sU7GO^;dq(0hqK##HfK<&UOxV~=rYtxtCRc<6`HSdgz92U!gJD^`G)hYt^U2M(7Cm?-l&{;Bt@;;K# z@~$Yz;elssLwWbRc`^5=v&ud3SKX7ma(@tD3^<2v?=>Ou>Ujlw+f~Ihi{5$rv4-Cm zz6T$^7Vyy^EdsYDR6c%>8a`b0?RkB(?Krm3DOsa+K9+3A$W~;DvkH$z1#XBviRc#Z z)O|$U>5K$h+W4Aj$X(rLwk2jhHf*|(X6xs)aZ)PS05Wf&i2sBO)aE~2^R}Dh7J7J* zt`ik$M!rc9vnV zS}bz6*kr*isu`g**Sgx~g65P8?BU@7=CJox3f30TTYFH&Tb+yM+uTo4^jI#Nbh{?Q z$i}-!>7s7E@{(5eC{6RV(f)xNasYYQc+uPldFV1@-rb;mo|!d}(+s3{FB(Pdnn`|d z?|x=KZT;|(bLGJ-rNf*TxLA+&rdGPFwYTh$I;h6ZXWgBs-6CoYO7@5Q(E`?f zH#bHrVWsQ?)P*7qQ)=&U?(NkUlx+^qd3UfJzTX(|Nucfb+KIP%{#SBk<-`3c*O?_G zTd0x8_nrd9wNx!kTK4wyIa7hX6$c<)v@Cu&feNm6MBUIH;SbzQ1e`R+7)HY2eOx_S z_WOEfV7rHBJ)m7;sw?i3o5OQPGmwXn!FsfRVfB9B!3V*Z%X)sI2Y6VA>Ee`xaS*bJ zi^D^Gf54ERS$%UBxF$yM1$jIeJx67flswqivlGMH1Z=!DEohwqc8Z&aXalz9!2cq3 zJoJXLq1xARFA(uQLg2O`Vz28rXLXWdwB2rx4%fbDyE?c0BZO@LYnkUBBgFLq=9t{VL0&Lb<8 zwef46(1r+io-3>%3MUjDoI>o(b+9j$sa2NOB5IItTK#+i9I4 zlyO0^!DRIudtubPQ=9FV$Z2xL94j+;O)Dm3%VlO&UvPY9gE>$6Y!$XV+ppg=&$xBA z2_>FOt#kHIr5rp1b1NBBkzg3?oH5~nLT47dW2I6h%*B0+TAa{UX_{*nOqAm4pllWy zP8^tg3z#>6>reSn1Y*$b%WrY1A-ltQoO!!NgYc}7Z!vkQgXx-AwMg8*@8zslq?>(M z&+!l0m^mbsBkLlKjz0wNO#Az32OSO}4)S*}#TYf%j^p8H%jb%&ifr(-xVoY|;+r{>grIQQi2=&XS8$*8YC|ia(SS9V?IxLYy7c9Y7hVJ2zNGU+@t3AL(KOM$VqO!u`A6Q&b?>T3SU3 zzZDM-cB?i(h<^Q8nudnAe;WlX!&va{h<9Y3Uq3r6b&%6k&>WtgdKvc=E2Z(|3CHFH zZZk%*$>-i|;N%Azjco5VicRXHytXH!d_Dfi3Eo!qd4k6(D9Um0I+I3z071-d@?16I z1;+evsMtx`|H!?!Z{T2YH(hkTX%Niw#u+$FJK%=_cSSryC}++6$1cA+Ko%#qk7B~v zu^o?=p(tk<8H=o;4EiILD@l$tg?Z3kD;2Fdi?v&(g3Xp|F^3zYUSG@)_Fz26derdE zlDY%Y@ad<44=i0L8!4%uWj5fjd2_}C?Tm-Ap{hxk^KTQOKTN1~of?^m@f)|cfoX$m zulZViJC;g7PmHkXKT_I$Ilf&R-L}H+J@%0NsGsLnR8j!Ki#UkWwgJA*Wbf#-rrhQjSk)p znavmw_eR^eL{}Q92U1##@tZ__du=B2E6KLiex$<@ctY+iXI*DfsZuE>QTj{;lLa9z zt9|MlVb1z}bwE}qH z8K%CcpeGEg+H7;FZ93H&{DAK|Q&M`=&cZ*fw@9DW@| zC&T_Q+utK1{qqib9rk+K)CC*Uh<1?SULg8=SiwiWEu8kl0B$iR)FfTmHVK~{sFfJe zy^4&~*d_Ov=23Tw6X97po$=m^V2!h)16p%kY2Y^j-BEo;ah+2I7BvrURnIeFxL~9q zqqCF-#JsqX)wJq*>u}w~IuxC!zn2Zhs-ESY4%zU-y`f+9M&R&mJ7=NQ;m%b4pwG0& z1v}DK8y%Is6lYGTaX!2zg|P7^Y~kLdC{Afu*>MFuW=RSs7*Glx5Zb_JQ63B)ZikAWEdK-frPq%U2MxlJ&k4oc}7HMdIx8wPfm zpO4ZvK7e%WEIlUmZQZXdwh95yD;ixnR4_RHd(+^Kj>bX4O|0P__AD1?X`%a28>T`p z^6Cd-u)+*erv1h{59Z*p=si^&0<)~Y8TSJ`Dq2MYQqXz8*@;BLZGvV_g2$TnHxfW>2Bex6L7dWG_&qQeCde_y&heHigheq-j5Z3^0lp(+e> zQ&4@pzTI_%5CG-q@9_$(VXA&t^0@l_(+pXPs->TM-k^oTIRkpUk~Tl z{oy(oADvAb$~;&jn{njr!4FrH58_}{;@o}Hg1az0Ja;T(8xPz1-S9$_9AtTr$rK(Q z-|g)u1rP#u#>ryv)Q3XeZz`$e!(I$A1<%#-t$ckg)+blNn^R-8ZunI(Yh%K)D@DR% zFJ|sTj1(bg@r=^}Ne+A`8^}zgz$&5hn3%ny+;zRVjJ8DjFSh>sXEv&XY=G#d#}2l{ zK3s@nW>%f6qE-jsMhKZOz<)TO*P4@?CWR3vI&z{?6h#iGCP7)J-WHO=>+@hw!|#%|(p%0ElPp`|&fGS5i){)(50 zHV2-yv#nyLju+Y^Q4S;E1;(|9YYcx_GFq$+Jt_`Qs;_|CZwC=z?W6U;3gQ#a2ClYS zHSxl;Lr8i1H=X&yyL5ODJ`+g-?MvjC+Jkw7ll}H|AeytNW3MuX+=h3{ae{hx*6bGs zAMA5woADk&X04NwV$;N4bHgk&p$q6aWTLjVpmX-{;B93}jBGTeq_jeUANqx%$Hrw% zvRzfahSHmrnFq&696|-~Zi6+wud=W=QPzVjQ&hjxd(6T@_?MeW}$d(6Or z?dx4eN`ChebN9p3xJ1BnLJgukgcvE9X?yNb#k5wbqY+?Q;r$x zG*ZKnPig-qHSOLKuIUZ08&tq>2v9m(7?zmlc%SyaactZPEd8vei9ys#AR^$Z_`?XE^0bwrARR?5cI`Dp5J7f+TI4 z^_=;%C)w^$(Wt|~bKO|)<&0(!ugGPYk{uvcwi)gpI51sA1#L49hj*i%E>?1N?DG`) zM91?CUo!Ck-3ISQ!2pn#w^ue3KT%&P6};gcqm#4w{V^#3>A}R-LD{NISAYRvFMT^B z^!v$~z}>pXBl5U+C!!9in~GeK5@4Unnzj(7t!F4nCXJ;~)Ykn8cowRrdfo96Le2RUH|Z-{Fuc%pcd@>`y%{cVre8qq3zLf zM%9}bsivSwi6Vwm?7YZtpNxX6A=qjiUtx&YJiz(z)#!V1Cp=WS6Ju$=Jox%%46$#F ztX4!G&ZBB7zpih-Ks#p?v>+7|Or9u}b3xo}Q*0@}ikqEi+%Ieno(QZ|k1-0`8JIKA zBO>Z2WIpU(Uk^0j08$~rpMwh9)<&howlV(ek4aV3S}@dL;{$$wt?t{XtZD91CSaHj zSxu*9*|7IUL`?xObE9hM0bim-^pCfqL`w>?9dE9li#pipQx_Hv< zNngqcg~`ht-joU$OUBu0d@N{o?Z`W)Ma?vAAe3q6`@h5(*BUah;nh?#_I?TNZm!tIYTlbchxS-|2{6NOfaLdb_>^r~M)Te6SH<~z8iDJUEc)x0H+W~PJoX^f2uanm*pem{ zBHW5!jhT{a6aAKV9zM;*f-NT2KKaCzG>sZH~7@Kd|cX1NLFjE!O8+ns2%=6LCgMto?8)XopvHx z(Q3z#_PDS`XySXdN+pq8d20i6Y#rpDXF3HG*_2YRHCK|F_rLEx3kyKC6wzd}teIWl z|8S&RhXqu4GI8R`zLym>t&8Ta{U~pnFM65-bBe>*9!bw4zLYt za~9`+G{t{a1i~PU07n5{^)u=ZP}yfcq8@#?+CBGYKEgkRr}LaZRO)`8^h1a9hZKeN z0gk|LcVG1UcomLHmc0_d_}CANKcQ>^S!w-r0NFE?vc&wTyzsAi$65j7H$EE#$a6o? zs-v|^mj*K0>}D4^b$$epk*Ruzlg{G{84vc{CQ3V*v}?v zZ2#eiOk?rPg7@c-VuA7dBwP>vNs52^tn;@cGELQ~YNbDY{7*H2&c-7$jnz2y2P*n+ z?PEA%Y282T0Qt${0|mGL-z%zpDa50by2mx%2yv_idZf)w2d+yTvE*DEJ+4;+(c*!e z*O{M4TwA>A5#nag7{=8GV8Atw1Kl2h{A4Oe45!5T4YL9Ff2jjj=KyF!C}vi9&ju)& z@CBFQUjIbs1KBjkcoi!!7A%W82 z++Of0KQRDPV%AImR$_wj{JmJadsv_`F-}TyyccLai#YaTw(8c8w{F-i_F08t ztXC$Ot_u`r4nbp8&C-quGVrbg<)3)T+`8z}3q%fFxdTynnrD7at?+eXetwm|z zXyucfJgt7RPNxMsX+8pE$)a(mw44~AtkuMO0Xis;<~Ye8kY8PS-@dPPE!7b@lb(H9 z>VJ9xtnz@kZ}Do)0o=Y~rOnUR)s4@WW&ea>Nui6C7GAu$8WZ-a`@@T<3}I8i%!!XO z!m}1>zG_-FxK4H%*kyrX2t%Dt_xUlfUOnt!i)EY~{v0i$Vl_)|3264qcOUuiM{gf~)+Nx8&+i0$ z^`~THD6NA-cdsrha8vp1#vT=5O?|IAX8NtSkyDkWENdRpX zUnH1KbiYOqkql4?NuXA z+&*8)tErnmnJ|a(1ppl( z&$s^^$8d%MN^#uy)7tBwNZ^#{%OfWEnVFxHLK!uIO-uLgfxpTBqT>IL$n_CerP}A~ zKX|gJ8(7 z5qMay=KfcrYnTZD-ua)+{f65}%A>wnt>S-Ukr1>X2n5UH4lnQGw^Ybe{&w#LqBJ{y z(aTbBTV6f<1>;ZlUj)&ISO;&7;$#{ge0t>o@ceU|;!s@@`{gvuX@)$j;ZEik7v>qjE>fT63v55254=u(-B8?@tO?f4?p{ zj6K`9GZ*uR;ZyYlUivH`4l9(Z_mKlroC)%7q8{w6+wppK(yCXgeqx*ksQfwWc^m3M z3JAR_m5>2#nk@V3zxv>?CP_dgE70}^6ov;(lNrC*-{%c;2&WmU{w0xFWhvhQA<;&H z8f+=w3Og>QspC&&nZL*k{b+%_$;d#CS6y2IMNN2UYYqD3r(f8TfMajhcpTLy0c4{n zRhoZ3-?0uaRA{^6Sh0$F7Yn@hj4U_LOio2Bv7Z=ub!s^g*EFx24X0;GxT-rqmI+Zx zIyP>w7YI;NIKc-S2{A`)@C>of?gK56kAt5GZA9dU(7iS|LAtL(xH4&9S8Pj;5xCdW z3^F&EXPw1=5*k5voqU18@^?RYd@{<~TToqOR|5Vwh$f9XAK{Tdpz@}h6@dC^OLa2qDJgo!;Y=8?#K87x3CGQ+i+?d_(osiTgg?%o0S=sw z?ZUezz_K9u19|$GdOAG<+@HJwj&)s}PKAg0ZJI`QDW1}2pH-e#9uMp}YtG>d-?*&Q z{b)`X6t}~3(_sM&&yMeAJ~%Qx#O(%U-GLwtc9xKxE&+{lEGTBBpAWiQ5=z&_urS}Y zreoQKTbulo+3d8Z!#<7$^QXx@a+vz){8`P~W9^98wKh5to06V*m5lW^YEoAb6dY$6 zp64v|ageQCGs`-{;HqL!zk7z*hYGp-eIL1q@AN>|#+fLdV!93Qbw#wh>(%Ge4YA|G z2+|6S4L-Hd6yTHZ<$}4dGgh4=Dgfu50?H|>m=a9h^0od(AZGA7Y;+Ui8?kt4UdbS+ zk~*Q7J#M!g5mb`XZveerR4uT|4s`|k7XUYmmJciBcp3M#zVgb*9El)`2Oh&o5i1b|k7r&7n{tj7?HlJl zi6-3K|DqvHkUw)+U? zzc}?7s^FS1SX`zz-sPO{=%#uYPA5gF;@mkT?1iq9H{)`U)ssTJmEmwMH$k=JPTSOG z7yowE*a{=&?E$nx6<#WA0?RfhL2?t+Cx5kC_O+a}ZEs(23Er~)Sj~ZtP;xdtF^wmf zpdcaTsWPcfSqpvDeG1A_ci?GgC-q6TD4cS;Zk=4a-R!fAlsoW2wRnosn8k^XhV^`> zMfL-u3dQKml?wE0#|TC|SCGs?!qac^i6{G1eBTGg`x5ijB7}gpg73VSWhZqozGcP1 zRUQV1&L2^ioX>4{xbZ}+#;p7~$FqOxdwPjW5nY@AG}M{$v1Hk?v>DgeOig403^Z{R z#`zpTKhI21#|W2mbOCo|+a#GI(ea(J!#G!T?t#f*vVAvju}q=U{29aEh}6{eO6nJyJqbdc4^qfRgJMS4?BuJZe1N`X7k@1}55-wMTvTJAyPexiV^H6wz9^G$MG}AzC#N*h zhD=A}v8{0dM(5`IvO(TUOGRf!QsSuTd8(s=c>!>Oww?#P38%_Q9$)_}OHA2pBg>QQOkwi0mLOCs^r%iz_tH=YMhbK)V$I5+fID>Z-p zF|)O5Q~RsAfG8_t?=$utf)E1t-Ov|)c_=bfy2Q#@vHv~g1&x(8;&CM;n>`fKH~mQ@ z4;z3iJPoz1=wY>pbKCf}Ws!Fx+4B>;V=<5&k9gt$BWWCBQTy14_*>giB>i@?g&?JM zuLGcUVK8w4pbz-}(!Mtu99iH7VcdA-U75MV)K77&cxw_Q#l5&^9594z9P@N9 zBfXW}+j!e%iZZ}!S8y$uqf<)>5u``(L%Huckt83gKqR_(9ai1Z{^!_O?xLn0PjC}; zsB-J5hehR}_TgQtJR7@n&EK%i))^-(`h&mtEK4f{q-& zm2M=$$5-J7@cIf0vXB+rXZz~111iod3{_|2H*$k5s~vjkhBQW<=Dfu*BZ&yF0l^M> z3!wE}WnGcU6f!~K7puotd2*6{&iB<`DaGBdV+>0zydZ-}I*9?+UOs`E@$_+tl}cej z#48mf-;y%(pIuf*10_KWwvJ7aPQ`-L;41YWwiw9+mzjp zn0reAQ7H_}{p%ge*U8?Rxs@5bE8N1S_D^k%C|_=TlW0j+|Dky4?6FdG26jM-yN$^{)#YTSj;_k#!ETQIm7YYm60UUjrKdM@#+7L# z=rtyn5#p#@+#VY5`t&wuv_ZnVF|^0Lu6wQL#q2GBMNesaMl5tU!bMcR1SR^+Gja`K zYuOkKbH_gm;GBNMU_wmo!l`Fb9qNsqd_ap5)!eLicDxOhx|=#GvI5;dXxHa*2O#7O z`XJZBX9^n@q72J8^!ue_(^_u1IF_75yTCO;6~~z5Sp^FJ8jA_l+b?(BU%x&_{@J;^ zD8j+y`nn5!M~r7J%f`}}&z?KwXvCR?iQx{I~X#^+af?quON zo$o{q4h#&mFO*$ZdH4rm#AXF+-8=6N&oP}DqF*%Ra6F9`45nog|vd<2%A?jxgGp&$#vEoeA}{MK^?rpmiJ8s7i; zdq?_SUN4wG9WVBrC4vL|`ueiR%)MX*nADd zxP3EO)7oBK#T=4PM=wbmldI7=!*DtHEa~(?qLImVnHe=ClvP zlweH~;fQ4iTLG}pW6{A7?kBIedzpy?>9dB(4-M|8UfX-D^H!XM)}P?>?sr%4s|red zbIb^L-Aufy67E^-I$@tb)KBwME4iKS^5`{Mc}Da#K^3s0pzjakRU#z<9`l?N^%#G_0? z(~kuIT7l0F??B4Q14WBS@!iP&OAspn4)d8`+pkmsr1a_e%6)aPvYK~qUOsTV#YsRxh2Z2 zaqVSZ$xuYVzyp0IulB}g+NmT{3)+3PgVHy0UjLz0zvB8EWMDxK6Ys@1Gqh0Gp_wRM zX2l~&iM#xzU6xChdwZxnnW|1JZ5c!ei{*5bWcfV_kNb5DRIQR zR&h!lQxgo(GbMf4u!{553xB92w3(<`?CP?9@Ru(+ldef%Q?HoS*lBG%WsiXGVioXT z(d^+Li`sz#+=Tgi@4^DlK9xKhV|2< znTFqTdi~>6O?qpt54&(if!Xtpg|=B_>=HOq9oE8B?^*U*Eu!ouAKVsB_a{XTmGBMA z9^mfxe8qjI!gdU4TtA86iJ6uBva)q^T7j#z054#IK^~`nHHIS%kLpcU-cE;V>`RsN z)_m=#Po|ruhZ~qmloun;AVUiGzdxrdNbAm~N>Ebe2zB;g!a6o*8*}G!nZC^e15GS0 zLQJ}VIpRH6%5Cg{UriypTX*&jsxB#DCEhESq@XF$v)HvjQ>a{#q9}^q#!^=sOrsHP0P6PxF zvCxXeHqL@mM-A44BfRG&I)^iITrHWJjuZPAV;e9M$=_4hT^@{kio2h?wWb1KE*DB(%{3fx;(jO4RC4&JoGbwvZEOyS<=&a+?HK=dJ6uD$q%e|v z#GvcCjMidA46l4myXwaEru?A_rGbIOgPk#Nqc6=SK6?Jo3P&P;^~_&Qj&O3OC)8l^B2I6bn9TC^*_dPkqIdmxB;9fX6T!{-*f_#b zy+0C#0|g+OzSv=!j_KG%3+&rD!n`cSI$GJ6KJ}DCd{7`dR6-g_M*ZPASgS zu-Q_8q8_3q_yOEPoI~29n5H4~ws=M~_f>;QK))#S$)H|$PPtFdsC{qGaXeVp6YR0r z8xS1Qpd7!JZ&ZmM!h~AYD3)i`DOHMkX04{{vZk%(l|&KCNO7%qGtip4FJgFf5-G?f z-K-nL6`50{@-f->%0p(}+Jo|b$x?KF+}g;qA}sOr2;-?KT4HH)_lg~TqAJiYqRFAE z{1IkY2>Pf{6aMHZ=LEns@kqimjAYLym^9Z++t%*cI{zhKta`4E{hXTvCYfw=hj{K; z2kQJ$uT+AZy+2ClyEqZ`S>L(7y$Z{Jyga07UXm-!0TQxw;);WT@3ko2#_e>6#kL=- zk6tYEIxi%4&1VDFa|Ox4Utd|TbX}A>a4*HkQBacpO`X**A!Y|YE=Z+YuM|b~tId=9 zEonKbP4fI2?^$8G7qIv)b!2nLrS()8&%m2Bm?iGg`skNPA^Zk?RPYxmj8>_ViRffN z{Zb8bT80>e0_)(evzD}$wDa>;$m(4*aNWg%o$fJ7xr|zVr?)Ez^z++x7r}|1fqh}5 zyeA*u#bxAO>hKwL+iPOpm~Xkssz;6fb^lIl2Fbl+$-T?vG*ln9ZlZ?b3UQ(gnIpmB{o1_vmaY2I3x12q zGW5Xz+yQl8{ZRJWrKg=E6~}*b7MD*fR+^sJ)}j2vPKyQ)ntg&ry4+MgE=nMEDY=J7+avuScvhAf@L1^&fb)>}Yema%$_= z#B)R)g@1ISXLvXzMN{-s&SLWK4pud*R)ss3(Aoq=BG=|eMZFtyA2N<`ZyeH?g#Sn zn&rPv-Znnz`DCk$&!bGT>}HVg2=ndKnmkkUAh9TK?3Y?GJTvHOf2lQ}5%GMYw0~x3 zjHMkYbiU-g5iy`52BuQ(wsM(6DNcoJbjH&qEgc3cSZZ=+0IZ;D- z&RvA6-ve#={aeV^(L%^n2%DfQsX6Ml87N5JxcgJ-Ct2~pKBoQp!_`wa*@v;SY;E*B z#>WJIKgIh1JB0z%fG7bchYoad=ROb+V|a$W8LwAgs3{P$)e5gB4u3hT8C5~{o|Dwi z6kkiLv7Bh#P4Ba+U95()lMQ4QYbT?}64i?xGBWXl#W``PF^D13zG=;+mDW%nQ&nw> zQh)7b=W>72-YkRcwD2#d_!D17Ok)D-;cKvFuaOm}p$ALPUg?Z?slz)B+y2#t;KVF? z$S!nydrF5CkPAcl#R`ow)&j6rtypynzh!g$aV+xQ=+h(RS}RdIMMuBlE~zq8v2Sj+ zsrY5Xz}}L-T8`4Qr1zx=rmdt=-`!fV(t!uJQv#1P%p+~8Gst^xfWedy8Jq`Z1Kt(2 zcZ1_{;gL^n>~0R#G?K>%yNJ_%nwY9!uEqS_y$g6xFKV1u=Ci)gqSO+_7^~brDv z>_Iy79rh9?=w68J$IBN>o>g(BlTLkzIYt?6qh3TMVsKYl@*l@I5gwo3EEi>VDJTWJ zbvG;x@)Sww*D_e!O8rqoaP^DtraIl4WFUC}xXE0e&6*0Q@k!8`AX-UFg`pYnn&|D4 z6H}H20mH%0txIuJA!D$XJmV!5n_9NKtuam6bLmoXqP}a;p~eHOzXQ(SDm( zFni}@M_C$mb2CbzWFe+!&SHaeYV-uFvpg~M*yoyE4NBk1*#|sz{ay6TPtm58->?v^ z`DDk#khRMRinTC<22gC2r*)`Io&cJBuS~(R<4vt6lHGIr_Tx z6gC`NsZuf-ui8gvW-78K25`CWr9ub05dOBXLU+LJu&?(g2?gezhN~66O{@ITJ21iBh}C8GhuuVIY?gie1RH}wnu zE$|WHn%sA)07IYJ4I6t^mC{l9D8(GKa4}*3#pN)FK}w%*4@%P``16*(g+^$VCdVsI z<$|1`x)B+%fr(L{3?n5kGZ$a`7g`OT`HNrQkA-9=VpI(b$u+4-(G2FPR29z}`?1VV z-z;P}Iz=l3%+1GWuXQkxn6{IYVz_iWovBP@YcO8_Vlyl>_WL#0{a05fYQVJ}kY1N6 zwFA!mdZH2t@oXK|>ARM-#qgo|r-<6%1$C@M)LdigW0uX+%qm#}_#fLUhbcP!n30ZB zWYCdocUx=c+@-Eu>~Hj}ohyO+<>??>rg1!6x$H+YY+qe$Q2ev|5MGo)x#5tzjyN4L zT7^cXe1qiARIvA1pp(ea4g1x=i=NL9K7U^WXdC)I0GC>Gf*W)!Pugygp z5pdRHkcs)u%@^bfWbJ19&sF^1s$baQvU$f3IMxu{IE`5=+x?#9MBVi)oN?{W z`^1c3_rMqG0S_+9zds}D(U>bwc|k30Z87s1^&g^JG{$~i+T3c$h>d0PtUw{FWBoR! zF-?Z>XqC|6QcfY@!n7sh8m%+gfu0M!yWd3OYDIsWug~MmUMi7LMTn>R%oa>ZVnczu zqKq0*8~iX)(?C^7VnderjToBqH60=;y#PyZs$rlhGdm}6VuY)!vL=Spkf+r}M8UG& z1C?$8r0w^{r7S~{Ij*C-uei%*+pEvg>Q@`6FL(tRH0|7-_Ka`83L?c*B9S-jg4K*f z{&}?YI60WsG@(|%iglW%zT!0?bh`G+NFSrvr(Vx z&L=YKJ5;V#Bc5aXBHAP*z*A;^s4EZW8A$OnRXP=BTaK&|wV|$&6+T*u`-KC<>zwnm zu;1#;yugS|;o(b^_8=O8kn6M&fO_kJ%GmDZ4m8kIEl6)g+m*)I!Zk~RChR>xtu*bp z`+@3GavIk&nbroHc-HIM(`!f@4_4Ui3#~8hE6G_?Lk63{a#$#>zPLo^tMjgKF+8MS zffzbbF`L}Qr{vu;r?~8IWCoP}>Hp@cNtukkUyq$*5d)fU*B1Sj`!28H5 zTWhZATaJJBhtUtZ%2%mnPv`t*FXgBRJFcp-_kwYwlg%zz00nADd_=2mM+H0ZOtp#E?SA(~j zrkcOF-7SFHqYtAto+Y4r>kN9dM;wEhfpNadT(|L14ZPDFrN94Nh*~0B{aPt?w?3KC zN7&5yFh|w#@|uTSU@J!N(Oyo~$Iy_r43!Js#$-Zm6Z}Sql8u4Qg?>hrGc-JVx4(ri zp%h=231P*=tx0SWjqAnmPMwT7T7)JUT`r2Px<+*d|4xYqtd4Ox`__WUo{JJ zJUO|KOX@!P>mn1oH} zK`=a|v#)x`{waCMASgxzRMK!>Yu2AE5@u>VZn(NU?;&^`)YXNNa8OiO5(K4Ey*}!{ zl!N=jOceU%10&T0J{O!C=Q}!MzFK}l4;deueUtxMEK{uKKz_(Ox6PNuk1!f*b?f4u zCyAy_#3h0zukaZD!@l>Ea_curr+U^V%9iDlzeX{P&Ce|bwex@k>3%q5aAPMcTl4~N zIM9CywhLORUIdOL-LP~BMl}GPjZwa9??9nC0Rzptk!(dw?nULT;(bG(zQB3z(;HOjMC)P0B`C6~|lSISddtxh%8dS&bbQ($+ z>#z>C&~d(iW4;J5E_Yxy(RVDgVQchpRi)*|ROu@8xc3v~NeS{+KU3JNuU3>Q^~joi?+>~!16(pib1|_6t%LG%B%C^e z*0Jnem%i_F>!A8wod`%gJF#~rXeEf{)`gI?MKRkpQ*zV7Rsq)>fV>LG?=*$7xNj*gD+}iNE-g6q}ix?_Z?h6=bG<`%DA1L}6Ya;t~nPS+n$v5Ze!ZfsJjL69l1W z`8`jxn;VEB!3boXZGO}3khqG6s}1S0qsrorxHv^Px4~m+;NlJJ&df!6jc;HavYY6+ zF&GEico*${`AfK8V{T>*tcj3qX~PMX(tjc~8ID-#WL+#7va(!FI7M_AuUdisiu0e# z1S5Eoyrd{J>-EXV3aReI(josi-EDyOB8ksT3r8{KjK_oC&o;j-ebz1bAp$s|R&++K zM1Hzc2+^0sh!mOHJP`Bv0@vN8AOZFBwM|N+lC8HLgX_cZf*o?m+~+BdI8+8Sd^Vh8 zAQOD?GJ@1Wf?&2{86H)p^(!I?8AScDM6+ED86A=AqOt|!U_9BUN|XraWb|Aa+Wuf# zW!aP@ecjgUlv6@@&&s>yPCE&rDY#DLN?OSaMNF$Z(>h|8ycN-Qv&v?_D9-o1gG@(=X2sj%17%*G^j7 za;;~`@5|s^%sC5sJ``x!qm6M>Pj&=}p}-(IU8amc6Xs1^vs^aokhmmhIF->?I@7AodR=G3Hm^V^brv-Y%HOt)WH^8mo&}Zbgcu4;As+@GOTtb#f_v~#$(|`Qk}WBi ze=3PqV;fR+3;p6oHHY?|{qfvt63b+lCsLeAHU*PZAtj63jmNK$9?{bwto_qYos{5R z6RG@dQ+*|cl2>)7W=ECskL146>!CI_+U}3@h(!KxB>fLaJ!XDDJBrHV4;=R5L4 z3J%O3lCur-J3>H#;|RY=L`{!8hdi*}b!SCDxAF4uM1$k!wUD15&BlUOhtEAlAj}%~ z55wd6H@nFlJY(;WF7og$&+^VwBWaCbFU}rP)6%mJ#KDj(PR49t5udaEV<`M3C5`-` z?vqT~+|Vk6Vo?R#m^;rGD?)VQG|k4@jTb4mdd)(Lv88pI+~hYNc#u66zpxl2+f&#& zNp*l_igE{aD%hJ29aXOBWIZUZ{&ExOlbn#!-g+1W9!Z*52hLZc&6$V^+R^!yz`Zdq80vzTcADEHQ+Ce2n+@!TlFn0X(#ds7 zv#Ql0|9aw&F-XH<{7IUodqHxO?uZr8j%XzEJYR~~W)$PvFu14Zh235=<`Sv$e12Wa zY~|UIw&PQJJyQduAcs8`bSt_c9v^{1D4Altr$PPFEwc&GXelpregvV#d(uh*r4U-u z32}X+YqhlzWL?<8r*BfdZ`~Uz)N_Tjy*je6BVX%^UL6T#cRO{u{LWmnUKTpM*}Q@$ zqg?8>lq+-*esgo@{$t<%p>1Kl0t=I3i@t@;J8FUvk~mTxt%Xdc@h}XWlIZHA;#5-C z{qdKW`(9e!ZdG36!cm!r5PDVUA-T2J&n_bbs!hM=+Bye-*QO>Y-lxfA19E(AQWA2q zEJo6Bd@d%fkSP;7DNaBfyv%Wt{2&iM=4c~)?Q>oT`zr|E--YDd^fq~GYDYRe29Pw{ zjms(3-q5k&>>%-jCcNFujXk1wJgP@5I>DU*35-uHgCPA0s@G1}mUq3KomYKRe?L7LKC-j2TI_;xB)a#p6f0ouoZM(^GoMtAt#C;YQ;E; z@qjZ3StkEfc_8mqHf(Y}&C8zityI}Pc1YZlD^+IW5o_o5TQrQ(UGbr6cUoHT_|2|O zGUzqv!fd6YPPDgjk2e^h*oiqy8hFPQG^Z$$-(;&KtDFRgn7);%gJXgBLkw=u`D2D1 z{MJtkYQMx2^!}uDPI0 zq$q}N`6QM$6*t*B*!wwvf_4qaiBwai{B+QTAey$~XiYwliU}rJ$FyEq-^<<~jcZL> zPdPQEOoAN)ClFLhqbb~j@E={dtlvsr?FeqhQKiLW1Jjc5!2OO%W+7ss0{E$DNW&LA z%Sz_Y+I@gy<^09B-UrcF8ZAohZU*|0KF*+swmCtd0UOk=t#L2?v^>1x!lFEfC^OGv z!+T-5%Wy7QJ}gdqfhBaL2&DbKTge{c~@w*HOCl_ zV_<$dLN-9i*%L{HED2)!#P$QQC|v!SRJ1=ppYiBc%FTa zoV=!OWNKXaVm3MrsongII7DA8-U}MFZ@ydF-{YwtG$&FLE%0)o2wtw(6PKx1FKe># zIePMuU2bNTKI}t8<62AOsDf&snV!h6i| z0tPbO1!mlg3R$m}DH)rr^9?HOw@Vl+pu64;+Q?tm4U|BQlw^2fBnWEsgQ@*Ja$eEu z?wx+&a{2jFVvwDf3q(+&&3bK7OYgZ&lmejZpt|MB0`KhJI&qsldwJqzWti7a`r#h) z^Yty6NB%3K0tBPm`~)0D)o=Qt&tW=3$tI`aI$Xyc-+kiV-+)EEi9kl34RM?t6h5UI zD61b8*m?0i(23m|{lF4ADLh!wC?9A-7Swa;iVceVC2aMGzXXbK3(mnAD05tUf&O_# zWxSCY88q+e^&@`!&t+H$1>6o+E6?zc!0uL)?uJ|x80~1kBFuaFX{T79#CV#O3gQ0v zb{|s4p!Cf+Qtk!0w_PZSdVKQ&S2BJKViT9v2S)jYkeaYA9Jd;tFb4hzIiwNF5v=gN z?I52ypD;b})}P-ese3=`=z^)%XB(3}VD=ZUbN&eA#6E7cuvQi3&#ht1#(aj1%r+jE z+>U+P0i?-7MjyBHq>0~bwRHx!Re$_dAs%#3MmOumfwdf&H@{2kX5lypT6DvxMHQ*e z>r&cczRkFkcYb!Qjz0=}BHpQ}YtH#2cHOX*d|@A#<-Q>kCwL%hKe-k(uMwk*=rtA9 zj?M{39s`Mmk&;;#mjsbPUC7(>w=n@=$L^j5BkkmQ!U{7Xi<6`8sK3NzuXC~oIJ5i9bL6|3u>dxss1kp6DX*awo6- z^^5=f#g7kRVP_>xv@VX`Q2bf<{vt8S&zBB_krGYMNBs4RpRx9!o^F=^{i@g2Cw_X% zzjFkC`b(A*kS0*fF+cv-FMb~D;TOOZBx;7r-n2*M^reAH`7lc+|1|5J+mg60@1i2M z8>!m~7;uGwvNA>(oUQS(=w4+Y^Lmu*&%^!cS)au@!0L&PPAjGg)yL&l+uiREzccL| zaPCbW2#2BMeln9k&E?0C_her>I)W{}gkDC>RRCkYmy?u~bk8}3#66T}L?uhh1J{`T z?f>franF6Vic6@56%Sv{s^cpSBPZIFXpC+w4wTE?~FRSr7UEs zF+b#9+nHSM7$t3-0lC}^T)!EN_6e8%C-H$e_>ZPRP!F;+aNRre-Eo64z*AIjD)v@x z4BpBwcf_4Kr?cK6pm^b)=}&V1%QHt|lDLmj>ZepLy>ABWh4n_;!VWb;>br=X@7(}e zHMjJ5MF03FajqS?cI?8#2mF$8Uz3MJ07csymMJW42|NSwc@zjVH*-d_-t&ki{{17e zHaYlzixal$_$VbKQ=JdY0$P1*H;_bD4h^_$1+ZB#(kkn)qaPpqgUJLc9Y>yk^(p{; zG^d=q7c(2TERM7X72YTNDwg$L!oLO1L}_XR-CX)a7MBG2VsRZQxkt3iaq!M9#Szp2 z@Baeizi1jDxdoT7dzH?)@@9E&-<##1x*R$Ec;mNIzpYYUERw!+blUy0xi#IRyz-~A6Rfd3=e|53fatk3_b-p|JLKS=i%ILP`Rr27ew|38Uz|Gz-D zw(nadnLqZ``|F&m0qBqhMkI3)s1lw$)Iq&266dhZO6I}w-4ZEMui9z&x)A!aVgA`M zICl+*w~n|z%D-{{27knp?@qP7?>Qrc4!@uhG)oDD@;xIP%ypIrdA4k_$New*{GZRc z|NYm9m}8SQD|FSgOHOV(_z_~hk$wGx(MRo@JGu8auY#Q2VHlLstz8e z7Qg=D;I*RW+<9`eq9ykH=Es=Ru9)Dh+sfuk5{qdrXq{%?MoUo7C=CU;|3*kkzhaM= z4%+{WN;Y*0+M5Pa?#pbxXrkfq^|PAyTz4X|u=eOK%cwR_TF3h;BKf2Ou_vKbyR??? zTMZfZa=S38ui%UF({`4}9%Wozg?bLXq;a#gDb-e5 zf%{+xm)PWkml|#$)RO9A2LUY~yprZ(8PuETF4p|WZ!D;FX||s|ZZ|rw ztx=;LhSMqcGB$OtPeRzQrTop{fFzF<=Z4&Bc;YU8s^k_9>*Lne2=}V?LLaQkbOpj5w=)MB#!Eko<7jFpnFHRqvs=xkHqbAnD({`sf5|`(<%>E$dzOm3u{xUXH zSQ~gZ>XM96$oalZZCN2REr4L)ZPnn{$WGc#N8H1DX+zYH_A7r+c9~k zm3s}TMRZ!#>ZtPV0qHs&ZPqe!F{NjAT~>C)BXIjwr0e$g=Slr%3QH3KUK0zC(B*a6 z^LoE4r-S^eZ|=e3giR|QLiqKZr%;UGPOugQ?M>7wMoX=yA2YUs2 z56-DMcJ)^E^eE(Re#tP=0+~$#foop4cz~KVWPSZ6E9M5ZMD=Vj1Ws1_z5At-ALc>% zS_@-;qeI~6*fiO_*mT{xa%sxxfU>hc>C>mhFfIw>*VxoQ3@k&^%SH=k2NfL3L#kJn z@B#2q`iT7;K2AKNE6%RG`_g*3>|m>cUOH^VZxYd1HGHRPTZfiRU zWQ%5Ng5aZ7snR9YT}zis@yb$90=trojfWhPO^f2D5pSvRI{9Eyhq54^%+By$Iz5tv z)!2&IDDi80zXRd$cVU00EV@wat?VrABgFgigZKZX=G*)6$$bw^RyZY(uH00WUTihV zXydy}+#|`loYXVIMS+@UgGV-@+x?b9r_#Lc1ZN0^Y*O(Gj0y^vd3TeK`Ha^eh7asWqVP>V-u)bY(9bLEVb zmc0oHP|?+1v%MKiW*uPAmtfP(2u%7*fE~Ez1#3Qm0EIT+>B>KII8ypHXD}2+OBx&@ zPA2=gZtohBcfhN!vBgJDB&q6{FJ|nQFhqY~IxU2%3B3suN8U+=hK2+XJMCK-kx4y^ z1t5gzZ%^A**pIbRZ;4iv9n$Fe$UoJS|!P{2E7rw%@sr@g_L!-(LmHy4- zLiOU3*0K{>iVU|2`ex(XK2WezIr}nkPkUnHVVep^I8>Wqxi}{fC%-kZg3>g9!DC-6 zT(M0|$gBnjZ*=pShwH?hjZeE_xlc}&x}Huiu}UFqqcmG)4y6Go^V$I(}EH zYnQ7i*xUJ`E9|?}rk(A(LjXN{dNyM_!*Aks$u7QnWSWi%^$iIeOPew|b~)28VzM4* zajZu^kRRZ*_jMhuTGW@+ej$aaS(58~{0*zIKD;rlhcT~gTIXrZty7XQ3Ov8}-g}Si zi9?g(H-^@*DeB++bKE9rSJNGh5fdI&GlB9?Kj#=I+7<109!eVKe3b)Y$&`*?)HXd` zz*=&L7g_J^<%Q;nu-6Gyd+s&;or+t?V?1K=^8%~If*(lF^(+Wf??=?hf@j`;pId4^ z2P7A4Dl+1Oy05TKw!>hhgVzUMs|WfTM9ri+R5QEub-mJx9q#gUpAvD{b?801yBa)o zrx#?)E4WrZ##X#Ewf^+b(LqAU`rzXs82bD4#y+E^(R%y=bpff{^LTx>?5V zAYAaMT)5Z7qWa!c5k$RJV55!JMcL6$w)t9v49>M_)mYUMCL8nhcqf%tHs=o(2JEB8 zGA@Uauxwg(YG#MTPFv96n{ZK+zR-_2j3G>nwI~=yj2o<{>@_&EKrK7yU5?$Vb&3D zL55zmXW2evk&C@nxOfnTE)srPXZsu=M)r$>ZK%bbEpX%=^K0?M-Xu(}lRb=-VSMwg zdyCm_`)~BZEt2RT)28ZRPD&&ITo>-{4zu&j52k@{-_9&;oS9FZXx$mAvma)WrYrQ^=GyX~DBS9mEQlTcK>(<_84gUHUkR5n6~JYYD`swuN{I zt9w{MHY;)Ye&Ua6>fV@(;H&wa{%t5E^)Sg3qL+HnN!HImP|*Z#{E#>&u;g1U99&MmW2 zkwf8*xCI_avv&-)OwtfgKEf*pHK8e?tM3!}uMEcZ&uX(;r8{$4hl&%#Tfm)V1A6B_ zY*O;Hv6gqSvP~MjenoYAag7Dk_R2{$uXGhh=g!^dJxyImT0u4!a#P>wjn*)IGHAtG zY!GM)d_j(T)2Q6AT}vM6qybEzmSPqViw-`4Gb3y#Wt$e~DfT2$!$kD3*q+H*j(!o( zGW%*5+6}C>_5_L$3V>c_$3VX{Y-95C=gSFtW%d2}#?>ahGn&4gZkd@%bIfN!*2bzV zTscrd{ePbOleYT;K%V#YDKQY7EQm>_Y$fOL8mWY@CbjIWq)z67_GhutC*@jZr~LA7 zr*A(WPWmu&iApEjE^V=`_Ks^#u{fhC<5{t8U?(p1v`#G7pxH~a$HpG&vO1p4wX)pv zxYs3w6`nBDq^#%#cMG7+jsCKKQz3ToaqC?RxW{SaRb}JUdy6Yg8uEeap?e!|U*TNb+%vG8SR6qkArE_l+K3!I3=W{`DT#IDNaRieRDB)gzPm_h{e89Z{y1 zGt3w8ux{E58{ihm3JJG3*i^eujq5>odM#UcN)e~~Y)f}R;$ z42LqEj9zNIaGvx{Q&d3imZSr{&-GQy3r>T6EsV;g$9?3FM5|_Br6n9Td3XjrARrOc z)Z~t0I^mTq?c$z0V)pN!X&tRgMKL>%CQN_7l#{4rUsAb+r!T>wYgO&;ceGaP8T&pf zn{2uHZ!I%xwZrlBXHj^dG?u^%_HRwyjHW+qUvIzDD>j}+=VH&#KW~5b-!A;>?5WrX zxnBDe&rVZmxw^G6#$wrF6*d*{oYVLH$x~5Uud#uy+!S8Fg0aVVs8{JdyU6R0WMBw@UGdS!?=v%iEMH#TEAEq?>UD_%U0F z^GQb9Y-T9{g)7V*yTSMQq7Hz6zHjb21Y`@8TZ?pH-$OcY-Va_He%=m)exH_hr5ici zo3OW!06!PWRdI9+_YYxOyT28ynaVT$l2{sDUpvdc`24i$phW4m4TQRkH7Cf<&%*Vy z{GW~3pW%f(?JO1-->I;fxA5%26_ix{mvi@jg}Bok%KA2kwf}6j|l)h8KvI}nw^`=x7 zl+r)8V+;-XeLUk69pK*tJu`;x=C+~ildz6fG&Ha3DaVCHjj08FQB1gah?33OP}=yy5MxGZA9B+US0=Z{Kw2zqt#LK8sR}mK01AORQ2!o1;BG-?=Ox2}ZZDyBn_Z9ED)d%FR;&<^KJ8Vw42Sy_HeHE8X5v zlW)QLpH(?x$bxiS*LX!n?m)tuKk$oQ#Br05Z)BQ9Sbxa{kZ*n3#iB z$i@&o%1EWkCtymUqye&aM+>*xDVs?nr}`FuG6~>s?ndr-o%?04bEx&6z=i#(`1~OR zcGtFl@H?}QwQOn^x=0Q+r0mj&BgBB2Kk)k|$8Kd3W`I-x*~kdWuDm3=v%j|Yc-^1X z2pm$LcswM<(t)h!ic=p5wBN3kTB#g~$WC4`wOu_hK2(uYr1f?)t;XIzCNt4B^lzb9 zxABN@U<1bH_r1+8#zAZK?mH_I`?jJ?sNY1|mgd6un~xvAu3Ci%3|8QkUW@xr&-ftE zJ+a|EQCdB}8ZIH$MN-Pa>i`7h5lpm$lXzrW+9_@Zk+{Ux~UaAVxIm$h+a3ij^rTiG0Q|g;%&&3=XJ^M_){Rp{9 zJ0W+^ebo-#f9lhR3RRxJ(BIff_Y~u+&ZMic(t}RIOhEGjac8qNse7!wcKcuV{d{6< zEj%!i>pC{fh~@5{z`t9^`}cv?t#Q5;KU#j=4RBd9Untd+Z7gJu4%qobP`++y`P7{( ziRe{k4N0J-_1mg-#I{#!M~Zs%G@-lQAMMlm@9W%et+k+cY;lW+tidRNW6CA*y?<%T z;HXOcy3JITWnZ4~)$J%KaH9PEG3{}~r~vm?dY|&h@3Fd0oaqwig;z^E$yf%BCy%K)-rEHt-J0e)8+gG-KIj=oy5A3Elt3T+EIRwc(7k zeBT=;iWKsT&(3J2b>1e%^9STG{CH2mF7AEJfqpgRm*@C`1+RatS}Ck8nvYc1WPiLh zbemA0C39UADT(zaAx6*>_k@9jw9}NMd!Q-Igf*-js+zHt;5EV-nUh54;sa-CcKW4& z80X_+9GX2Ws%0N1X{vVv2Lt4!Wv`Gf%=~SdR5<3}naBcuTg{yay}^HnC|m!kh7^5R znzc0gcRWM94Lzq{R%1Hqjv?lj1~cQcYuy8aHq`qUe`IiQ5{wx^a)KGB6O^@x$z`-M zCdeZRlC#-Kod^l=1+jo_)+zZM(@D9afgRyN=CW6Cy^MbfpXi4nka<6O7aN+t;FL}T z(QU<_1}&87QzzfWc<&_^Y)Hc6vthoRwbURdN)!L`-ev&wdgJPyF&20}OoxG+vVUvz zu=Ohi<`Sy-BV(x~bGd;2wOql1V?ICMZr_XL*3wjFmMgRdhVEEcdref+3tqH?F`_Ej zCm!p_H?L*+jc|I?jS<@Doqu8S*6zlsx(yEopw_iKzLqUfxgmaur0`=xdo-3lhCwSE z3&2{oJbdP}*a*zQ)hj6_m(RMa>+BOHhSFJQ(9K0|Sx!A&zI6p6J?_N9-{oo03sjE+I zftugyxLVLwdi;4rvy;JPac9uHP9~5NbN%66Da4As?m0kq=2H*d-+kCWKGPnaEBwFO z`_8bYwry=iP!yy{RYDO|KspvW2?_*7stBkMnp>qCkdn{@1vEf{B1MoQ9g(7d^d?n$ z@4ffl>z#3*v&FryIX}KX_j$gfe|SPzbFI1N7~>u79kD2+I(dmCYU#GIW=uh1%2yM$ z@G|bWl(kn|%E2odWp=h_E2u*eg@CyQ0?uVPV7XhIE{6#jQII^4Xm8;H>eIlntYwX zF14;7MTqjQrt*7YvAg`ae<&-djF6tPyZXWgZj<`k0=363Kk^_v6~m4 z{DK`HLAh5K4=q_m`BAf3L>vCLWJ0h5l&lzP=WfA^U)tdFSwppVk7!S=0LU`bib*qZ}Ycx7`kNS zv*KQm1E=MeEa<;nE5RV(WFEV1D$ejbW%(xzy9~-Ldx+og622VcEg<{P6Rw1k{T7z| zxLhz5@Mwr`&=X#h`~rCB;+Eh#_ka6=e}{yBzA_~MbVOXw{f9M=`aOd!>YFHyU;E#$ z@~iou`{MT+ayZ}{3Gzwn`uchXphr7sqMtY3D=|dgCjhMd`-PO&mB$b*X`k#o=bam7?v`OF=e!Mr4MP36$(#&$p_H z3qyZAa}HtPPS-S*n9`?m1*uKwUZohc@%5lbnaq6I*XX##Z^_@fWugK^!})RG#0_Ai zMusXAlfK@gKF1VJ^T9@1Tks4GMgcyQAXHM|26vsdMTfnH5$e~=R0S1t^`Zi zSs>pm9tG+DngOW6BLd}ksj|mM7(4J)rz@8Ox>SHDuy#Ed_197cY2I`ZIcSH(x!-F< zSU>5AKC&NDtCqu~KQTFn$>1u~Q%)yU7a}NfPw}OR-4AU4cf0kkQOn&B>A|Fm+T{3!&8sAy)HAoU=^~;~Y z_D|p^O>wiFaxJ3^l5(&OzHHnyG-m<*Rel@h!IHau#1{`dE==!)>@$17xX;5f%g7rB z`cKe%o8!&a^-_WKyxI-I!G6l(+~)WmN+V>ox!h^wwdL`86-wyNldWOa{h&JO?U|v? z*Kii&$<43i$uctQ{VpzDhF*M4fZijdPITD&s2JL^l}sSdcSs1}-B`*~?HV(efH1_a z8^?KrJ}L)fuk#28t8tu3!Y|O_XkO96VV9%IznqrmvTQ{0U_P`pdgv7==2Fgp8(9;W zmD2ns&0*}_qo%jqw=rMXRVBfpvvbs5LS08O7Vn5Pj1$0>WVpdX7*#YZ)P z0`?r;!h)_rki$-C+rQtxr6~@0p_&>#Q12_{$k%&jx$ zup9`h@8_MY2c$)B5+`tEZI-a^rJi|^1f2nm8_n+?g}no$wCgdTM8C~ad3mf>5%9t* zf`fq{wspAWoUJjTd58v#d(q&_&< zwF|R6q2st%cBO28h@ns+d~YSL8PxTgv{9xrP&@hz7VK~L@6+GGg(X$6IGA;T7OG~v z)c})b0@xMy{8JobI-FNqwc>CtJHwO5pc^u^3Gz=aqs!&*8Tni>`I<2$cd6p#*|?Of zAq=gQE*M?5t;u{(?4a~8wh%2^%KX`)kt(Hm_YZ$*$>8e;khh>OF=(B6?Yvp7$FDvL z-QTRA^s#gtW}iORgfUIdjs=FwG${k~A&vtEf~HO!8#(O=hZv`|&bSGrBT)IV-~Bk_ zYs2fRCd3;7qiB&*#t@MER=AMxmvu54AY63i zPJ_8&_tkLnCE@LXgI5k}zN23NcsY{2+7kK`^}EV&s8W!EVPSof^Dg@i)aUl5_4b1= zCgP2CbcmjF7mFs$g%OXc-iru%C0{E0g{6e9r+rvjHpMxRrN zbD87E992u^t=!Sl^~cjCzmKuUe)uTT`M_@~2S{mr4unjo#)iDx3D66)eZn!MIP^|8 zf%C6v)D>zZE4Z4G;$*Hi2ljW+Flc7$xg?NIIftE%3TSe_#(TY*&w0JJ-F#n?3N=eN z!=1M#x!pQP)4^;Sy5j946rvG%IS<9qF5zIqS`CVq%kk|+Ql%=Ai=~$hOJC;YjENCt z=xsFxHi?bD2T3K5XsExncjxSy!I;3iPUgRqJ3d!};f9LaBI`U4Z0})CD%Sy0QaHN1 zyDtEg&b?b16DiELXWp+a{xpTjaVK4H9wgJ#KBoZbWVc@f&6-fT`R{e=JXI6%J6icu zcUqMSK}b?O3A9M+?M`Vi;7HZ^m2*bkC^pUI&i4r_Kt|N+H+V*>JjJl-2PDKt>*AC-K|wI^nF&V%ClY|9HrjKBSMr9-R`VE z)3$gieDl^;(J|@NAB89lYn;%{8)~-1FQ#v=3`qk$N{>&uPJ}{SyD!yCra5QIA>d(I zd(+QLUkjm+dD=*}$60mDMdU)xCD%_MJafad?;U@C3beg{J_2p{U|m9o@2Eghs>)hx42n<(zdL<+6c%l_rCbqz|^ z!^8MFKp+^~R7TY@NY_39`mWwbdfKg&*KvDb_>biRtiO?7Mt?urU-Mo9qs4Zj-mVEX z3_Cp?>BWDuk$!X~$#EvHQ)7csbY`X zeya8<02T~o@Q90)#*o7;z|YAf4JZ?=5~{b{g~oDlotDzF79V^;(*$Oj1?bUVhZOG4 zTB}#c3H@sZard~N_99d$WDwX9&jO;gE)@kdY5TRdaLGrI^1fz9`xjC;RmnP z19WXW*&l0#W!lyR37tHGYP=+|e@)6PQp-dc49v8ke(Td$`lHi1+l*ra;yN}5;RtM> z*mNV~R|rI)jX!hONp#r)ILVEuK$TM-j;we}jNsVa8Heh9f*~1g>16pRR5$hZG9NpK zh7b4P2fB>~auelJx6~PfS!M#sAHdkNRBf9UH8`C$)p(KPN)MKHmN6e4#bBolhI%0| ztfV7@8NczIqgW`I5hphwRp`7st*1vW z5a#({zh_)pYu5l)=NI07&on9Wk+tM6&CuHFAZ8NyHo!LQr=>lb4I_41@3t)DuNQVJ z#hZ1Equ$<9U^Bi`a8$Z6Dl^lqjQS83_9U9oHb*jWnyG)Id}^VL`fVa5EfnzHwMIN9 zZmW_hbX0F7yNuQ;UugCL3g@wg#cocJ=~cI)Mz5>KtcJzL9@Z^ig6m)D5<~~sUg5NM z+T64u4qHxXSx5WvrvW7vS{-?wlDfw$%!DiI)_1tcoAc=q|1Y~n`P{+#rg5|B99k313 z0`xL-6QR1{Sm~_^!}vu;wD?q`8m6zot(19ETjeL}EYhcA6xUX%K^@dWpyCaWpIS}k zK)FThsP;<($7RWUp%)g}aC0*p;++^!%<}n2qito0*|%wYOFbB2dhYb+bkr}s^?UC= z3cW*)(+%ev3-U1+%0rUB>S5wWHwN4$ztj7XcGk~x6N~uTXq}~oZ#kvz@MdL`#SP6J zeZ}OTEjn$bYt-MfoT(;;WFC78CKipB@rRdic)Z5Gb%FTWZPmN%`7xM!q$MXiBF}=B z`>tf)l;zmcL+*>VL`3!pQ#QH1VOJ5nd5oPlvI-- z>0Te4C2lq<0vmGmM8VfcZ!EU^@h-rbO<+>4t|TB>(YJ3_5{bdvNI8N24$d|>Wjkr4 zWoq$JAIzkc?n;)ASg>W*pogKw0O=`dVBo!od2@N=nNANeikTN|jxisdvknWBtk7uz zezeJLxl-Q-N-&*^9aSyUjB}bXngo;I?w?cn#1*c!LX2IjHZFD}k=u5+-XUcha|M_n z&UEcNs=8jN5UDJLvxX<6O^c%+jMKI&!V`4%v-+4;HE}5a^9S)euQBkT6z_RS=>7ck zxpqY7GYb|A@gI^u*aS%LDTI0y@sy-Tk%_st{}b3e~dBibJ(+F?5?%=BKYauRp{f zz6*KVsx-AE+%^Raej27%-y$S|`_Zbk80EoubqnvuB%9g}f&qvEU&?LnvsmOi@6&1N ze0ILGYJ-{^eK^G;fp4I0-Q_zFYHY{^I2;CWIC?FmYE3-wYl>1-c0yM{)5QZ}7Ff#h zx8rH%V{FvEUWjM`ht6uu%xG8;PHIgXqyGe+5QTC85D#84Gq*LH*%q2ixNCD{D z6dPjEW#Ip)J}guo3weic;Wi1A+_(Jnl1aMJ0twO4B~FN)LA~L zl4YF8NB6?4hP#JE_k2PfZo+fn3wdgo!Ez(gsgAzKEW`5K+6S6tVbdfRZUr@LUGwkY z0D&u(MH@?yfk$LJNK`5v&;Sv1_P}tOpbeb?2Aq5nKQzG{a>Njr|9P)xIz@RybSd#$ zrK;7;t5_^d$IS9AKpIh2%Q=jQ?F#4~5)%e8H3ew&FjIe!$Q8%BdqWt-SZ7Ve|Nx0?6PQlAV ztJOg5%Pnck{CqBeJefhashye8FOD}FWHvxa+DL!70FclJb>a6RG?JMnMN0Y%r zt7o4s6dw~IKUE9npuh~_5)gtV>(v8b@ zT3!o`an4Qp_NrkI#$sHH*&6J1VvC>?{Rv;6fY|&FNQ%)sN9-i6#L+P?TJ41Pm+t@oq@YT3Yz78Q$(C zrc{#&jhmq?1_1+Pa$fcYQWmM`Y0vWr^kmHKu*A*m@vy{q`(g~VQkiLb2rXy+^+Og< zy{-D>t1Vr&yAuHy0^b&8Y^Ac{?l-fu7nkVAO_Q(MAAAoek2*+ zu5&Zqy?a?8RbqS9k&H6#AR|SW1{;xoW*^#AEtfRNfBg7dd=>Sxnr+!bvtZ+q0bGGMlo56h8do@Ood{Y51Ck~6@ZG*FA#{KaW{j91yGjEg+ukv}mlu1?# ztuGvtUT{q>KFUOUxB?$21rS7c5&&b{)mWJtltY8eu~u4U8=tT^B=&VA=d=WH=+ev2 zn9`0zriXesvKS8++vQK|%J~Yq&EUgbIZIVMrXJhaoz;YQMlJJzGfHM36t0~wd!LP@ zIoCA?Ft}+Keik<(o0dBw!sJfI;`_}5pjt$gx9kM+&yZG2;BD%@L5iA2eu;P!TqXAGl9V`7U2XeHJ7aswBkFx^ z?8wQ6l-hxTzh1Lq7Hnmx$FgwuggZXIR)TVJ@B^Gdl~gjy`-Ug1NFal<%hK}qj=hY^n5l4WiSL?+_rLhMZBPG^6TuV0-1P(lgK6DPKb zl0@}_*p1$bh56;T3j)rIUXh-rbs^!)Cp|?oCyIAUX;em(uGBXkkr=yy*Q577;Ucyl zhL~&m#U&cudN2B!G`3tBKgD^glv7$Q&4rwgIX^4SuC~0%YeI^-c;?}4Vhg(-WCiilRN-P0b-6 zjgLW9zUk%~$8}TVhe*a99`sX@77Im|pO$Nos*~~%N>N!n@(LdXSWPrVzz=PKP=&#n zD<&E?yZE35?e(v4469`+j`(27DO!bX-yyni7|&xhBQ-gucy;CJvZmDmM+^p7yGB`J zvPB+gkyKR5qQZtq-Nsz>E1ap#yuon{7NgHD(&coCQKGJ zRS6TEvYS-<>P>2IZ$x+_S&o{o=-Ml^ysr~*_>47VF^Jf~TmY2$zDRxI=s1nCx=48? zwB5$Wdr8`7EZ1ii7^!xa(mIhthiJQQ$jWFm2~|MSDznS8;YG_sUIKHPpkGl_+G6iD zEZO=|eKuC;vRxEyerP?g^B&OPms|4ls}ols`R>)+V|v(SE=rEj%nBS)0gw`g<eJ-H~^U*i_nn3V2 zG(gXJ^Rn~KM(}{)r3w(_u7z-VgcOX2IjGkt=1qM}{DE?lzEoYTow-*_b6uhywG$)vs~agyO#Cv?+t&71hNuQluLI;Zb|&A zK`R9FGlF$6fcRrYfE;fH34Um=(p(q<32cF|Zo}COQEOhlHrJVhX7(R{c*s?m*u(Zq z?S+Hy^#E@D_cJoSz~@|2)j|oT&}>irzvNs5;ebV|Y~;J2SJb=2EXqb^LQj0iRSwuY z#Ea4sgzuc(;zy>mpM;Elr9_RHASF>h*-M|*eAFd4EtiFSf*cCpRQVQPJw)#x&q!#Q zS-wB>b$uDSRwkUPJ)Y+IE$WJN@DL#*c~+Lz4}Uuqzr2N&@S>f~)fMVjOFE`IfLmcLB@*Ic||8fQJx&XN{J518x+Y5Ycux-K{v<9&( z@AiVQwlnBmezo1uP3RW7;6*xRv(5BsG}l$u4IE0#Saqj=@y7ms9u(ieN-F`sPGT7e z5PvBT^3&=kX;s3drIbRhCQ;(%2$O#)4*;d2OewXZipkO@#&G-~vicCWoyzAy!F!9-T=_~GtTc~(&Hs5RH4Sja z5SlF)E2_4#F%T@=D!@)qVc<+bQ8~Mx2O#gCQVk=eC?iBtPUcu zf1H9Onk)co6Wx6PSvts6F}{N#{bTv)0h?CVX)v~if~hc!5g4*y;XiH)=`Gg0yJSjK zhN@ZHx~Lo^c-9BxWpnNoxYO#;V;A-E1ra&}-_G-sw7Y^D3SW#dt0Di}t&!cqAOBxN z8Ss@Be-=^{Upu^hnqGV(cVONPUv+{1;ab5_{1+q(n}CBr&IVt6(XGML$0erSbJbs+nvmG>0UlhW z#Oei$iYx=N?A$wVrTakFjP98I^-`wmg!1!(j3ixr3X+UI4pzz@VRNq#lr2;B(GD0iuvo_hlwer&Nq{$KL70$|AbPTRy0`;rk6)lA#Bzb!HSv*Mwb7U)Nj;s=DNz+lgcV1Utz zWr8n$kHCF^Uroe<@`(Rl?Jtx1e|+DxBZMDDgr6!P z+m^@4XxpzAY7sgO0pzX_o|?}k=Z8N@NLQZ+d40!?)NA^6J=x5P9Ry4HiX!MLKYxQr zK+-w8MqcqzyAHkPkf((JP6UfLa#sn7F7Z)67?y6VV^Ci_vrH@yESlG%&OsIeDD&b_ z%i{uAKPN(RNW~aIM9tx?Z4eC>kBDpfVj=(*2fP$_p5f^gYTsRo`@{YOAr%0N*d<|? zXZ-w4LCm&eFJrFv3_rL6=*xe$w*1?f@8 zF`gwdA+;0+i#u}#Ng~}U9GmQaM+v_e(N*Ff7r|DZ;JEY@LX5q@g{{utxMEt;7-8Pl zZEGztzsA#-MM>~2Y`7A=CXv6u*s1^E#cMio1PS&cxJtp!ZsKZ=O+_Fs(YCjjF>&tb zXY$qQlY`!WN!vevfc@cGxyOFQn^#b{xbi3m$J0E92^&>-=G}R#Y!jChi>~Imgp(e- z7JN%-S3|w;7rH#n-rUYD<|K5++)2K6_d2T#XOQzRS}|G z!*`Ywr@Ox*m;=JYlq+U;9J`VfrOkF}To>DWEIuCe1%KH%@WLt?I_7M3_cAT)?7Y5e z^R7dws+B52O4rWsb!$igv2`mZp>w7stW@9(YdFM76~en{3tsex-hKgT%_S{D zyFgr!U)0%|-fg?qa=$b@=xxNtNp(q^RC^tkFwqHwQoq^PZ71)3$1-k$r==gpw#Ldr zzGdEG&d20HrNTM9Yf{`Fk#UNT6r<>U!;g@D0sBQG`kg0+kn4b7T!V^=Rx9WY3n55G z#pcV!#-&`ULC`liOhiXaTt*ig4JWuWq~}hMVrYc?3F?VwFp|SENO$hZ*Z(}V2S0g5 z^=gm3=8+x4&gf&F*8{*>MSp00mO-+zVi gPlfz{S9fh3(~Rr&S%fLyIt2dRkW-dTzNYW?KL+iGfdBvi literal 0 HcmV?d00001 diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 3793aec037f..11effc82fe7 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -541,7 +541,14 @@ const sidebars = { }, "realtime", "rerank", - "response_api", + { + type: "category", + label: "/responses", + items: [ + "response_api", + "response_api_compact", + ] + }, { type: "category", label: "/search", diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 7ffbe95be13..a1fa4ef8440 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -10,6 +10,7 @@ from pathlib import Path from typing import Optional from litellm_proxy_extras._logging import logger +from litellm.caching.redis_cache import RedisCache def str_to_bool(value: Optional[str]) -> bool: @@ -18,6 +19,103 @@ def str_to_bool(value: Optional[str]) -> bool: return value.lower() in ("true", "1", "t", "y", "yes") +class MigrationLockManager: + """Redis-based lock manager for database migrations""" + + MIGRATION_LOCK_KEY = "migration_lock" + LOCK_TTL_SECONDS = 300 # 5 minutes TTL + + def __init__(self, redis_cache: Optional[RedisCache] = None): + self.redis_cache = redis_cache + self.lock_acquired = False + self.pod_id = f"pod_{os.getpid()}_{int(time.time())}" + + def _get_redis_lock_key(self) -> str: + """Get Redis lock key for migration""" + return f"migration_lock:{self.MIGRATION_LOCK_KEY}" + + def acquire_lock(self) -> bool: + """Acquire migration lock""" + if self.redis_cache is None: + logger.warning( + "Redis cache is not available, running migration without lock protection" + ) + self.lock_acquired = True + return True + + try: + lock_key = self._get_redis_lock_key() + + # Redis SET with NX (only if not exists) and EX (expiration) + acquired = self.redis_cache.set_cache( + key=lock_key, value=self.pod_id, nx=True, ttl=self.LOCK_TTL_SECONDS + ) + + if acquired: + self.lock_acquired = True + logger.info(f"Migration lock acquired by pod {self.pod_id}") + return True + else: + logger.info("Migration lock is already held by another pod") + return False + + except Exception as e: + logger.warning(f"Failed to acquire migration lock: {e}") + return False + + def wait_for_lock_release( + self, check_interval: int = 5, max_wait: int = 300 + ) -> bool: + """Wait for another process to release the lock""" + if self.redis_cache is None: + logger.warning("Redis cache is not available, cannot wait for lock") + return False + + logger.info(f"Waiting for migration lock to be released (max {max_wait}s)...") + start_time = time.time() + + while time.time() - start_time < max_wait: + # Try to acquire lock using the public acquire_lock method + if self.acquire_lock(): + logger.info( + f"Migration lock acquired after waiting by pod {self.pod_id}" + ) + return True + + time.sleep(check_interval) + + logger.warning(f"Failed to acquire migration lock within {max_wait} seconds") + return False + + def release_lock(self): + """Release migration lock""" + if not self.lock_acquired or self.redis_cache is None: + return + + try: + lock_key = self._get_redis_lock_key() + + # Verify current pod owns the lock + current_value = self.redis_cache.get_cache(lock_key) + if current_value and str(current_value) == self.pod_id: + self.redis_cache.delete_cache(lock_key) + logger.info(f"Migration lock released by pod {self.pod_id}") + else: + logger.warning(f"Pod {self.pod_id} cannot release lock (not owner)") + + except Exception as e: + logger.warning(f"Failed to release migration lock: {e}") + finally: + self.lock_acquired = False + + def __enter__(self): + """Context manager entry - acquire lock when entering with statement""" + self.acquire_lock() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Context manager exit - release lock when exiting with statement""" + self.release_lock() def _get_prisma_env() -> dict: """Get environment variables for Prisma, handling offline mode if configured.""" @@ -346,19 +444,50 @@ class ProxyExtrasDBManager: ) @staticmethod - def setup_database(use_migrate: bool = False) -> bool: + def setup_database( + use_migrate: bool = False, redis_cache: Optional[RedisCache] = None + ) -> bool: """ Set up the database using either prisma migrate or prisma db push - Uses migrations from litellm-proxy-extras package + Uses migrations from litellm-proxy-extras package. + In multi-instance environment, use redis lock to prevent concurrent execution. Args: schema_path (str): Path to the Prisma schema file use_migrate (bool): Whether to use prisma migrate instead of db push + redis_cache: Redis cache instance for distributed locking Returns: bool: True if setup was successful, False otherwise """ schema_path = ProxyExtrasDBManager._get_prisma_dir() + "/schema.prisma" + + database_url = os.getenv("DATABASE_URL") + if not database_url: + logger.error("DATABASE_URL environment variable is not set") + return False + + # Use MigrationLockManager to prevent concurrent migration execution + with MigrationLockManager(redis_cache) as lock_manager: + # Lock is already acquired in __enter__, check if it was successful + if not lock_manager.lock_acquired: + # Cannot acquire lock, another process is running migration + logger.info( + "Another pod is running migration, waiting for completion..." + ) + + # Wait for other process to complete migration + if not lock_manager.wait_for_lock_release(): + logger.error("Failed to acquire migration lock after waiting") + return False + + # Successfully acquired lock, proceed with migration + logger.info("Acquired migration lock, proceeding with migration") + return ProxyExtrasDBManager._execute_migration(use_migrate, schema_path) + + @staticmethod + def _execute_migration(use_migrate: bool, schema_path: str) -> bool: + """Execute the actual migration""" for attempt in range(4): original_dir = os.getcwd() migrations_dir = ProxyExtrasDBManager._get_prisma_dir() diff --git a/litellm/__init__.py b/litellm/__init__.py index dfe959bf747..7f7ee21f692 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -197,6 +197,7 @@ retry = True api_key: Optional[str] = None openai_key: Optional[str] = None groq_key: Optional[str] = None +gigachat_key: Optional[str] = None databricks_key: Optional[str] = None openai_like_key: Optional[str] = None azure_key: Optional[str] = None @@ -275,6 +276,7 @@ banned_keywords_list: Optional[Union[str, List]] = None llm_guard_mode: Literal["all", "key-specific", "request-specific"] = "all" guardrail_name_config_map: Dict[str, GuardrailItem] = {} include_cost_in_streaming_usage: bool = False +reasoning_auto_summary: bool = False ### PROMPTS #### from litellm.types.prompts.init_prompts import PromptSpec @@ -1440,6 +1442,8 @@ if TYPE_CHECKING: from .llms.github_copilot.chat.transformation import GithubCopilotConfig as GithubCopilotConfig from .llms.github_copilot.responses.transformation import GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig + from .llms.gigachat.chat.transformation import GigaChatConfig as GigaChatConfig + from .llms.gigachat.embedding.transformation import GigaChatEmbeddingConfig as GigaChatEmbeddingConfig from .llms.nebius.chat.transformation import NebiusConfig as NebiusConfig from .llms.wandb.chat.transformation import WandbConfig as WandbConfig from .llms.dashscope.chat.transformation import DashScopeChatConfig as DashScopeChatConfig diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 8c8266d5ca4..26133ebc222 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -255,6 +255,8 @@ LLM_CONFIG_NAMES = ( "GithubCopilotEmbeddingConfig", "NebiusConfig", "WandbConfig", + "GigaChatConfig", + "GigaChatEmbeddingConfig", "DashScopeChatConfig", "MoonshotChatConfig", "DockerModelRunnerChatConfig", @@ -644,6 +646,8 @@ _LLM_CONFIGS_IMPORT_MAP = { "GithubCopilotEmbeddingConfig": (".llms.github_copilot.embedding.transformation", "GithubCopilotEmbeddingConfig"), "NebiusConfig": (".llms.nebius.chat.transformation", "NebiusConfig"), "WandbConfig": (".llms.wandb.chat.transformation", "WandbConfig"), + "GigaChatConfig": (".llms.gigachat.chat.transformation", "GigaChatConfig"), + "GigaChatEmbeddingConfig": (".llms.gigachat.embedding.transformation", "GigaChatEmbeddingConfig"), "DashScopeChatConfig": (".llms.dashscope.chat.transformation", "DashScopeChatConfig"), "MoonshotChatConfig": (".llms.moonshot.chat.transformation", "MoonshotChatConfig"), "DockerModelRunnerChatConfig": (".llms.docker_model_runner.chat.transformation", "DockerModelRunnerChatConfig"), diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 5b206317b29..a89efc4e82b 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -3,6 +3,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req """ import json +import os from typing import ( TYPE_CHECKING, Any, @@ -22,6 +23,7 @@ from typing import ( from openai.types.responses.tool_param import FunctionToolParam from pydantic import BaseModel +import litellm from litellm import ModelResponse from litellm._logging import verbose_logger from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator @@ -691,19 +693,26 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if isinstance(reasoning_effort, dict): return Reasoning(**reasoning_effort) # type: ignore[typeddict-item] - # If string is passed, map with summary="detailed" + # Check if auto-summary is enabled via flag or environment variable + # Priority: litellm.reasoning_auto_summary flag > LITELLM_REASONING_AUTO_SUMMARY env var + auto_summary_enabled = ( + litellm.reasoning_auto_summary + or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true" + ) + + # If string is passed, map with optional summary based on flag/env var if reasoning_effort == "none": - return Reasoning(effort="none", summary="detailed") # type: ignore + return Reasoning(effort="none", summary="detailed") if auto_summary_enabled else Reasoning(effort="none") # type: ignore elif reasoning_effort == "high": - return Reasoning(effort="high", summary="detailed") + return Reasoning(effort="high", summary="detailed") if auto_summary_enabled else Reasoning(effort="high") elif reasoning_effort == "xhigh": - return Reasoning(effort="xhigh", summary="detailed") # type: ignore[typeddict-item] + return Reasoning(effort="xhigh", summary="detailed") if auto_summary_enabled else Reasoning(effort="xhigh") # type: ignore[typeddict-item] elif reasoning_effort == "medium": - return Reasoning(effort="medium", summary="detailed") + return Reasoning(effort="medium", summary="detailed") if auto_summary_enabled else Reasoning(effort="medium") elif reasoning_effort == "low": - return Reasoning(effort="low", summary="detailed") + return Reasoning(effort="low", summary="detailed") if auto_summary_enabled else Reasoning(effort="low") elif reasoning_effort == "minimal": - return Reasoning(effort="minimal", summary="detailed") + return Reasoning(effort="minimal", summary="detailed") if auto_summary_enabled else Reasoning(effort="minimal") return None def _transform_response_format_to_text_format( diff --git a/litellm/constants.py b/litellm/constants.py index e8524a87c41..1cd2da549ca 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -375,6 +375,7 @@ LITELLM_CHAT_PROVIDERS = [ "perplexity", "mistral", "groq", + "gigachat", "nvidia_nim", "cerebras", "baseten", diff --git a/litellm/integrations/callback_configs.json b/litellm/integrations/callback_configs.json index 88f7908e9a2..6b30b6b736e 100644 --- a/litellm/integrations/callback_configs.json +++ b/litellm/integrations/callback_configs.json @@ -187,6 +187,12 @@ "ui_name": "Sampling Rate", "description": "Sampling rate for logging (0.0 to 1.0, default: 1.0)", "required": false + }, + "langsmith_tenant_id": { + "type": "text", + "ui_name": "Tenant ID", + "description": "LangSmith tenant ID for organization-scoped API keys (required when using org-scoped keys)", + "required": false } }, "description": "Langsmith Logging Integration" diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index f0f355b4895..7e62613a7e4 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -50,6 +50,42 @@ else: Langfuse = Any +def _extract_cache_read_input_tokens(usage_obj) -> int: + """ + Extract cache_read_input_tokens from usage object. + + Checks both: + 1. Top-level cache_read_input_tokens (Anthropic format) + 2. prompt_tokens_details.cached_tokens (Gemini, OpenAI format) + + See: https://github.com/BerriAI/litellm/issues/18520 + + Args: + usage_obj: Usage object from LLM response + + Returns: + int: Number of cached tokens read, defaults to 0 + """ + cache_read_input_tokens = usage_obj.get("cache_read_input_tokens") or 0 + + # Check prompt_tokens_details.cached_tokens (used by Gemini and other providers) + if hasattr(usage_obj, "prompt_tokens_details"): + prompt_tokens_details = getattr(usage_obj, "prompt_tokens_details", None) + if ( + prompt_tokens_details is not None + and hasattr(prompt_tokens_details, "cached_tokens") + ): + cached_tokens = getattr(prompt_tokens_details, "cached_tokens", None) + if ( + cached_tokens is not None + and isinstance(cached_tokens, (int, float)) + and cached_tokens > 0 + ): + cache_read_input_tokens = cached_tokens + + return cache_read_input_tokens + + class LangFuseLogger: # Class variables or attributes def __init__( @@ -757,8 +793,8 @@ class LangFuseLogger: cache_creation_input_tokens = ( _usage_obj.get("cache_creation_input_tokens") or 0 ) - cache_read_input_tokens = ( - _usage_obj.get("cache_read_input_tokens") or 0 + cache_read_input_tokens = _extract_cache_read_input_tokens( + _usage_obj ) usage = { diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index cc9b361b69d..570b78f2927 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -40,6 +40,7 @@ class LangsmithLogger(CustomBatchLogger): langsmith_project: Optional[str] = None, langsmith_base_url: Optional[str] = None, langsmith_sampling_rate: Optional[float] = None, + langsmith_tenant_id: Optional[str] = None, **kwargs, ): self.flush_lock = asyncio.Lock() @@ -48,6 +49,7 @@ class LangsmithLogger(CustomBatchLogger): langsmith_api_key=langsmith_api_key, langsmith_project=langsmith_project, langsmith_base_url=langsmith_base_url, + langsmith_tenant_id=langsmith_tenant_id, ) self.sampling_rate: float = ( langsmith_sampling_rate @@ -76,6 +78,7 @@ class LangsmithLogger(CustomBatchLogger): langsmith_api_key: Optional[str] = None, langsmith_project: Optional[str] = None, langsmith_base_url: Optional[str] = None, + langsmith_tenant_id: Optional[str] = None, ) -> LangsmithCredentialsObject: _credentials_api_key = langsmith_api_key or os.getenv("LANGSMITH_API_KEY") _credentials_project = ( @@ -86,11 +89,13 @@ class LangsmithLogger(CustomBatchLogger): or os.getenv("LANGSMITH_BASE_URL") or "https://api.smith.langchain.com" ) + _credentials_tenant_id = langsmith_tenant_id or os.getenv("LANGSMITH_TENANT_ID") return LangsmithCredentialsObject( LANGSMITH_API_KEY=_credentials_api_key, LANGSMITH_BASE_URL=_credentials_base_url, LANGSMITH_PROJECT=_credentials_project, + LANGSMITH_TENANT_ID=_credentials_tenant_id, ) def _prepare_log_data( @@ -365,8 +370,11 @@ class LangsmithLogger(CustomBatchLogger): """ langsmith_api_base = credentials["LANGSMITH_BASE_URL"] langsmith_api_key = credentials["LANGSMITH_API_KEY"] + langsmith_tenant_id = credentials.get("LANGSMITH_TENANT_ID") url = self._add_endpoint_to_url(langsmith_api_base, "runs/batch") headers = {"x-api-key": langsmith_api_key} + if langsmith_tenant_id: + headers["x-tenant-id"] = langsmith_tenant_id elements_to_log = [queue_object["data"] for queue_object in queue_objects] try: @@ -418,6 +426,7 @@ class LangsmithLogger(CustomBatchLogger): api_key=credentials["LANGSMITH_API_KEY"], project=credentials["LANGSMITH_PROJECT"], base_url=credentials["LANGSMITH_BASE_URL"], + tenant_id=credentials.get("LANGSMITH_TENANT_ID"), ) if key not in log_queue_by_credentials: @@ -466,6 +475,9 @@ class LangsmithLogger(CustomBatchLogger): langsmith_base_url=standard_callback_dynamic_params.get( "langsmith_base_url", None ), + langsmith_tenant_id=standard_callback_dynamic_params.get( + "langsmith_tenant_id", None + ), ) else: credentials = self.default_credentials @@ -491,13 +503,16 @@ class LangsmithLogger(CustomBatchLogger): def get_run_by_id(self, run_id): langsmith_api_key = self.default_credentials["LANGSMITH_API_KEY"] - langsmith_api_base = self.default_credentials["LANGSMITH_BASE_URL"] + langsmith_tenant_id = self.default_credentials.get("LANGSMITH_TENANT_ID") url = f"{langsmith_api_base}/runs/{run_id}" + headers = {"x-api-key": langsmith_api_key} + if langsmith_tenant_id: + headers["x-tenant-id"] = langsmith_tenant_id response = litellm.module_level_client.get( url=url, - headers={"x-api-key": langsmith_api_key}, + headers=headers, ) return response.json() diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 12e60bc25bb..a7d2326d938 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -196,50 +196,88 @@ class OpenTelemetry(CustomLogger): litellm.service_callback.append(self) setattr(proxy_server, "open_telemetry_logger", self) + def _get_or_create_provider( + self, + provider, + provider_name: str, + get_existing_provider_fn, + sdk_provider_class, + create_new_provider_fn, + set_provider_fn, + ): + """ + Generic helper to get or create an OpenTelemetry provider (Tracer, Meter, or Logger). + + Args: + provider: The provider instance passed to the init function (can be None) + provider_name: Name for logging (e.g., "TracerProvider") + get_existing_provider_fn: Function to get the existing global provider + sdk_provider_class: The SDK provider class to check for (e.g., TracerProvider from SDK) + create_new_provider_fn: Function to create a new provider instance + set_provider_fn: Function to set the provider globally + + Returns: + The provider to use (either existing, new, or explicitly provided) + """ + if provider is not None: + # Provider explicitly provided (e.g., for testing) + # Do NOT call set_provider_fn - the caller is responsible for managing global state + # If they want it to be global, they've already set it before passing it to us + verbose_logger.debug( + "OpenTelemetry: Using provided TracerProvider: %s", + type(provider).__name__, + ) + return provider + + # Check if a provider is already set globally + try: + existing_provider = get_existing_provider_fn() + + # If a real SDK provider exists (set by another SDK like Langfuse), use it + # This uses a positive check for SDK providers instead of a negative check for proxy providers + if isinstance(existing_provider, sdk_provider_class): + verbose_logger.debug( + "OpenTelemetry: Using existing %s: %s", + provider_name, + type(existing_provider).__name__, + ) + provider = existing_provider + # Don't call set_provider to preserve existing context + else: + # Default proxy provider or unknown type, create our own + verbose_logger.debug("OpenTelemetry: Creating new %s", provider_name) + provider = create_new_provider_fn() + set_provider_fn(provider) + except Exception as e: + # Fallback: create a new provider if something goes wrong + verbose_logger.debug( + "OpenTelemetry: Exception checking existing %s, creating new one: %s", + provider_name, + str(e), + ) + provider = create_new_provider_fn() + set_provider_fn(provider) + + return provider + def _init_tracing(self, tracer_provider): from opentelemetry import trace from opentelemetry.sdk.trace import TracerProvider from opentelemetry.trace import SpanKind - # use provided tracer or create a new one - if tracer_provider is None: - # Check if a TracerProvider is already set globally (e.g., by Langfuse SDK) - try: - from opentelemetry.trace import ProxyTracerProvider + def create_tracer_provider(): + provider = TracerProvider(resource=_get_litellm_resource()) + provider.add_span_processor(self._get_span_processor()) + return provider - existing_provider = trace.get_tracer_provider() - - # If an actual provider exists (not the default proxy), use it - if not isinstance(existing_provider, ProxyTracerProvider): - verbose_logger.debug( - "OpenTelemetry: Using existing TracerProvider: %s", - type(existing_provider).__name__, - ) - tracer_provider = existing_provider - # Don't call set_tracer_provider to preserve existing context - else: - # No real provider exists yet, create our own - verbose_logger.debug("OpenTelemetry: Creating new TracerProvider") - tracer_provider = TracerProvider(resource=_get_litellm_resource()) - tracer_provider.add_span_processor(self._get_span_processor()) - trace.set_tracer_provider(tracer_provider) - except Exception as e: - # Fallback: create a new provider if something goes wrong - verbose_logger.debug( - "OpenTelemetry: Exception checking existing provider, creating new one: %s", - str(e), - ) - tracer_provider = TracerProvider(resource=_get_litellm_resource()) - tracer_provider.add_span_processor(self._get_span_processor()) - trace.set_tracer_provider(tracer_provider) - else: - # Tracer provider explicitly provided (e.g., for testing) - # Do NOT call set_tracer_provider - the caller is responsible for managing global state - # If they want it to be global, they've already set it before passing it to us - verbose_logger.debug( - "OpenTelemetry: Using provided TracerProvider: %s", - type(tracer_provider).__name__, - ) + tracer_provider = self._get_or_create_provider( + provider=tracer_provider, + provider_name="TracerProvider", + get_existing_provider_fn=trace.get_tracer_provider, + sdk_provider_class=TracerProvider, + create_new_provider_fn=create_tracer_provider, + set_provider_fn=trace.set_tracer_provider, + ) # Grab our tracer from the TracerProvider (not from global context) # This ensures we use the provided TracerProvider (e.g., for testing) @@ -257,39 +295,24 @@ class OpenTelemetry(CustomLogger): return from opentelemetry import metrics - from opentelemetry.sdk.metrics import Histogram, MeterProvider + from opentelemetry.sdk.metrics import MeterProvider - # Only create OTLP infrastructure if no custom meter provider is provided - if meter_provider is None: - from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( - OTLPMetricExporter, - ) - from opentelemetry.sdk.metrics.export import ( - AggregationTemporality, - PeriodicExportingMetricReader, + def create_meter_provider(): + metric_reader = self._get_metric_reader() + return MeterProvider( + metric_readers=[metric_reader], resource=_get_litellm_resource() ) - normalized_endpoint = self._normalize_otel_endpoint( - self.config.endpoint, "metrics" - ) - _metric_exporter = OTLPMetricExporter( - endpoint=normalized_endpoint, - headers=OpenTelemetry._get_headers_dictionary(self.config.headers), - preferred_temporality={Histogram: AggregationTemporality.DELTA}, - ) - _metric_reader = PeriodicExportingMetricReader( - _metric_exporter, export_interval_millis=10000 - ) + meter_provider = self._get_or_create_provider( + provider=meter_provider, + provider_name="MeterProvider", + get_existing_provider_fn=metrics.get_meter_provider, + sdk_provider_class=MeterProvider, + create_new_provider_fn=create_meter_provider, + set_provider_fn=metrics.set_meter_provider, + ) - meter_provider = MeterProvider( - metric_readers=[_metric_reader], resource=_get_litellm_resource() - ) - meter = meter_provider.get_meter(__name__) - else: - # Use the provided meter provider as-is, without creating additional OTLP infrastructure - meter = meter_provider.get_meter(__name__) - - metrics.set_meter_provider(meter_provider) + meter = meter_provider.get_meter(__name__) self._operation_duration_histogram = meter.create_histogram( name="gen_ai.client.operation.duration", # Replace with semconv constant in otel 1.38 @@ -327,22 +350,26 @@ class OpenTelemetry(CustomLogger): if not self.config.enable_events: return - from opentelemetry._logs import set_logger_provider + from opentelemetry._logs import get_logger_provider, set_logger_provider from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider from opentelemetry.sdk._logs.export import BatchLogRecordProcessor - # set up log pipeline - if logger_provider is None: - litellm_resource = _get_litellm_resource() - logger_provider = OTLoggerProvider(resource=litellm_resource) - # Only add OTLP exporter if we created the logger provider ourselves + def create_logger_provider(): + provider = OTLoggerProvider(resource=_get_litellm_resource()) log_exporter = self._get_log_exporter() - if log_exporter: - logger_provider.add_log_record_processor( - BatchLogRecordProcessor(log_exporter) # type: ignore[arg-type] - ) + provider.add_log_record_processor( + BatchLogRecordProcessor(log_exporter) # type: ignore[arg-type] + ) + return provider - set_logger_provider(logger_provider) + self._get_or_create_provider( + provider=logger_provider, + provider_name="LoggerProvider", + get_existing_provider_fn=get_logger_provider, + sdk_provider_class=OTLoggerProvider, + create_new_provider_fn=create_logger_provider, + set_provider_fn=set_logger_provider, + ) def log_success_event(self, kwargs, response_obj, start_time, end_time): self._handle_success(kwargs, response_obj, start_time, end_time) @@ -944,6 +971,15 @@ class OpenTelemetry(CustomLogger): if not self.config.enable_events: return + # NOTE: Semantic logs (gen_ai.content.prompt/completion events) have compatibility issues + # with OTEL SDK >= 1.39.0 due to breaking changes in PR #4676: + # - LogRecord moved from opentelemetry.sdk._logs to opentelemetry.sdk._logs._internal + # - LogRecord constructor no longer accepts 'resource' parameter (now inherited from LoggerProvider) + # - LogData class was removed entirely + # These logs work correctly in OTEL SDK < 1.39.0 but may fail in >= 1.39.0. + # See: https://github.com/open-telemetry/opentelemetry-python/pull/4676 + # TODO: Refactor to use the proper OTEL Logs API instead of directly creating SDK LogRecords + from opentelemetry._logs import SeverityNumber, get_logger, get_logger_provider from opentelemetry.sdk._logs import LogRecord as SdkLogRecord @@ -1807,7 +1843,8 @@ class OpenTelemetry(CustomLogger): ) return self.OTEL_EXPORTER - if self.OTEL_EXPORTER == "console": + otel_logs_exporter = os.getenv("OTEL_LOGS_EXPORTER") + if self.OTEL_EXPORTER == "console" or otel_logs_exporter == "console": from opentelemetry.sdk._logs.export import ConsoleLogExporter verbose_logger.debug( @@ -1854,6 +1891,67 @@ class OpenTelemetry(CustomLogger): return ConsoleLogExporter() + def _get_metric_reader(self): + """ + Get the appropriate metric reader based on the configuration. + """ + from opentelemetry.sdk.metrics import Histogram + from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + ConsoleMetricExporter, + PeriodicExportingMetricReader, + ) + + verbose_logger.debug( + "OpenTelemetry Logger, initializing metric reader\nself.OTEL_EXPORTER: %s\nself.OTEL_ENDPOINT: %s\nself.OTEL_HEADERS: %s", + self.OTEL_EXPORTER, + self.OTEL_ENDPOINT, + self.OTEL_HEADERS, + ) + + _split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS) + normalized_endpoint = self._normalize_otel_endpoint(self.OTEL_ENDPOINT, "metrics") + + if self.OTEL_EXPORTER == "console": + exporter = ConsoleMetricExporter() + return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) + + elif ( + self.OTEL_EXPORTER == "otlp_http" + or self.OTEL_EXPORTER == "http/protobuf" + or self.OTEL_EXPORTER == "http/json" + ): + from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter, + ) + + exporter = OTLPMetricExporter( + endpoint=normalized_endpoint, + headers=_split_otel_headers, + preferred_temporality={Histogram: AggregationTemporality.DELTA}, + ) + return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) + + elif self.OTEL_EXPORTER == "otlp_grpc" or self.OTEL_EXPORTER == "grpc": + from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, + ) + + exporter = OTLPMetricExporter( + endpoint=normalized_endpoint, + headers=_split_otel_headers, + preferred_temporality={Histogram: AggregationTemporality.DELTA}, + ) + return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) + + else: + verbose_logger.warning( + "OpenTelemetry: Unknown metric exporter '%s', defaulting to console. Supported: console, otlp_http, otlp_grpc", + self.OTEL_EXPORTER, + ) + exporter = ConsoleMetricExporter() + return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) + def _normalize_otel_endpoint( self, endpoint: Optional[str], signal_type: str ) -> Optional[str]: diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index d92af417175..6baaae7ae3f 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2000,24 +2000,56 @@ class CustomStreamWrapper: ) ## Map to OpenAI Exception try: - raise exception_type( + mapped_exception = exception_type( model=self.model, custom_llm_provider=self.custom_llm_provider, original_exception=e, completion_kwargs={}, extra_kwargs={}, ) - except Exception as e: - from litellm.exceptions import MidStreamFallbackError + except Exception as mapping_error: + mapped_exception = mapping_error - raise MidStreamFallbackError( - message=str(e), - model=self.model, - llm_provider=self.custom_llm_provider or "anthropic", - original_exception=e, - generated_content=self.response_uptil_now, - is_pre_first_chunk=not self.sent_first_chunk, - ) + def _normalize_status_code(exc: Exception) -> Optional[int]: + """ + Best-effort status_code extraction. + Uses status_code on the exception, then falls back to the response. + """ + try: + code = getattr(exc, "status_code", None) + if code is not None: + return int(code) + except Exception: + pass + + response = getattr(exc, "response", None) + if response is not None: + try: + status_code = getattr(response, "status_code", None) + if status_code is not None: + return int(status_code) + except Exception: + pass + return None + + mapped_status_code = _normalize_status_code(mapped_exception) + original_status_code = _normalize_status_code(e) + + if mapped_status_code is not None and 400 <= mapped_status_code < 500: + raise mapped_exception + if original_status_code is not None and 400 <= original_status_code < 500: + raise mapped_exception + + from litellm.exceptions import MidStreamFallbackError + + raise MidStreamFallbackError( + message=str(mapped_exception), + model=self.model, + llm_provider=self.custom_llm_provider or "anthropic", + original_exception=mapped_exception, + generated_content=self.response_uptil_now, + is_pre_first_chunk=not self.sent_first_chunk, + ) @staticmethod def _strip_sse_data_from_chunk(chunk: Optional[str]) -> Optional[str]: diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index facabbda72a..7a4da985528 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -242,3 +242,30 @@ class BaseResponsesAPIConfig(ABC): ######################################################### ########## END CANCEL RESPONSE API TRANSFORMATION ####### ######################################################### + + ######################################################### + ########## COMPACT RESPONSE API TRANSFORMATION ########## + ######################################################### + @abstractmethod + def transform_compact_response_api_request( + self, + model: str, + input: Union[str, ResponseInputParam], + response_api_optional_request_params: Dict, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + pass + + @abstractmethod + def transform_compact_response_api_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + pass + + ######################################################### + ########## END COMPACT RESPONSE API TRANSFORMATION ###### + ######################################################### diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 34ea598a655..ea740400664 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -91,6 +91,7 @@ from litellm.types.rerank import RerankResponse from litellm.types.responses.main import DeleteResponseResult from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( + CallTypes, EmbeddingResponse, FileTypes, LiteLLMBatch, @@ -850,7 +851,9 @@ class BaseLLMHTTPHandler: ) if client is None or not isinstance(client, HTTPHandler): - sync_httpx_client = _get_httpx_client() + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) else: sync_httpx_client = client @@ -896,7 +899,8 @@ class BaseLLMHTTPHandler: ) -> EmbeddingResponse: if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider) + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: async_httpx_client = client @@ -2004,6 +2008,10 @@ class BaseLLMHTTPHandler: """ Handles responses API requests. When _is_async=True, returns a coroutine instead of making the call directly. + + Keeps the pre-transform request context for streaming so post-call hooks/metadata + (added for Responses API parity with chat) receive the original params instead of + the provider-shaped body that caused them to be skipped before. """ if _is_async: @@ -2060,6 +2068,18 @@ class BaseLLMHTTPHandler: if extra_body: data.update(extra_body) + # Preserve the OpenAI-style request context (not sent to the provider) for streaming + # hooks/metadata; the streaming iterator now consumes this to run deployment hooks + # with the same info as chat, including litellm_params. + request_context: Dict[str, Any] = {"input": input} + try: + request_context.update(response_api_optional_request_params) + except Exception: + pass + # Needed by streaming callbacks/metadata helpers to reconstruct api_base/model_id + # but never included in the outbound provider payload. + request_context["litellm_params"] = dict(litellm_params) + ## LOGGING logging_obj.pre_call( input=input, @@ -2097,6 +2117,8 @@ class BaseLLMHTTPHandler: responses_api_provider_config=responses_api_provider_config, litellm_metadata=litellm_metadata, custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, ) return SyncResponsesAPIStreamingIterator( @@ -2106,6 +2128,8 @@ class BaseLLMHTTPHandler: responses_api_provider_config=responses_api_provider_config, litellm_metadata=litellm_metadata, custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, ) else: # For non-streaming requests @@ -2189,6 +2213,18 @@ class BaseLLMHTTPHandler: if extra_body: data.update(extra_body) + # Preserve the OpenAI-style request context (not sent to the provider) for streaming + # hooks/metadata; the streaming iterator now consumes this to run deployment hooks + # with the same info as chat, including litellm_params. + request_context: Dict[str, Any] = {"input": input} + try: + request_context.update(response_api_optional_request_params) + except Exception: + pass + # Needed by streaming callbacks/metadata helpers to reconstruct api_base/model_id + # but never included in the outbound provider payload. + request_context["litellm_params"] = dict(litellm_params) + ## LOGGING logging_obj.pre_call( input=input, @@ -2227,6 +2263,8 @@ class BaseLLMHTTPHandler: responses_api_provider_config=responses_api_provider_config, litellm_metadata=litellm_metadata, custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, ) # Return the streaming iterator @@ -2237,6 +2275,8 @@ class BaseLLMHTTPHandler: responses_api_provider_config=responses_api_provider_config, litellm_metadata=litellm_metadata, custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, ) else: # For non-streaming, proceed as before @@ -3526,6 +3566,174 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, ) + def compact_response_api_handler( + self, + model: str, + input: Union[str, "ResponseInputParam"], + responses_api_provider_config: BaseResponsesAPIConfig, + response_api_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str], + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + _is_async: bool = False, + shared_session: Optional["ClientSession"] = None, + ) -> Union[ResponsesAPIResponse, Coroutine[Any, Any, ResponsesAPIResponse]]: + """ + Handler for the compact responses API. + """ + if _is_async: + return self.async_compact_response_api_handler( + model=model, + input=input, + responses_api_provider_config=responses_api_provider_config, + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, + client=client, + shared_session=shared_session, + ) + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = responses_api_provider_config.validate_environment( + headers=extra_headers or {}, model=model, litellm_params=litellm_params + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = responses_api_provider_config.get_complete_url( + api_base=litellm_params.api_base, + litellm_params=dict(litellm_params), + ) + + url, data = responses_api_provider_config.transform_compact_response_api_request( + model=model, + input=input, + response_api_optional_request_params=response_api_optional_request_params, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input=input, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + }, + ) + + try: + response = sync_httpx_client.post( + url=url, headers=headers, json=data, timeout=timeout + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=responses_api_provider_config, + ) + + return responses_api_provider_config.transform_compact_response_api_response( + raw_response=response, + logging_obj=logging_obj, + ) + + async def async_compact_response_api_handler( + self, + model: str, + input: Union[str, "ResponseInputParam"], + responses_api_provider_config: BaseResponsesAPIConfig, + response_api_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str], + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + _is_async: bool = False, + shared_session: Optional["ClientSession"] = None, + ) -> ResponsesAPIResponse: + """ + Async version of the compact response API handler. + """ + if client is None or not isinstance(client, AsyncHTTPHandler): + verbose_logger.debug( + f"Creating HTTP client for compact_response with shared_session: {id(shared_session) if shared_session else None}" + ) + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + shared_session=shared_session, + ) + else: + async_httpx_client = client + + headers = responses_api_provider_config.validate_environment( + headers=extra_headers or {}, model=model, litellm_params=litellm_params + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = responses_api_provider_config.get_complete_url( + api_base=litellm_params.api_base, + litellm_params=dict(litellm_params), + ) + + url, data = responses_api_provider_config.transform_compact_response_api_request( + model=model, + input=input, + response_api_optional_request_params=response_api_optional_request_params, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input=input, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + }, + ) + + try: + response = await async_httpx_client.post( + url=url, headers=headers, json=data, timeout=timeout + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=responses_api_provider_config, + ) + + return responses_api_provider_config.transform_compact_response_api_response( + raw_response=response, + logging_obj=logging_obj, + ) + def list_files(self): """ Lists all files @@ -8288,4 +8496,4 @@ class BaseLLMHTTPHandler: return skills_api_provider_config.transform_delete_skill_response( raw_response=response, logging_obj=logging_obj, - ) \ No newline at end of file + ) diff --git a/litellm/llms/gigachat/__init__.py b/litellm/llms/gigachat/__init__.py new file mode 100644 index 00000000000..3ddbd7864d9 --- /dev/null +++ b/litellm/llms/gigachat/__init__.py @@ -0,0 +1,23 @@ +""" +GigaChat Provider for LiteLLM + +GigaChat is Sber AI's large language model (Russia's leading LLM). +Supports: +- Chat completions (sync/async) +- Streaming (sync/async) +- Function calling / Tools +- Structured output via JSON schema (emulated through function calls) +- Image input (base64 and URL) +- Embeddings + +API Documentation: https://developers.sber.ru/docs/ru/gigachat/api/overview +""" + +from .chat.transformation import GigaChatConfig, GigaChatError +from .embedding.transformation import GigaChatEmbeddingConfig + +__all__ = [ + "GigaChatConfig", + "GigaChatEmbeddingConfig", + "GigaChatError", +] diff --git a/litellm/llms/gigachat/authenticator.py b/litellm/llms/gigachat/authenticator.py new file mode 100644 index 00000000000..e61015a4a21 --- /dev/null +++ b/litellm/llms/gigachat/authenticator.py @@ -0,0 +1,241 @@ +""" +GigaChat OAuth Authenticator + +Handles OAuth 2.0 token management for GigaChat API. +Based on official GigaChat SDK authentication flow. +""" + +import time +import uuid +from typing import Optional, Tuple + +import httpx + +from litellm._logging import verbose_logger +from litellm.caching.caching import InMemoryCache +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.custom_httpx.http_handler import ( + HTTPHandler, + _get_httpx_client, + get_async_httpx_client, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.utils import LlmProviders + +# GigaChat OAuth endpoint +GIGACHAT_AUTH_URL = "https://ngw.devices.sberbank.ru:9443/api/v2/oauth" + +# Default scope for personal API access +GIGACHAT_SCOPE = "GIGACHAT_API_PERS" + +# Token expiry buffer in milliseconds (refresh token 60s before expiry) +TOKEN_EXPIRY_BUFFER_MS = 60000 + +# Cache for access tokens +_token_cache = InMemoryCache() + + +class GigaChatAuthError(BaseLLMException): + """GigaChat authentication error.""" + + pass + + +def _get_credentials() -> Optional[str]: + """Get GigaChat credentials from environment.""" + return get_secret_str("GIGACHAT_CREDENTIALS") or get_secret_str("GIGACHAT_API_KEY") + + +def _get_auth_url() -> str: + """Get GigaChat auth URL from environment or use default.""" + return get_secret_str("GIGACHAT_AUTH_URL") or GIGACHAT_AUTH_URL + + +def _get_scope() -> str: + """Get GigaChat scope from environment or use default.""" + return get_secret_str("GIGACHAT_SCOPE") or GIGACHAT_SCOPE + + +def _get_http_client() -> HTTPHandler: + """Get cached httpx client with SSL verification disabled.""" + return _get_httpx_client(params={"ssl_verify": False}) + + +def get_access_token( + credentials: Optional[str] = None, + scope: Optional[str] = None, + auth_url: Optional[str] = None, +) -> str: + """ + Get valid access token, using cache if available. + + Args: + credentials: Base64-encoded credentials (client_id:client_secret) + scope: API scope (GIGACHAT_API_PERS, GIGACHAT_API_CORP, etc.) + auth_url: OAuth endpoint URL + + Returns: + Access token string + + Raises: + GigaChatAuthError: If authentication fails + """ + credentials = credentials or _get_credentials() + if not credentials: + raise GigaChatAuthError( + status_code=401, + message="GigaChat credentials not provided. Set GIGACHAT_CREDENTIALS or GIGACHAT_API_KEY environment variable.", + ) + + scope = scope or _get_scope() + auth_url = auth_url or _get_auth_url() + + # Check cache + cache_key = f"gigachat_token:{credentials[:16]}" + cached = _token_cache.get_cache(cache_key) + if cached: + token, expires_at = cached + # Check if token is still valid (with buffer) + if time.time() * 1000 < expires_at - TOKEN_EXPIRY_BUFFER_MS: + verbose_logger.debug("Using cached GigaChat access token") + return token + + # Request new token + token, expires_at = _request_token_sync(credentials, scope, auth_url) + + # Cache token + ttl_seconds = max(0, (expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000) + if ttl_seconds > 0: + _token_cache.set_cache(cache_key, (token, expires_at), ttl=ttl_seconds) + + return token + + +async def get_access_token_async( + credentials: Optional[str] = None, + scope: Optional[str] = None, + auth_url: Optional[str] = None, +) -> str: + """Async version of get_access_token.""" + credentials = credentials or _get_credentials() + if not credentials: + raise GigaChatAuthError( + status_code=401, + message="GigaChat credentials not provided. Set GIGACHAT_CREDENTIALS or GIGACHAT_API_KEY environment variable.", + ) + + scope = scope or _get_scope() + auth_url = auth_url or _get_auth_url() + + # Check cache + cache_key = f"gigachat_token:{credentials[:16]}" + cached = _token_cache.get_cache(cache_key) + if cached: + token, expires_at = cached + if time.time() * 1000 < expires_at - TOKEN_EXPIRY_BUFFER_MS: + verbose_logger.debug("Using cached GigaChat access token") + return token + + # Request new token + token, expires_at = await _request_token_async(credentials, scope, auth_url) + + # Cache token + ttl_seconds = max(0, (expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000) + if ttl_seconds > 0: + _token_cache.set_cache(cache_key, (token, expires_at), ttl=ttl_seconds) + + return token + + +def _request_token_sync( + credentials: str, + scope: str, + auth_url: str, +) -> Tuple[str, int]: + """ + Request new access token from GigaChat OAuth endpoint (sync). + + Returns: + Tuple of (access_token, expires_at_ms) + """ + headers = { + "Authorization": f"Basic {credentials}", + "RqUID": str(uuid.uuid4()), + "Content-Type": "application/x-www-form-urlencoded", + } + data = {"scope": scope} + + verbose_logger.debug(f"Requesting GigaChat access token from {auth_url}") + + try: + client = _get_http_client() + response = client.post(auth_url, headers=headers, data=data, timeout=30) + response.raise_for_status() + return _parse_token_response(response) + except httpx.HTTPStatusError as e: + raise GigaChatAuthError( + status_code=e.response.status_code, + message=f"GigaChat authentication failed: {e.response.text}", + ) + except httpx.RequestError as e: + raise GigaChatAuthError( + status_code=500, + message=f"GigaChat authentication request failed: {str(e)}", + ) + + +async def _request_token_async( + credentials: str, + scope: str, + auth_url: str, +) -> Tuple[str, int]: + """Async version of _request_token_sync.""" + headers = { + "Authorization": f"Basic {credentials}", + "RqUID": str(uuid.uuid4()), + "Content-Type": "application/x-www-form-urlencoded", + } + data = {"scope": scope} + + verbose_logger.debug(f"Requesting GigaChat access token from {auth_url}") + + try: + client = get_async_httpx_client( + llm_provider=LlmProviders.GIGACHAT, + params={"ssl_verify": False}, + ) + response = await client.post(auth_url, headers=headers, data=data, timeout=30) + response.raise_for_status() + return _parse_token_response(response) + except httpx.HTTPStatusError as e: + raise GigaChatAuthError( + status_code=e.response.status_code, + message=f"GigaChat authentication failed: {e.response.text}", + ) + except httpx.RequestError as e: + raise GigaChatAuthError( + status_code=500, + message=f"GigaChat authentication request failed: {str(e)}", + ) + + +def _parse_token_response(response: httpx.Response) -> Tuple[str, int]: + """Parse OAuth token response.""" + data = response.json() + + # GigaChat returns either 'tok'/'exp' or 'access_token'/'expires_at' + access_token = data.get("tok") or data.get("access_token") + expires_at = data.get("exp") or data.get("expires_at") + + if not access_token: + raise GigaChatAuthError( + status_code=500, + message=f"Invalid token response: {data}", + ) + + # expires_at is in milliseconds + if isinstance(expires_at, str): + expires_at = int(expires_at) + + verbose_logger.debug("GigaChat access token obtained successfully") + return access_token, expires_at diff --git a/litellm/llms/gigachat/chat/__init__.py b/litellm/llms/gigachat/chat/__init__.py new file mode 100644 index 00000000000..3e030497a1a --- /dev/null +++ b/litellm/llms/gigachat/chat/__init__.py @@ -0,0 +1,12 @@ +""" +GigaChat Chat Module +""" + +from .transformation import GigaChatConfig, GigaChatError +from .streaming import GigaChatModelResponseIterator + +__all__ = [ + "GigaChatConfig", + "GigaChatError", + "GigaChatModelResponseIterator", +] diff --git a/litellm/llms/gigachat/chat/streaming.py b/litellm/llms/gigachat/chat/streaming.py new file mode 100644 index 00000000000..3565559e43c --- /dev/null +++ b/litellm/llms/gigachat/chat/streaming.py @@ -0,0 +1,134 @@ +""" +GigaChat Streaming Response Handler +""" + +import json +import uuid +from typing import Any, Optional + +from litellm.types.llms.openai import ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk +from litellm.types.utils import GenericStreamingChunk + + +class GigaChatModelResponseIterator: + """Iterator for GigaChat streaming responses.""" + + def __init__( + self, + streaming_response: Any, + sync_stream: bool, + json_mode: Optional[bool] = False, + ): + self.streaming_response = streaming_response + self.response_iterator = self.streaming_response + self.json_mode = json_mode + + def chunk_parser(self, chunk: dict) -> GenericStreamingChunk: + """Parse a single streaming chunk from GigaChat.""" + text = "" + tool_use: Optional[ChatCompletionToolCallChunk] = None + is_finished = False + finish_reason: Optional[str] = None + + choices = chunk.get("choices", []) + if not choices: + return GenericStreamingChunk( + text="", + tool_use=None, + is_finished=False, + finish_reason="", + usage=None, + index=0, + ) + + choice = choices[0] + delta = choice.get("delta", {}) + finish_reason = choice.get("finish_reason") + + # Extract text content + text = delta.get("content", "") or "" + + # Handle function_call in stream + if finish_reason == "function_call" and delta.get("function_call"): + func_call = delta["function_call"] + args = func_call.get("arguments", {}) + + if isinstance(args, dict): + args = json.dumps(args, ensure_ascii=False) + + tool_use = ChatCompletionToolCallChunk( + id=f"call_{uuid.uuid4().hex[:24]}", + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=func_call.get("name", ""), + arguments=args, + ), + index=0, + ) + finish_reason = "tool_calls" + + if finish_reason is not None: + is_finished = True + + return GenericStreamingChunk( + text=text, + tool_use=tool_use, + is_finished=is_finished, + finish_reason=finish_reason or "", + usage=None, + index=choice.get("index", 0), + ) + + def __iter__(self): + return self + + def __next__(self) -> GenericStreamingChunk: + try: + chunk = self.response_iterator.__next__() + if isinstance(chunk, str): + # Parse SSE format: data: {...} + if chunk.startswith("data: "): + chunk = chunk[6:] + if chunk.strip() == "[DONE]": + raise StopIteration + try: + chunk = json.loads(chunk) + except json.JSONDecodeError: + return GenericStreamingChunk( + text="", + tool_use=None, + is_finished=False, + finish_reason="", + usage=None, + index=0, + ) + return self.chunk_parser(chunk) + except StopIteration: + raise + + def __aiter__(self): + return self + + async def __anext__(self) -> GenericStreamingChunk: + try: + chunk = await self.response_iterator.__anext__() + if isinstance(chunk, str): + # Parse SSE format + if chunk.startswith("data: "): + chunk = chunk[6:] + if chunk.strip() == "[DONE]": + raise StopAsyncIteration + try: + chunk = json.loads(chunk) + except json.JSONDecodeError: + return GenericStreamingChunk( + text="", + tool_use=None, + is_finished=False, + finish_reason="", + usage=None, + index=0, + ) + return self.chunk_parser(chunk) + except StopAsyncIteration: + raise diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py new file mode 100644 index 00000000000..4ce333a1309 --- /dev/null +++ b/litellm/llms/gigachat/chat/transformation.py @@ -0,0 +1,473 @@ +""" +GigaChat Chat Transformation + +Transforms OpenAI-format requests to GigaChat format and back. +""" + +import json +import time +import uuid +from typing import TYPE_CHECKING, Any, AsyncIterator, Iterator, List, Optional, Union + +import httpx + +from litellm._logging import verbose_logger +from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import Choices, Message, ModelResponse, Usage + +from ..authenticator import get_access_token +from ..file_handler import upload_file_sync + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + +# GigaChat API endpoint +GIGACHAT_BASE_URL = "https://gigachat.devices.sberbank.ru/api/v1" + + +class GigaChatError(BaseLLMException): + """GigaChat API error.""" + + pass + + +class GigaChatConfig(BaseConfig): + """ + Configuration class for GigaChat API. + + GigaChat is Sber's (Russia's largest bank) LLM API. + + Supported parameters: + temperature: Sampling temperature (0-2, default 0.87) + top_p: Nucleus sampling parameter + max_tokens: Maximum tokens to generate + repetition_penalty: Repetition penalty factor + profanity_check: Enable content filtering + stream: Enable streaming + """ + + temperature: Optional[float] = None + top_p: Optional[float] = None + max_tokens: Optional[int] = None + repetition_penalty: Optional[float] = None + profanity_check: Optional[bool] = None + + def __init__( + self, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + max_tokens: Optional[int] = None, + repetition_penalty: Optional[float] = None, + profanity_check: Optional[bool] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + # Instance variables for current request context + self._current_credentials: Optional[str] = None + self._current_api_base: Optional[str] = None + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """Get complete API URL for chat completions.""" + base = api_base or get_secret_str("GIGACHAT_API_BASE") or GIGACHAT_BASE_URL + return f"{base}/chat/completions" + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Set up headers with OAuth token. + """ + # Get access token + credentials = api_key or get_secret_str("GIGACHAT_CREDENTIALS") or get_secret_str("GIGACHAT_API_KEY") + access_token = get_access_token(credentials=credentials) + + # Store credentials for image uploads + self._current_credentials = credentials + self._current_api_base = api_base + + headers["Authorization"] = f"Bearer {access_token}" + headers["Content-Type"] = "application/json" + headers["Accept"] = "application/json" + + return headers + + def get_supported_openai_params(self, model: str) -> List[str]: + """Return list of supported OpenAI parameters.""" + return [ + "stream", + "temperature", + "top_p", + "max_tokens", + "max_completion_tokens", + "stop", + "tools", + "tool_choice", + "functions", + "function_call", + "response_format", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """Map OpenAI parameters to GigaChat parameters.""" + for param, value in non_default_params.items(): + if param == "stream": + optional_params["stream"] = value + elif param == "temperature": + # GigaChat: temperature 0 means use top_p=0 instead + if value == 0: + optional_params["top_p"] = 0 + else: + optional_params["temperature"] = value + elif param == "top_p": + optional_params["top_p"] = value + elif param in ("max_tokens", "max_completion_tokens"): + optional_params["max_tokens"] = value + elif param == "stop": + # GigaChat doesn't support stop sequences + pass + elif param == "tools": + # Convert tools to functions format + optional_params["functions"] = self._convert_tools_to_functions(value) + elif param == "tool_choice": + if isinstance(value, dict) and value.get("function"): + optional_params["function_call"] = {"name": value["function"]["name"]} + elif value == "auto": + pass # Default behavior + elif value == "required": + # GigaChat doesn't have 'required', handled differently + pass + elif param == "functions": + optional_params["functions"] = value + elif param == "function_call": + optional_params["function_call"] = value + elif param == "response_format": + # Handle structured output via function calling + if value.get("type") == "json_schema": + json_schema = value.get("json_schema", {}) + schema_name = json_schema.get("name", "structured_output") + schema = json_schema.get("schema", {}) + + function_def = { + "name": schema_name, + "description": f"Output structured response: {schema_name}", + "parameters": schema, + } + + if "functions" not in optional_params: + optional_params["functions"] = [] + optional_params["functions"].append(function_def) + optional_params["function_call"] = {"name": schema_name} + optional_params["_structured_output"] = True + + return optional_params + + def _convert_tools_to_functions(self, tools: List[dict]) -> List[dict]: + """Convert OpenAI tools format to GigaChat functions format.""" + functions = [] + for tool in tools: + if tool.get("type") == "function": + func = tool.get("function", {}) + functions.append({ + "name": func.get("name", ""), + "description": func.get("description", ""), + "parameters": func.get("parameters", {}), + }) + return functions + + def _upload_image(self, image_url: str) -> Optional[str]: + """ + Upload image to GigaChat and return file_id. + + Args: + image_url: URL or base64 data URL of the image + + Returns: + file_id string or None if upload failed + """ + try: + return upload_file_sync( + image_url=image_url, + credentials=self._current_credentials, + api_base=self._current_api_base, + ) + except Exception as e: + verbose_logger.error(f"Failed to upload image: {e}") + return None + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """Transform OpenAI request to GigaChat format.""" + # Transform messages + giga_messages = self._transform_messages(messages) + + # Build request + request_data = { + "model": model.replace("gigachat/", ""), + "messages": giga_messages, + } + + # Add optional params + for key in ["temperature", "top_p", "max_tokens", "stream", + "repetition_penalty", "profanity_check"]: + if key in optional_params: + request_data[key] = optional_params[key] + + # Add functions if present + if "functions" in optional_params: + request_data["functions"] = optional_params["functions"] + if "function_call" in optional_params: + request_data["function_call"] = optional_params["function_call"] + + return request_data + + def _transform_messages(self, messages: List[AllMessageValues]) -> List[dict]: + """Transform OpenAI messages to GigaChat format.""" + transformed = [] + + for i, msg in enumerate(messages): + message = dict(msg) + + # Remove unsupported fields + message.pop("name", None) + + # Transform roles + role = message.get("role", "user") + if role == "developer": + message["role"] = "system" + elif role == "system" and i > 0: + # GigaChat only allows system message as first message + message["role"] = "user" + elif role == "tool": + message["role"] = "function" + content = message.get("content", "") + if not isinstance(content, str): + message["content"] = json.dumps(content, ensure_ascii=False) + + # Handle None content + if message.get("content") is None: + message["content"] = "" + + # Handle list content (multimodal) - extract text and images + content = message.get("content") + if isinstance(content, list): + texts = [] + attachments = [] + for part in content: + if isinstance(part, dict): + if part.get("type") == "text": + texts.append(part.get("text", "")) + elif part.get("type") == "image_url": + # Extract image URL and upload to GigaChat + image_url = part.get("image_url", {}) + if isinstance(image_url, str): + url = image_url + else: + url = image_url.get("url", "") + if url: + file_id = self._upload_image(url) + if file_id: + attachments.append(file_id) + message["content"] = "\n".join(texts) if texts else "" + if attachments: + message["attachments"] = attachments + + # Transform tool_calls to function_call + tool_calls = message.get("tool_calls") + if tool_calls and isinstance(tool_calls, list) and len(tool_calls) > 0: + tool_call = tool_calls[0] + func = tool_call.get("function", {}) + args = func.get("arguments", "{}") + if isinstance(args, str): + try: + args = json.loads(args) + except json.JSONDecodeError: + args = {} + message["function_call"] = { + "name": func.get("name", ""), + "arguments": args, + } + message.pop("tool_calls", None) + + transformed.append(message) + + # Collapse consecutive user messages + return self._collapse_user_messages(transformed) + + def _collapse_user_messages(self, messages: List[dict]) -> List[dict]: + """Collapse consecutive user messages into one.""" + collapsed: List[dict] = [] + prev_user_msg: Optional[dict] = None + content_parts: List[str] = [] + + for msg in messages: + if msg.get("role") == "user" and prev_user_msg is not None: + content_parts.append(msg.get("content", "")) + else: + if content_parts and prev_user_msg: + prev_user_msg["content"] = "\n".join( + [prev_user_msg.get("content", "")] + content_parts + ) + content_parts = [] + collapsed.append(msg) + prev_user_msg = msg if msg.get("role") == "user" else None + + if content_parts and prev_user_msg: + prev_user_msg["content"] = "\n".join( + [prev_user_msg.get("content", "")] + content_parts + ) + + return collapsed + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + """Transform GigaChat response to OpenAI format.""" + try: + response_json = raw_response.json() + except Exception: + raise GigaChatError( + status_code=raw_response.status_code, + message=f"Invalid JSON response: {raw_response.text}", + ) + + is_structured_output = optional_params.get("_structured_output", False) + + choices = [] + for choice in response_json.get("choices", []): + message_data = choice.get("message", {}) + finish_reason = choice.get("finish_reason", "stop") + + # Transform function_call to tool_calls or content + if finish_reason == "function_call" and message_data.get("function_call"): + func_call = message_data["function_call"] + args = func_call.get("arguments", {}) + + if is_structured_output: + # Convert to content for structured output + if isinstance(args, dict): + content = json.dumps(args, ensure_ascii=False) + else: + content = str(args) + message_data["content"] = content + message_data.pop("function_call", None) + message_data.pop("functions_state_id", None) + finish_reason = "stop" + else: + # Convert to tool_calls format + if isinstance(args, dict): + args = json.dumps(args, ensure_ascii=False) + message_data["tool_calls"] = [{ + "id": f"call_{uuid.uuid4().hex[:24]}", + "type": "function", + "function": { + "name": func_call.get("name", ""), + "arguments": args, + } + }] + message_data.pop("function_call", None) + finish_reason = "tool_calls" + + # Clean up GigaChat-specific fields + message_data.pop("functions_state_id", None) + + choices.append( + Choices( + index=choice.get("index", 0), + message=Message( + role=message_data.get("role", "assistant"), + content=message_data.get("content"), + tool_calls=message_data.get("tool_calls"), + ), + finish_reason=finish_reason, + ) + ) + + # Build usage + usage_data = response_json.get("usage", {}) + usage = Usage( + prompt_tokens=usage_data.get("prompt_tokens", 0), + completion_tokens=usage_data.get("completion_tokens", 0), + total_tokens=usage_data.get("total_tokens", 0), + ) + + model_response.id = response_json.get("id", f"chatcmpl-{uuid.uuid4().hex[:12]}") + model_response.created = response_json.get("created", int(time.time())) + model_response.model = model + model_response.choices = choices # type: ignore + setattr(model_response, "usage", usage) + + return model_response + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: Union[dict, httpx.Headers], + ) -> BaseLLMException: + """Return GigaChat error class.""" + return GigaChatError( + status_code=status_code, + message=error_message, + headers=headers, + ) + + def get_model_response_iterator( + self, + streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse], + sync_stream: bool, + json_mode: Optional[bool] = False, + ): + """Return streaming response iterator.""" + from .streaming import GigaChatModelResponseIterator + + return GigaChatModelResponseIterator( + streaming_response=streaming_response, + sync_stream=sync_stream, + json_mode=json_mode, + ) diff --git a/litellm/llms/gigachat/embedding/__init__.py b/litellm/llms/gigachat/embedding/__init__.py new file mode 100644 index 00000000000..af237e49aab --- /dev/null +++ b/litellm/llms/gigachat/embedding/__init__.py @@ -0,0 +1,7 @@ +""" +GigaChat Embedding Module +""" + +from .transformation import GigaChatEmbeddingConfig + +__all__ = ["GigaChatEmbeddingConfig"] diff --git a/litellm/llms/gigachat/embedding/transformation.py b/litellm/llms/gigachat/embedding/transformation.py new file mode 100644 index 00000000000..0da6565050e --- /dev/null +++ b/litellm/llms/gigachat/embedding/transformation.py @@ -0,0 +1,212 @@ +""" +GigaChat Embedding Transformation + +Transforms OpenAI /v1/embeddings format to GigaChat format. +API Documentation: https://developers.sber.ru/docs/ru/gigachat/api/reference/rest/post-embeddings +""" + +import types +from typing import List, Optional, Tuple, Union + +import httpx + +from litellm import LlmProviders +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues +from litellm.types.utils import EmbeddingResponse + +from ..authenticator import get_access_token + +# GigaChat API endpoint +GIGACHAT_BASE_URL = "https://gigachat.devices.sberbank.ru/api/v1" + + +class GigaChatEmbeddingError(BaseLLMException): + """GigaChat Embedding API error.""" + + pass + + +class GigaChatEmbeddingConfig(BaseEmbeddingConfig): + """ + Configuration class for GigaChat Embeddings API. + + GigaChat embeddings endpoint: POST /api/v1/embeddings + """ + + def __init__(self) -> None: + pass + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self, model: str) -> List[str]: + """GigaChat embeddings don't support additional parameters.""" + return [] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """Map OpenAI params to GigaChat format (no special mapping needed).""" + return optional_params + + def _get_openai_compatible_provider_info( + self, + api_base: Optional[str], + api_key: Optional[str], + ) -> Tuple[str, Optional[str], Optional[str]]: + """ + Returns provider info for GigaChat. + + Returns: + Tuple of (custom_llm_provider, api_base, dynamic_api_key) + """ + api_base = api_base or GIGACHAT_BASE_URL + return LlmProviders.GIGACHAT.value, api_base, api_key + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """Get the complete URL for embeddings endpoint.""" + base = api_base or GIGACHAT_BASE_URL + return f"{base}/embeddings" + + def transform_embedding_request( + self, + model: str, + input: AllEmbeddingInputValues, + optional_params: dict, + headers: dict, + ) -> dict: + """ + Transform OpenAI embedding request to GigaChat format. + + GigaChat format: + { + "model": "Embeddings", + "input": ["text1", "text2", ...] + } + """ + # Normalize input to list + if isinstance(input, str): + input_list: list = [input] + elif isinstance(input, list): + input_list = input + else: + input_list = [input] + + # Remove gigachat/ prefix from model if present + if model.startswith("gigachat/"): + model = model[9:] + + return { + "model": model, + "input": input_list, + } + + def transform_embedding_response( + self, + model: str, + raw_response: httpx.Response, + model_response: EmbeddingResponse, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + request_data: dict, + optional_params: dict, + litellm_params: dict, + ) -> EmbeddingResponse: + """ + Transform GigaChat embedding response to OpenAI format. + + GigaChat returns: + { + "object": "list", + "data": [{"object": "embedding", "embedding": [...], "index": 0, "usage": {...}}], + "model": "Embeddings" + } + """ + response_json = raw_response.json() + + # Log response + logging_obj.post_call( + input=request_data.get("input"), + api_key=api_key, + additional_args={"complete_input_dict": request_data}, + original_response=response_json, + ) + + # Calculate total tokens from individual embeddings + total_tokens = 0 + if "data" in response_json: + for emb in response_json["data"]: + if "usage" in emb and "prompt_tokens" in emb["usage"]: + total_tokens += emb["usage"]["prompt_tokens"] + # Remove usage from individual embeddings (not part of OpenAI format) + if "usage" in emb: + del emb["usage"] + + # Set overall usage + response_json["usage"] = { + "prompt_tokens": total_tokens, + "total_tokens": total_tokens, + } + + return EmbeddingResponse(**response_json) + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Set up headers with OAuth token for GigaChat. + """ + # Get access token via OAuth + access_token = get_access_token(api_key) + + default_headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {access_token}", + } + return {**default_headers, **headers} + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + """Return GigaChat-specific error class.""" + return GigaChatEmbeddingError( + status_code=status_code, + message=error_message, + ) diff --git a/litellm/llms/gigachat/file_handler.py b/litellm/llms/gigachat/file_handler.py new file mode 100644 index 00000000000..200428a747a --- /dev/null +++ b/litellm/llms/gigachat/file_handler.py @@ -0,0 +1,211 @@ +""" +GigaChat File Handler + +Handles file uploads to GigaChat API for image processing. +GigaChat requires files to be uploaded first, then referenced by file_id. +""" + +import base64 +import hashlib +import re +import uuid +from typing import Dict, Optional, Tuple + +from litellm._logging import verbose_logger +from litellm.llms.custom_httpx.http_handler import ( + _get_httpx_client, + get_async_httpx_client, +) +from litellm.types.utils import LlmProviders + +from .authenticator import get_access_token, get_access_token_async + +# GigaChat API endpoint +GIGACHAT_BASE_URL = "https://gigachat.devices.sberbank.ru/api/v1" + +# Simple in-memory cache for file IDs +_file_cache: Dict[str, str] = {} + + +def _get_url_hash(url: str) -> str: + """Generate hash for URL to use as cache key.""" + return hashlib.sha256(url.encode()).hexdigest() + + +def _parse_data_url(data_url: str) -> Optional[Tuple[bytes, str, str]]: + """ + Parse data URL (base64 image). + + Returns: + Tuple of (content_bytes, content_type, extension) or None + """ + match = re.match(r"data:([^;]+);base64,(.+)", data_url) + if not match: + return None + + content_type = match.group(1) + base64_data = match.group(2) + content_bytes = base64.b64decode(base64_data) + ext = content_type.split("/")[-1].split(";")[0] or "jpg" + + return content_bytes, content_type, ext + + +def _download_image_sync(url: str) -> Tuple[bytes, str, str]: + """Download image from URL synchronously.""" + client = _get_httpx_client(params={"ssl_verify": False}) + response = client.get(url) + response.raise_for_status() + + content_type = response.headers.get("content-type", "image/jpeg") + ext = content_type.split("/")[-1].split(";")[0] or "jpg" + + return response.content, content_type, ext + + +async def _download_image_async(url: str) -> Tuple[bytes, str, str]: + """Download image from URL asynchronously.""" + client = get_async_httpx_client( + llm_provider=LlmProviders.GIGACHAT, + params={"ssl_verify": False}, + ) + response = await client.get(url) + response.raise_for_status() + + content_type = response.headers.get("content-type", "image/jpeg") + ext = content_type.split("/")[-1].split(";")[0] or "jpg" + + return response.content, content_type, ext + + +def upload_file_sync( + image_url: str, + credentials: Optional[str] = None, + api_base: Optional[str] = None, +) -> Optional[str]: + """ + Upload file to GigaChat and return file_id (sync). + + Args: + image_url: URL or base64 data URL of the image + credentials: GigaChat credentials for auth + api_base: Optional custom API base URL + + Returns: + file_id string or None if upload failed + """ + url_hash = _get_url_hash(image_url) + + # Check cache + if url_hash in _file_cache: + verbose_logger.debug(f"Image found in cache: {url_hash[:16]}...") + return _file_cache[url_hash] + + try: + # Get image data + parsed = _parse_data_url(image_url) + if parsed: + content_bytes, content_type, ext = parsed + verbose_logger.debug("Decoded base64 image") + else: + verbose_logger.debug(f"Downloading image from URL: {image_url[:80]}...") + content_bytes, content_type, ext = _download_image_sync(image_url) + + filename = f"{uuid.uuid4()}.{ext}" + + # Get access token + access_token = get_access_token(credentials) + + # Upload to GigaChat + base_url = api_base or GIGACHAT_BASE_URL + upload_url = f"{base_url}/files" + + client = _get_httpx_client(params={"ssl_verify": False}) + response = client.post( + upload_url, + headers={"Authorization": f"Bearer {access_token}"}, + files={"file": (filename, content_bytes, content_type)}, + data={"purpose": "general"}, + timeout=60, + ) + response.raise_for_status() + result = response.json() + + file_id = result.get("id") + if file_id: + _file_cache[url_hash] = file_id + verbose_logger.debug(f"File uploaded successfully, file_id: {file_id}") + + return file_id + + except Exception as e: + verbose_logger.error(f"Error uploading file to GigaChat: {e}") + return None + + +async def upload_file_async( + image_url: str, + credentials: Optional[str] = None, + api_base: Optional[str] = None, +) -> Optional[str]: + """ + Upload file to GigaChat and return file_id (async). + + Args: + image_url: URL or base64 data URL of the image + credentials: GigaChat credentials for auth + api_base: Optional custom API base URL + + Returns: + file_id string or None if upload failed + """ + url_hash = _get_url_hash(image_url) + + # Check cache + if url_hash in _file_cache: + verbose_logger.debug(f"Image found in cache: {url_hash[:16]}...") + return _file_cache[url_hash] + + try: + # Get image data + parsed = _parse_data_url(image_url) + if parsed: + content_bytes, content_type, ext = parsed + verbose_logger.debug("Decoded base64 image") + else: + verbose_logger.debug(f"Downloading image from URL: {image_url[:80]}...") + content_bytes, content_type, ext = await _download_image_async(image_url) + + filename = f"{uuid.uuid4()}.{ext}" + + # Get access token + access_token = await get_access_token_async(credentials) + + # Upload to GigaChat + base_url = api_base or GIGACHAT_BASE_URL + upload_url = f"{base_url}/files" + + client = get_async_httpx_client( + llm_provider=LlmProviders.GIGACHAT, + params={"ssl_verify": False}, + ) + response = await client.post( + upload_url, + headers={"Authorization": f"Bearer {access_token}"}, + files={"file": (filename, content_bytes, content_type)}, + data={"purpose": "general"}, + timeout=60, + ) + response.raise_for_status() + result = response.json() + + file_id = result.get("id") + if file_id: + _file_cache[url_hash] = file_id + verbose_logger.debug(f"File uploaded successfully, file_id: {file_id}") + + return file_id + + except Exception as e: + verbose_logger.error(f"Error uploading file to GigaChat: {e}") + return None diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 96598c1dfe6..cc2439b431a 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -500,3 +500,69 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): response._hidden_params["headers"] = raw_response_headers return response + + ######################################################### + ########## COMPACT RESPONSE API TRANSFORMATION ########## + ######################################################### + def transform_compact_response_api_request( + self, + model: str, + input: Union[str, ResponseInputParam], + response_api_optional_request_params: Dict, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """ + Transform the compact response API request into a URL and data + + OpenAI API expects the following request + - POST /v1/responses/compact + """ + url = f"{api_base}/compact" + + input = self._validate_input_param(input) + data = dict( + ResponsesAPIRequestParams( + model=model, input=input, **response_api_optional_request_params + ) + ) + + return url, data + + def transform_compact_response_api_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + """ + Transform the compact response API response into a ResponsesAPIResponse + """ + try: + logging_obj.post_call( + original_response=raw_response.text, + additional_args={"complete_input_dict": {}}, + ) + raw_response_json = raw_response.json() + raw_response_json["created_at"] = _safe_convert_created_field( + raw_response_json["created_at"] + ) + except Exception: + raise OpenAIError( + message=raw_response.text, status_code=raw_response.status_code + ) + raw_response_headers = dict(raw_response.headers) + processed_headers = process_response_headers(raw_response_headers) + + try: + response = ResponsesAPIResponse(**raw_response_json) + except Exception: + verbose_logger.debug( + f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct" + ) + response = ResponsesAPIResponse.model_construct(**raw_response_json) + + response._hidden_params["additional_headers"] = processed_headers + response._hidden_params["headers"] = raw_response_headers + + return response diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 826f151df35..d23c698cd7a 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -6,7 +6,7 @@ Handles Authentication and generating request urls for Vertex AI and Google AI S import json import os -from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple, cast import litellm from litellm._logging import verbose_logger @@ -168,7 +168,6 @@ class VertexBase: ) def _credentials_from_default_auth(self, scopes): - import google.auth as google_auth return google_auth.default(scopes=scopes) @@ -392,7 +391,7 @@ class VertexBase: Returns token, url """ - version: Optional[Literal["v1beta1", "v1"]] = None + version: Optional[Literal["v1", "v1beta1"]] = None if custom_llm_provider == "gemini": url, endpoint = _get_gemini_url( mode=mode, @@ -415,7 +414,7 @@ class VertexBase: stream=stream, vertex_project=vertex_project, vertex_location=vertex_location, - vertex_api_version=version, + vertex_api_version=cast(Literal["v1", "v1beta1"], version), ) return self._check_custom_proxy( diff --git a/litellm/main.py b/litellm/main.py index f4f27eb5841..e8a8b504d96 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2141,6 +2141,49 @@ def completion( # type: ignore # noqa: PLR0915 logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, ) + elif custom_llm_provider == "gigachat": + # GigaChat - Sber AI's LLM (Russia) + api_key = ( + api_key + or litellm.api_key + or litellm.gigachat_key + or get_secret("GIGACHAT_API_KEY") + or get_secret("GIGACHAT_CREDENTIALS") + ) + + headers = headers or litellm.headers or {} + + ## COMPLETION CALL + try: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + elif custom_llm_provider == "sap": headers = headers or litellm.headers ## LOAD CONFIG - if set @@ -5224,6 +5267,28 @@ def embedding( # noqa: PLR0915 aembedding=aembedding, litellm_params={}, ) + elif custom_llm_provider == "gigachat": + api_key = ( + api_key + or litellm.api_key + or litellm.gigachat_key + or get_secret_str("GIGACHAT_CREDENTIALS") + or get_secret_str("GIGACHAT_API_KEY") + ) + response = base_llm_http_handler.embedding( + model=model, + input=input, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + logging_obj=logging, + timeout=timeout, + model_response=EmbeddingResponse(), + optional_params=optional_params, + client=client, + aembedding=aembedding, + litellm_params={"ssl_verify": kwargs.get("ssl_verify", None)}, + ) else: raise LiteLLMUnknownProvider( model=model, custom_llm_provider=custom_llm_provider diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e823dd5dc6b..c7a2f60856d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -15831,6 +15831,68 @@ "max_tokens": 8191, "mode": "embedding" }, + "gigachat/GigaChat-2-Lite": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0.0, + "supports_function_calling": true, + "supports_system_messages": true + }, + "gigachat/GigaChat-2-Max": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0.0, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_vision": true + }, + "gigachat/GigaChat-2-Pro": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0.0, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_vision": true + }, + "gigachat/Embeddings": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024 + }, + "gigachat/Embeddings-2": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024 + }, + "gigachat/EmbeddingsGigaR": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 4096, + "max_tokens": 4096, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 2560 + }, "google.gemma-3-12b-it": { "input_cost_per_token": 9e-08, "litellm_provider": "bedrock_converse", @@ -32092,3 +32154,4 @@ "mode": "chat" } } + diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index ffa17a5b7c4..ded591a8f53 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -15,6 +15,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( ) from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.types.mcp_server.mcp_server_manager import MCPServer +from litellm.proxy.utils import get_server_root_path router = APIRouter( tags=["mcp"], @@ -381,13 +382,30 @@ async def callback(code: str, state: str): # ------------------------------ # Optional .well-known endpoints for MCP + OAuth discovery # ------------------------------ -@router.get("/.well-known/oauth-protected-resource/{mcp_server_name}/mcp") +""" + Per SEP-985, the client MUST: + 1. Try resource_metadata from WWW-Authenticate header (if present) + 2. Fall back to path-based well-known URI: /.well-known/oauth-protected-resource/{path} + ( + If the resource identifier value contains a path or query component, any terminating slash (/) + following the host component MUST be removed before inserting /.well-known/ and the well-known + URI path suffix between the host component and the path(include root path) and/or query components. + https://datatracker.ietf.org/doc/html/rfc9728#section-3.1) + 3. Fall back to root-based well-known URI: /.well-known/oauth-protected-resource +""" +@router.get(f"/.well-known/oauth-protected-resource{'' if get_server_root_path() == '/' else get_server_root_path()}/{{mcp_server_name}}/mcp") @router.get("/.well-known/oauth-protected-resource") async def oauth_protected_resource_mcp( request: Request, mcp_server_name: Optional[str] = None ): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) # Get the correct base URL considering X-Forwarded-* headers request_base_url = get_request_base_url(request) + mcp_server: Optional[MCPServer] = None + if mcp_server_name: + mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name) return { "authorization_servers": [ ( @@ -401,14 +419,25 @@ async def oauth_protected_resource_mcp( if mcp_server_name else f"{request_base_url}/mcp" ), # this is what Claude will call + "scopes_supported": mcp_server.scopes if mcp_server else [], } - -@router.get("/.well-known/oauth-authorization-server/{mcp_server_name}") +""" + https://datatracker.ietf.org/doc/html/rfc8414#section-3.1 + RFC 8414: Path-aware OAuth discovery + If the issuer identifier value contains a path component, any + terminating "/" MUST be removed before inserting "/.well-known/" and + the well-known URI suffix between the host component and the path(include root path) + component. +""" +@router.get(f"/.well-known/oauth-authorization-server{'' if get_server_root_path() == '/' else get_server_root_path()}/{{mcp_server_name}}") @router.get("/.well-known/oauth-authorization-server") async def oauth_authorization_server_mcp( request: Request, mcp_server_name: Optional[str] = None ): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) # Get the correct base URL considering X-Forwarded-* headers request_base_url = get_request_base_url(request) @@ -423,16 +452,21 @@ async def oauth_authorization_server_mcp( else f"{request_base_url}/token" ) + mcp_server: Optional[MCPServer] = None + if mcp_server_name: + mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name) + return { "issuer": request_base_url, # point to your proxy "authorization_endpoint": authorization_endpoint, "token_endpoint": token_endpoint, "response_types_supported": ["code"], - "grant_types_supported": ["authorization_code"], + "scopes_supported": mcp_server.scopes if mcp_server else [], + "grant_types_supported": ["authorization_code", "refresh_token"], "code_challenge_methods_supported": ["S256"], "token_endpoint_auth_methods_supported": ["client_secret_post"], # Claude expects a registration endpoint, even if we just fake it - "registration_endpoint": f"{request_base_url}/{mcp_server_name}/register", + "registration_endpoint": f"{request_base_url}/{mcp_server_name}/register" if mcp_server_name else f"{request_base_url}/register", } diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index a5ac966062e..3a548e203c5 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -660,14 +660,14 @@ class MCPServerManager: """ allowed_mcp_servers = await self.get_allowed_mcp_servers(user_api_key_auth) - list_tools_result: List[MCPTool] = [] verbose_logger.debug("SERVER MANAGER LISTING TOOLS") - for server_id in allowed_mcp_servers: + async def _fetch_server_tools(server_id: str) -> List[MCPTool]: + """Fetch tools from a single server with error handling.""" server = self.get_mcp_server_by_id(server_id) if server is None: verbose_logger.warning(f"MCP Server {server_id} not found") - continue + return [] # Get server-specific auth header if available server_auth_header = None @@ -685,15 +685,21 @@ class MCPServerManager: server=server, mcp_auth_header=server_auth_header, ) - list_tools_result.extend(tools) - verbose_logger.info( - f"Successfully fetched {len(tools)} tools from server {server.name}" - ) + return tools except Exception as e: verbose_logger.warning( f"Failed to list tools from server {server.name}: {str(e)}. Continuing with other servers." ) - # Continue with other servers instead of failing completely + return [] + + # Fetch tools from all servers in parallel + tasks = [_fetch_server_tools(server_id) for server_id in allowed_mcp_servers] + results = await asyncio.gather(*tasks) + + # Flatten results into single list + list_tools_result: List[MCPTool] = [ + tool for tools in results for tool in tools + ] verbose_logger.info( f"Successfully fetched {len(list_tools_result)} tools total from all servers" @@ -2003,6 +2009,9 @@ class MCPServerManager: Note: This now handles prefixed tool names """ for server in self.get_registry().values(): + if server.auth_type == MCPAuth.oauth2: + # Skip OAuth2 servers for now as they may require user-specific tokens + continue tools = await self._get_tools_from_server(server) for tool in tools: # The tool.name here is already prefixed from _get_tools_from_server @@ -2284,14 +2293,7 @@ class MCPServerManager: # Check all accessible servers target_server_ids = allowed_server_ids - # Run health checks concurrently - tasks = [self.health_check_server(server_id) for server_id in target_server_ids] - results = await asyncio.gather(*tasks) - - # Filter out None results (servers that were not found) - list_mcp_servers = [server for server in results if server is not None] - - return list_mcp_servers + return await self._run_health_checks(target_server_ids) async def get_all_allowed_mcp_servers( self, @@ -2306,8 +2308,6 @@ class MCPServerManager: Returns: List of MCP server objects without health status """ - from datetime import datetime - # Get allowed server IDs allowed_server_ids = await self.get_allowed_mcp_servers(user_api_key_auth) @@ -2319,40 +2319,56 @@ class MCPServerManager: verbose_logger.warning(f"MCP Server {server_id} not found in registry") continue - # Build LiteLLM_MCPServerTable without health check - mcp_server_table = LiteLLM_MCPServerTable( - server_id=server.server_id, - server_name=server.server_name, - alias=server.alias, - description=( - server.mcp_info.get("description") if server.mcp_info else None - ), - url=server.url, - transport=server.transport, - auth_type=server.auth_type, - created_at=datetime.now(), - updated_at=datetime.now(), - teams=[], - mcp_access_groups=server.access_groups or [], - allowed_tools=server.allowed_tools or [], - extra_headers=server.extra_headers or [], - mcp_info=server.mcp_info, - static_headers=server.static_headers, - status=None, # No health check performed - last_health_check=None, # No health check performed - health_check_error=None, - command=getattr(server, "command", None), - args=getattr(server, "args", None) or [], - env=getattr(server, "env", None) or {}, - authorization_url=server.authorization_url, - token_url=server.token_url, - registration_url=server.registration_url, - allow_all_keys=server.allow_all_keys, - ) + mcp_server_table = self._build_mcp_server_table(server) list_mcp_servers.append(mcp_server_table) return list_mcp_servers + def _build_mcp_server_table(self, server: MCPServer) -> LiteLLM_MCPServerTable: + from datetime import datetime + + return LiteLLM_MCPServerTable( + server_id=server.server_id, + server_name=server.server_name, + alias=server.alias, + description=( + server.mcp_info.get("description") if server.mcp_info else None + ), + url=server.url, + transport=server.transport, + auth_type=server.auth_type, + created_at=datetime.now(), + updated_at=datetime.now(), + teams=[], + mcp_access_groups=server.access_groups or [], + allowed_tools=server.allowed_tools or [], + extra_headers=server.extra_headers or [], + mcp_info=server.mcp_info, + static_headers=server.static_headers, + status=None, # No health check performed + last_health_check=None, # No health check performed + health_check_error=None, + command=getattr(server, "command", None), + args=getattr(server, "args", None) or [], + env=getattr(server, "env", None) or {}, + authorization_url=server.authorization_url, + token_url=server.token_url, + registration_url=server.registration_url, + allow_all_keys=server.allow_all_keys, + ) + + async def get_all_mcp_servers_unfiltered(self) -> List[LiteLLM_MCPServerTable]: + """Return all MCP servers from registry without applying access controls.""" + + registry = self.get_registry() + if not registry: + return [] + + servers: List[LiteLLM_MCPServerTable] = [] + for server in registry.values(): + servers.append(self._build_mcp_server_table(server)) + return servers + async def reload_servers_from_database(self): """ Public method to reload all MCP servers from database into registry. @@ -2360,5 +2376,34 @@ class MCPServerManager: """ await self._add_mcp_servers_from_db_to_in_memory_registry() + async def get_all_mcp_servers_with_health_unfiltered( + self, server_ids: Optional[List[str]] = None + ) -> List[LiteLLM_MCPServerTable]: + """Return health info for all servers in registry regardless of user access.""" + + registry = self.get_registry() + if not registry: + return [] + + if server_ids: + target_server_ids = [sid for sid in server_ids if sid in registry] + else: + target_server_ids = list(registry.keys()) + + if not target_server_ids: + return [] + + return await self._run_health_checks(target_server_ids) + + async def _run_health_checks( + self, target_server_ids: List[str] + ) -> List[LiteLLM_MCPServerTable]: + if not target_server_ids: + return [] + + tasks = [self.health_check_server(server_id) for server_id in target_server_ids] + results = await asyncio.gather(*tasks) + return [server for server in results if server is not None] + global_mcp_server_manager: MCPServerManager = MCPServerManager() diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index e00fdbfb930..9c7001266f0 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -709,7 +709,8 @@ if MCP_AVAILABLE: extra_headers: Optional[Dict[str, str]] = None if server.auth_type == MCPAuth.oauth2: - extra_headers = oauth2_headers + # Copy to avoid mutating the original dict (important for parallel fetching) + extra_headers = oauth2_headers.copy() if oauth2_headers else None if server.extra_headers and raw_headers: if extra_headers is None: @@ -755,11 +756,10 @@ if MCP_AVAILABLE: # Decide whether to add prefix based on number of allowed servers add_prefix = not (len(allowed_mcp_servers) == 1) - # Get tools from each allowed server - all_tools = [] - for server in allowed_mcp_servers: + async def _fetch_and_filter_server_tools(server: MCPServer) -> List[MCPTool]: + """Fetch and filter tools from a single server with error handling.""" if server is None: - continue + return [] server_auth_header, extra_headers = _prepare_mcp_server_headers( server=server, @@ -786,16 +786,24 @@ if MCP_AVAILABLE: user_api_key_auth=user_api_key_auth, ) - all_tools.extend(filtered_tools) - verbose_logger.debug( f"Successfully fetched {len(tools)} tools from server {server.name}, {len(filtered_tools)} after filtering" ) + return filtered_tools except Exception as e: verbose_logger.exception( f"Error getting tools from server {server.name}: {str(e)}" ) - # Continue with other servers instead of failing completely + return [] + + # Fetch tools from all servers in parallel + tasks = [ + _fetch_and_filter_server_tools(server) for server in allowed_mcp_servers + ] + results = await asyncio.gather(*tasks) + + # Flatten results into single list + all_tools: List[MCPTool] = [tool for tools in results for tool in tools] verbose_logger.info( f"Successfully fetched {len(all_tools)} tools total from all MCP servers" diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b77cc40d6dc..954c26e2cb2 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1908,6 +1908,9 @@ class UserHeaderMapping(LiteLLMPydanticObjectBase): } +UserMCPManagementMode = Literal["restricted", "view_all"] + + class ConfigGeneralSettings(LiteLLMPydanticObjectBase): """ Documents all the fields supported by `general_settings` in config.yaml @@ -2025,6 +2028,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="Fine-grained control over which object types to load from the database when store_model_in_db is True. Available types: 'models', 'mcp', 'guardrails', 'vector_stores', 'pass_through_endpoints', 'prompts', 'model_cost_map'. If not set, all objects are loaded (default behavior).", ) + user_mcp_management_mode: Optional[UserMCPManagementMode] = Field( + None, + description="Controls how non-admin users interact with MCP servers in the dashboard. 'restricted' shows only accessible servers, 'view_all' lists every server in read-only mode.", + ) class ConfigYAML(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 34049a44c8c..537b48f06ed 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -319,6 +319,7 @@ class ProxyBaseLLMRequestProcessing: "aget_responses", "adelete_responses", "acancel_responses", + "acompact_responses", "acreate_batch", "aretrieve_batch", "alist_batches", @@ -457,6 +458,7 @@ class ProxyBaseLLMRequestProcessing: "aget_responses", "adelete_responses", "acancel_responses", + "acompact_responses", "atext_completion", "aimage_edit", "alist_input_items", diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index 406ddceabf5..fe27af78d58 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -154,7 +154,11 @@ class PrismaManager: prisma_dir = PrismaManager._get_prisma_dir() - return ProxyExtrasDBManager.setup_database(use_migrate=use_migrate) + from litellm.proxy.proxy_server import redis_usage_cache + + return ProxyExtrasDBManager.setup_database( + use_migrate=use_migrate, redis_cache=redis_usage_cache + ) else: # Use prisma db push with increased timeout subprocess.run( diff --git a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py index ea8f1b0a97f..5850103132c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py @@ -118,7 +118,7 @@ class LassoGuardrail(CustomGuardrail): Falls back to UUID if ULID library is not available. """ if ULID_AVAILABLE and ulid is not None: - return str(ulid.new()) # type: ignore + return str(ulid.ULID()) # type: ignore else: verbose_proxy_logger.debug("ULID library not available, using UUID") return str(uuid.uuid4()) diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index a871a6637a2..47793c8fc8e 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -32,8 +32,8 @@ from fastapi import ( from fastapi.responses import JSONResponse import litellm -from litellm._uuid import uuid from litellm._logging import verbose_logger, verbose_proxy_logger +from litellm._uuid import uuid from litellm.constants import LITELLM_PROXY_ADMIN_NAME from litellm.proxy._experimental.mcp_server.utils import ( validate_and_normalize_mcp_server_payload, @@ -67,7 +67,6 @@ if MCP_AVAILABLE: from litellm.proxy._experimental.mcp_server.ui_session_utils import ( build_effective_auth_contexts, ) - from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy._types import ( LiteLLM_MCPServerTable, LitellmUserRoles, @@ -76,8 +75,10 @@ if MCP_AVAILABLE: SpecialMCPServerName, UpdateMCPServerRequest, UserAPIKeyAuth, + UserMCPManagementMode, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view from litellm.proxy.management_helpers.utils import management_endpoint_wrapper from litellm.types.mcp import MCPCredentials @@ -302,6 +303,20 @@ if MCP_AVAILABLE: return {"access_groups": access_groups_list} ## FastAPI Routes + def _get_user_mcp_management_mode() -> UserMCPManagementMode: + proxy_general_settings: dict = {} + try: + from litellm.proxy.proxy_server import ( + general_settings as proxy_general_settings, + ) + except Exception: + pass + + mode = proxy_general_settings.get("user_mcp_management_mode") + if mode == "view_all": + return "view_all" + return "restricted" + @router.get( "/server", description="Returns the mcp server list with associated teams", @@ -319,18 +334,26 @@ if MCP_AVAILABLE: ``` """ - auth_contexts = await build_effective_auth_contexts(user_api_key_dict) + user_mcp_management_mode = _get_user_mcp_management_mode() - aggregated_servers: Dict[str, LiteLLM_MCPServerTable] = {} - for auth_context in auth_contexts: - servers = await global_mcp_server_manager.get_all_allowed_mcp_servers( - user_api_key_auth=auth_context + if user_mcp_management_mode == "view_all": + servers = await global_mcp_server_manager.get_all_mcp_servers_unfiltered() + redacted_mcp_servers = _redact_mcp_credentials_list(servers) + else: + auth_contexts = await build_effective_auth_contexts(user_api_key_dict) + + aggregated_servers: Dict[str, LiteLLM_MCPServerTable] = {} + for auth_context in auth_contexts: + servers = await global_mcp_server_manager.get_all_allowed_mcp_servers( + user_api_key_auth=auth_context + ) + for server in servers: + if server.server_id not in aggregated_servers: + aggregated_servers[server.server_id] = server + + redacted_mcp_servers = _redact_mcp_credentials_list( + aggregated_servers.values() ) - for server in servers: - if server.server_id not in aggregated_servers: - aggregated_servers[server.server_id] = server - - redacted_mcp_servers = _redact_mcp_credentials_list(aggregated_servers.values()) # augment the mcp servers with public status if litellm.public_mcp_servers is not None: @@ -372,6 +395,17 @@ if MCP_AVAILABLE: --header 'Authorization: Bearer your_api_key_here' ``` """ + user_mcp_management_mode = _get_user_mcp_management_mode() + + if user_mcp_management_mode == "view_all": + servers = await global_mcp_server_manager.get_all_mcp_servers_with_health_unfiltered( + server_ids=server_ids + ) + return [ + {"server_id": server.server_id, "status": server.status} + for server in servers + ] + auth_contexts = await build_effective_auth_contexts(user_api_key_dict) server_status_map: Dict[ diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 623e8408862..ec1bc5497bd 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -698,6 +698,88 @@ async def get_response_input_items( ) +@router.post( + "/v1/responses/compact", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], +) +@router.post( + "/responses/compact", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], +) +@router.post( + "/openai/v1/responses/compact", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], +) +async def compact_response( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Compact a response by running a compaction pass over a conversation. + + Returns encrypted, opaque items that can be used to reduce context size. + + Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/compact + + ```bash + curl -X POST http://localhost:4000/v1/responses/compact \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-4o", + "input": [{"role": "user", "content": "Hello"}] + }' + ``` + """ + from litellm.proxy.proxy_server import ( + _read_request_body, + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + data = await _read_request_body(request=request) + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="acompact_responses", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=None, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + @router.post( "/v1/responses/{response_id}/cancel", dependencies=[Depends(user_api_key_auth)], diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index fd00cfc1c0a..a321e25a9a5 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -25,6 +25,7 @@ ROUTE_ENDPOINT_MAPPING = { "alist_input_items": "/responses/{response_id}/input_items", "aimage_edit": "/images/edits", "acancel_responses": "/responses/{response_id}/cancel", + "acompact_responses": "/responses/compact", "aocr": "/ocr", "asearch": "/search", "avideo_generation": "/videos", @@ -116,6 +117,7 @@ async def route_request( "aget_responses", "adelete_responses", "acancel_responses", + "acompact_responses", "acreate_response_reply", "alist_input_items", "_arealtime", # private function for realtime API diff --git a/litellm/responses/main.py b/litellm/responses/main.py index e837346df23..8177b177fe6 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -1361,3 +1361,205 @@ def cancel_responses( completion_kwargs=local_vars, extra_kwargs=kwargs, ) + + +@client +async def acompact_responses( + input: Union[str, ResponseInputParam], + model: str, + instructions: Optional[str] = None, + previous_response_id: Optional[str] = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + # LiteLLM specific params, + custom_llm_provider: Optional[str] = None, + **kwargs, +) -> ResponsesAPIResponse: + """ + Async version of the POST Compact Responses API + + POST /v1/responses/compact endpoint in the responses API + + Runs a compaction pass over a conversation, returning encrypted, opaque items. + """ + local_vars = locals() + try: + loop = asyncio.get_event_loop() + kwargs["acompact_responses"] = True + + # get custom llm provider so we can use this for mapping exceptions + if custom_llm_provider is None: + _, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=model, api_base=local_vars.get("base_url", None) + ) + + func = partial( + compact_responses, + input=input, + model=model, + instructions=instructions, + previous_response_id=previous_response_id, + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + + # Update the responses_api_response_id with the model_id + if isinstance(response, ResponsesAPIResponse): + response = ResponsesAPIRequestUtils._update_responses_api_response_id_with_model_id( + responses_api_response=response, + litellm_metadata=kwargs.get("litellm_metadata", {}), + custom_llm_provider=custom_llm_provider, + ) + + return response + except Exception as e: + raise litellm.exception_type( + model=model, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +@client +def compact_responses( + input: Union[str, ResponseInputParam], + model: str, + instructions: Optional[str] = None, + previous_response_id: Optional[str] = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + # LiteLLM specific params, + custom_llm_provider: Optional[str] = None, + **kwargs, +) -> Union[ResponsesAPIResponse, Coroutine[Any, Any, ResponsesAPIResponse]]: + """ + Synchronous version of the POST Compact Responses API + + POST /v1/responses/compact endpoint in the responses API + + Runs a compaction pass over a conversation, returning encrypted, opaque items. + """ + local_vars = locals() + try: + litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore + litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) + _is_async = kwargs.pop("acompact_responses", False) is True + + # get llm provider logic + litellm_params = GenericLiteLLMParams(**kwargs) + + ( + model, + custom_llm_provider, + dynamic_api_key, + dynamic_api_base, + ) = litellm.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + + if custom_llm_provider is None: + raise ValueError("custom_llm_provider is required but passed as None") + + # get provider config + responses_api_provider_config: Optional[BaseResponsesAPIConfig] = ( + ProviderConfigManager.get_provider_responses_api_config( + model=model, + provider=litellm.LlmProviders(custom_llm_provider), + ) + ) + + if responses_api_provider_config is None: + raise ValueError( + f"COMPACT responses is not supported for {custom_llm_provider}" + ) + + local_vars.update(kwargs) + + # Build optional params for compact endpoint + response_api_optional_params: ResponsesAPIOptionalRequestParams = ( + ResponsesAPIRequestUtils.get_requested_response_api_optional_param( + local_vars + ) + ) + + # Get optional parameters for the responses API + responses_api_request_params: Dict = ( + ResponsesAPIRequestUtils.get_optional_params_responses_api( + model=model, + responses_api_provider_config=responses_api_provider_config, + response_api_optional_params=response_api_optional_params, + allowed_openai_params=None, + ) + ) + + # Pre Call logging + litellm_logging_obj.update_environment_variables( + model=model, + optional_params=dict(responses_api_request_params), + litellm_params={ + **responses_api_request_params, + "litellm_call_id": litellm_call_id, + }, + custom_llm_provider=custom_llm_provider, + ) + + # Call the handler with _is_async flag instead of directly calling the async handler + response = base_llm_http_handler.compact_response_api_handler( + model=model, + input=input, + responses_api_provider_config=responses_api_provider_config, + response_api_optional_request_params=responses_api_request_params, + litellm_params=litellm_params, + logging_obj=litellm_logging_obj, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout or request_timeout, + _is_async=_is_async, + client=kwargs.get("client"), + shared_session=kwargs.get("shared_session"), + ) + + # Update the responses_api_response_id with the model_id + if isinstance(response, ResponsesAPIResponse): + response = ResponsesAPIRequestUtils._update_responses_api_response_id_with_model_id( + responses_api_response=response, + litellm_metadata=kwargs.get("litellm_metadata", {}), + custom_llm_provider=custom_llm_provider, + ) + + return response + except Exception as e: + raise litellm.exception_type( + model=model, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 0407776029d..0b838f916e2 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -1,5 +1,6 @@ import asyncio import json +import traceback from datetime import datetime from typing import Any, Dict, Optional @@ -11,6 +12,9 @@ from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base +from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + update_response_metadata, +) from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.responses.utils import ResponsesAPIRequestUtils @@ -22,7 +26,8 @@ from litellm.types.llms.openai import ( ResponsesAPIStreamEvents, ResponsesAPIStreamingResponse, ) -from litellm.utils import CustomStreamWrapper +from litellm.types.utils import CallTypes +from litellm.utils import CustomStreamWrapper, async_post_call_success_deployment_hook class BaseResponsesAPIStreamingIterator: @@ -40,6 +45,8 @@ class BaseResponsesAPIStreamingIterator: logging_obj: LiteLLMLoggingObj, litellm_metadata: Optional[Dict[str, Any]] = None, custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict[str, Any]] = None, + call_type: Optional[str] = None, ): self.response = response self.model = model @@ -47,21 +54,25 @@ class BaseResponsesAPIStreamingIterator: self.finished = False self.responses_api_provider_config = responses_api_provider_config self.completed_response: Optional[ResponsesAPIStreamingResponse] = None - self.start_time = datetime.now() + self.start_time = getattr(logging_obj, "start_time", datetime.now()) - # set request kwargs + # track request context for hooks self.litellm_metadata = litellm_metadata self.custom_llm_provider = custom_llm_provider + self.request_data: Dict[str, Any] = request_data or {} + self.call_type: Optional[str] = call_type # set hidden params for response headers (e.g., x-litellm-model-id) - # This matches ths stream wrapper in litellm/litellm_core_utils/streaming_handler.py + # This matches the stream wrapper in litellm/litellm_core_utils/streaming_handler.py _api_base = get_api_base( model=model or "", optional_params=self.logging_obj.model_call_details.get( "litellm_params", {} ), ) - _model_info: Dict = litellm_metadata.get("model_info", {}) if litellm_metadata else {} + _model_info: Dict = ( + litellm_metadata.get("model_info", {}) if litellm_metadata else {} + ) self._hidden_params = { "model_id": _model_info.get("id", None), "api_base": _api_base, @@ -102,13 +113,21 @@ class BaseResponsesAPIStreamingIterator: # if "response" in parsed_chunk, then encode litellm specific information like custom_llm_provider response_object = getattr(openai_responses_api_chunk, "response", None) if response_object: - response = ResponsesAPIRequestUtils._update_responses_api_response_id_with_model_id( - responses_api_response=response_object, - litellm_metadata=self.litellm_metadata, - custom_llm_provider=self.custom_llm_provider, + response = ( + ResponsesAPIRequestUtils._update_responses_api_response_id_with_model_id( + responses_api_response=response_object, + litellm_metadata=self.litellm_metadata, + custom_llm_provider=self.custom_llm_provider, + ) ) setattr(openai_responses_api_chunk, "response", response) + # Allow callbacks to modify chunk before returning + openai_responses_api_chunk = run_async_function( + async_function=self._call_post_streaming_deployment_hook, + chunk=openai_responses_api_chunk, + ) + # Store the completed response if ( openai_responses_api_chunk @@ -149,11 +168,159 @@ class BaseResponsesAPIStreamingIterator: except json.JSONDecodeError: # If we can't parse the chunk, continue return None + except Exception as e: + # Ensure failures trigger failure hooks + self._handle_failure(e) + raise def _handle_logging_completed_response(self): """Base implementation - should be overridden by subclasses""" pass + async def _call_post_streaming_deployment_hook(self, chunk): + """ + Allow callbacks to modify streaming chunks before returning (parity with chat). + """ + try: + # Align with chat pipeline: use logging_obj model_call_details + call_type + typed_call_type: Optional[CallTypes] = None + if self.call_type is not None: + try: + typed_call_type = CallTypes(self.call_type) + except ValueError: + typed_call_type = None + if typed_call_type is None: + try: + typed_call_type = CallTypes(getattr(self.logging_obj, "call_type", None)) + except Exception: + typed_call_type = None + + request_data = self.request_data or getattr( + self.logging_obj, "model_call_details", {} + ) + callbacks = getattr(litellm, "callbacks", None) or [] + hooks_ran = False + for callback in callbacks: + if hasattr(callback, "async_post_call_streaming_deployment_hook"): + hooks_ran = True + result = await callback.async_post_call_streaming_deployment_hook( + request_data=request_data, + response_chunk=chunk, + call_type=typed_call_type, + ) + if result is not None: + chunk = result + if hooks_ran: + setattr(chunk, "_post_streaming_hooks_ran", True) + return chunk + except Exception: + return chunk + + async def call_post_streaming_hooks_for_testing(self, chunk): + """ + Helper to invoke streaming deployment hooks explicitly (used in tests). + """ + return await self._call_post_streaming_deployment_hook(chunk) + + def _run_post_success_hooks(self, end_time: datetime): + """ + Run post-call deployment hooks and update metadata similar to chat pipeline. + """ + if self.completed_response is None: + return + + request_payload: Dict[str, Any] = {} + if isinstance(self.request_data, dict): + request_payload.update(self.request_data) + try: + if hasattr(self.logging_obj, "model_call_details"): + request_payload.update(self.logging_obj.model_call_details) + except Exception: + pass + if "litellm_params" not in request_payload: + try: + request_payload["litellm_params"] = getattr( + self.logging_obj, "model_call_details", {} + ).get("litellm_params", {}) + except Exception: + request_payload["litellm_params"] = {} + + try: + update_response_metadata( + result=self.completed_response, + logging_obj=self.logging_obj, + model=self.model, + kwargs=request_payload, + start_time=self.start_time, + end_time=end_time, + ) + except Exception: + # Non-blocking + pass + + try: + typed_call_type: Optional[CallTypes] = None + if self.call_type is not None: + try: + typed_call_type = CallTypes(self.call_type) + except ValueError: + typed_call_type = None + except Exception: + typed_call_type = None + if typed_call_type is None: + try: + typed_call_type = CallTypes.responses + except Exception: + typed_call_type = None + + try: + # Call synchronously; async hook will be executed via asyncio.run in a new loop + run_async_function( + async_function=async_post_call_success_deployment_hook, + request_data=request_payload, + response=self.completed_response, + call_type=typed_call_type, + ) + except Exception: + pass + + def _handle_failure(self, exception: Exception): + """ + Trigger failure handlers before bubbling the exception. + """ + traceback_exception = traceback.format_exc() + try: + run_async_function( + async_function=self.logging_obj.async_failure_handler, + exception=exception, + traceback_exception=traceback_exception, + start_time=self.start_time, + end_time=datetime.now(), + ) + except Exception: + pass + + try: + executor.submit( + self.logging_obj.failure_handler, + exception, + traceback_exception, + self.start_time, + datetime.now(), + ) + except Exception: + pass + + +async def call_post_streaming_hooks_for_testing(iterator, chunk): + """ + Module-level helper for tests to ensure hooks can be invoked even if the iterator is wrapped. + """ + hook_fn = getattr(iterator, "_call_post_streaming_deployment_hook", None) + if hook_fn is None: + return chunk + return await hook_fn(chunk) + class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): """ @@ -168,6 +335,8 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): logging_obj: LiteLLMLoggingObj, litellm_metadata: Optional[Dict[str, Any]] = None, custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict[str, Any]] = None, + call_type: Optional[str] = None, ): super().__init__( response, @@ -176,6 +345,8 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): logging_obj, litellm_metadata, custom_llm_provider, + request_data, + call_type, ) self.stream_iterator = response.aiter_lines() @@ -203,16 +374,21 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): except httpx.HTTPError as e: # Handle HTTP errors self.finished = True + self._handle_failure(e) + raise e + except Exception as e: + self.finished = True + self._handle_failure(e) raise e def _handle_logging_completed_response(self): """Handle logging for completed responses in async context""" # Create a deep copy for logging to avoid modifying the response object that will be returned to the user - # The logging handlers may transform usage from Responses API format (input_tokens/output_tokens) + # The logging handlers may transform usage from Responses API format (input_tokens/output_tokens) # to chat completion format (prompt_tokens/completion_tokens) for internal logging import copy logging_response = copy.deepcopy(self.completed_response) - + asyncio.create_task( self.logging_obj.async_success_handler( result=logging_response, @@ -229,6 +405,7 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): start_time=self.start_time, end_time=datetime.now(), ) + self._run_post_success_hooks(end_time=datetime.now()) class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): @@ -244,6 +421,8 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): logging_obj: LiteLLMLoggingObj, litellm_metadata: Optional[Dict[str, Any]] = None, custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict[str, Any]] = None, + call_type: Optional[str] = None, ): super().__init__( response, @@ -252,6 +431,8 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): logging_obj, litellm_metadata, custom_llm_provider, + request_data, + call_type, ) self.stream_iterator = response.iter_lines() @@ -279,16 +460,21 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): except httpx.HTTPError as e: # Handle HTTP errors self.finished = True + self._handle_failure(e) + raise e + except Exception as e: + self.finished = True + self._handle_failure(e) raise e def _handle_logging_completed_response(self): """Handle logging for completed responses in sync context""" # Create a deep copy for logging to avoid modifying the response object that will be returned to the user - # The logging handlers may transform usage from Responses API format (input_tokens/output_tokens) + # The logging handlers may transform usage from Responses API format (input_tokens/output_tokens) # to chat completion format (prompt_tokens/completion_tokens) for internal logging import copy logging_response = copy.deepcopy(self.completed_response) - + run_async_function( async_function=self.logging_obj.async_success_handler, result=logging_response, @@ -304,6 +490,7 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): start_time=self.start_time, end_time=datetime.now(), ) + self._run_post_success_hooks(end_time=datetime.now()) class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): @@ -324,6 +511,8 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): logging_obj: LiteLLMLoggingObj, litellm_metadata: Optional[Dict[str, Any]] = None, custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict[str, Any]] = None, + call_type: Optional[str] = None, ): super().__init__( response=response, @@ -332,6 +521,8 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): logging_obj=logging_obj, litellm_metadata=litellm_metadata, custom_llm_provider=custom_llm_provider, + request_data=request_data, + call_type=call_type, ) # one-time transform diff --git a/litellm/router.py b/litellm/router.py index 6821ab9e6c6..d980b5f74d8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -713,6 +713,23 @@ class Router: self, routing_strategy: Union[RoutingStrategy, str], routing_strategy_args: dict ): verbose_router_logger.info(f"Routing strategy: {routing_strategy}") + + # Validate routing_strategy value to fail fast with helpful error + # See: https://github.com/BerriAI/litellm/issues/11330 + # Derive valid strategies from RoutingStrategy enum + "simple-shuffle" (default, not in enum) + valid_strategy_strings = ["simple-shuffle"] + [s.value for s in RoutingStrategy] + + if routing_strategy is not None: + is_valid_string = isinstance(routing_strategy, str) and routing_strategy in valid_strategy_strings + is_valid_enum = isinstance(routing_strategy, RoutingStrategy) + if not is_valid_string and not is_valid_enum: + raise ValueError( + f"Invalid routing_strategy: '{routing_strategy}'. " + f"Valid options: {valid_strategy_strings}. " + f"Check 'router_settings.routing_strategy' in your config.yaml " + f"or the 'routing_strategy' parameter if using the Router SDK directly." + ) + if ( routing_strategy == RoutingStrategy.LEAST_BUSY.value or routing_strategy == RoutingStrategy.LEAST_BUSY @@ -812,6 +829,9 @@ class Router: self.acancel_responses = self.factory_function( litellm.acancel_responses, call_type="acancel_responses" ) + self.acompact_responses = self.factory_function( + litellm.acompact_responses, call_type="acompact_responses" + ) self.adelete_responses = self.factory_function( litellm.adelete_responses, call_type="adelete_responses" ) @@ -3924,6 +3944,7 @@ class Router: "anthropic_messages", "aresponses", "acancel_responses", + "acompact_responses", "responses", "aget_responses", "adelete_responses", @@ -4152,6 +4173,7 @@ class Router: elif call_type in ( "aget_responses", "acancel_responses", + "acompact_responses", "adelete_responses", "alist_input_items", ): diff --git a/litellm/types/integrations/langsmith.py b/litellm/types/integrations/langsmith.py index 23f760ecf32..9c026a117fd 100644 --- a/litellm/types/integrations/langsmith.py +++ b/litellm/types/integrations/langsmith.py @@ -31,6 +31,7 @@ class LangsmithCredentialsObject(TypedDict): LANGSMITH_API_KEY: Optional[str] LANGSMITH_PROJECT: Optional[str] LANGSMITH_BASE_URL: str + LANGSMITH_TENANT_ID: Optional[str] class LangsmithQueueObject(TypedDict): @@ -52,6 +53,7 @@ class CredentialsKey(NamedTuple): api_key: str project: str base_url: str + tenant_id: Optional[str] @dataclass diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3eec67d9d26..784c8403c3f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2677,6 +2677,7 @@ class StandardCallbackDynamicParams(TypedDict, total=False): langsmith_project: Optional[str] langsmith_base_url: Optional[str] langsmith_sampling_rate: Optional[float] + langsmith_tenant_id: Optional[str] # Humanloop dynamic params humanloop_api_key: Optional[str] @@ -2946,6 +2947,7 @@ class LlmProviders(str, Enum): MISTRAL = "mistral" MILVUS = "milvus" GROQ = "groq" + GIGACHAT = "gigachat" NVIDIA_NIM = "nvidia_nim" CEREBRAS = "cerebras" AI21_CHAT = "ai21_chat" diff --git a/litellm/utils.py b/litellm/utils.py index 6b9aeca0934..fbbaa94f7a1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7521,6 +7521,8 @@ class ProviderConfigManager: return litellm.CompactifAIChatConfig() elif litellm.LlmProviders.GITHUB_COPILOT == provider: return litellm.GithubCopilotConfig() + elif litellm.LlmProviders.GIGACHAT == provider: + return litellm.GigaChatConfig() elif litellm.LlmProviders.RAGFLOW == provider: return litellm.RAGFlowConfig() elif ( @@ -7716,6 +7718,8 @@ class ProviderConfigManager: return litellm.CometAPIEmbeddingConfig() elif litellm.LlmProviders.GITHUB_COPILOT == provider: return litellm.GithubCopilotEmbeddingConfig() + elif litellm.LlmProviders.GIGACHAT == provider: + return litellm.GigaChatEmbeddingConfig() elif litellm.LlmProviders.SAGEMAKER == provider: from litellm.llms.sagemaker.embedding.transformation import ( SagemakerEmbeddingConfig, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e823dd5dc6b..c7a2f60856d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -15831,6 +15831,68 @@ "max_tokens": 8191, "mode": "embedding" }, + "gigachat/GigaChat-2-Lite": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0.0, + "supports_function_calling": true, + "supports_system_messages": true + }, + "gigachat/GigaChat-2-Max": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0.0, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_vision": true + }, + "gigachat/GigaChat-2-Pro": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0.0, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_vision": true + }, + "gigachat/Embeddings": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024 + }, + "gigachat/Embeddings-2": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024 + }, + "gigachat/EmbeddingsGigaR": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 4096, + "max_tokens": 4096, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 2560 + }, "google.gemma-3-12b-it": { "input_cost_per_token": 9e-08, "litellm_provider": "bedrock_converse", @@ -32092,3 +32154,4 @@ "mode": "chat" } } + diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 45ee47c01bc..bc5dea7b97c 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -28,7 +28,8 @@ "list_container_files": "Supports GET /containers/{id}/files endpoint", "retrieve_container_file": "Supports GET /containers/{id}/files/{file_id} endpoint", "retrieve_container_file_content": "Supports GET /containers/{id}/files/{file_id}/content endpoint", - "delete_container_file": "Supports DELETE /containers/{id}/files/{file_id} endpoint" + "delete_container_file": "Supports DELETE /containers/{id}/files/{file_id} endpoint", + "compact": "Supports /responses/compact endpoint" } } }, @@ -1519,6 +1520,7 @@ "retrieve_container_file": true, "retrieve_container_file_content": true, "delete_container_file": true, + "compact": true, "a2a": true, "interactions": true } diff --git a/requirements.txt b/requirements.txt index 06a7c17336c..249b899b86b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ google-cloud-aiplatform==1.47.0 # for vertex ai calls google-cloud-iam==2.19.1 # for GCP IAM Redis authentication google-genai==1.22.0 anthropic[vertex]==0.54.0 -mcp==1.23.0 ; python_version >= "3.10" # for MCP server +mcp==1.25.0 ; python_version >= "3.10" # for MCP server google-generativeai==0.5.0 # for vertex ai calls async_generator==1.10.0 # for async ollama calls langfuse==2.59.7 # for langfuse self-hosted logging diff --git a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py index 9a96919da87..60d4f479733 100644 --- a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py +++ b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py @@ -61,13 +61,19 @@ async def test_bedrock_apply_guardrail_blocked(): guardrailVersion="DRAFT", ) - # Mock the make_bedrock_api_request method + # Mock the make_bedrock_api_request method to raise an exception for blocked content with patch.object( - guardrail, "make_bedrock_api_request", new_callable=AsyncMock + guardrail, "make_bedrock_api_request", new_callable=AsyncMock ) as mock_api_request: - # Mock a blocked response from Bedrock - mock_response = {"action": "BLOCKED", "reason": "Content violates policy"} - mock_api_request.return_value = mock_response + # Mock the method to raise an HTTPException as it would for blocked content + from fastapi import HTTPException + mock_api_request.side_effect = HTTPException( + status_code=400, + detail={ + "error": "Violated guardrail policy", + "bedrock_guardrail_response": "", + }, + ) # Test the apply_guardrail method should raise an exception with pytest.raises(Exception) as exc_info: @@ -77,8 +83,9 @@ async def test_bedrock_apply_guardrail_blocked(): input_type="request", ) - assert "Content blocked by Bedrock guardrail" in str(exc_info.value) - assert "Content violates policy" in str(exc_info.value) + # The apply_guardrail method wraps the original exception in a generic Exception + assert "Bedrock guardrail failed:" in str(exc_info.value) + assert "Violated guardrail policy" in str(exc_info.value) @pytest.mark.asyncio @@ -253,7 +260,15 @@ async def test_bedrock_apply_guardrail_filters_request_messages_when_flag_enable with patch.object( guardrail, "make_bedrock_api_request", new_callable=AsyncMock ) as mock_api: - mock_api.return_value = {"action": "BLOCKED", "reason": "policy"} + # Mock the method to raise an HTTPException as it would for blocked content + from fastapi import HTTPException + mock_api.side_effect = HTTPException( + status_code=400, + detail={ + "error": "Violated guardrail policy", + "bedrock_guardrail_response": "policy", + }, + ) with pytest.raises(Exception, match="policy") as exc_info: await guardrail.apply_guardrail( @@ -265,7 +280,8 @@ async def test_bedrock_apply_guardrail_filters_request_messages_when_flag_enable assert mock_api.called _, kwargs = mock_api.call_args assert kwargs["messages"] == [request_messages[-1]] - assert "Content blocked by Bedrock guardrail" in str(exc_info.value) + # The apply_guardrail method wraps the original exception in a generic Exception + assert "Bedrock guardrail failed:" in str(exc_info.value) def test_bedrock_guardrail_filters_latest_user_message_when_enabled(): diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py index 5714cd5c487..c981ccd8dac 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py @@ -1,11 +1,14 @@ import os import sys +import time +from unittest.mock import Mock, patch sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path -from litellm_proxy_extras.utils import ProxyExtrasDBManager +from litellm_proxy_extras.utils import ProxyExtrasDBManager, MigrationLockManager + def test_custom_prisma_dir(monkeypatch): @@ -27,101 +30,279 @@ def test_custom_prisma_dir(monkeypatch): assert os.path.exists(migrations_dir) -class TestPermissionErrorDetection: - """Test cases for permission error detection in Prisma migrations""" +class TestMigrationLockManager: + """Test cases for MigrationLockManager""" - def test_is_permission_error_postgres_42501(self): - """Test detection of PostgreSQL 42501 error code (insufficient privilege)""" - error_message = "Database error code: 42501 - permission denied for table users" - assert ProxyExtrasDBManager._is_permission_error(error_message) is True + def test_acquire_lock_without_redis(self): + """Test lock acquisition when Redis is not available""" + lock_manager = MigrationLockManager() + result = lock_manager.acquire_lock() + assert result is True # Should return True when Redis is not available + assert lock_manager.lock_acquired is True # Redis 없을 때도 lock_acquired는 True - def test_is_permission_error_must_be_owner(self): - """Test detection of 'must be owner of table' error""" - error_message = "ERROR: must be owner of table my_table" - assert ProxyExtrasDBManager._is_permission_error(error_message) is True + def test_acquire_lock_with_redis_success(self): + """Test successful lock acquisition with Redis""" + mock_redis = Mock() + mock_redis.set_cache.return_value = True + lock_manager = MigrationLockManager(mock_redis) - def test_is_permission_error_permission_denied_schema(self): - """Test detection of 'permission denied for schema' error""" - error_message = "permission denied for schema public" - assert ProxyExtrasDBManager._is_permission_error(error_message) is True + result = lock_manager.acquire_lock() - def test_is_permission_error_permission_denied_table(self): - """Test detection of 'permission denied for table' error""" - error_message = "permission denied for table my_table" - assert ProxyExtrasDBManager._is_permission_error(error_message) is True + assert result is True + assert lock_manager.lock_acquired is True + mock_redis.set_cache.assert_called_once() - def test_is_permission_error_must_be_owner_schema(self): - """Test detection of 'must be owner of schema' error""" - error_message = "must be owner of schema public" - assert ProxyExtrasDBManager._is_permission_error(error_message) is True + def test_acquire_lock_with_redis_failure(self): + """Test failed lock acquisition with Redis""" + mock_redis = Mock() + mock_redis.set_cache.return_value = False + lock_manager = MigrationLockManager(mock_redis) - def test_is_permission_error_case_insensitive(self): - """Test that permission error detection is case insensitive""" - error_message = "PERMISSION DENIED FOR TABLE my_table" - assert ProxyExtrasDBManager._is_permission_error(error_message) is True + result = lock_manager.acquire_lock() - def test_is_permission_error_negative(self): - """Test that non-permission errors are not detected as permission errors""" - error_message = "column 'id' already exists" - assert ProxyExtrasDBManager._is_permission_error(error_message) is False + assert result is False + assert lock_manager.lock_acquired is False + mock_redis.set_cache.assert_called_once() + + def test_acquire_lock_with_redis_exception(self): + """Test lock acquisition with Redis exception""" + mock_redis = Mock() + mock_redis.set_cache.side_effect = Exception("Redis error") + lock_manager = MigrationLockManager(mock_redis) + + result = lock_manager.acquire_lock() + + assert result is False + assert lock_manager.lock_acquired is False + + def test_wait_for_lock_release_success(self): + """Test successful waiting for lock release""" + mock_redis = Mock() + # First call returns False (lock held), second call returns True (lock acquired) + mock_redis.set_cache.side_effect = [False, True] + lock_manager = MigrationLockManager(mock_redis) + + result = lock_manager.wait_for_lock_release(check_interval=0.1, max_wait=1) + + assert result is True + assert lock_manager.lock_acquired is True + assert mock_redis.set_cache.call_count == 2 + + def test_wait_for_lock_release_timeout(self): + """Test timeout while waiting for lock release""" + mock_redis = Mock() + mock_redis.set_cache.return_value = False # Lock always held + lock_manager = MigrationLockManager(mock_redis) + + result = lock_manager.wait_for_lock_release(check_interval=0.1, max_wait=0.2) + + assert result is False + assert lock_manager.lock_acquired is False + + def test_release_lock_not_acquired(self): + """Test releasing lock when not acquired""" + mock_redis = Mock() + lock_manager = MigrationLockManager(mock_redis) + + lock_manager.release_lock() + + mock_redis.get_cache.assert_not_called() + mock_redis.delete_cache.assert_not_called() + + def test_release_lock_success(self): + """Test successful lock release""" + mock_redis = Mock() + mock_redis.get_cache.return_value = "pod_123_456" + lock_manager = MigrationLockManager(mock_redis) + lock_manager.pod_id = "pod_123_456" + lock_manager.lock_acquired = True + + lock_manager.release_lock() + + mock_redis.get_cache.assert_called_once() + mock_redis.delete_cache.assert_called_once() + assert lock_manager.lock_acquired is False + + def test_release_lock_wrong_owner(self): + """Test releasing lock when not the owner""" + mock_redis = Mock() + mock_redis.get_cache.return_value = "pod_999_999" # Different pod + lock_manager = MigrationLockManager(mock_redis) + lock_manager.pod_id = "pod_123_456" + lock_manager.lock_acquired = True + + lock_manager.release_lock() + + mock_redis.get_cache.assert_called_once() + mock_redis.delete_cache.assert_not_called() + assert lock_manager.lock_acquired is False + + def test_context_manager(self): + """Test MigrationLockManager as context manager""" + mock_redis = Mock() + mock_redis.set_cache.return_value = True + # Mock get_cache to return the same pod_id for successful release + mock_redis.get_cache.return_value = "pod_123_456" + + lock_manager = MigrationLockManager(mock_redis) + lock_manager.pod_id = "pod_123_456" # Set consistent pod_id + + with lock_manager: + assert lock_manager.lock_acquired is True + + # Should call release_lock when exiting context + mock_redis.get_cache.assert_called_once() + mock_redis.delete_cache.assert_called_once() -class TestIdempotentErrorDetection: - """Test cases for idempotent error detection in Prisma migrations""" +class TestProxyExtrasDBManagerMigrationLock: + """Test cases for ProxyExtrasDBManager with migration locking""" - def test_is_idempotent_error_already_exists(self): - """Test detection of generic 'already exists' error""" - error_message = "object already exists" - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + @patch("litellm_proxy_extras.utils.ProxyExtrasDBManager._resolve_all_migrations") + @patch("litellm_proxy_extras.utils.subprocess.run") + @patch("litellm_proxy_extras.utils.ProxyExtrasDBManager._get_prisma_dir") + @patch("os.chdir") + def test_setup_database_with_redis_lock_success( + self, mock_chdir, mock_get_prisma_dir, mock_subprocess, mock_resolve_migrations + ): + """Test successful database setup with Redis lock""" + # Setup mocks + mock_get_prisma_dir.return_value = "/test/prisma" + mock_subprocess.return_value = Mock(stdout="Migration completed", stderr="") + mock_resolve_migrations.return_value = None - def test_is_idempotent_error_column_already_exists(self): - """Test detection of 'column already exists' error""" - error_message = "column 'email' already exists" - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + # Mock Redis cache + mock_redis = Mock() + mock_redis.set_cache.return_value = True # Lock acquired successfully + mock_redis.get_cache.return_value = "pod_123_456" - def test_is_idempotent_error_duplicate_key(self): - """Test detection of duplicate key violation error""" - error_message = "duplicate key value violates unique constraint" - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + # Set DATABASE_URL + with patch.dict( + os.environ, {"DATABASE_URL": "postgresql://test:test@localhost/test"} + ): + result = ProxyExtrasDBManager.setup_database( + use_migrate=True, redis_cache=mock_redis + ) - def test_is_idempotent_error_relation_already_exists(self): - """Test detection of 'relation already exists' error""" - error_message = "relation 'users_pkey' already exists" - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + assert result is True + # set_cache is called once in acquire_lock (__enter__ calls acquire_lock) + assert mock_redis.set_cache.call_count == 1 + mock_subprocess.assert_called_once() - def test_is_idempotent_error_constraint_already_exists(self): - """Test detection of 'constraint already exists' error""" - error_message = "constraint 'fk_user_id' already exists" - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + @patch("litellm_proxy_extras.utils.ProxyExtrasDBManager._resolve_all_migrations") + @patch("litellm_proxy_extras.utils.subprocess.run") + @patch("litellm_proxy_extras.utils.ProxyExtrasDBManager._get_prisma_dir") + @patch("os.chdir") + def test_setup_database_with_redis_lock_wait_and_skip( + self, mock_chdir, mock_get_prisma_dir, mock_subprocess, mock_resolve_migrations + ): + """Test database setup when lock is held by another pod, then acquired after waiting""" + # Setup mocks + mock_get_prisma_dir.return_value = "/test/prisma" + mock_resolve_migrations.return_value = None - def test_is_idempotent_error_case_insensitive(self): - """Test that idempotent error detection is case insensitive""" - error_message = "COLUMN 'ID' ALREADY EXISTS" - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + # Mock Redis cache - first call fails, second call succeeds + mock_redis = Mock() + mock_redis.set_cache.side_effect = [False, True] # First fails, then succeeds + mock_redis.get_cache.return_value = "pod_123_456" - def test_is_idempotent_error_negative(self): - """Test that non-idempotent errors are not detected as idempotent errors""" - error_message = "Database error code: 42501 - permission denied" - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is False + # Set DATABASE_URL + with patch.dict( + os.environ, {"DATABASE_URL": "postgresql://test:test@localhost/test"} + ): + result = ProxyExtrasDBManager.setup_database( + use_migrate=True, redis_cache=mock_redis + ) + assert result is True # Should return True after waiting and acquiring lock + # set_cache is called 2 times: once in __enter__, once in wait_for_lock_release + assert mock_redis.set_cache.call_count == 2 + # Proceed for case handling in case of migration failure + mock_subprocess.assert_called_once() -class TestErrorClassificationPriority: - """Test cases to ensure errors are correctly classified""" + @patch("litellm_proxy_extras.utils.ProxyExtrasDBManager._resolve_all_migrations") + @patch("litellm_proxy_extras.utils.subprocess.run") + @patch("litellm_proxy_extras.utils.ProxyExtrasDBManager._get_prisma_dir") + @patch("os.chdir") + def test_setup_database_without_redis( + self, mock_chdir, mock_get_prisma_dir, mock_subprocess, mock_resolve_migrations + ): + """Test database setup without Redis cache""" + # Setup mocks + mock_get_prisma_dir.return_value = "/test/prisma" + mock_subprocess.return_value = Mock(stdout="Migration completed", stderr="") + mock_resolve_migrations.return_value = None - def test_permission_error_not_classified_as_idempotent(self): - """Ensure permission errors are not mistakenly classified as idempotent""" - error_message = "Database error code: 42501 - must be owner of table users" - assert ProxyExtrasDBManager._is_permission_error(error_message) is True - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is False + # Set DATABASE_URL + with patch.dict( + os.environ, {"DATABASE_URL": "postgresql://test:test@localhost/test"} + ): + result = ProxyExtrasDBManager.setup_database( + use_migrate=True, redis_cache=None + ) - def test_idempotent_error_not_classified_as_permission(self): - """Ensure idempotent errors are not mistakenly classified as permission errors""" - error_message = "column 'created_at' already exists" - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True - assert ProxyExtrasDBManager._is_permission_error(error_message) is False + assert result is True + # Redis가 없을 때는 락 보호 없이 마이그레이션을 실행해야 함 + mock_subprocess.assert_called_once() - def test_unknown_error_classified_as_neither(self): - """Ensure unknown errors are classified as neither permission nor idempotent""" - error_message = "connection timeout" - assert ProxyExtrasDBManager._is_permission_error(error_message) is False - assert ProxyExtrasDBManager._is_idempotent_error(error_message) is False + def test_setup_database_no_database_url(self): + """Test database setup without DATABASE_URL""" + with patch.dict(os.environ, {}, clear=True): + result = ProxyExtrasDBManager.setup_database( + use_migrate=True, redis_cache=None + ) + + assert result is False + + @patch("litellm_proxy_extras.utils.subprocess.run") + @patch("litellm_proxy_extras.utils.ProxyExtrasDBManager._get_prisma_dir") + @patch("os.chdir") + @patch.object( + MigrationLockManager, "LOCK_TTL_SECONDS", 1 + ) # Set short TTL for testing + def test_setup_database_lock_timeout( + self, mock_chdir, mock_get_prisma_dir, mock_subprocess + ): + """Test database setup when lock acquisition times out""" + # Setup mocks + mock_get_prisma_dir.return_value = "/test/prisma" + + # Mock Redis cache - always fails to acquire lock + mock_redis = Mock() + mock_redis.set_cache.return_value = False # Always fails + + # Set DATABASE_URL + with patch.dict( + os.environ, {"DATABASE_URL": "postgresql://test:test@localhost/test"} + ): + # Patch the wait_for_lock_release method to use shorter timeout + with patch.object( + MigrationLockManager, "wait_for_lock_release" + ) as mock_wait: + mock_wait.return_value = False # Simulate timeout + + result = ProxyExtrasDBManager.setup_database( + use_migrate=True, redis_cache=mock_redis + ) + + assert result is False # Should return False after timeout + mock_subprocess.assert_not_called() # Should not run migration + # Verify that wait_for_lock_release was called with default parameters + mock_wait.assert_called_once_with() + + def test_wait_for_lock_release_actual_timeout(self): + """Test actual timeout behavior of wait_for_lock_release with real timing""" + mock_redis = Mock() + mock_redis.set_cache.return_value = False # Always fails to acquire lock + lock_manager = MigrationLockManager(mock_redis) + + # Test with very short timeout to verify actual timeout behavior + start_time = time.time() + result = lock_manager.wait_for_lock_release(check_interval=0.1, max_wait=0.5) + end_time = time.time() + + assert result is False # Should timeout + assert end_time - start_time >= 0.5 # Should wait at least the max_wait time + assert end_time - start_time < 1.0 # But not too much longer + # Should have called set_cache multiple times during the wait + assert mock_redis.set_cache.call_count > 1 diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 7553c670774..5f35d6837c0 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -1814,3 +1814,49 @@ async def test_extra_body_merges_with_request_data(extra_body_mock_response_data assert "temperature" in request_body assert "custom_field" in request_body assert request_body["custom_field"] == "custom_value" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_openai_compact_responses_api(sync_mode): + """ + Test the compact_responses API for OpenAI. + + This test verifies that the compact_responses endpoint works correctly + for compressing conversation history. + """ + litellm._turn_on_debug() + litellm.set_verbose = True + + input_messages = [ + {"role": "user", "content": "Hello, how are you?"}, + {"role": "assistant", "content": "I'm doing well, thank you for asking!"}, + {"role": "user", "content": "What is the weather like today?"}, + ] + + try: + if sync_mode: + response = litellm.compact_responses( + model="openai/gpt-4o", + input=input_messages, + instructions="Be helpful and concise", + ) + else: + response = await litellm.acompact_responses( + model="openai/gpt-4o", + input=input_messages, + instructions="Be helpful and concise", + ) + except litellm.InternalServerError: + pytest.skip("Skipping test due to InternalServerError") + except litellm.BadRequestError as e: + # compact_responses may not be available for all models/accounts + pytest.skip(f"Skipping test due to BadRequestError: {e}") + + print("compact_responses response=", json.dumps(response, indent=4, default=str)) + + # Validate response structure + assert response is not None + assert "id" in response, "Response should have an 'id' field" + assert "output" in response, "Response should have an 'output' field" + assert isinstance(response["output"], list), "Output should be a list" diff --git a/tests/llm_responses_api_testing/test_responses_hooks.py b/tests/llm_responses_api_testing/test_responses_hooks.py new file mode 100644 index 00000000000..8c0f7dab2af --- /dev/null +++ b/tests/llm_responses_api_testing/test_responses_hooks.py @@ -0,0 +1,165 @@ +import asyncio +from datetime import datetime +from types import SimpleNamespace + +import httpx +import pytest + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.responses import streaming_iterator as streaming_module +from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator +from litellm.types.llms.openai import ResponsesAPIStreamEvents +from litellm.types.utils import CallTypes + + +class _FakeLoggingObj: + def __init__(self): + self.success_calls = 0 + self.async_success_calls = 0 + self.failure_calls = 0 + self.async_failure_calls = 0 + self.start_time = datetime.now() + self.model_call_details = {"litellm_params": {}} + + # Signature alignment with Logging handlers + def success_handler(self, *args, **kwargs): + self.success_calls += 1 + + async def async_success_handler(self, *args, **kwargs): + self.async_success_calls += 1 + + def failure_handler(self, *args, **kwargs): + self.failure_calls += 1 + + async def async_failure_handler(self, *args, **kwargs): + self.async_failure_calls += 1 + + +@pytest.mark.asyncio +async def test_responses_streaming_triggers_hooks(monkeypatch): + """ + Ensure streaming iterator fires success + post-call hooks for responses API. + """ + hook_calls = {"post_call": 0, "metadata": 0} + seen = {} + + async def fake_post_call(request_data, response, call_type): + hook_calls["post_call"] += 1 + seen["request_data"] = request_data + seen["call_type"] = call_type + + def fake_update_metadata(**kwargs): + hook_calls["metadata"] += 1 + + monkeypatch.setattr( + streaming_module, + "async_post_call_success_deployment_hook", + fake_post_call, + ) + monkeypatch.setattr( + streaming_module, + "update_response_metadata", + fake_update_metadata, + ) + + logging_obj = _FakeLoggingObj() + + iterator = ResponsesAPIStreamingIterator( + response=httpx.Response(200), + model="test-model", + responses_api_provider_config=SimpleNamespace(), # not used in this test + logging_obj=logging_obj, + request_data={"foo": "bar", "litellm_params": {}}, + call_type=CallTypes.responses.value, + ) + + # Simulate completed streaming event + iterator.completed_response = SimpleNamespace( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=SimpleNamespace() + ) + + iterator._handle_logging_completed_response() + await asyncio.sleep(0.2) # allow async tasks to run + + assert logging_obj.success_calls == 1 + assert logging_obj.async_success_calls == 1 + assert hook_calls["post_call"] == 1 + assert hook_calls["metadata"] == 1 + assert seen["request_data"]["foo"] == "bar" + assert seen["request_data"].get("litellm_params") is not None + assert seen["call_type"] == CallTypes.responses + + +@pytest.mark.asyncio +async def test_responses_streaming_calls_post_streaming_deployment_hook(monkeypatch): + """ + Ensure per-chunk streaming deployment hook can modify chunks. + """ + + class _HookLogger(CustomLogger): + async def async_post_call_streaming_deployment_hook( + self, request_data, response_chunk, call_type + ): + response_chunk.tagged = True + return response_chunk + + # Set callbacks to our fake hook + original_callbacks = litellm.callbacks + litellm.callbacks = [_HookLogger()] + + logging_obj = _FakeLoggingObj() + + class _StubConfig: + def transform_streaming_response(self, **kwargs): + return SimpleNamespace( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, response=None + ) + + iterator = ResponsesAPIStreamingIterator( + response=httpx.Response(200), + model="test-model", + responses_api_provider_config=_StubConfig(), + logging_obj=logging_obj, + request_data={"foo": "bar"}, + call_type=CallTypes.responses.value, + ) + + # Call hook helper directly to verify chunk is modified/flagged + chunk = SimpleNamespace(type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, response=None) + chunk = await streaming_module.call_post_streaming_hooks_for_testing(iterator, chunk) + assert getattr(chunk, "_post_streaming_hooks_ran", False) is True + assert getattr(chunk, "tagged", False) is True + + # reset callbacks + litellm.callbacks = original_callbacks + + +@pytest.mark.asyncio +async def test_responses_streaming_failure_triggers_failure_handlers(): + """ + If transform raises, failure handlers should be called. + """ + + class _FailConfig: + def transform_streaming_response(self, **kwargs): + raise ValueError("boom") + + logging_obj = _FakeLoggingObj() + + iterator = ResponsesAPIStreamingIterator( + response=httpx.Response(200), + model="test-model", + responses_api_provider_config=_FailConfig(), + logging_obj=logging_obj, + request_data={"foo": "bar"}, + call_type=CallTypes.responses.value, + ) + + with pytest.raises(ValueError): + iterator._process_chunk('{"delta": "chunk"}') + + # allow failure callbacks to run + await asyncio.sleep(0.2) + assert logging_obj.failure_calls >= 1 + assert logging_obj.async_failure_calls >= 1 diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 7c849650bf6..ab5709cd72d 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -385,7 +385,7 @@ def test_anthropic_tool_use(tool_type, tool_config, message_content): "computer_tool_used, prompt_caching_set, expected_beta_header", [ (True, False, True), - (False, True, True), + (False, True, False), (True, True, True), (False, False, False), ], diff --git a/tests/llm_translation/test_databricks.py b/tests/llm_translation/test_databricks.py index 40fc712f2b7..3013d00288f 100644 --- a/tests/llm_translation/test_databricks.py +++ b/tests/llm_translation/test_databricks.py @@ -15,6 +15,7 @@ import litellm from litellm.exceptions import BadRequestError from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.utils import CustomStreamWrapper +from litellm._version import version from base_llm_unit_tests import BaseLLMChatTest, BaseAnthropicChatTest try: @@ -725,6 +726,7 @@ def test_embeddings_with_sync_http_handler(monkeypatch): headers={ "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", + "User-Agent": f"litellm/{version}", }, data=json.dumps( { @@ -767,6 +769,7 @@ def test_embeddings_with_async_http_handler(monkeypatch): headers={ "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", + "User-Agent": f"litellm/{version}", }, data=json.dumps( { @@ -823,6 +826,7 @@ def test_embeddings_uses_databricks_sdk_if_api_key_and_base_not_specified(monkey headers={ "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", + "User-Agent": f"litellm/{version}", }, data=json.dumps( { @@ -895,6 +899,7 @@ async def test_databricks_embeddings(sync_mode, monkeypatch): headers={ "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", + "User-Agent": f"litellm/{version}", }, data=json.dumps( { @@ -923,6 +928,7 @@ async def test_databricks_embeddings(sync_mode, monkeypatch): headers={ "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", + "User-Agent": f"litellm/{version}", }, data=json.dumps( { diff --git a/tests/llm_translation/test_gigachat.py b/tests/llm_translation/test_gigachat.py new file mode 100644 index 00000000000..80bf51b4646 --- /dev/null +++ b/tests/llm_translation/test_gigachat.py @@ -0,0 +1,349 @@ +""" +Tests for GigaChat LiteLLM Provider + +Tests message transformation, parameter handling, and response transformation. +Run with: pytest tests/llm_translation/test_gigachat.py -v +""" + +import json +import pytest +from unittest.mock import Mock, MagicMock + + +class TestGigaChatMessageTransformation: + """Tests for message transformation (OpenAI -> GigaChat format)""" + + @pytest.fixture + def config(self): + from litellm.llms.gigachat.chat.transformation import GigaChatConfig + return GigaChatConfig() + + def test_simple_user_message(self, config): + """Basic user message should pass through""" + messages = [{"role": "user", "content": "Hello"}] + result = config._transform_messages(messages) + + assert len(result) == 1 + assert result[0]["role"] == "user" + assert result[0]["content"] == "Hello" + + def test_developer_role_to_system(self, config): + """Developer role should be converted to system""" + messages = [{"role": "developer", "content": "You are helpful"}] + result = config._transform_messages(messages) + + assert result[0]["role"] == "system" + + def test_system_after_first_becomes_user(self, config): + """System message after first position should become user""" + messages = [ + {"role": "assistant", "content": "Response"}, + {"role": "system", "content": "Additional instruction"}, + ] + result = config._transform_messages(messages) + + assert result[0]["role"] == "assistant" + assert result[1]["role"] == "user" # system after first becomes user + + def test_tool_role_to_function(self, config): + """Tool role should be converted to function""" + messages = [{"role": "tool", "content": "result data"}] + result = config._transform_messages(messages) + + assert result[0]["role"] == "function" + + def test_tool_calls_to_function_call(self, config): + """tool_calls should be converted to function_call""" + messages = [{ + "role": "assistant", + "content": "", + "tool_calls": [{ + "id": "call_123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Moscow"}' + } + }] + }] + result = config._transform_messages(messages) + + assert "function_call" in result[0] + assert result[0]["function_call"]["name"] == "get_weather" + assert result[0]["function_call"]["arguments"] == {"city": "Moscow"} + assert "tool_calls" not in result[0] + + def test_none_content_becomes_empty_string(self, config): + """None content should become empty string""" + messages = [{"role": "assistant", "content": None}] + result = config._transform_messages(messages) + + assert result[0]["content"] == "" + + def test_name_field_removed(self, config): + """name field should be removed (not supported by GigaChat)""" + messages = [{"role": "user", "content": "Hi", "name": "John"}] + result = config._transform_messages(messages) + + assert "name" not in result[0] + + +class TestGigaChatCollapseUserMessages: + """Tests for collapsing consecutive user messages""" + + @pytest.fixture + def config(self): + from litellm.llms.gigachat.chat.transformation import GigaChatConfig + return GigaChatConfig() + + def test_no_collapse_single_message(self, config): + """Single message should not be changed""" + messages = [{"role": "user", "content": "Hello"}] + result = config._collapse_user_messages(messages) + + assert len(result) == 1 + assert result[0]["content"] == "Hello" + + def test_collapse_consecutive_user_messages(self, config): + """Consecutive user messages should be collapsed""" + messages = [ + {"role": "user", "content": "First"}, + {"role": "user", "content": "Second"}, + {"role": "user", "content": "Third"}, + ] + result = config._collapse_user_messages(messages) + + assert len(result) == 1 + assert "First" in result[0]["content"] + assert "Second" in result[0]["content"] + assert "Third" in result[0]["content"] + + def test_no_collapse_with_assistant_between(self, config): + """Messages with assistant between should not be collapsed""" + messages = [ + {"role": "user", "content": "First"}, + {"role": "assistant", "content": "Response"}, + {"role": "user", "content": "Second"}, + ] + result = config._collapse_user_messages(messages) + + assert len(result) == 3 + + +class TestGigaChatToolsTransformation: + """Tests for tools -> functions conversion""" + + @pytest.fixture + def config(self): + from litellm.llms.gigachat.chat.transformation import GigaChatConfig + return GigaChatConfig() + + def test_single_tool_conversion(self, config): + """Single tool should be converted correctly""" + tools = [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a city", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"} + } + } + } + }] + result = config._convert_tools_to_functions(tools) + + assert len(result) == 1 + assert result[0]["name"] == "get_weather" + assert result[0]["description"] == "Get weather for a city" + + def test_multiple_tools_conversion(self, config): + """Multiple tools should all be converted""" + tools = [ + {"type": "function", "function": {"name": "func1", "description": "First", "parameters": {"type": "object", "properties": {}}}}, + {"type": "function", "function": {"name": "func2", "description": "Second", "parameters": {"type": "object", "properties": {}}}}, + ] + result = config._convert_tools_to_functions(tools) + + assert len(result) == 2 + assert result[0]["name"] == "func1" + assert result[1]["name"] == "func2" + + +class TestGigaChatParamsTransformation: + """Tests for parameter transformation""" + + @pytest.fixture + def config(self): + from litellm.llms.gigachat.chat.transformation import GigaChatConfig + return GigaChatConfig() + + def test_temperature_zero_becomes_top_p_zero(self, config): + """temperature=0 should become top_p=0""" + params = {"temperature": 0} + result = config.map_openai_params( + non_default_params=params, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + + assert "top_p" in result + assert result["top_p"] == 0 + assert "temperature" not in result + + def test_temperature_nonzero_preserved(self, config): + """Non-zero temperature should be preserved""" + params = {"temperature": 0.7} + result = config.map_openai_params( + non_default_params=params, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + + assert result["temperature"] == 0.7 + + def test_max_completion_tokens_to_max_tokens(self, config): + """max_completion_tokens should become max_tokens""" + params = {"max_completion_tokens": 100} + result = config.map_openai_params( + non_default_params=params, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + + assert result["max_tokens"] == 100 + + def test_structured_output_via_json_schema(self, config): + """json_schema response_format should trigger structured output mode""" + params = { + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "person", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"} + } + } + } + } + } + result = config.map_openai_params( + non_default_params=params, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + + assert "_structured_output" in result + assert result["_structured_output"] is True + assert "function_call" in result + assert result["function_call"]["name"] == "person" + + +class TestGigaChatProviderRegistration: + """Tests for provider registration in LiteLLM""" + + def test_gigachat_in_provider_list(self): + """GigaChat should be in provider list""" + from litellm.types.utils import LlmProviders + + assert hasattr(LlmProviders, "GIGACHAT") + assert LlmProviders.GIGACHAT.value == "gigachat" + + def test_gigachat_in_chat_providers(self): + """GigaChat should be in LITELLM_CHAT_PROVIDERS""" + from litellm.constants import LITELLM_CHAT_PROVIDERS + + assert "gigachat" in LITELLM_CHAT_PROVIDERS + + def test_gigachat_key_exists(self): + """gigachat_key should be available""" + import litellm + + assert hasattr(litellm, "gigachat_key") + + def test_gigachat_config_exists(self): + """GigaChatConfig should be available""" + import litellm + + assert hasattr(litellm, "GigaChatConfig") + + +class TestGigaChatTransformRequest: + """Tests for request transformation""" + + @pytest.fixture + def config(self): + from litellm.llms.gigachat.chat.transformation import GigaChatConfig + return GigaChatConfig() + + def test_basic_request(self, config): + """Basic request should be transformed correctly""" + messages = [{"role": "user", "content": "Hello"}] + result = config.transform_request( + model="gigachat/GigaChat", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert result["model"] == "GigaChat" + assert len(result["messages"]) == 1 + assert result["messages"][0]["role"] == "user" + + def test_request_with_temperature(self, config): + """Request with temperature should include it""" + messages = [{"role": "user", "content": "Hello"}] + result = config.transform_request( + model="gigachat/GigaChat", + messages=messages, + optional_params={"temperature": 0.7}, + litellm_params={}, + headers={}, + ) + + assert result["temperature"] == 0.7 + + def test_request_with_functions(self, config): + """Request with functions should include them""" + messages = [{"role": "user", "content": "Hello"}] + functions = [{"name": "test", "description": "Test", "parameters": {}}] + result = config.transform_request( + model="gigachat/GigaChat", + messages=messages, + optional_params={"functions": functions}, + litellm_params={}, + headers={}, + ) + + assert "functions" in result + assert len(result["functions"]) == 1 + + +class TestGigaChatSupportedParams: + """Tests for supported parameters""" + + @pytest.fixture + def config(self): + from litellm.llms.gigachat.chat.transformation import GigaChatConfig + return GigaChatConfig() + + def test_supported_params(self, config): + """Check supported parameters list""" + supported = config.get_supported_openai_params("GigaChat") + + assert "temperature" in supported + assert "max_tokens" in supported + assert "max_completion_tokens" in supported + assert "tools" in supported + assert "response_format" in supported + assert "stream" in supported diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index 2b01b4c2a12..5e92c10fbdc 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -286,7 +286,7 @@ def test_completion_claude_3_empty_response(): }, ] try: - response = litellm.completion(model="claude-3-opus-20240229", messages=messages) + response = litellm.completion(model="claude-3-7-sonnet-20250219", messages=messages) print(response) except litellm.InternalServerError as e: pytest.skip(f"InternalServerError - {str(e)}") @@ -313,7 +313,7 @@ def test_completion_claude_3(): try: # test without max tokens response = completion( - model="anthropic/claude-3-opus-20240229", + model="anthropic/claude-3-7-sonnet-20250219", messages=messages, ) # Add any assertions, here to check response args @@ -326,7 +326,7 @@ def test_completion_claude_3(): @pytest.mark.parametrize( "model", - ["anthropic/claude-3-opus-20240229", "anthropic.claude-3-sonnet-20240229-v1:0"], + ["anthropic/claude-3-7-sonnet-20250219", "anthropic.claude-3-sonnet-20240229-v1:0"], ) def test_completion_claude_3_function_call(model): litellm.set_verbose = True @@ -411,7 +411,7 @@ def test_completion_claude_3_function_call(model): "model, api_key, api_base", [ ("gpt-3.5-turbo", None, None), - ("claude-3-opus-20240229", None, None), + ("claude-3-7-sonnet-20250219", None, None), ("anthropic.claude-3-sonnet-20240229-v1:0", None, None), # ( # "azure_ai/command-r-plus", @@ -512,7 +512,7 @@ async def test_anthropic_no_content_error(): try: litellm.drop_params = True response = await litellm.acompletion( - model="anthropic/claude-3-opus-20240229", + model="anthropic/claude-3-7-sonnet-20250219", api_key=os.getenv("ANTHROPIC_API_KEY"), messages=[ { @@ -630,7 +630,7 @@ def test_completion_claude_3_multi_turn_conversations(): ] try: response = completion( - model="anthropic/claude-3-opus-20240229", + model="anthropic/claude-3-7-sonnet-20250219", messages=messages, ) print(response) @@ -644,7 +644,7 @@ def test_completion_claude_3_stream(): try: # test without max tokens response = completion( - model="anthropic/claude-3-opus-20240229", + model="anthropic/claude-3-7-sonnet-20250219", messages=messages, max_tokens=10, stream=True, @@ -669,7 +669,7 @@ def encode_image(image_path): [ "gpt-4o", "azure/gpt-4.1-mini", - "anthropic/claude-3-opus-20240229", + "anthropic/claude-3-7-sonnet-20250219", ], ) # def test_completion_base64(model): diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index 0d9f84a301c..b9b5d0fdb07 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -1418,7 +1418,7 @@ def test_bedrock_claude_3_streaming(): @pytest.mark.parametrize( "model", [ - "claude-3-opus-20240229", + "claude-3-7-sonnet-20250219", "cohere.command-r-plus-v1:0", # bedrock "gpt-3.5-turbo", ], @@ -2914,7 +2914,7 @@ def test_completion_claude_3_function_call_with_streaming(): try: # test without max tokens response = completion( - model="claude-3-opus-20240229", + model="claude-3-7-sonnet-20250219", messages=messages, tools=tools, tool_choice="required", @@ -2946,7 +2946,7 @@ def test_completion_claude_3_function_call_with_streaming(): "model", [ "gemini/gemini-2.5-flash-lite", - ], # "claude-3-opus-20240229" + ], ) # @pytest.mark.asyncio async def test_acompletion_function_call_with_streaming(model): diff --git a/tests/logging_callback_tests/test_langsmith_unit_test.py b/tests/logging_callback_tests/test_langsmith_unit_test.py index e63ce9f8b38..bde2b944579 100644 --- a/tests/logging_callback_tests/test_langsmith_unit_test.py +++ b/tests/logging_callback_tests/test_langsmith_unit_test.py @@ -47,6 +47,19 @@ async def test_get_credentials_from_env(): credentials = logger.get_credentials_from_env() assert credentials["LANGSMITH_BASE_URL"] == "https://api.smith.langchain.com" + # Test with tenant_id + credentials = logger.get_credentials_from_env( + langsmith_tenant_id="test-tenant-id" + ) + assert credentials["LANGSMITH_TENANT_ID"] == "test-tenant-id" + + # Test tenant_id from environment variable + import os + os.environ["LANGSMITH_TENANT_ID"] = "env-tenant-id" + credentials = logger.get_credentials_from_env() + assert credentials["LANGSMITH_TENANT_ID"] == "env-tenant-id" + del os.environ["LANGSMITH_TENANT_ID"] + @pytest.mark.asyncio async def test_group_batches_by_credentials(): @@ -60,6 +73,7 @@ async def test_group_batches_by_credentials(): "LANGSMITH_API_KEY": "key1", "LANGSMITH_PROJECT": "proj1", "LANGSMITH_BASE_URL": "url1", + "LANGSMITH_TENANT_ID": None, }, ) @@ -69,6 +83,7 @@ async def test_group_batches_by_credentials(): "LANGSMITH_API_KEY": "key1", "LANGSMITH_PROJECT": "proj1", "LANGSMITH_BASE_URL": "url1", + "LANGSMITH_TENANT_ID": None, }, ) @@ -95,6 +110,7 @@ async def test_group_batches_by_credentials_multiple_credentials(): "LANGSMITH_API_KEY": "key1", "LANGSMITH_PROJECT": "proj1", "LANGSMITH_BASE_URL": "url1", + "LANGSMITH_TENANT_ID": None, }, ) @@ -104,6 +120,7 @@ async def test_group_batches_by_credentials_multiple_credentials(): "LANGSMITH_API_KEY": "key2", # Different API key "LANGSMITH_PROJECT": "proj1", "LANGSMITH_BASE_URL": "url1", + "LANGSMITH_TENANT_ID": None, }, ) @@ -113,6 +130,7 @@ async def test_group_batches_by_credentials_multiple_credentials(): "LANGSMITH_API_KEY": "key1", "LANGSMITH_PROJECT": "proj2", # Different project "LANGSMITH_BASE_URL": "url1", + "LANGSMITH_TENANT_ID": None, }, ) @@ -127,6 +145,57 @@ async def test_group_batches_by_credentials_multiple_credentials(): assert len(batch_group.queue_objects) == 1 # Each group should have one object +@pytest.mark.asyncio +async def test_group_batches_by_credentials_with_tenant_id(): + + # Test that different tenant_ids create separate groups + logger = LangsmithLogger(langsmith_api_key="test-key") + + queue_obj1 = LangsmithQueueObject( + data={"test": "data1"}, + credentials={ + "LANGSMITH_API_KEY": "key1", + "LANGSMITH_PROJECT": "proj1", + "LANGSMITH_BASE_URL": "url1", + "LANGSMITH_TENANT_ID": "tenant1", + }, + ) + + queue_obj2 = LangsmithQueueObject( + data={"test": "data2"}, + credentials={ + "LANGSMITH_API_KEY": "key1", + "LANGSMITH_PROJECT": "proj1", + "LANGSMITH_BASE_URL": "url1", + "LANGSMITH_TENANT_ID": "tenant2", # Different tenant_id + }, + ) + + queue_obj3 = LangsmithQueueObject( + data={"test": "data3"}, + credentials={ + "LANGSMITH_API_KEY": "key1", + "LANGSMITH_PROJECT": "proj1", + "LANGSMITH_BASE_URL": "url1", + "LANGSMITH_TENANT_ID": "tenant1", # Same as queue_obj1 + }, + ) + + logger.log_queue = [queue_obj1, queue_obj2, queue_obj3] + + grouped = logger._group_batches_by_credentials() + + # Should have two groups: one for tenant1 (queue_obj1 and queue_obj3), one for tenant2 (queue_obj2) + assert len(grouped) == 2 + for key, batch_group in grouped.items(): + assert isinstance(key, CredentialsKey) + assert key.tenant_id in ["tenant1", "tenant2"] + if key.tenant_id == "tenant1": + assert len(batch_group.queue_objects) == 2 + else: + assert len(batch_group.queue_objects) == 1 + + # Test make_dot_order @pytest.mark.asyncio async def test_make_dot_order(): @@ -201,10 +270,43 @@ async def test_async_send_batch(): call_args = logger.async_httpx_client.post.call_args assert "runs/batch" in call_args[1]["url"] assert "x-api-key" in call_args[1]["headers"] + # tenant_id should not be in headers if not provided + assert "x-tenant-id" not in call_args[1]["headers"] @pytest.mark.asyncio -async def test_langsmith_key_based_logging(mocker): +async def test_async_send_batch_with_tenant_id(): + logger = LangsmithLogger( + langsmith_api_key="test-key", + langsmith_tenant_id="test-tenant-id" + ) + + # Mock the httpx client + mock_response = AsyncMock() + mock_response.status_code = 200 + logger.async_httpx_client = AsyncMock() + logger.async_httpx_client.post.return_value = mock_response + + # Add test data to queue + logger.log_queue = [ + LangsmithQueueObject( + data={"test": "data"}, credentials=logger.default_credentials + ) + ] + + await logger.async_send_batch() + + # Verify the API call includes tenant_id header + logger.async_httpx_client.post.assert_called_once() + call_args = logger.async_httpx_client.post.call_args + assert "runs/batch" in call_args[1]["url"] + assert "x-api-key" in call_args[1]["headers"] + assert "x-tenant-id" in call_args[1]["headers"] + assert call_args[1]["headers"]["x-tenant-id"] == "test-tenant-id" + + +@pytest.mark.asyncio +async def test_langsmith_key_based_logging(): """ In key based logging langsmith_api_key and langsmith_project are passed directly to litellm.acompletion """ @@ -219,10 +321,11 @@ async def test_langsmith_key_based_logging(mocker): mock_response.text = "" mock_async_httpx_handler.post = AsyncMock(return_value=mock_response) - mock_get_client = mocker.patch( + mock_get_client = patch( "litellm.integrations.langsmith.get_async_httpx_client", return_value=mock_async_httpx_handler ) + mock_get_client.start() litellm.set_verbose = True litellm.DEFAULT_FLUSH_INTERVAL_SECONDS = 1 @@ -253,6 +356,8 @@ async def test_langsmith_key_based_logging(mocker): # Check headers contain the correct API key assert call_args[1]["headers"]["x-api-key"] == "fake_key_project2" + # tenant_id should not be in headers if not provided + assert "x-tenant-id" not in call_args[1]["headers"] # Verify the request body contains the expected data request_body = call_args[1]["json"] @@ -344,6 +449,8 @@ async def test_langsmith_key_based_logging(mocker): actual_body["post"][0]["session_name"] == expected_body["post"][0]["session_name"] ) + + mock_get_client.stop() except Exception as e: pytest.fail(f"Error occurred: {e}") diff --git a/tests/logging_callback_tests/test_opentelemetry_unit_tests.py b/tests/logging_callback_tests/test_opentelemetry_unit_tests.py index 3d0682d9033..04f8abe64de 100644 --- a/tests/logging_callback_tests/test_opentelemetry_unit_tests.py +++ b/tests/logging_callback_tests/test_opentelemetry_unit_tests.py @@ -65,31 +65,6 @@ class TestOpentelemetryUnitTests(BaseLoggingCallbackTest): # External spans should only be closed by their creators parent_otel_span.end.assert_not_called() - def test_init_tracing_respects_existing_tracer_provider(self): - """ - Unit test: _init_tracing() should respect existing TracerProvider. - - When a TracerProvider already exists (e.g., set by Langfuse SDK), - LiteLLM should use it instead of creating a new one. - """ - from opentelemetry import trace - from opentelemetry.sdk.trace import TracerProvider - from litellm.integrations.opentelemetry import OpenTelemetry - - # Setup: Create and set an existing TracerProvider - tracer_provider = TracerProvider() - trace.set_tracer_provider(tracer_provider) - existing_provider = trace.get_tracer_provider() - - # Act: Initialize OpenTelemetry integration (should detect existing provider) - otel_integration = OpenTelemetry() - - # Assert: The existing provider should still be active - current_provider = trace.get_tracer_provider() - assert current_provider is existing_provider, ( - "Existing TracerProvider should be respected and not overridden" - ) - def test_get_span_context_detects_active_span(self): """ Unit test: _get_span_context() should auto-detect active spans from global context. diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 40e223ffe07..45aae3b9aee 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -79,6 +79,73 @@ def test_routing_strategy_init(model_list): ) +def test_routing_strategy_init_invalid_strategy(model_list): + """Test that invalid routing_strategy raises ValueError with helpful message. + + See: https://github.com/BerriAI/litellm/issues/11330 + Invalid strategies like 'simple' (without '-shuffle') should fail fast + with a clear error, not silently cause 'No deployments available' errors. + """ + router = Router(model_list=model_list) + + # Test common mistake: "simple" instead of "simple-shuffle" + with pytest.raises(ValueError) as exc_info: + router.routing_strategy_init( + routing_strategy="simple", + routing_strategy_args={} + ) + + # Verify error message is helpful + error_msg = str(exc_info.value) + assert "Invalid routing_strategy" in error_msg + assert "simple" in error_msg + assert "simple-shuffle" in error_msg # Suggests the correct option + # Verify error message tells user WHERE to fix it + assert "config.yaml" in error_msg + assert "router_settings.routing_strategy" in error_msg + assert "Router SDK" in error_msg + + # Test completely invalid strategy + with pytest.raises(ValueError) as exc_info: + router.routing_strategy_init( + routing_strategy="not-a-real-strategy", + routing_strategy_args={} + ) + assert "Invalid routing_strategy" in str(exc_info.value) + + +def test_routing_strategy_init_valid_string_strategies(model_list): + """Test that all valid string routing strategies work without error. + + Valid strategies are derived from RoutingStrategy enum values plus 'simple-shuffle'. + """ + from litellm.types.router import RoutingStrategy + + router = Router(model_list=model_list) + + # All strategies from enum + simple-shuffle (default, not in enum) + valid_strategies = ["simple-shuffle"] + [s.value for s in RoutingStrategy] + + for strategy in valid_strategies: + # Should not raise + router.routing_strategy_init( + routing_strategy=strategy, routing_strategy_args={} + ) + + +def test_routing_strategy_init_valid_enum_strategies(model_list): + """Test that RoutingStrategy enum values work without error.""" + from litellm.types.router import RoutingStrategy + + router = Router(model_list=model_list) + + for strategy in RoutingStrategy: + # Should not raise when passing enum directly + router.routing_strategy_init( + routing_strategy=strategy, routing_strategy_args={} + ) + + def test_print_deployment(model_list): """Test if the api key is masked correctly""" diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 6490352c39b..596398e639f 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -1011,39 +1011,87 @@ def test_multiple_tool_calls_in_single_choice(): def test_map_reasoning_effort_adds_summary_detailed(): """ - Test that _map_reasoning_effort adds summary="detailed" when user provides reasoning_effort as a string. + Test that _map_reasoning_effort behavior with reasoning_auto_summary flag. - This ensures that when users pass reasoning_effort in the completions API for OpenAI responses/models, - the transformation automatically includes summary="detailed" in the reasoning parameter. + By default (flag=False), summary should NOT be added to avoid: + 1. Breaking for users without verified OpenAI orgs (400 errors) + 2. Making requests more expensive by including summary reasoning tokens + + When flag is enabled (flag=True or env var), summary="detailed" is added. """ + import os + + import litellm from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, ) handler = LiteLLMResponsesTransformationHandler() - # Test all string effort levels + # Test all string effort levels - DEFAULT BEHAVIOR (no summary) effort_levels = ["none", "low", "medium", "high", "xhigh", "minimal"] - for effort in effort_levels: - result = handler._map_reasoning_effort(effort) + # Save original flag value + original_flag = litellm.reasoning_auto_summary + original_env = os.environ.get("LITELLM_REASONING_AUTO_SUMMARY") + + try: + # Test 1: Default behavior (flag=False, no env var) - NO summary + litellm.reasoning_auto_summary = False + if "LITELLM_REASONING_AUTO_SUMMARY" in os.environ: + del os.environ["LITELLM_REASONING_AUTO_SUMMARY"] - assert result is not None, f"Result should not be None for effort={effort}" - assert result["effort"] == effort, f"Effort should be {effort}" - assert result["summary"] == "detailed", f"Summary should be 'detailed' for effort={effort}" + for effort in effort_levels: + result = handler._map_reasoning_effort(effort) + + assert result is not None, f"Result should not be None for effort={effort}" + assert result["effort"] == effort, f"Effort should be {effort}" + assert "summary" not in result, f"Summary should NOT be present by default for effort={effort}" + + print(f"✓ reasoning_effort='{effort}' correctly maps to effort='{effort}' (no summary by default)") - print(f"✓ reasoning_effort='{effort}' correctly maps to effort='{effort}', summary='detailed'") + # Test 2: With flag enabled - summary IS added + litellm.reasoning_auto_summary = True + + for effort in effort_levels: + result = handler._map_reasoning_effort(effort) + + assert result is not None, f"Result should not be None for effort={effort}" + assert result["effort"] == effort, f"Effort should be {effort}" + assert result["summary"] == "detailed", f"Summary should be 'detailed' when flag is enabled for effort={effort}" + + print(f"✓ reasoning_effort='{effort}' correctly maps to effort='{effort}', summary='detailed' (flag enabled)") + + # Test 3: With env var enabled (flag disabled) - summary IS added + litellm.reasoning_auto_summary = False + os.environ["LITELLM_REASONING_AUTO_SUMMARY"] = "true" + + result = handler._map_reasoning_effort("high") + assert result["summary"] == "detailed", "Summary should be 'detailed' when env var is enabled" + print("✓ LITELLM_REASONING_AUTO_SUMMARY env var works correctly") + + # Test 4: Dict input is passed through as-is (no modification) + litellm.reasoning_auto_summary = False + if "LITELLM_REASONING_AUTO_SUMMARY" in os.environ: + del os.environ["LITELLM_REASONING_AUTO_SUMMARY"] + + dict_input = {"effort": "high", "summary": "custom_summary"} + result_dict = handler._map_reasoning_effort(dict_input) + assert result_dict["effort"] == "high" + assert result_dict["summary"] == "custom_summary" + print("✓ Dict input is passed through without modification") + + # Test 5: None/unknown values return None + result_unknown = handler._map_reasoning_effort("unknown_value") + assert result_unknown is None + print("✓ Unknown reasoning_effort values return None") + + print("✓ All reasoning_effort behaviors work correctly with flag/env var control") - # Test that dict input is passed through as-is (no modification) - dict_input = {"effort": "high", "summary": "custom_summary"} - result_dict = handler._map_reasoning_effort(dict_input) - assert result_dict["effort"] == "high" - assert result_dict["summary"] == "custom_summary" - print("✓ Dict input is passed through without modification") - - # Test that None/unknown values return None - result_unknown = handler._map_reasoning_effort("unknown_value") - assert result_unknown is None - print("✓ Unknown reasoning_effort values return None") - - print("✓ All reasoning_effort string values correctly map to summary='detailed'") + finally: + # Restore original values + litellm.reasoning_auto_summary = original_flag + if original_env is not None: + os.environ["LITELLM_REASONING_AUTO_SUMMARY"] = original_env + elif "LITELLM_REASONING_AUTO_SUMMARY" in os.environ: + del os.environ["LITELLM_REASONING_AUTO_SUMMARY"] diff --git a/tests/test_litellm/integrations/langfuse/test_gemini_cached_tokens.py b/tests/test_litellm/integrations/langfuse/test_gemini_cached_tokens.py new file mode 100644 index 00000000000..e717840ec95 --- /dev/null +++ b/tests/test_litellm/integrations/langfuse/test_gemini_cached_tokens.py @@ -0,0 +1,90 @@ +""" +Test for Langfuse integration with Gemini cached_tokens bug +https://github.com/BerriAI/litellm/issues/18520 +""" +import pytest +from litellm.types.utils import PromptTokensDetailsWrapper, Usage + + +def test_cached_tokens_extraction(): + """ + Test that we can extract cached_tokens from prompt_tokens_details. + This is the core logic fix for https://github.com/BerriAI/litellm/issues/18520 + """ + # Create usage object like Gemini returns + usage = Usage( + prompt_tokens=20209, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=20203, + text_tokens=6, + ), + completion_tokens=541, + ) + + # Simulate the logic from langfuse.py lines 745-757 (after the fix) + cache_read_input_tokens = 0 # Default value + + # Check prompt_tokens_details.cached_tokens (the fix) + if hasattr(usage, "prompt_tokens_details"): + prompt_tokens_details = getattr(usage, "prompt_tokens_details", None) + if ( + prompt_tokens_details is not None + and hasattr(prompt_tokens_details, "cached_tokens") + ): + cached_tokens = getattr(prompt_tokens_details, "cached_tokens", None) + if cached_tokens is not None and cached_tokens > 0: + cache_read_input_tokens = cached_tokens + + # Verify the fix works + assert cache_read_input_tokens == 20203, f"Expected 20203, got {cache_read_input_tokens}" + + +def test_cached_tokens_not_present(): + """Test backward compatibility when cached_tokens is not present""" + # Usage without prompt_tokens_details + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + ) + + cache_read_input_tokens = 0 + + if hasattr(usage, "prompt_tokens_details"): + prompt_tokens_details = getattr(usage, "prompt_tokens_details", None) + if ( + prompt_tokens_details is not None + and hasattr(prompt_tokens_details, "cached_tokens") + ): + cached_tokens = getattr(prompt_tokens_details, "cached_tokens", None) + if cached_tokens is not None and cached_tokens > 0: + cache_read_input_tokens = cached_tokens + + # Should remain 0 + assert cache_read_input_tokens == 0 + + +def test_cached_tokens_is_zero(): + """Test when cached_tokens is explicitly 0""" + usage = Usage( + prompt_tokens=100, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=0, + text_tokens=100, + ), + completion_tokens=50, + ) + + cache_read_input_tokens = 0 + + if hasattr(usage, "prompt_tokens_details"): + prompt_tokens_details = getattr(usage, "prompt_tokens_details", None) + if ( + prompt_tokens_details is not None + and hasattr(prompt_tokens_details, "cached_tokens") + ): + cached_tokens = getattr(prompt_tokens_details, "cached_tokens", None) + if cached_tokens is not None and cached_tokens > 0: + cache_read_input_tokens = cached_tokens + + # Should remain 0 when cached_tokens is 0 + assert cache_read_input_tokens == 0 diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 5d648f601f6..6c17570e135 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -172,6 +172,86 @@ class TestOpenTelemetryCostBreakdown(unittest.TestCase): assert ("gen_ai.cost.original_cost", 0.004) not in call_args_list +class TestOpenTelemetryProviderInitialization(unittest.TestCase): + """Test suite for verifying provider initialization respects existing providers""" + + def test_init_tracing_respects_existing_tracer_provider(self): + """ + Unit test: _init_tracing() should respect existing TracerProvider. + + When a TracerProvider already exists (e.g., set by Langfuse SDK), + LiteLLM should use it instead of creating a new one. + """ + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + + # Setup: Create and set an existing TracerProvider + tracer_provider = TracerProvider() + trace.set_tracer_provider(tracer_provider) + existing_provider = trace.get_tracer_provider() + + # Act: Initialize OpenTelemetry integration (should detect existing provider) + otel_integration = OpenTelemetry() + + # Assert: The existing provider should still be active + current_provider = trace.get_tracer_provider() + assert current_provider is existing_provider, ( + "Existing TracerProvider should be respected and not overridden" + ) + + @patch.dict(os.environ, {"LITELLM_OTEL_INTEGRATION_ENABLE_METRICS": "true"}, clear=True) + def test_init_metrics_respects_existing_meter_provider(self): + """ + Unit test: _init_metrics() should respect existing MeterProvider. + + When a MeterProvider already exists (e.g., set by Langfuse SDK), + LiteLLM should use it instead of creating a new one. + """ + from opentelemetry import metrics + from opentelemetry.sdk.metrics import MeterProvider + + # Create and set an existing MeterProvider + meter_provider = MeterProvider() + metrics.set_meter_provider(meter_provider) + existing_provider = metrics.get_meter_provider() + + # Act: Initialize OpenTelemetry integration (should detect existing provider) + config = OpenTelemetryConfig.from_env() + otel_integration = OpenTelemetry(config=config) + + # Assert: The existing provider should still be active + current_provider = metrics.get_meter_provider() + assert current_provider is existing_provider, ( + "Existing MeterProvider should be respected and not overridden" + ) + + @patch.dict(os.environ, {"LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS": "true"}, clear=True) + def test_init_logs_respects_existing_logger_provider(self): + """ + Unit test: _init_logs() should respect existing LoggerProvider. + + When a LoggerProvider already exists (e.g., set by Langfuse SDK), + LiteLLM should use it instead of creating a new one. + """ + from opentelemetry._logs import get_logger_provider, set_logger_provider + from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider + + # Create and set an existing LoggerProvider + logger_provider = OTLoggerProvider() + set_logger_provider(logger_provider) + existing_provider = get_logger_provider() + + # Act: Initialize OpenTelemetry integration (should detect existing provider) + config = OpenTelemetryConfig.from_env() + otel_integration = OpenTelemetry(config=config) + + # Assert: The existing provider should still be active + current_provider = get_logger_provider() + assert current_provider is existing_provider, ( + "Existing LoggerProvider should be respected and not overridden" + ) + + class TestOpenTelemetry(unittest.TestCase): POLL_INTERVAL = 0.05 POLL_TIMEOUT = 2.0 @@ -620,7 +700,6 @@ class TestOpenTelemetry(unittest.TestCase): self.assertEqual(attributes.get("extra.attr"), "extra-value") - def test_handle_success_spans_only(self): # make sure neither events nor metrics is on os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", None) @@ -687,11 +766,8 @@ class TestOpenTelemetry(unittest.TestCase): logs = log_exporter.get_finished_logs() self.assertFalse(logs, "Did not expect any logs") + @patch.dict(os.environ, {"LITELLM_OTEL_INTEGRATION_ENABLE_METRICS": "true"}, clear=True) def test_handle_success_spans_and_metrics(self): - # only metrics on - os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", None) - os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_METRICS"] = "true" - # ─── build in‐memory OTEL providers/exporters ───────────────────────────── span_exporter = InMemorySpanExporter() tracer_provider = TracerProvider() @@ -1320,6 +1396,23 @@ class TestOpenTelemetryProtocolSelection(unittest.TestCase): ) self.assertEqual(normalized, "http://collector:4317/v1/logs") + def test_get_metric_reader_uses_http_exporter_for_http_protobuf(self): + """Test that http/protobuf protocol uses OTLPMetricExporterHTTP""" + from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter, + ) + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + + config = OpenTelemetryConfig( + exporter="http/protobuf", endpoint="http://collector:4318" + ) + otel = OpenTelemetry(config=config) + + reader = otel._get_metric_reader() + + self.assertIsInstance(reader, PeriodicExportingMetricReader) + self.assertIsInstance(reader._exporter, OTLPMetricExporter) + class TestOpenTelemetryExternalSpan(unittest.TestCase): """ diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 6a528fef8f0..ec2f528a35d 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -691,6 +691,29 @@ async def test_streaming_completion_start_time(logging_obj: Logging): ) +@pytest.mark.asyncio +async def test_vertex_streaming_bad_request_not_midstream(logging_obj: Logging): + """Ensure Vertex bad request errors surface as 400, not mid-stream fallbacks.""" + from litellm.llms.vertex_ai.common_utils import VertexAIError + + async def _raise_bad_request(**kwargs): + raise VertexAIError(status_code=400, message="invalid maxOutputTokens", headers=None) + + response = CustomStreamWrapper( + completion_stream=None, + model="gemini-3-pro-preview", + logging_obj=logging_obj, + custom_llm_provider="vertex_ai_beta", + make_call=_raise_bad_request, + ) + + with pytest.raises(litellm.BadRequestError) as excinfo: + await response.__anext__() + + assert getattr(excinfo.value, "status_code", None) == 400 + assert "invalid maxOutputTokens" in str(excinfo.value) + + def test_streaming_handler_with_created_time_propagation( initialized_custom_stream_wrapper: CustomStreamWrapper, logging_obj: Logging ): diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_function_call_args_serialization.py b/tests/test_litellm/llms/vertex_ai/gemini/test_function_call_args_serialization.py new file mode 100644 index 00000000000..0f369fbb8b9 --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_function_call_args_serialization.py @@ -0,0 +1,355 @@ +""" +Test cases for functionCall args serialization in Vertex AI Gemini. + +This test file specifically tests the edge cases where Vertex AI might return +functionCall args in unexpected formats that could lead to invalid JSON strings +like: {"x":"x"}{"a":"a"} +""" +import json +from typing import List, Optional + +import pytest + +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, +) +from litellm.types.llms.vertex_ai import HttpxPartType + + +class TestFunctionCallArgsSerialization: + """Test cases for functionCall args serialization edge cases.""" + + def test_normal_dict_args(self): + """Test normal case: args is a dict.""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": {"location": "Boston", "unit": "celsius"}, + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + assert tools[0]["function"]["name"] == "get_weather" + + # Verify arguments is a valid JSON string + arguments = tools[0]["function"]["arguments"] + assert isinstance(arguments, str) + # Should be valid JSON + parsed = json.loads(arguments) + assert parsed == {"location": "Boston", "unit": "celsius"} + + def test_none_args(self): + """Test case: args is None.""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": None, + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + # Should serialize None to "null" or empty dict + assert isinstance(arguments, str) + parsed = json.loads(arguments) + # json.dumps(None) returns "null" + assert parsed is None or parsed == {} + + def test_args_as_string_valid_json(self): + """Test case: args is already a valid JSON string.""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": '{"location": "Boston"}', # String, not dict + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + # If args is a string, json.dumps will double-encode it + # This would result in: "{\"location\": \"Boston\"}" + assert isinstance(arguments, str) + # This is the problematic case - string gets double-encoded + # The result would be a JSON string containing a JSON string + parsed = json.loads(arguments) + # If it's double-encoded, parsed would be a string, not a dict + if isinstance(parsed, str): + # Double-encoded case + inner_parsed = json.loads(parsed) + assert inner_parsed == {"location": "Boston"} + else: + # Normal case (shouldn't happen if args is string) + assert parsed == {"location": "Boston"} + + def test_args_as_string_invalid_json_concatenated(self): + """Test case: args is a string with concatenated JSON objects (the bug case). + + When args is a string like '{"x":"x"}{"a":"a"}', json.dumps() will serialize it + as a JSON string, resulting in: "{\"x\":\"x\"}{\"a\":\"a\"}" + This is a valid JSON string (the outer quotes), but the content inside is invalid JSON. + When you try to parse the inner content, it fails. + """ + # This simulates the case where Vertex might return something like: + # args = '{"x":"x"}{"a":"a"}' # Two JSON objects concatenated + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": '{"x":"x"}{"a":"a"}', # Invalid concatenated JSON + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + assert isinstance(arguments, str) + + # json.dumps() on a string will escape it, so we get: + # arguments = '"{\\"x\\":\\"x\\"}{\\"a\\":\\"a\\"}"' + # This is a valid JSON string (the outer quotes), but the inner content is invalid + parsed_outer = json.loads(arguments) + assert isinstance(parsed_outer, str) + + # The inner string is invalid JSON (two objects concatenated) + # This is the bug: the inner content cannot be parsed as valid JSON + with pytest.raises(json.JSONDecodeError): + json.loads(parsed_outer) + + # The arguments string would be: "{\"x\":\"x\"}{\"a\":\"a\"}" + # Which when parsed gives: '{"x":"x"}{"a":"a"}' (invalid JSON) + + def test_args_as_array(self): + """Test case: args is an array (unexpected but possible).""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": [{"x": "x"}, {"a": "a"}], # Array of objects + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + assert isinstance(arguments, str) + # Should serialize array correctly + parsed = json.loads(arguments) + assert parsed == [{"x": "x"}, {"a": "a"}] + + def test_args_missing_key(self): + """Test case: args key is missing from functionCall. + + This will raise a KeyError because the code directly accesses part["functionCall"]["args"] + without checking if the key exists. This is a bug that should be fixed. + """ + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + # args key missing + } + } + ] + + # This should raise KeyError because args key is missing + with pytest.raises(KeyError): + VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + def test_multiple_function_calls(self): + """Test case: multiple function calls in parts.""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": {"location": "Boston"}, + } + }, + { + "functionCall": { + "name": "get_time", + "args": {"timezone": "EST"}, + } + }, + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 2 + assert tools[0]["function"]["name"] == "get_weather" + assert tools[1]["function"]["name"] == "get_time" + + # Both should have valid JSON arguments + args1 = json.loads(tools[0]["function"]["arguments"]) + args2 = json.loads(tools[1]["function"]["arguments"]) + assert args1 == {"location": "Boston"} + assert args2 == {"timezone": "EST"} + + def test_args_with_vertex_protobuf_format(self): + """Test case: args in Vertex protobuf format with string_value, etc.""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": { + "location": {"string_value": "Boston, MA"}, + "unit": {"string_value": "celsius"}, + }, + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + assert isinstance(arguments, str) + # Should serialize the nested structure correctly + parsed = json.loads(arguments) + assert "location" in parsed + assert "unit" in parsed + + def test_args_as_empty_dict(self): + """Test case: args is an empty dict.""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": {}, + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + assert isinstance(arguments, str) + parsed = json.loads(arguments) + assert parsed == {} + + def test_args_with_special_characters(self): + """Test case: args contains special characters that need escaping.""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": { + "location": 'Boston, MA "downtown"', + "note": "Line 1\nLine 2", + }, + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + assert isinstance(arguments, str) + # Should handle special characters correctly + parsed = json.loads(arguments) + assert parsed["location"] == 'Boston, MA "downtown"' + assert parsed["note"] == "Line 1\nLine 2" + + def test_args_as_list_of_strings_that_look_like_json(self): + """Test case: args is a list containing strings that look like JSON objects.""" + # This could potentially cause issues if not handled correctly + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "get_weather", + "args": ['{"x":"x"}', '{"a":"a"}'], # List of JSON strings + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + assert isinstance(arguments, str) + # Should serialize list correctly + parsed = json.loads(arguments) + assert isinstance(parsed, list) + assert parsed == ['{"x":"x"}', '{"a":"a"}'] + + def test_args_as_dict_with_nested_structures(self): + """Test case: args contains nested dicts and lists.""" + parts: List[HttpxPartType] = [ + { + "functionCall": { + "name": "complex_function", + "args": { + "nested": {"key": "value"}, + "list": [1, 2, 3], + "mixed": [{"a": 1}, {"b": 2}], + }, + } + } + ] + + function, tools, idx = VertexGeminiConfig._transform_parts( + parts=parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + assert tools is not None + assert len(tools) == 1 + arguments = tools[0]["function"]["arguments"] + assert isinstance(arguments, str) + parsed = json.loads(arguments) + assert parsed["nested"] == {"key": "value"} + assert parsed["list"] == [1, 2, 3] + assert parsed["mixed"] == [{"a": 1}, {"b": 2}] + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) + diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 91a28ee6ec9..d09de3a0f26 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -10,11 +10,13 @@ from pydantic import BaseModel import litellm from litellm import ModelResponse, completion +from litellm.llms.vertex_ai.common_utils import VertexAIError from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, ) from litellm.types.llms.vertex_ai import UsageMetadata from litellm.types.utils import ChoiceLogprobs, Usage +from litellm.utils import CustomStreamWrapper def test_top_logprobs(): @@ -1605,6 +1607,39 @@ def test_vertex_ai_annotation_streaming_events(): assert "Weather information" in annotation["url_citation"]["title"] +@pytest.mark.asyncio +async def test_vertex_ai_streaming_bad_request_is_not_wrapped(): + class DummyLogging: + def __init__(self): + self.model_call_details = {"litellm_params": {}} + self.optional_params = {} + self.messages = [] + self.completion_start_time = None + self.stream_options = None + + def failure_handler(self, *args, **kwargs): + return None + + async def async_failure_handler(self, *args, **kwargs): + return None + + async def failing_make_call(client=None, **kwargs): + raise VertexAIError(status_code=400, message="bad input", headers={}) + + stream = CustomStreamWrapper( + completion_stream=None, + make_call=failing_make_call, + model="gemini-3-pro-preview", + logging_obj=DummyLogging(), + custom_llm_provider="vertex_ai_beta", + ) + + with pytest.raises(litellm.BadRequestError) as exc_info: + await stream.__anext__() + + assert getattr(exc_info.value, "status_code", None) == 400 + + def test_vertex_ai_annotation_conversion(): """ Test the conversion of Vertex AI grounding metadata to OpenAI annotations. diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 6df9abd3fee..4c5723b8284 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -354,7 +354,7 @@ async def test_register_client_remote_registration_success(): request_payload = { "client_name": "Litellm Proxy", - "grant_types": ["authorization_code"], + "grant_types": ["authorization_code", "refresh_token"], "response_types": ["code"], "token_endpoint_auth_method": "client_secret_post", } @@ -556,9 +556,33 @@ async def test_oauth_protected_resource_respects_x_forwarded_proto(): from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( oauth_protected_resource_mcp, ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + from litellm.proxy._types import MCPTransport from fastapi import Request except ImportError: pytest.skip("MCP discoverable endpoints not available") + # Clear registry + global_mcp_server_manager.registry.clear() + + # Create mock OAuth2 server + oauth2_server = MCPServer( + server_id="test_oauth_server", + name="test_oauth", + server_name="test_oauth", + alias="test_oauth", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id="test_client_id", + client_secret="test_client_secret", + authorization_url="https://provider.com/oauth/authorize", + token_url="https://provider.com/oauth/token", + scopes=["read", "write"], + ) + global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server # Mock request with http base_url but X-Forwarded-Proto: https mock_request = MagicMock(spec=Request) @@ -568,13 +592,14 @@ async def test_oauth_protected_resource_respects_x_forwarded_proto(): # Call the endpoint response = await oauth_protected_resource_mcp( request=mock_request, - mcp_server_name="test_server", + mcp_server_name="test_oauth", ) # Verify response uses HTTPS URLs assert response["authorization_servers"][0].startswith( "https://litellm.example.com/" ) + assert response["scopes_supported"] == oauth2_server.scopes @pytest.mark.asyncio @@ -584,9 +609,33 @@ async def test_oauth_authorization_server_respects_x_forwarded_proto(): from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( oauth_authorization_server_mcp, ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + from litellm.proxy._types import MCPTransport from fastapi import Request except ImportError: pytest.skip("MCP discoverable endpoints not available") + # Clear registry + global_mcp_server_manager.registry.clear() + + # Create mock OAuth2 server + oauth2_server = MCPServer( + server_id="test_oauth_server", + name="test_oauth", + server_name="test_oauth", + alias="test_oauth", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id="test_client_id", + client_secret="test_client_secret", + authorization_url="https://provider.com/oauth/authorize", + token_url="https://provider.com/oauth/token", + scopes=["read", "write"], + ) + global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server # Mock request with http base_url but X-Forwarded-Proto: https mock_request = MagicMock(spec=Request) @@ -596,13 +645,15 @@ async def test_oauth_authorization_server_respects_x_forwarded_proto(): # Call the endpoint response = await oauth_authorization_server_mcp( request=mock_request, - mcp_server_name="test_server", + mcp_server_name="test_oauth", ) # Verify response uses HTTPS URLs assert response["authorization_endpoint"].startswith("https://litellm.example.com/") assert response["token_endpoint"].startswith("https://litellm.example.com/") assert response["registration_endpoint"].startswith("https://litellm.example.com/") + assert response["grant_types_supported"] == ["authorization_code", "refresh_token"] + assert response["scopes_supported"] == oauth2_server.scopes @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 6491e11024a..d59b3f04ef5 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -594,7 +594,26 @@ class TestMCPServerManager: assert ( server.registration_url == "https://discovered.example.com/register" ) + @pytest.mark.asyncio + async def test_config_oauth_initialize_tool_name_to_mcp_server_name_mapping(self): + manager = MCPServerManager() + config = { + "example": { + "url": "https://example.com/mcp", + "transport": MCPTransport.http, + "auth_type": MCPAuth.oauth2, + "scopes": ["config"], + "authorization_url": "https://config.example.com/auth", + } + } + + await manager.load_servers_from_config(config) + + # Initialize the tool mapping + await manager._initialize_tool_name_to_mcp_server_name_mapping() + assert manager.tool_name_to_mcp_server_name_mapping == {} + @pytest.mark.asyncio async def test_list_tools_handles_missing_server_alias(self): """Test that list_tools handles servers without alias gracefully""" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_qualifire.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_qualifire.py index 6d6129b17bb..35ed49a84ed 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_qualifire.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_qualifire.py @@ -2,6 +2,7 @@ Unit tests for Qualifire guardrail integration. """ +import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -139,76 +140,98 @@ class TestQualifireGuardrailEvaluateKwargs: @pytest.mark.asyncio async def test_evaluate_called_with_prompt_injections(self): """Test that evaluate is called with prompt_injections enabled.""" - from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( - QualifireGuardrail, - ) + # Mock the qualifire module and its types + mock_qualifire_types = MagicMock() + mock_llm_message = MagicMock() + mock_llm_tool_call = MagicMock() + mock_message_instance = MagicMock() + mock_llm_message.return_value = mock_message_instance + + mock_qualifire_types.LLMMessage = mock_llm_message + mock_qualifire_types.LLMToolCall = mock_llm_tool_call + + with patch.dict('sys.modules', {'qualifire': MagicMock(), 'qualifire.types': mock_qualifire_types}): + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) - guardrail = QualifireGuardrail( - api_key="test_key", - prompt_injections=True, - guardrail_name="test_guardrail", - ) + guardrail = QualifireGuardrail( + api_key="test_key", + prompt_injections=True, + guardrail_name="test_guardrail", + ) - # Mock the client - mock_client = MagicMock() - mock_result = MagicMock() - mock_result.score = 100 - mock_result.status = "completed" - mock_result.evaluationResults = [] - mock_client.evaluate.return_value = mock_result - guardrail._client = mock_client + # Mock the client + mock_client = MagicMock() + mock_result = MagicMock() + mock_result.score = 100 + mock_result.status = "completed" + mock_result.evaluationResults = [] + mock_client.evaluate.return_value = mock_result + guardrail._client = mock_client - messages = [{"role": "user", "content": "Hello, world!"}] + messages = [{"role": "user", "content": "Hello, world!"}] - await guardrail._run_qualifire_check( - messages=messages, output=None, dynamic_params={} - ) + await guardrail._run_qualifire_check( + messages=messages, output=None, dynamic_params={} + ) - # Verify evaluate was called with correct kwargs - mock_client.evaluate.assert_called_once() - call_kwargs = mock_client.evaluate.call_args[1] - assert call_kwargs["prompt_injections"] is True - assert "messages" in call_kwargs + # Verify evaluate was called with correct kwargs + mock_client.evaluate.assert_called_once() + call_kwargs = mock_client.evaluate.call_args[1] + assert call_kwargs["prompt_injections"] is True + assert "messages" in call_kwargs @pytest.mark.asyncio async def test_evaluate_called_with_multiple_checks(self): """Test that evaluate is called with multiple checks enabled.""" - from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( - QualifireGuardrail, - ) + # Mock the qualifire module and its types + mock_qualifire_types = MagicMock() + mock_llm_message = MagicMock() + mock_llm_tool_call = MagicMock() + mock_message_instance = MagicMock() + mock_llm_message.return_value = mock_message_instance + + mock_qualifire_types.LLMMessage = mock_llm_message + mock_qualifire_types.LLMToolCall = mock_llm_tool_call + + with patch.dict('sys.modules', {'qualifire': MagicMock(), 'qualifire.types': mock_qualifire_types}): + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) - guardrail = QualifireGuardrail( - api_key="test_key", - prompt_injections=True, - pii_check=True, - hallucinations_check=True, - assertions=["Output must be valid JSON"], - guardrail_name="test_guardrail", - ) + guardrail = QualifireGuardrail( + api_key="test_key", + prompt_injections=True, + pii_check=True, + hallucinations_check=True, + assertions=["Output must be valid JSON"], + guardrail_name="test_guardrail", + ) - # Mock the client - mock_client = MagicMock() - mock_result = MagicMock() - mock_result.score = 100 - mock_result.status = "completed" - mock_result.evaluationResults = [] - mock_client.evaluate.return_value = mock_result - guardrail._client = mock_client + # Mock the client + mock_client = MagicMock() + mock_result = MagicMock() + mock_result.score = 100 + mock_result.status = "completed" + mock_result.evaluationResults = [] + mock_client.evaluate.return_value = mock_result + guardrail._client = mock_client - messages = [{"role": "user", "content": "Hello, world!"}] + messages = [{"role": "user", "content": "Hello, world!"}] - await guardrail._run_qualifire_check( - messages=messages, output="Test output", dynamic_params={} - ) + await guardrail._run_qualifire_check( + messages=messages, output="Test output", dynamic_params={} + ) - # Verify evaluate was called with correct kwargs - mock_client.evaluate.assert_called_once() - call_kwargs = mock_client.evaluate.call_args[1] - assert call_kwargs["prompt_injections"] is True - assert call_kwargs["pii_check"] is True - assert call_kwargs["hallucinations_check"] is True - assert call_kwargs["assertions"] == ["Output must be valid JSON"] - assert call_kwargs["output"] == "Test output" + # Verify evaluate was called with correct kwargs + mock_client.evaluate.assert_called_once() + call_kwargs = mock_client.evaluate.call_args[1] + assert call_kwargs["prompt_injections"] is True + assert call_kwargs["pii_check"] is True + assert call_kwargs["hallucinations_check"] is True + assert call_kwargs["assertions"] == ["Output must be valid JSON"] + assert call_kwargs["output"] == "Test output" class TestQualifireGuardrailCheckIfFlagged: diff --git a/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py b/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py index 011031c1e4f..97c1733a935 100644 --- a/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py +++ b/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py @@ -40,6 +40,7 @@ class TestKeyManagementEventHooksIndependentOperations: mock_data = MagicMock() mock_data.key_alias = "test-key-alias" mock_data.team_id = None + mock_data.send_invite_email = True mock_response = MagicMock() mock_response.model_dump.return_value = {"key": "sk-test", "token": "test-token"} @@ -59,6 +60,10 @@ class TestKeyManagementEventHooksIndependentOperations: KeyManagementEventHooks, "_store_virtual_key_in_secret_manager", side_effect=mock_store_secret, + ), patch.object( + KeyManagementEventHooks, + "_is_email_sending_enabled", + return_value=True, ), patch( "litellm.store_audit_logs", False ), patch( @@ -96,6 +101,7 @@ class TestKeyManagementEventHooksIndependentOperations: mock_data = MagicMock() mock_data.key_alias = "test-key-alias" mock_data.team_id = None + mock_data.send_invite_email = True mock_response = MagicMock() mock_response.model_dump.return_value = {"key": "sk-test", "token": "test-token"} @@ -115,6 +121,10 @@ class TestKeyManagementEventHooksIndependentOperations: KeyManagementEventHooks, "_store_virtual_key_in_secret_manager", side_effect=mock_store_secret_raises, + ), patch.object( + KeyManagementEventHooks, + "_is_email_sending_enabled", + return_value=True, ), patch( "litellm.store_audit_logs", False ), patch( diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index cc268ab9925..f2bae2cb14a 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -231,6 +231,40 @@ class TestListMCPServers: assert server.url == "https://mcp.deepwiki.com/mcp" assert server.transport == "http" + @pytest.mark.asyncio + async def test_list_mcp_servers_view_all_mode(self): + """Users should see all MCP servers when view_all mode is enabled.""" + + mock_user_auth = generate_mock_user_api_key_auth( + user_role=LitellmUserRoles.INTERNAL_USER + ) + + mock_servers = [ + generate_mock_mcp_server_db_record(server_id="server-1", alias="One"), + generate_mock_mcp_server_db_record(server_id="server-2", alias="Two"), + ] + + mock_manager = MagicMock() + mock_manager.get_all_mcp_servers_unfiltered = AsyncMock( + return_value=mock_servers + ) + + with patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._get_user_mcp_management_mode", + return_value="view_all", + ), patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ): + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + fetch_all_mcp_servers, + ) + + result = await fetch_all_mcp_servers(user_api_key_dict=mock_user_auth) + + assert len(result) == 2 + assert {server.server_id for server in result} == {"server-1", "server-2"} + @pytest.mark.asyncio async def test_list_mcp_servers_combined_config_and_db(self): """ @@ -1096,6 +1130,51 @@ class TestHealthCheckServers: assert result[0]["server_id"] == "server-1" assert result[0]["status"] == "healthy" + @pytest.mark.asyncio + async def test_health_check_view_all_mode(self): + """view_all mode should return health info for all MCP servers.""" + + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + health_check_servers, + ) + + mock_user_auth = generate_mock_user_api_key_auth( + user_role=LitellmUserRoles.INTERNAL_USER + ) + + health_result_one = generate_mock_mcp_server_db_record( + server_id="server-1", alias="One" + ) + health_result_one.status = "healthy" + + health_result_two = generate_mock_mcp_server_db_record( + server_id="server-2", alias="Two" + ) + health_result_two.status = "unhealthy" + + mock_manager = MagicMock() + mock_manager.get_all_mcp_servers_with_health_unfiltered = AsyncMock( + return_value=[health_result_one, health_result_two] + ) + + with patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._get_user_mcp_management_mode", + return_value="view_all", + ), patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ): + result = await health_check_servers( + server_ids=None, + user_api_key_dict=mock_user_auth, + ) + + assert len(result) == 2 + assert result[0]["server_id"] == "server-1" + assert result[0]["status"] == "healthy" + assert result[1]["server_id"] == "server-2" + assert result[1]["status"] == "unhealthy" + @pytest.mark.asyncio async def test_health_check_unauthorized_servers(self): """ diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index 8fdfd6897a8..ad4f53dac4b 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -742,18 +742,16 @@ class TestProxySettingEndpoints: ): """Test updating UI settings with an allowlisted field""" from unittest.mock import AsyncMock, MagicMock + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy._types import UserAPIKeyAuth - class MockUser: - def __init__(self, user_role): - self.user_role = user_role - - async def mock_admin_auth(): - return MockUser(LitellmUserRoles.PROXY_ADMIN) - - monkeypatch.setattr( - "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.user_api_key_auth", - mock_admin_auth, + # Override the FastAPI dependency with a proper mock + mock_user_auth = UserAPIKeyAuth( + user_id="test-user-123", + user_role=LitellmUserRoles.PROXY_ADMIN, ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) mock_prisma = MagicMock() mock_prisma.db.litellm_uisettings.upsert = AsyncMock() @@ -761,7 +759,11 @@ class TestProxySettingEndpoints: payload = {"disable_model_add_for_internal_users": True} - response = client.patch("/update/ui_settings", json=payload) + try: + response = client.patch("/update/ui_settings", json=payload) + finally: + # Clean up the dependency override + app.dependency_overrides.clear() assert response.status_code == 200 data = response.json() @@ -780,18 +782,16 @@ class TestProxySettingEndpoints: ): """Test non-allowlisted UI settings are ignored on update""" from unittest.mock import AsyncMock, MagicMock + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy._types import UserAPIKeyAuth - class MockUser: - def __init__(self, user_role): - self.user_role = user_role - - async def mock_admin_auth(): - return MockUser(LitellmUserRoles.PROXY_ADMIN) - - monkeypatch.setattr( - "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.user_api_key_auth", - mock_admin_auth, + # Override the FastAPI dependency with a proper mock + mock_user_auth = UserAPIKeyAuth( + user_id="test-user-123", + user_role=LitellmUserRoles.PROXY_ADMIN, ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) mock_prisma = MagicMock() mock_prisma.db.litellm_uisettings.upsert = AsyncMock() @@ -802,7 +802,11 @@ class TestProxySettingEndpoints: "unsupported_flag": True, } - response = client.patch("/update/ui_settings", json=payload) + try: + response = client.patch("/update/ui_settings", json=payload) + finally: + # Clean up the dependency override + app.dependency_overrides.clear() assert response.status_code == 200 data = response.json() diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useSSOSettings.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useSSOSettings.ts index 3e09c3c2ca8..f03f3977115 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useSSOSettings.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useSSOSettings.ts @@ -1,7 +1,7 @@ +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { getSSOSettings } from "@/components/networking"; import { useQuery, UseQueryResult } from "@tanstack/react-query"; import { createQueryKeys } from "../common/queryKeysFactory"; -import { getSSOSettings } from "@/components/networking"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; export interface SSOFieldSchema { description: string; @@ -27,13 +27,15 @@ export interface SSOSettingsValues { proxy_base_url: string | null; user_email: string | null; ui_access_mode: string | null; - role_mappings: { - provider: string; - group_claim: string; - default_role: "internal_user" | "internal_user_viewer" | "proxy_admin" | "proxy_admin_viewer"; - roles: { - [key: string]: string[]; - }; + role_mappings: RoleMappings; +} + +export interface RoleMappings { + provider: string; + group_claim: string; + default_role: "internal_user" | "internal_user_viewer" | "proxy_admin" | "proxy_admin_viewer"; + roles: { + [key: string]: string[]; }; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx index 65418179595..a8b1d2cddc9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx @@ -18,6 +18,7 @@ import { useQueryClient } from "@tanstack/react-query"; import { Col, Grid, Icon, Tab, TabGroup, TabList, TabPanel, TabPanels, Text } from "@tremor/react"; import type { UploadProps } from "antd"; import { Form, Typography } from "antd"; +import { PlusCircleOutlined } from "@ant-design/icons"; import React, { useEffect, useMemo, useState } from "react"; import AddModelTab from "../../../components/add_model/add_model_tab"; import HealthCheckComponent from "../../../components/model_dashboard/HealthCheckComponent"; @@ -274,6 +275,30 @@ const ModelsAndEndpointsView: React.FC = ({ premiumUser, te )}
+ + {/* Missing Provider Banner */} + {selectedModelId && !isLoading ? ( = ({ form, onFormS prevValues.use_role_mappings !== currentValues.use_role_mappings} + shouldUpdate={(prevValues, currentValues) => + prevValues.use_role_mappings !== currentValues.use_role_mappings || + prevValues.sso_provider !== currentValues.sso_provider + } > {({ getFieldValue }) => { const useRoleMappings = getFieldValue("use_role_mappings"); - return useRoleMappings ? ( + const provider = getFieldValue("sso_provider"); + const supportsRoleMappings = provider === "okta" || provider === "generic"; + return useRoleMappings && supportsRoleMappings ? ( = ({ form, onFormS prevValues.use_role_mappings !== currentValues.use_role_mappings} + shouldUpdate={(prevValues, currentValues) => + prevValues.use_role_mappings !== currentValues.use_role_mappings || + prevValues.sso_provider !== currentValues.sso_provider + } > {({ getFieldValue }) => { const useRoleMappings = getFieldValue("use_role_mappings"); - return useRoleMappings ? ( + const provider = getFieldValue("sso_provider"); + const supportsRoleMappings = provider === "okta" || provider === "generic"; + return useRoleMappings && supportsRoleMappings ? ( <> diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.test.tsx index 7d8a35b7f44..ef6ec6c7055 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.test.tsx @@ -1,60 +1,20 @@ -import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { render, screen } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; import DeleteSSOSettingsModal from "./DeleteSSOSettingsModal"; -vi.mock("@/app/(dashboard)/hooks/sso/useSSOSettings", () => ({ - useSSOSettings: vi.fn(() => ({ - data: { - values: { - google_client_id: "test-client-id", - }, - }, - })), -})); - -vi.mock("@/app/(dashboard)/hooks/sso/useEditSSOSettings", () => ({ - useEditSSOSettings: vi.fn(() => ({ - mutateAsync: vi.fn(), - isPending: false, - })), -})); - -vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ - default: vi.fn(() => ({ - accessToken: "test-token", - userId: "test-user-id", - userRole: "proxy_admin", - })), -})); - -const createQueryClient = () => - new QueryClient({ - defaultOptions: { - queries: { - retry: false, - gcTime: 0, - }, - }, - }); - describe("DeleteSSOSettingsModal", () => { it("should render", () => { const onCancel = vi.fn(); const onSuccess = vi.fn(); - const queryClient = createQueryClient(); render( - - - , + , ); expect(screen.getByText("Confirm Clear SSO Settings")).toBeInTheDocument(); expect( - screen.getByText( - "Are you sure you want to clear all SSO settings? Users will no longer be able to login using SSO after this change.", - ), + screen.getByText("Are you sure you want to clear all SSO settings? This action cannot be undone."), ).toBeInTheDocument(); + expect(screen.getByText("Users will no longer be able to login using SSO after this change.")).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.tsx index 44cbf0020eb..6a28b6490e0 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.tsx @@ -1,66 +1,79 @@ -import { useEditSSOSettings } from "@/app/(dashboard)/hooks/sso/useEditSSOSettings"; -import { useSSOSettings } from "@/app/(dashboard)/hooks/sso/useSSOSettings"; +import { Modal } from "antd"; import React from "react"; -import DeleteResourceModal from "../../../../common_components/DeleteResourceModal"; import NotificationsManager from "../../../../molecules/notifications_manager"; +import { updateSSOSettings } from "../../../../networking"; import { parseErrorMessage } from "../../../../shared/errorUtils"; -import { detectSSOProvider } from "../utils"; interface DeleteSSOSettingsModalProps { isVisible: boolean; onCancel: () => void; onSuccess: () => void; + accessToken: string | null; } -const DeleteSSOSettingsModal: React.FC = ({ isVisible, onCancel, onSuccess }) => { - const { data: ssoSettings } = useSSOSettings(); - const { mutateAsync: editSSOSettings, isPending: isEditingSSOSettings } = useEditSSOSettings(); - +const DeleteSSOSettingsModal: React.FC = ({ + isVisible, + onCancel, + onSuccess, + accessToken, +}) => { // Handle clearing SSO settings const handleClearSSO = async () => { - const clearSettings = { - google_client_id: null, - google_client_secret: null, - microsoft_client_id: null, - microsoft_client_secret: null, - microsoft_tenant: null, - generic_client_id: null, - generic_client_secret: null, - generic_authorization_endpoint: null, - generic_token_endpoint: null, - generic_userinfo_endpoint: null, - proxy_base_url: null, - user_email: null, - sso_provider: null, - role_mappings: null, - }; + if (!accessToken) { + NotificationsManager.fromBackend("No access token available"); + return; + } - await editSSOSettings(clearSettings, { - onSuccess: () => { - NotificationsManager.success("SSO settings cleared successfully"); - onCancel(); - onSuccess(); - }, - onError: (error) => { - NotificationsManager.fromBackend("Failed to clear SSO settings: " + parseErrorMessage(error)); - }, - }); + try { + // Clear all SSO settings + const clearSettings = { + google_client_id: null, + google_client_secret: null, + microsoft_client_id: null, + microsoft_client_secret: null, + microsoft_tenant: null, + generic_client_id: null, + generic_client_secret: null, + generic_authorization_endpoint: null, + generic_token_endpoint: null, + generic_userinfo_endpoint: null, + proxy_base_url: null, + user_email: null, + sso_provider: null, + }; + + await updateSSOSettings(accessToken, clearSettings); + + NotificationsManager.success("SSO settings cleared successfully"); + + // Close modal and trigger success callback + onCancel(); + onSuccess(); + } catch (error) { + console.error("Failed to clear SSO settings:", error); + NotificationsManager.fromBackend("Failed to clear SSO settings: " + parseErrorMessage(error)); + } }; return ( - + onCancel={onCancel} + okText="Yes, Clear" + cancelText="Cancel" + okButtonProps={{ + danger: true, + style: { + backgroundColor: "#dc2626", + borderColor: "#dc2626", + }, + }} + > +

Are you sure you want to clear all SSO settings? This action cannot be undone.

+

Users will no longer be able to login using SSO after this change.

+ ); }; diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RoleMappings.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RoleMappings.test.tsx deleted file mode 100644 index f4b7b9aadd4..00000000000 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RoleMappings.test.tsx +++ /dev/null @@ -1,92 +0,0 @@ -import type { RoleMappings as RoleMappingsType } from "@/app/(dashboard)/hooks/sso/useSSOSettings"; -import { screen } from "@testing-library/react"; -import { describe, expect, it } from "vitest"; -import { renderWithProviders } from "../../../../../tests/test-utils"; -import RoleMappings from "./RoleMappings"; - -describe("RoleMappings", () => { - it("should render successfully", () => { - const roleMappings: RoleMappingsType = { - provider: "generic", - group_claim: "groups", - default_role: "internal_user", - roles: { - proxy_admin: ["admin-group"], - proxy_admin_viewer: [], - internal_user: ["user-group"], - internal_user_viewer: [], - }, - }; - - renderWithProviders(); - - expect(screen.getByText("Role Mappings")).toBeInTheDocument(); - }); - - it("should return null when roleMappings is undefined", () => { - const { container } = renderWithProviders(); - - expect(container.firstChild).toBeNull(); - }); - - it("should display Group Claim and Default Role with correct values and display names", () => { - const testCases: Array<{ role: RoleMappingsType["default_role"]; displayName: string; groupClaim: string }> = [ - { role: "internal_user_viewer", displayName: "Internal Viewer", groupClaim: "custom-groups-1" }, - { role: "internal_user", displayName: "Internal User", groupClaim: "custom-groups-2" }, - { role: "proxy_admin_viewer", displayName: "Proxy Admin Viewer", groupClaim: "custom-groups-3" }, - { role: "proxy_admin", displayName: "Proxy Admin", groupClaim: "custom-groups-4" }, - ]; - - testCases.forEach(({ role, displayName, groupClaim }) => { - const roleMappings: RoleMappingsType = { - provider: "generic", - group_claim: groupClaim, - default_role: role, - roles: { - proxy_admin: [], - proxy_admin_viewer: [], - internal_user: [], - internal_user_viewer: [], - }, - }; - - const { unmount } = renderWithProviders(); - - expect(screen.getByText("Group Claim")).toBeInTheDocument(); - expect(screen.getByText(groupClaim)).toBeInTheDocument(); - expect(screen.getByText("Default Role")).toBeInTheDocument(); - const displayNameElements = screen.getAllByText(displayName); - expect(displayNameElements.length).toBeGreaterThan(0); - unmount(); - }); - }); - - it("should display table with roles, groups as Tags when mapped, and 'No groups mapped' when empty", () => { - const roleMappings: RoleMappingsType = { - provider: "generic", - group_claim: "groups", - default_role: "internal_user", - roles: { - proxy_admin: ["admin-group-1", "admin-group-2", "admin-group-3"], - proxy_admin_viewer: ["viewer-group"], - internal_user: ["user-group"], - internal_user_viewer: [], - }, - }; - - renderWithProviders(); - - expect(screen.getByText("Role")).toBeInTheDocument(); - expect(screen.getByText("Mapped Groups")).toBeInTheDocument(); - expect(screen.getAllByText("Proxy Admin").length).toBeGreaterThan(0); - expect(screen.getAllByText("Proxy Admin Viewer").length).toBeGreaterThan(0); - expect(screen.getAllByText("Internal User").length).toBeGreaterThan(0); - expect(screen.getAllByText("Internal Viewer").length).toBeGreaterThan(0); - expect(screen.getByText("admin-group-1")).toBeInTheDocument(); - expect(screen.getByText("admin-group-2")).toBeInTheDocument(); - expect(screen.getByText("admin-group-3")).toBeInTheDocument(); - expect(screen.getByText("viewer-group")).toBeInTheDocument(); - expect(screen.getByText("user-group")).toBeInTheDocument(); - expect(screen.getByText("No groups mapped")).toBeInTheDocument(); - }); -}); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RoleMappings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RoleMappings.tsx deleted file mode 100644 index 3750ee88183..00000000000 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RoleMappings.tsx +++ /dev/null @@ -1,74 +0,0 @@ -import type { RoleMappings as RoleMappingsType } from "@/app/(dashboard)/hooks/sso/useSSOSettings"; -import { Card, Divider, Table, Tag, Typography } from "antd"; -import { Users } from "lucide-react"; -import { defaultRoleDisplayNames } from "./constants"; -const { Title, Text } = Typography; - -export default function RoleMappings({ roleMappings }: { roleMappings: RoleMappingsType | undefined }) { - if (!roleMappings) { - return null; - } - - const roleMappingsColumns = [ - { - title: "Role", - dataIndex: "role", - key: "role", - render: (text: string) => {defaultRoleDisplayNames[text]}, - }, - { - title: "Mapped Groups", - dataIndex: "groups", - key: "groups", - render: (groups: string[]) => ( - <> - {groups.length > 0 ? ( - groups.map((group, index) => ( - - {group} - - )) - ) : ( - No groups mapped - )} - - ), - }, - ]; - return ( - -
- - Role Mappings -
-
-
-
- Group Claim -
- {roleMappings.group_claim} -
-
-
- Default Role -
- {defaultRoleDisplayNames[roleMappings.default_role]} -
-
-
- - ({ - role, - groups, - }))} - pagination={false} - bordered - size="small" - className="w-full" - /> - - - ); -} diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx index adc1251cde2..27ff96af05f 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx @@ -1,23 +1,23 @@ "use client"; import { useSSOSettings, type SSOSettingsValues } from "@/app/(dashboard)/hooks/sso/useSSOSettings"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { Button, Card, Descriptions, Space, Typography } from "antd"; import { Edit, Shield, Trash2 } from "lucide-react"; import { useState } from "react"; -import { ssoProviderDisplayNames, ssoProviderLogoMap } from "./constants"; import AddSSOSettingsModal from "./Modals/AddSSOSettingsModal"; import DeleteSSOSettingsModal from "./Modals/DeleteSSOSettingsModal"; import EditSSOSettingsModal from "./Modals/EditSSOSettingsModal"; import RedactableField from "./RedactableField"; -import RoleMappings from "./RoleMappings"; import SSOSettingsEmptyPlaceholder from "./SSOSettingsEmptyPlaceholder"; import SSOSettingsLoadingSkeleton from "./SSOSettingsLoadingSkeleton"; -import { detectSSOProvider } from "./utils"; +import { ssoProviderDisplayNames, ssoProviderLogoMap } from "./constants"; const { Title, Text } = Typography; export default function SSOSettings() { const { data: ssoSettings, refetch, isLoading } = useSSOSettings(); + const { accessToken } = useAuthorized(); const [isDeleteModalVisible, setIsDeleteModalVisible] = useState(false); const [isAddModalVisible, setIsAddModalVisible] = useState(false); const [isEditModalVisible, setIsEditModalVisible] = useState(false); @@ -26,8 +26,24 @@ export default function SSOSettings() { Boolean(ssoSettings?.values.microsoft_client_id) || Boolean(ssoSettings?.values.generic_client_id); + // Determine the SSO provider based on the configuration + const detectSSOProvider = (values: SSOSettingsValues): string | null => { + if (values.google_client_id) return "google"; + if (values.microsoft_client_id) return "microsoft"; + if (values.generic_client_id) { + // Check if it looks like Okta/Auth0 based on endpoints + if ( + values.generic_authorization_endpoint?.includes("okta") || + values.generic_authorization_endpoint?.includes("auth0") + ) { + return "okta"; + } + return "generic"; + } + return null; + }; + const selectedProvider = ssoSettings?.values ? detectSSOProvider(ssoSettings.values) : null; - const isRoleMappingsEnabled = Boolean(ssoSettings?.values.role_mappings); const renderEndpointValue = (value?: string | null) => ( @@ -169,52 +185,46 @@ export default function SSOSettings() { {isLoading ? ( ) : ( - - - - {/* Header Section */} -
-
- -
- SSO Configuration - Manage Single Sign-On authentication settings -
-
- -
- {isSSOConfigured && ( - <> - - - - )} + + + {/* Header Section */} +
+
+ +
+ SSO Configuration + Manage Single Sign-On authentication settings
- {isSSOConfigured ? ( - renderSSOSettings() - ) : ( - setIsAddModalVisible(true)} /> - )} - - - {isRoleMappingsEnabled && } - +
+ {isSSOConfigured && ( + <> + + + + )} +
+
+ + {isSSOConfigured ? ( + renderSSOSettings() + ) : ( + setIsAddModalVisible(true)} /> + )} + + )} setIsDeleteModalVisible(false)} onSuccess={() => refetch()} + accessToken={accessToken} /> = { okta: "Okta / Auth0 SSO", generic: "Generic SSO", }; - -export const defaultRoleDisplayNames: Record = { - internal_user_viewer: "Internal Viewer", - internal_user: "Internal User", - proxy_admin_viewer: "Proxy Admin Viewer", - proxy_admin: "Proxy Admin", -}; diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.test.ts b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.test.ts index 718302d35fe..1c878d7b7b3 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.test.ts +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.test.ts @@ -55,7 +55,6 @@ describe("processSSOSettingsPayload", () => { default_role: "proxy_admin", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", other_field: "value", }; @@ -84,7 +83,6 @@ describe("processSSOSettingsPayload", () => { default_role: "internal_user", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -102,7 +100,6 @@ describe("processSSOSettingsPayload", () => { default_role: "internal_user_viewer", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -124,7 +121,6 @@ describe("processSSOSettingsPayload", () => { default_role: "proxy_admin_viewer", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -146,7 +142,6 @@ describe("processSSOSettingsPayload", () => { default_role: "internal_user", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -165,7 +160,6 @@ describe("processSSOSettingsPayload", () => { default_role: "internal_user", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -180,7 +174,6 @@ describe("processSSOSettingsPayload", () => { default_role: "internal_user_viewer", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -193,7 +186,6 @@ describe("processSSOSettingsPayload", () => { default_role: "internal_user", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -206,7 +198,6 @@ describe("processSSOSettingsPayload", () => { default_role: "proxy_admin_viewer", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -219,7 +210,6 @@ describe("processSSOSettingsPayload", () => { default_role: "proxy_admin", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -232,7 +222,6 @@ describe("processSSOSettingsPayload", () => { default_role: "unknown_role", group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); @@ -244,7 +233,6 @@ describe("processSSOSettingsPayload", () => { const formValues = { group_claim: "groups", use_role_mappings: true, - sso_provider: "generic", }; const result = processSSOSettingsPayload(formValues); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.ts b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.ts index c199048df3e..3533e1226c2 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.ts +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.ts @@ -1,5 +1,3 @@ -import { SSOSettingsValues } from "@/app/(dashboard)/hooks/sso/useSSOSettings"; - /** * Processes SSO settings form values and transforms them into the payload format expected by the API * Handles role mappings transformation and field extraction @@ -20,7 +18,7 @@ export const processSSOSettingsPayload = (formValues: Record): Reco ...rest, }; - // Add role mappings only if use_role_mappings is checked AND provider supports role mappings + // Add role mappings if use_role_mappings is checked if (use_role_mappings) { // Helper function to split comma-separated string into array const splitTeams = (teams: string | undefined): string[] => { @@ -54,20 +52,3 @@ export const processSSOSettingsPayload = (formValues: Record): Reco return payload; }; - -// Determine the SSO provider based on the configuration -export const detectSSOProvider = (values: SSOSettingsValues): string | null => { - if (values.google_client_id) return "google"; - if (values.microsoft_client_id) return "microsoft"; - if (values.generic_client_id) { - // Check if it looks like Okta/Auth0 based on endpoints - if ( - values.generic_authorization_endpoint?.includes("okta") || - values.generic_authorization_endpoint?.includes("auth0") - ) { - return "okta"; - } - return "generic"; - } - return null; -}; diff --git a/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx b/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx index a62d8baa75b..9600c962564 100644 --- a/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx +++ b/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx @@ -136,7 +136,7 @@ export const useMcpOAuthFlow = ({ if (!hasPreconfiguredCredentials) { const registration = await registerMcpOAuthClient(accessToken, serverId, { client_name: temporaryPayload.alias || temporaryPayload.server_name || serverId, - grant_types: ["authorization_code", "refresh_token"], + grant_types: ["authorization_code"], response_types: ["code"], token_endpoint_auth_method: temporaryPayload.credentials && temporaryPayload.credentials.client_secret ? "client_secret_post" : "none", From 0756b48963c0065199cbb61f81596531ba3e698e Mon Sep 17 00:00:00 2001 From: Felipe Peter Date: Tue, 6 Jan 2026 10:33:25 -0800 Subject: [PATCH 009/121] Add Anthropic cache control option to image tool call results (#18674) --- .../prompt_templates/factory.py | 7 +- ...llm_core_utils_prompt_templates_factory.py | 246 +++++++++++++++--- 2 files changed, 211 insertions(+), 42 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 12570a02de7..cbb31be146c 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1645,9 +1645,12 @@ def convert_to_anthropic_tool_result( ) elif content["type"] == "image_url": format = content["image_url"].get("format") if isinstance(content["image_url"], dict) else None - anthropic_content_list.append( - create_anthropic_image_param(content["image_url"], format=format) + _anthropic_image_param = create_anthropic_image_param(content["image_url"], format=format) + _anthropic_image_param = add_cache_control_to_content( + anthropic_content_element=_anthropic_image_param, + original_content_element=content, ) + anthropic_content_list.append(_anthropic_image_param) anthropic_content = anthropic_content_list anthropic_tool_result: Optional[AnthropicMessagesToolResultParam] = None diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index c8fe6efeaa1..a4188f03cde 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -620,26 +620,26 @@ def test_bedrock_tools_unpack_defs(): def test_bedrock_image_processor_content_type_fallback_url_extension(): """ - Test that _post_call_image_processing falls back to URL extension + Test that _post_call_image_processing falls back to URL extension when content-type is binary/octet-stream or application/octet-stream """ import base64 - + # Create mock response with binary/octet-stream content-type mock_response = MagicMock() mock_response.headers.get.return_value = "binary/octet-stream" - + # Create a simple PNG header (magic bytes) png_header = b"\x89\x50\x4e\x47\x0d\x0a\x1a\x0a" png_content = png_header + b"\x00" * 100 # Add some padding mock_response.content = png_content - + # Test with .png URL image_url = "https://example.com/test-image.png" base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( mock_response, image_url ) - + assert content_type == "image/png" assert base64_bytes == base64.b64encode(png_content).decode("utf-8") @@ -650,22 +650,22 @@ def test_bedrock_image_processor_content_type_fallback_binary_detection(): when content-type is missing and URL extension is not recognized """ import base64 - + # Create mock response with no content-type mock_response = MagicMock() mock_response.headers.get.return_value = None - + # Create a JPEG header (magic bytes) jpeg_header = b"\xff\xd8\xff" jpeg_content = jpeg_header + b"\x00" * 100 # Add some padding mock_response.content = jpeg_content - + # Test with URL without extension image_url = "https://example.com/test-image-without-extension" base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( mock_response, image_url ) - + assert content_type == "image/jpeg" assert base64_bytes == base64.b64encode(jpeg_content).decode("utf-8") @@ -675,22 +675,22 @@ def test_bedrock_image_processor_content_type_fallback_application_octet_stream( Test that _post_call_image_processing handles application/octet-stream correctly """ import base64 - + # Create mock response with application/octet-stream content-type mock_response = MagicMock() mock_response.headers.get.return_value = "application/octet-stream" - + # Create a GIF header (magic bytes) gif_header = b"GIF8" + b"\x00" + b"a" gif_content = gif_header + b"\x00" * 100 # Add some padding mock_response.content = gif_content - + # Test with .gif URL image_url = "https://s3.amazonaws.com/bucket/image.gif" base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( mock_response, image_url ) - + assert content_type == "image/gif" assert base64_bytes == base64.b64encode(gif_content).decode("utf-8") @@ -700,22 +700,22 @@ def test_bedrock_image_processor_content_type_with_query_params(): Test that _post_call_image_processing correctly extracts extension from URL with query parameters """ import base64 - + # Create mock response with binary/octet-stream content-type mock_response = MagicMock() mock_response.headers.get.return_value = "binary/octet-stream" - + # Create a WebP header (magic bytes) webp_header = b"RIFF" + b"\x00\x00\x00\x00" + b"WEBP" webp_content = webp_header + b"\x00" * 100 # Add some padding mock_response.content = webp_content - + # Test with URL containing query parameters (common in S3 signed URLs) image_url = "https://s3.amazonaws.com/bucket/image.webp?AWSAccessKeyId=123&Expires=456&Signature=789" base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( mock_response, image_url ) - + assert content_type == "image/webp" assert base64_bytes == base64.b64encode(webp_content).decode("utf-8") @@ -725,21 +725,21 @@ def test_bedrock_image_processor_content_type_normal_header(): Test that _post_call_image_processing works normally when content-type is correctly set """ import base64 - + # Create mock response with correct content-type mock_response = MagicMock() mock_response.headers.get.return_value = "image/png" - + # Create a PNG header png_header = b"\x89\x50\x4e\x47\x0d\x0a\x1a\x0a" png_content = png_header + b"\x00" * 100 mock_response.content = png_content - + image_url = "https://example.com/test-image.png" base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( mock_response, image_url ) - + assert content_type == "image/png" assert base64_bytes == base64.b64encode(png_content).decode("utf-8") @@ -751,16 +751,16 @@ def test_bedrock_image_processor_content_type_fallback_failure(): # Create mock response with binary/octet-stream content-type mock_response = MagicMock() mock_response.headers.get.return_value = "binary/octet-stream" - + # Create content with unrecognizable image format mock_response.content = b"\x00" * 100 - + # Test with URL without recognizable extension image_url = "https://example.com/unknown-file" - + with pytest.raises(ValueError) as excinfo: BedrockImageProcessor._post_call_image_processing(mock_response, image_url) - + assert "Unable to determine content type" in str(excinfo.value) @@ -771,18 +771,18 @@ def test_bedrock_image_processor_content_type_jpeg_variants(): # Create mock response with binary/octet-stream mock_response = MagicMock() mock_response.headers.get.return_value = "binary/octet-stream" - + jpeg_header = b"\xff\xd8\xff" jpeg_content = jpeg_header + b"\x00" * 100 mock_response.content = jpeg_content - + # Test with .jpg extension image_url_jpg = "https://example.com/photo.jpg" _, content_type_jpg = BedrockImageProcessor._post_call_image_processing( mock_response, image_url_jpg ) assert content_type_jpg == "image/jpeg" - + # Test with .jpeg extension image_url_jpeg = "https://example.com/photo.jpeg" _, content_type_jpeg = BedrockImageProcessor._post_call_image_processing( @@ -797,22 +797,22 @@ def test_bedrock_image_processor_content_type_pdf_document(): when content-type is binary/octet-stream """ import base64 - + # Create mock response with binary/octet-stream content-type mock_response = MagicMock() mock_response.headers.get.return_value = "binary/octet-stream" - + # Create a PDF header (magic bytes: %PDF) pdf_header = b"%PDF-1.4" pdf_content = pdf_header + b"\x00" * 100 mock_response.content = pdf_content - + # Test with .pdf URL pdf_url = "https://s3.amazonaws.com/bucket/document.pdf" base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( mock_response, pdf_url ) - + assert content_type == "application/pdf" assert base64_bytes == base64.b64encode(pdf_content).decode("utf-8") @@ -822,12 +822,12 @@ def test_bedrock_image_processor_content_type_document_formats(): Test that _post_call_image_processing handles various document formats """ import base64 - + # Create mock response mock_response = MagicMock() mock_response.headers.get.return_value = "application/octet-stream" mock_response.content = b"\x00" * 100 - + # Test various document formats test_cases = [ ("https://example.com/doc.pdf", "application/pdf"), @@ -837,7 +837,7 @@ def test_bedrock_image_processor_content_type_document_formats(): ("https://example.com/page.html", "text/html"), ("https://example.com/readme.txt", "text/plain"), ] - + for url, expected_mime in test_cases: _, content_type = BedrockImageProcessor._post_call_image_processing( mock_response, url @@ -850,21 +850,21 @@ def test_bedrock_image_processor_content_type_s3_pdf_with_query(): Test that _post_call_image_processing handles S3 PDF with query parameters """ import base64 - + # Create mock response mock_response = MagicMock() mock_response.headers.get.return_value = "binary/octet-stream" - + pdf_content = b"%PDF-1.4" + b"\x00" * 100 mock_response.content = pdf_content - + # S3 signed URL with query parameters s3_url = "https://my-bucket.s3.us-east-1.amazonaws.com/documents/report.pdf?AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Expires=1234567890&Signature=abcdef123456" - + base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( mock_response, s3_url ) - + assert content_type == "application/pdf" assert base64_bytes == base64.b64encode(pdf_content).decode("utf-8") @@ -1137,3 +1137,169 @@ def test_bedrock_create_bedrock_block_different_document_formats(): assert f"DocumentPDFmessages_" in block["document"]["name"] assert block["document"]["name"].endswith(f"_{format_type}") assert block["document"]["format"] == format_type + + +def test_convert_to_anthropic_tool_result_image_with_cache_control(): + """ + Test that cache_control is properly applied to image content in tool results. + This tests the functionality added in the uncommitted changes where + add_cache_control_to_content is called for image_url content types. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + # Test with base64 image data URI + message = { + "role": "tool", + "tool_call_id": "call_test_123", + "content": [ + { + "type": "text", + "text": "Here is the image you requested:", + }, + { + "type": "image_url", + "image_url": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQ", + "cache_control": {"type": "ephemeral"}, + }, + ], + } + + result = convert_to_anthropic_tool_result(message) + + # Verify the result structure + assert result["type"] == "tool_result" + assert result["tool_use_id"] == "call_test_123" + assert isinstance(result["content"], list) + assert len(result["content"]) == 2 + + # Verify text content + assert result["content"][0]["type"] == "text" + assert result["content"][0]["text"] == "Here is the image you requested:" + + # Verify image content with cache_control + assert result["content"][1]["type"] == "image" + assert result["content"][1]["source"]["type"] == "base64" + assert result["content"][1]["source"]["media_type"] == "image/jpeg" + assert "cache_control" in result["content"][1] + assert result["content"][1]["cache_control"]["type"] == "ephemeral" + + +def test_convert_to_anthropic_tool_result_image_without_cache_control(): + """ + Test that images without cache_control in tool results work correctly. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + message = { + "role": "tool", + "tool_call_id": "call_test_456", + "content": [ + { + "type": "image_url", + "image_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA", + }, + ], + } + + result = convert_to_anthropic_tool_result(message) + + # Verify the result structure + assert result["type"] == "tool_result" + assert result["tool_use_id"] == "call_test_456" + assert isinstance(result["content"], list) + assert len(result["content"]) == 1 + + # Verify image content without cache_control (cache_control will be None if not set) + assert result["content"][0]["type"] == "image" + assert result["content"][0]["source"]["type"] == "base64" + assert result["content"][0]["source"]["media_type"] == "image/png" + assert result["content"][0].get("cache_control") is None + + +def test_convert_to_anthropic_tool_result_mixed_content_with_cache_control(): + """ + Test tool results with mixed content types (text and image) where only some have cache_control. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + message = { + "role": "tool", + "tool_call_id": "call_test_789", + "content": [ + { + "type": "text", + "text": "First image:", + "cache_control": {"type": "ephemeral"}, + }, + { + "type": "image_url", + "image_url": "data:image/jpeg;base64,/9j/4AAQSkZJRg", + "cache_control": {"type": "ephemeral"}, + }, + { + "type": "text", + "text": "Second image (no cache):", + }, + { + "type": "image_url", + "image_url": "data:image/png;base64,iVBORw0KGgo", + }, + ], + } + + result = convert_to_anthropic_tool_result(message) + + assert result["type"] == "tool_result" + assert isinstance(result["content"], list) + assert len(result["content"]) == 4 + + # First text with cache_control + assert result["content"][0]["type"] == "text" + assert result["content"][0]["cache_control"]["type"] == "ephemeral" + + # First image with cache_control + assert result["content"][1]["type"] == "image" + assert result["content"][1]["cache_control"]["type"] == "ephemeral" + + # Second text without cache_control (cache_control will be None if not set) + assert result["content"][2]["type"] == "text" + assert result["content"][2].get("cache_control") is None + + # Second image without cache_control (cache_control will be None if not set) + assert result["content"][3]["type"] == "image" + assert result["content"][3].get("cache_control") is None + + +def test_convert_to_anthropic_tool_result_image_url_as_http(): + """ + Test that HTTP/HTTPS URLs with cache_control are handled correctly. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + message = { + "role": "tool", + "tool_call_id": "call_http_001", + "content": [ + { + "type": "image_url", + "image_url": "https://example.com/image.jpg", + "cache_control": {"type": "ephemeral"}, + }, + ], + } + + result = convert_to_anthropic_tool_result(message) + + # Verify image is passed as URL reference with cache_control + assert result["content"][0]["type"] == "image" + assert result["content"][0]["source"]["type"] == "url" + assert result["content"][0]["source"]["url"] == "https://example.com/image.jpg" + assert result["content"][0]["cache_control"]["type"] == "ephemeral" From b40a6a0f6c0f6c6998b8f9c1e4c4f0e0cd178696 Mon Sep 17 00:00:00 2001 From: Stephen Matta Date: Fri, 19 Dec 2025 00:57:28 -0500 Subject: [PATCH 010/121] [Fix] Nova model detection for Bedrock provider (#17910) Resolves issue #17910 where Amazon Nova models (like amazon.nova-pro-v1:0) were incorrectly identified as Amazon Titan models, causing requests to use textGenerationConfig instead of inferenceConfig. The fix moves the "nova" check before the initial provider check on the split model name. This ensures that models containing "nova" (like amazon.nova-pro-v1:0 or amazon.nova-2-lite-v1:0) are correctly identified as Nova models, rather than matching "amazon" first. --- litellm/llms/bedrock/base_aws_llm.py | 24 +- .../base_invoke_transformation.py | 10 +- .../test_bedrock_completion.py | 271 ++++++++++++------ 3 files changed, 203 insertions(+), 102 deletions(-) diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index 71d21001cc3..6ffe45624bc 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -314,6 +314,12 @@ class BaseAWSLLM: if model.startswith("invoke/"): model = model.replace("invoke/", "", 1) + # Special case: Check for "nova" in model name first (before "amazon") + # This handles amazon.nova-* models which would otherwise match "amazon" (Titan) + if "nova" in model.lower(): + if "nova" in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL): + return cast(BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova") + _split_model = model.split(".")[0] if _split_model in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL): return cast(BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model) @@ -323,13 +329,9 @@ class BaseAWSLLM: if provider is not None: return provider - # check if provider == "nova" - if "nova" in model: - return "nova" - else: - for provider in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL): - if provider in model: - return provider + for provider in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL): + if provider in model: + return provider return None @staticmethod @@ -364,7 +366,7 @@ class BaseAWSLLM: elif provider == "qwen3" and "qwen3/" in model_id: model_id = BaseAWSLLM._get_model_id_from_model_with_spec( model_id, spec="qwen3" - ) + ) elif provider == "stability" and "stability/" in model_id: model_id = BaseAWSLLM._get_model_id_from_model_with_spec( model_id, spec="stability" @@ -412,7 +414,7 @@ class BaseAWSLLM: if "nova" in model.lower(): if "nova" in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL): return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, "nova") - + # Handle regional models like us.twelvelabs.marengo-embed-2-7-v1:0 if "." in model: parts = model.split(".") @@ -958,7 +960,9 @@ class BaseAWSLLM: return endpoint_url, proxy_endpoint_url def _select_default_endpoint_url( - self, endpoint_type: Optional[Literal["runtime", "agent", "agentcore"]], aws_region_name: str + self, + endpoint_type: Optional[Literal["runtime", "agent", "agentcore"]], + aws_region_name: str, ) -> str: """ Select the default endpoint url based on the endpoint type diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index c602b71fe05..cf8aee6954b 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -524,6 +524,12 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): if model.startswith("invoke/"): model = model.replace("invoke/", "", 1) + # Special case: Check for "nova" in model name first (before "amazon") + # This handles amazon.nova-* models which would otherwise match "amazon" (Titan) + if "nova" in model.lower(): + if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): + return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova") + _split_model = model.split(".")[0] if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model) @@ -533,10 +539,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): if provider is not None: return provider - # check if provider == "nova" - if "nova" in model: - return "nova" - for provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): if provider in model: return provider diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 78c9f94239b..cb8b761b048 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -2841,6 +2841,34 @@ def test_bedrock_invoke_provider(): ) == "nova" ) + assert ( + litellm.AmazonInvokeConfig().get_bedrock_invoke_provider("amazon.nova-pro-v1:0") + == "nova" + ) + assert ( + litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( + "amazon.nova-lite-v1:0" + ) + == "nova" + ) + assert ( + litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( + "amazon.nova-micro-v1:0" + ) + == "nova" + ) + assert ( + litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( + "amazon.nova-premier-v1:0" + ) + == "nova" + ) + assert ( + litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( + "amazon.nova-2-lite-v1:0" + ) + == "nova" + ) def test_bedrock_description_param(): @@ -3494,7 +3522,9 @@ def test_bedrock_openai_imported_model(): url = mock_post.call_args.kwargs["url"] print(f"URL: {url}") assert "bedrock-runtime.us-east-1.amazonaws.com" in url - assert "arn:aws:bedrock:us-east-1:117159858402:imported-model/m4gc1mrfuddy" in url + assert ( + "arn:aws:bedrock:us-east-1:117159858402:imported-model/m4gc1mrfuddy" in url + ) assert "/invoke" in url # Validate request body follows OpenAI format @@ -3523,7 +3553,9 @@ def test_bedrock_openai_imported_model(): # Check image_url content assert user_msg["content"][1]["type"] == "image_url" assert "image_url" in user_msg["content"][1] - assert user_msg["content"][1]["image_url"]["url"].startswith("data:image/jpeg;base64,") + assert user_msg["content"][1]["image_url"]["url"].startswith( + "data:image/jpeg;base64," + ) assert user_msg["content"][2]["type"] == "image_url" assert "image_url" in user_msg["content"][2] @@ -3532,21 +3564,67 @@ def test_bedrock_openai_imported_model(): assert request_body["max_tokens"] == 300 assert request_body["temperature"] == 0.5 + +def test_bedrock_nova_provider_detection(): + """ + Test that Nova models are correctly detected even when prefixed with "amazon." + Regression test for issue #17910 where models like "amazon.nova-pro-v1:0" + were incorrectly identified as "amazon" (Titan) instead of "nova". + """ + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + # Test various Nova model formats + nova_test_cases = [ + ("us.amazon.nova-pro-v1:0", "nova"), + ("us.amazon.nova-lite-v1:0", "nova"), + ("us.amazon.nova-micro-v1:0", "nova"), + ("amazon.nova-pro-v1:0", "nova"), + ("amazon.nova-lite-v1:0", "nova"), + ("amazon.nova-micro-v1:0", "nova"), + ("amazon.nova-premier-v1:0", "nova"), + ("amazon.nova-2-lite-v1:0", "nova"), + ("bedrock/amazon.nova-pro-v1:0", "nova"), + ("bedrock/invoke/amazon.nova-pro-v1:0", "nova"), + ("amazon.Nova-pro-v1:0", "nova"), + ("amazon.NOVA-pro-v1:0", "nova"), + ] + + for model, expected in nova_test_cases: + provider = BaseAWSLLM.get_bedrock_invoke_provider(model) + assert ( + provider == expected + ), f"Failed for model: {model}, expected: {expected}, got: {provider}" + + # Verify that Amazon Titan models still return "amazon" + titan_test_cases = [ + ("amazon.titan-text-express-v1", "amazon"), + ("us.amazon.titan-text-lite-v1", "amazon"), + ] + + for model, expected in titan_test_cases: + provider = BaseAWSLLM.get_bedrock_invoke_provider(model) + assert ( + provider == expected + ), f"Failed for model: {model}, expected: {expected}, got: {provider}" + + def test_bedrock_openai_provider_detection(): """ Test that the OpenAI provider is correctly detected from model strings. """ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM - + # Test various OpenAI model formats test_cases = [ "openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123", "bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/xyz789", ] - + for model in test_cases: provider = BaseAWSLLM.get_bedrock_invoke_provider(model) - assert provider == "openai", f"Failed for model: {model}, got provider: {provider}" + assert ( + provider == "openai" + ), f"Failed for model: {model}, got provider: {provider}" print(f"✓ Provider detection works for: {model}") @@ -3555,16 +3633,16 @@ def test_bedrock_openai_model_id_extraction(): Test that the model ID (ARN) is correctly extracted and encoded for OpenAI models. """ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM - - model = "openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/test-model-123" - provider = BaseAWSLLM.get_bedrock_invoke_provider(model) - - model_id = BaseAWSLLM.get_bedrock_model_id( - model=model, - provider=provider, - optional_params={} + + model = ( + "openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/test-model-123" ) - + provider = BaseAWSLLM.get_bedrock_invoke_provider(model) + + model_id = BaseAWSLLM.get_bedrock_model_id( + model=model, provider=provider, optional_params={} + ) + # The ARN should be double URL encoded assert "arn" in model_id assert "imported-model" in model_id @@ -3576,20 +3654,17 @@ def test_bedrock_openai_convert_messages_to_prompt(): Test that convert_messages_to_prompt returns empty string for OpenAI models. """ from litellm.llms.bedrock.chat.invoke_handler import BedrockLLM - + bedrock_llm = BedrockLLM() messages = [ {"role": "system", "content": "You are helpful"}, - {"role": "user", "content": "Hello"} + {"role": "user", "content": "Hello"}, ] - + prompt, chat_history = bedrock_llm.convert_messages_to_prompt( - model="test-model", - messages=messages, - provider="openai", - custom_prompt_dict={} + model="test-model", messages=messages, provider="openai", custom_prompt_dict={} ) - + # OpenAI models use messages directly, no prompt conversion assert prompt == "" assert chat_history is None @@ -3604,37 +3679,33 @@ def test_bedrock_openai_response_parsing(): from litellm import ModelResponse from unittest.mock import Mock import json - + bedrock_llm = BedrockLLM() - + # Mock OpenAI-style response openai_response = { "choices": [ { "message": { "content": "The capital of France is Paris.", - "role": "assistant" + "role": "assistant", }, "finish_reason": "stop", - "index": 0 + "index": 0, } ], - "usage": { - "prompt_tokens": 10, - "completion_tokens": 8, - "total_tokens": 18 - } + "usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18}, } - + mock_response = Mock() mock_response.json.return_value = openai_response mock_response.text = json.dumps(openai_response) mock_response.status_code = 200 mock_response.headers = {} - + model_response = ModelResponse() mock_logging = Mock() - + result = bedrock_llm.process_response( model="openai/arn:aws:bedrock:us-east-1:123:imported-model/test", response=mock_response, @@ -3646,18 +3717,18 @@ def test_bedrock_openai_response_parsing(): data={}, messages=[{"role": "user", "content": "What is the capital of France?"}], print_verbose=lambda x: None, - encoding=None + encoding=None, ) - + # Verify response content assert result.choices[0].message.content == "The capital of France is Paris." assert result.choices[0].finish_reason == "stop" - + # Verify usage assert result.usage.prompt_tokens == 10 assert result.usage.completion_tokens == 8 assert result.usage.total_tokens == 18 - + print("✓ OpenAI response parsing works correctly") @@ -3665,45 +3736,47 @@ def test_bedrock_openai_request_transformation(): """ Test that the request is correctly transformed for OpenAI models. """ - from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import AmazonInvokeConfig - + from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + AmazonInvokeConfig, + ) + config = AmazonInvokeConfig() - + model = "openai/arn:aws:bedrock:us-east-1:123:imported-model/test" messages = [ {"role": "system", "content": "You are helpful"}, - {"role": "user", "content": "Hello"} + {"role": "user", "content": "Hello"}, ] - + optional_params = { "max_tokens": 100, "temperature": 0.7, "top_p": 0.9, - "stream": False + "stream": False, } - + litellm_params = {} headers = {} - - with patch.object(config, 'get_bedrock_invoke_provider', return_value="openai"): + + with patch.object(config, "get_bedrock_invoke_provider", return_value="openai"): result = config.transform_request( model=model, messages=messages, optional_params=optional_params.copy(), litellm_params=litellm_params, - headers=headers + headers=headers, ) - + # Verify the request uses messages format (not prompt) assert "messages" in result assert len(result["messages"]) == 2 assert result["messages"][0]["role"] == "system" assert result["messages"][1]["role"] == "user" - + # Verify parameters are included assert "max_tokens" in result assert "temperature" in result - + print("✓ Request transformation works correctly") @@ -3711,20 +3784,22 @@ def test_bedrock_openai_parameter_filtering(): """ Test that only supported OpenAI parameters are included in the request. """ - from litellm.llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import AmazonBedrockOpenAIConfig - + from litellm.llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import ( + AmazonBedrockOpenAIConfig, + ) + config = AmazonBedrockOpenAIConfig() model = "test-model" - + supported_params = config.get_supported_openai_params(model=model) - + # Verify common OpenAI parameters are supported assert "max_tokens" in supported_params assert "temperature" in supported_params assert "top_p" in supported_params assert "stream" in supported_params assert "stop" in supported_params - + print(f"✓ Parameter filtering supports: {len(supported_params)} parameters") print(f" Supported params: {supported_params}") @@ -3734,12 +3809,12 @@ def test_bedrock_openai_route_detection(): Test that the OpenAI route is correctly detected. """ from litellm.llms.bedrock.common_utils import BedrockModelInfo - + test_cases = [ ("openai/arn:aws:bedrock:us-east-1:123:imported-model/test", "openai"), ("bedrock/openai/arn:aws:bedrock:us-east-1:123:imported-model/test", "openai"), ] - + for model, expected_route in test_cases: route = BedrockModelInfo.get_bedrock_route(model) assert route == expected_route, f"Failed for model: {model}, got route: {route}" @@ -3751,15 +3826,30 @@ def test_bedrock_openai_explicit_route_check(): Test the explicit OpenAI route checker helper method. """ from litellm.llms.bedrock.common_utils import BedrockModelInfo - + # Test with openai/ prefix - assert BedrockModelInfo._explicit_openai_route("openai/arn:aws:bedrock:us-east-1:123:imported-model/test") is True - assert BedrockModelInfo._explicit_openai_route("bedrock/openai/arn:aws:bedrock:us-east-1:123:imported-model/test") is True - + assert ( + BedrockModelInfo._explicit_openai_route( + "openai/arn:aws:bedrock:us-east-1:123:imported-model/test" + ) + is True + ) + assert ( + BedrockModelInfo._explicit_openai_route( + "bedrock/openai/arn:aws:bedrock:us-east-1:123:imported-model/test" + ) + is True + ) + # Test without openai/ prefix assert BedrockModelInfo._explicit_openai_route("anthropic.claude-3-sonnet") is False - assert BedrockModelInfo._explicit_openai_route("arn:aws:bedrock:us-east-1:123:imported-model/test") is False - + assert ( + BedrockModelInfo._explicit_openai_route( + "arn:aws:bedrock:us-east-1:123:imported-model/test" + ) + is False + ) + print("✓ Explicit route check works correctly") @@ -3767,16 +3857,18 @@ def test_bedrock_openai_config_initialization(): """ Test that AmazonBedrockOpenAIConfig can be properly initialized. """ - from litellm.llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import AmazonBedrockOpenAIConfig - + from litellm.llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import ( + AmazonBedrockOpenAIConfig, + ) + config = AmazonBedrockOpenAIConfig() - + # Verify it has the necessary methods - assert hasattr(config, 'get_supported_openai_params') - assert hasattr(config, 'transform_request') - assert hasattr(config, 'transform_response') - assert hasattr(config, 'map_openai_params') - + assert hasattr(config, "get_supported_openai_params") + assert hasattr(config, "transform_request") + assert hasattr(config, "transform_response") + assert hasattr(config, "map_openai_params") + print("✓ AmazonBedrockOpenAIConfig initializes correctly") @@ -3785,9 +3877,9 @@ def test_bedrock_openai_multiple_message_types(): Test that various message content types are handled correctly. """ from litellm.llms.custom_httpx.http_handler import HTTPHandler - + client = HTTPHandler() - + # Test with mixed content types messages = [ {"role": "system", "content": "You are helpful"}, @@ -3796,11 +3888,14 @@ def test_bedrock_openai_multiple_message_types(): "role": "user", "content": [ {"type": "text", "text": "Complex message with text"}, - {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,iVBORw0KGg"}} - ] - } + { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,iVBORw0KGg"}, + }, + ], + }, ] - + with patch.object(client, "post") as mock_post: try: response = completion( @@ -3811,18 +3906,18 @@ def test_bedrock_openai_multiple_message_types(): ) except Exception as e: pass - + # Verify the request was made if mock_post.called: request_body = json.loads(mock_post.call_args.kwargs["data"]) - + # Verify messages are preserved assert "messages" in request_body assert len(request_body["messages"]) == 3 - + # Verify mixed content is handled assert isinstance(request_body["messages"][2]["content"], list) - + print("✓ Multiple message types handled correctly") @@ -3835,18 +3930,18 @@ def test_bedrock_openai_error_handling(): from litellm.llms.bedrock.common_utils import BedrockError from unittest.mock import Mock import json - + bedrock_llm = BedrockLLM() - + # Mock error response mock_response = Mock() mock_response.json.side_effect = Exception("Invalid JSON") mock_response.text = "Invalid response" mock_response.status_code = 422 - + model_response = ModelResponse() mock_logging = Mock() - + with pytest.raises(BedrockError) as exc_info: bedrock_llm.process_response( model="openai/arn:aws:bedrock:us-east-1:123:imported-model/test", @@ -3859,8 +3954,8 @@ def test_bedrock_openai_error_handling(): data={}, messages=[], print_verbose=lambda x: None, - encoding=None + encoding=None, ) - + assert exc_info.value.status_code == 422 print("✓ Error handling works correctly") From cc454232940233f26e4c7a6521dc2bb9423ba9ff Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 7 Jan 2026 10:42:02 +0530 Subject: [PATCH 011/121] Fix: Nonetype object has no method .get() for call tool --- ...odel_prices_and_context_window_backup.json | 193 +++++++++++++++++- .../mcp_server/rest_endpoints.py | 9 +- litellm/proxy/utils.py | 2 +- 3 files changed, 200 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c7a2f60856d..90b73e4709c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -405,7 +405,23 @@ "supports_video_input": true, "supports_vision": true }, - + "amazon.nova-2-multimodal-embeddings-v1:0": { + "litellm_provider": "bedrock", + "max_input_tokens": 8172, + "max_tokens": 8172, + "mode": "embedding", + "input_cost_per_token": 1.35e-7, + "input_cost_per_image": 6e-5, + "input_cost_per_video_per_second": 0.0007, + "input_cost_per_audio_per_second": 0.00014, + "output_cost_per_token": 0.0, + "output_vector_size": 3072, + "source": "https://us-east-1.console.aws.amazon.com/bedrock/home?region=us-east-1#/model-catalog/serverless/amazon.nova-2-multimodal-embeddings-v1:0", + "supports_embedding_image_input": true, + "supports_image_input": true, + "supports_video_input": true, + "supports_audio_input": true + }, "amazon.nova-micro-v1:0": { "input_cost_per_token": 3.5e-08, "litellm_provider": "bedrock_converse", @@ -32152,6 +32168,181 @@ "output_cost_per_token": 2e-07, "litellm_provider": "fireworks_ai", "mode": "chat" + }, + "llamagate/llama-3.1-8b": { + "max_tokens": 8192, + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 3e-08, + "output_cost_per_token": 5e-08, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "llamagate/llama-3.2-3b": { + "max_tokens": 8192, + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 4e-08, + "output_cost_per_token": 8e-08, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "llamagate/mistral-7b-v0.3": { + "max_tokens": 8192, + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "llamagate/qwen3-8b": { + "max_tokens": 8192, + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 4e-08, + "output_cost_per_token": 1.4e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "llamagate/dolphin3-8b": { + "max_tokens": 8192, + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 8e-08, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "llamagate/deepseek-r1-8b": { + "max_tokens": 16384, + "max_input_tokens": 65536, + "max_output_tokens": 16384, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "llamagate/deepseek-r1-7b-qwen": { + "max_tokens": 16384, + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "input_cost_per_token": 8e-08, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "llamagate/openthinker-7b": { + "max_tokens": 8192, + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 8e-08, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true + }, + "llamagate/qwen2.5-coder-7b": { + "max_tokens": 8192, + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 1.2e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "llamagate/deepseek-coder-6.7b": { + "max_tokens": 4096, + "max_input_tokens": 16384, + "max_output_tokens": 4096, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 1.2e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "llamagate/codellama-7b": { + "max_tokens": 4096, + "max_input_tokens": 16384, + "max_output_tokens": 4096, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 1.2e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "llamagate/qwen3-vl-8b": { + "max_tokens": 8192, + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5.5e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "llamagate/llava-7b": { + "max_tokens": 2048, + "max_input_tokens": 4096, + "max_output_tokens": 2048, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_response_schema": true, + "supports_vision": true + }, + "llamagate/gemma3-4b": { + "max_tokens": 8192, + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 3e-08, + "output_cost_per_token": 8e-08, + "litellm_provider": "llamagate", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "llamagate/nomic-embed-text": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "input_cost_per_token": 2e-08, + "output_cost_per_token": 0, + "litellm_provider": "llamagate", + "mode": "embedding" + }, + "llamagate/qwen3-embedding-8b": { + "max_tokens": 40960, + "max_input_tokens": 40960, + "input_cost_per_token": 2e-08, + "output_cost_per_token": 0, + "litellm_provider": "llamagate", + "mode": "embedding" } } diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 4c947b99ba3..642cb0cec2d 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -218,10 +218,10 @@ if MCP_AVAILABLE: from fastapi import HTTPException from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException - from litellm.proxy.proxy_server import add_litellm_data_to_request, proxy_config from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) + from litellm.proxy.proxy_server import add_litellm_data_to_request, proxy_config try: data = await request.json() @@ -252,7 +252,12 @@ if MCP_AVAILABLE: if mcp_server_auth_headers: data["mcp_server_auth_headers"] = mcp_server_auth_headers data["raw_headers"] = raw_headers_from_request - + + # Extract user_api_key_auth from metadata and add to top level + # call_mcp_tool expects user_api_key_auth as a top-level parameter + if "metadata" in data and "user_api_key_auth" in data["metadata"]: + data["user_api_key_auth"] = data["metadata"]["user_api_key_auth"] + result = await call_mcp_tool(**data) return result except BlockedPiiEntityError as e: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index d1a78534dae..f16c115fed3 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1195,7 +1195,7 @@ class ProxyLogging: and _callback.__class__.async_pre_call_hook != CustomLogger.async_pre_call_hook ): - if call_type == "mcp_call" and user_api_key_dict is None: + if call_type == "call_mcp_tool" and user_api_key_dict is None: continue response = await _callback.async_pre_call_hook( From bf506378b88ad31072643fc54b8a70deeec0313c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 7 Jan 2026 11:01:31 +0530 Subject: [PATCH 012/121] Fix: tool content should be str --- litellm/llms/deepinfra/chat/transformation.py | 60 +++++- .../test_deepinfra_chat_transformation.py | 191 ++++++++++++++++++ 2 files changed, 250 insertions(+), 1 deletion(-) diff --git a/litellm/llms/deepinfra/chat/transformation.py b/litellm/llms/deepinfra/chat/transformation.py index 09cdabcdd82..490597a0e60 100644 --- a/litellm/llms/deepinfra/chat/transformation.py +++ b/litellm/llms/deepinfra/chat/transformation.py @@ -1,9 +1,11 @@ -from typing import Optional, Tuple, Union +import json +from typing import List, Optional, Tuple, Union import litellm from litellm.constants import MIN_NON_ZERO_TEMPERATURE from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues class DeepInfraConfig(OpenAIGPTConfig): @@ -117,6 +119,62 @@ class DeepInfraConfig(OpenAIGPTConfig): optional_params[param] = value return optional_params + def _transform_tool_message_content(self, messages: List[AllMessageValues]) -> List[AllMessageValues]: + """ + Transform tool message content from array to string format for DeepInfra compatibility. + + DeepInfra requires tool message content to be a string, not an array. + This method converts tool message content from array format to string format. + + Example transformation: + - Input: {"role": "tool", "content": [{"type": "text", "text": "20"}]} + - Output: {"role": "tool", "content": "20"} + + Or if content is complex: + - Input: {"role": "tool", "content": [{"type": "text", "text": "result"}]} + - Output: {"role": "tool", "content": "[{\"type\": \"text\", \"text\": \"result\"}]"} + """ + for message in messages: + if message.get("role") == "tool": + content = message.get("content") + + # If content is a list/array, convert it to string + if isinstance(content, list): + # Check if it's a simple single text item + if ( + len(content) == 1 + and isinstance(content[0], dict) + and content[0].get("type") == "text" + and "text" in content[0] + ): + # Extract just the text value for simple cases + message["content"] = content[0]["text"] + else: + # For complex content, serialize the entire array as JSON string + message["content"] = json.dumps(content) + + return messages + + def _transform_messages( + self, messages: List[AllMessageValues], model: str, is_async: bool = False + ): + """ + Transform messages for DeepInfra compatibility. + Handles both sync and async transformations. + """ + # First apply parent class transformations + parent_result = super()._transform_messages(messages=messages, model=model, is_async=is_async) + + if is_async: + # If parent returns a coroutine, we need to await it and then apply our transformations + async def _async_transform(): + transformed_messages = await parent_result + return self._transform_tool_message_content(transformed_messages) + return _async_transform() + else: + # For sync case, parent_result is already the transformed messages + return self._transform_tool_message_content(parent_result) + def _get_openai_compatible_provider_info( self, api_base: Optional[str], api_key: Optional[str] ) -> Tuple[Optional[str], Optional[str]]: diff --git a/tests/test_litellm/llms/deepinfra/test_deepinfra_chat_transformation.py b/tests/test_litellm/llms/deepinfra/test_deepinfra_chat_transformation.py index fc8cf6dc60f..49d55f920b5 100644 --- a/tests/test_litellm/llms/deepinfra/test_deepinfra_chat_transformation.py +++ b/tests/test_litellm/llms/deepinfra/test_deepinfra_chat_transformation.py @@ -24,3 +24,194 @@ def test_deepseek_supported_openai_params(): supported_openai_params = DeepInfraConfig().get_supported_openai_params(model="deepinfra/deepseek-ai/DeepSeek-V3.1") print(supported_openai_params) assert "reasoning_effort" in supported_openai_params + + +def test_deepinfra_tool_message_content_transformation(): + """ + Test that DeepInfra transforms tool message content from array to string. + + This fixes the issue where LibreChat sends tool messages with content as an array: + {"role": "tool", "content": [{"type": "text", "text": "20"}]} + + DeepInfra requires content to be a string, so we transform it to: + {"role": "tool", "content": "20"} + + Related to issue #13982 + """ + from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig + + config = DeepInfraConfig() + + # Test case 1: Simple single text item in array (common case from LibreChat) + messages_with_array_content = [ + { + "role": "user", + "content": "Calculate 10 + 10" + }, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "function": { + "name": "calculator", + "arguments": '{"input": "10 + 10"}' + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "call_123", + "name": "calculator", + "content": [{"type": "text", "text": "20"}] # Array format from LibreChat + } + ] + + transformed_messages = config._transform_messages( + messages=messages_with_array_content, + model="deepinfra/Qwen/Qwen3-235B-A22B" + ) + + # Verify the tool message content was converted to string + tool_message = transformed_messages[2] + assert tool_message["role"] == "tool" + assert isinstance(tool_message["content"], str) + assert tool_message["content"] == "20" + print(f"✓ Test case 1 passed: {tool_message['content']}") + + # Test case 2: Complex content array (multiple items) + messages_with_complex_content = [ + { + "role": "user", + "content": "Test" + }, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_456", + "type": "function", + "function": {"name": "test", "arguments": "{}"} + } + ] + }, + { + "role": "tool", + "tool_call_id": "call_456", + "content": [ + {"type": "text", "text": "Result 1"}, + {"type": "text", "text": "Result 2"} + ] + } + ] + + transformed_messages_complex = config._transform_messages( + messages=messages_with_complex_content, + model="deepinfra/Qwen/Qwen3-235B-A22B" + ) + + tool_message_complex = transformed_messages_complex[2] + assert tool_message_complex["role"] == "tool" + assert isinstance(tool_message_complex["content"], str) + # For complex content, it should be JSON stringified + parsed_content = json.loads(tool_message_complex["content"]) + assert len(parsed_content) == 2 + assert parsed_content[0]["text"] == "Result 1" + print(f"✓ Test case 2 passed: {tool_message_complex['content']}") + + # Test case 3: Tool message with string content (should remain unchanged) + messages_with_string_content = [ + { + "role": "user", + "content": "Test" + }, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_789", + "type": "function", + "function": {"name": "test", "arguments": "{}"} + } + ] + }, + { + "role": "tool", + "tool_call_id": "call_789", + "content": "Simple string result" # Already a string + } + ] + + transformed_messages_string = config._transform_messages( + messages=messages_with_string_content, + model="deepinfra/Qwen/Qwen3-235B-A22B" + ) + + tool_message_string = transformed_messages_string[2] + assert tool_message_string["role"] == "tool" + assert isinstance(tool_message_string["content"], str) + assert tool_message_string["content"] == "Simple string result" + print(f"✓ Test case 3 passed: {tool_message_string['content']}") + + print("\n✅ All DeepInfra tool message transformation tests passed!") + + +@pytest.mark.asyncio +async def test_deepinfra_tool_message_content_transformation_async(): + """ + Test that DeepInfra transforms tool message content from array to string in async mode. + + This ensures the async path works correctly when is_async=True. + + Related to issue #13982 + """ + from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig + + config = DeepInfraConfig() + + # Test async transformation with tool message containing array content + messages_with_array_content = [ + { + "role": "user", + "content": "Calculate 10 + 10" + }, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "function": { + "name": "calculator", + "arguments": '{"input": "10 + 10"}' + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "call_123", + "name": "calculator", + "content": [{"type": "text", "text": "20"}] # Array format from LibreChat + } + ] + + # Call with is_async=True + transformed_messages = await config._transform_messages( + messages=messages_with_array_content, + model="deepinfra/Qwen/Qwen3-235B-A22B", + is_async=True + ) + + # Verify the tool message content was converted to string + tool_message = transformed_messages[2] + assert tool_message["role"] == "tool" + assert isinstance(tool_message["content"], str) + assert tool_message["content"] == "20" + print(f"✓ Async test passed: {tool_message['content']}") + + print("\n✅ DeepInfra async tool message transformation test passed!") From d3e24ab9cdf15883a9bb8e77a9faf1d994c62a42 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 7 Jan 2026 12:58:54 +0530 Subject: [PATCH 013/121] Fix: Gemini generate content request with audio file id --- .../prompt_templates/common_utils.py | 17 +- .../llms/vertex_ai/gemini/transformation.py | 2 +- .../test_vertex_ai_gemini_transformation.py | 186 ++++++++++++++++++ 3 files changed, 199 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index b100b9b516b..2f8568db704 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -6,6 +6,7 @@ import io import mimetypes import re from os import PathLike +from pathlib import Path from typing import ( TYPE_CHECKING, Any, @@ -533,6 +534,12 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData: # Convert content to bytes if isinstance(file_content, (str, PathLike)): # If it's a path, open and read the file + # Extract filename from path if not already set + if filename is None: + if isinstance(file_content, PathLike): + filename = Path(file_content).name + else: + filename = Path(str(file_content)).name with open(file_content, "rb") as f: content = f.read() elif isinstance(file_content, io.IOBase): @@ -550,11 +557,11 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData: # Use provided content type or guess based on filename if not content_type: - content_type = ( - mimetypes.guess_type(filename)[0] - if filename - else "application/octet-stream" - ) + if filename: + guessed_type = mimetypes.guess_type(filename)[0] + content_type = guessed_type if guessed_type else "application/octet-stream" + else: + content_type = "application/octet-stream" return ExtractedFileData( filename=filename, diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 2bbdfa17cde..22042f7d641 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -115,7 +115,7 @@ def _process_gemini_image( and (image_type := format or _get_image_mime_type_from_url(image_url)) is not None ): - file_data = FileDataType(file_uri=image_url, mime_type=image_type) + file_data = FileDataType(mime_type=image_type, file_uri=image_url) part = {"file_data": file_data} if media_resolution_enum is not None and model is not None: diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 3c3d68e8be8..70e6e9452e5 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -721,3 +721,189 @@ def test_convert_tool_response_text_only(): # Check inline_data does NOT exist (no image provided) assert "inline_data" not in result + + +def test_file_data_field_order(): + """ + Test that file_data fields are in the correct order (mime_type before file_uri). + + The Gemini API is sensitive to field order in the file_data object. + This test verifies that mime_type comes before file_uri in both: + 1. Dictionary key order + 2. JSON serialization + + Related issue: Gemini API returns 400 INVALID_ARGUMENT when fields are in wrong order. + """ + import json + from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_image + + # Test with HTTPS URL and explicit format (audio file) + file_url = "https://generativelanguage.googleapis.com/v1beta/files/test123" + format = "audio/mpeg" + + result = _process_gemini_image(image_url=file_url, format=format) + + # Verify the result has file_data + assert "file_data" in result + file_data = result["file_data"] + + # Verify both fields are present + assert "mime_type" in file_data + assert "file_uri" in file_data + assert file_data["mime_type"] == "audio/mpeg" + assert file_data["file_uri"] == file_url + + # Verify field order by checking dictionary keys + # In Python 3.7+, dict maintains insertion order + file_data_keys = list(file_data.keys()) + assert file_data_keys.index("mime_type") < file_data_keys.index("file_uri"), \ + "mime_type must come before file_uri in the file_data dict" + + # Also verify by serializing to JSON string + json_str = json.dumps(file_data) + mime_type_pos = json_str.find('"mime_type"') + file_uri_pos = json_str.find('"file_uri"') + assert mime_type_pos < file_uri_pos, \ + "mime_type must appear before file_uri in JSON serialization" + + +def test_file_data_field_order_gcs_urls(): + """Test that GCS URLs also maintain correct field order.""" + import json + from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_image + + # Test with GCS URL + gcs_url = "gs://bucket/audio.mp3" + + result = _process_gemini_image(image_url=gcs_url) + + # Verify the result has file_data + assert "file_data" in result + file_data = result["file_data"] + + # Verify both fields are present + assert "mime_type" in file_data + assert "file_uri" in file_data + + # Verify field order + file_data_keys = list(file_data.keys()) + assert file_data_keys.index("mime_type") < file_data_keys.index("file_uri"), \ + "mime_type must come before file_uri in the file_data dict" + + +def test_extract_file_data_with_path_object(): + """ + Test that filename is correctly extracted from Path objects for MIME type detection. + + When uploading files using Path objects (e.g., Path("speech.mp3")), the filename + must be extracted to enable proper MIME type detection. Without this, files get + uploaded with 'application/octet-stream' instead of the correct MIME type. + + Related issue: Files uploaded with wrong MIME type cause Gemini API to reject + requests where the specified format doesn't match the uploaded file's MIME type. + """ + from pathlib import Path + from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data + import tempfile + import os + + # Create a temporary MP3 file + with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as tmp: + tmp.write(b"fake mp3 content") + tmp_path = tmp.name + + try: + # Test with Path object + path_obj = Path(tmp_path) + extracted = extract_file_data(path_obj) + + # Verify filename was extracted + assert extracted["filename"] is not None + assert extracted["filename"].endswith(".mp3") + + # Verify MIME type was correctly detected + assert extracted["content_type"] == "audio/mpeg", \ + f"Expected 'audio/mpeg' but got '{extracted['content_type']}'" + + # Verify content was read + assert extracted["content"] == b"fake mp3 content" + + finally: + # Clean up temporary file + os.unlink(tmp_path) + + +def test_extract_file_data_with_string_path(): + """Test that filename is correctly extracted from string paths.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data + import tempfile + import os + + # Create a temporary WAV file + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: + tmp.write(b"fake wav content") + tmp_path = tmp.name + + try: + # Test with string path + extracted = extract_file_data(tmp_path) + + # Verify filename was extracted + assert extracted["filename"] is not None + assert extracted["filename"].endswith(".wav") + + # Verify MIME type was correctly detected (can be audio/wav or audio/x-wav depending on system) + assert extracted["content_type"] in ["audio/wav", "audio/x-wav"], \ + f"Expected 'audio/wav' or 'audio/x-wav' but got '{extracted['content_type']}'" + + # Verify content was read + assert extracted["content"] == b"fake wav content" + + finally: + # Clean up temporary file + os.unlink(tmp_path) + + +def test_extract_file_data_with_tuple_format(): + """Test that tuple format (with explicit content_type) still works correctly.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data + + # Test with tuple format: (filename, content, content_type) + filename = "test_audio.mp3" + content = b"test audio content" + content_type = "audio/mpeg" + + extracted = extract_file_data((filename, content, content_type)) + + # Verify all fields are correct + assert extracted["filename"] == filename + assert extracted["content"] == content + assert extracted["content_type"] == content_type + + +def test_extract_file_data_fallback_to_octet_stream(): + """Test that unknown file types fall back to application/octet-stream.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data + import tempfile + import os + + # Create a temporary file with unknown extension + with tempfile.NamedTemporaryFile(suffix=".xyz123", delete=False) as tmp: + tmp.write(b"unknown content") + tmp_path = tmp.name + + try: + # Test with unknown file type + extracted = extract_file_data(tmp_path) + + # Verify filename was extracted + assert extracted["filename"] is not None + assert extracted["filename"].endswith(".xyz123") + + # Verify MIME type falls back to octet-stream + assert extracted["content_type"] == "application/octet-stream", \ + f"Expected 'application/octet-stream' for unknown type, got '{extracted['content_type']}'" + + finally: + # Clean up temporary file + os.unlink(tmp_path) From fdb96796574e3d28016c76455a4388ba1171f49d Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 7 Jan 2026 14:30:31 +0530 Subject: [PATCH 014/121] Add annotations to completions responses API bridge --- .../transformation.py | 47 +++++ litellm/proxy/hooks/litellm_skills/main.py | 4 +- ...responses_transformation_transformation.py | 183 ++++++++++++++++++ 3 files changed, 232 insertions(+), 2 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index a89efc4e82b..585454f944b 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -90,9 +90,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): content_type = content_item.get("type") if content_type == "output_text": response_text = content_item.get("text", "") + # Extract annotations from content if present + annotations = LiteLLMResponsesTransformationHandler._convert_annotations_to_chat_format( + content_item.get("annotations", None) + ) msg = Message( role=item.get("role", "assistant"), content=response_text if response_text else "", + annotations=annotations, ) choice = Choices(message=msg, finish_reason="stop", index=index) return choice, index + 1 @@ -364,10 +369,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): elif isinstance(item, ResponseOutputMessage): for content in item.content: response_text = getattr(content, "text", "") + # Extract annotations from content if present + raw_annotations = getattr(content, "annotations", None) + annotations = LiteLLMResponsesTransformationHandler._convert_annotations_to_chat_format( + raw_annotations + ) msg = Message( role=item.role, content=response_text if response_text else "", reasoning_content=reasoning_content, + annotations=annotations, ) choices.append( @@ -763,6 +774,42 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return {"format": {"type": "text"}} return None + + @staticmethod + def _convert_annotations_to_chat_format( + annotations: Optional[List[Any]], + ) -> Optional[List[Dict[str, Any]]]: + """ + Convert annotations from Responses API to Chat Completions format. + + Annotations are already in compatible format between both APIs, + so we just need to convert Pydantic models to dicts. + """ + if not annotations: + return None + + result: List[Dict[str, Any]] = [] + for annotation in annotations: + try: + # Convert Pydantic models to dicts (handles both v1 and v2) + if hasattr(annotation, "model_dump"): + annotation_dict = annotation.model_dump() + elif hasattr(annotation, "dict"): + annotation_dict = annotation.dict() + elif isinstance(annotation, dict): + annotation_dict = annotation + else: + # Skip unsupported annotation types + verbose_logger.debug(f"Skipping unsupported annotation type: {type(annotation)}") + continue + + result.append(annotation_dict) + except Exception as e: + # Skip malformed annotations + verbose_logger.debug(f"Skipping malformed annotation: {annotation}, error: {e}") + continue + + return result if result else None def _map_responses_status_to_finish_reason(self, status: Optional[str]) -> str: """Map responses API status to chat completion finish_reason""" diff --git a/litellm/proxy/hooks/litellm_skills/main.py b/litellm/proxy/hooks/litellm_skills/main.py index 26d4cbe1de7..c2ad1e29447 100644 --- a/litellm/proxy/hooks/litellm_skills/main.py +++ b/litellm/proxy/hooks/litellm_skills/main.py @@ -336,8 +336,8 @@ class SkillsInjectionHook(CustomLogger): ) # Check if code execution is enabled for this request - litellm_metadata = request_data.get("litellm_metadata", {}) - metadata = request_data.get("metadata", {}) + litellm_metadata = request_data.get("litellm_metadata") or {} + metadata = request_data.get("metadata") or {} code_exec_enabled = ( litellm_metadata.get("_litellm_code_execution_enabled") or diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 596398e639f..f8a082ee30c 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -1095,3 +1095,186 @@ def test_map_reasoning_effort_adds_summary_detailed(): os.environ["LITELLM_REASONING_AUTO_SUMMARY"] = original_env elif "LITELLM_REASONING_AUTO_SUMMARY" in os.environ: del os.environ["LITELLM_REASONING_AUTO_SUMMARY"] + + +def test_transform_response_preserves_annotations(): + """ + Test that annotations from Responses API are preserved when transforming to Chat Completions format. + + This is a regression test for the bug where annotations (like url_citation) were being + dropped during the transformation from ResponsesAPIResponse to ModelResponse. + + The fix ensures annotations are extracted from ResponseOutputText content items and + passed through to the Message object in the Chat Completions response. + """ + from unittest.mock import Mock + + from openai.types.responses import ResponseOutputMessage, ResponseOutputText + + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + from litellm.types.llms.openai import ( + InputTokensDetails, + OutputTokensDetails, + ResponseAPIUsage, + ResponsesAPIResponse, + ) + from litellm.types.utils import ModelResponse, Usage + + handler = LiteLLMResponsesTransformationHandler() + + # Create annotations similar to what OpenAI Responses API returns + annotations = [ + { + "type": "url_citation", + "start_index": 0, + "end_index": 100, + "title": "Example Article", + "url": "https://example.com/article", + }, + { + "type": "url_citation", + "start_index": 101, + "end_index": 200, + "title": "Another Source", + "url": "https://example.com/source", + }, + ] + + # Create output text with annotations + output_text = ResponseOutputText( + annotations=annotations, + text="Here is some information with citations.", + type="output_text", + logprobs=[], + ) + + # Create output message + output_message = ResponseOutputMessage( + id="msg_test123", + content=[output_text], + role="assistant", + status="completed", + type="message", + ) + + # Create usage information + usage = ResponseAPIUsage( + input_tokens=10, + input_tokens_details=InputTokensDetails( + audio_tokens=None, cached_tokens=0, text_tokens=None + ), + output_tokens=20, + output_tokens_details=OutputTokensDetails( + reasoning_tokens=0, text_tokens=None + ), + total_tokens=30, + cost=None, + ) + + # Create the full ResponsesAPIResponse + raw_response = ResponsesAPIResponse( + id="resp_test123", + created_at=1234567890, + error=None, + incomplete_details=None, + instructions=None, + metadata={}, + model="gpt-5.1", + object="response", + output=[output_message], + parallel_tool_calls=True, + temperature=1.0, + tool_choice="auto", + tools=[], + top_p=1.0, + max_output_tokens=None, + previous_response_id=None, + reasoning=None, + status="completed", + text={"format": {"type": "text"}, "verbosity": "medium"}, + truncation="disabled", + usage=usage, + user=None, + store=True, + background=False, + billing={"payer": "openai"}, + max_tool_calls=None, + prompt_cache_key=None, + safety_identifier=None, + service_tier="default", + top_logprobs=0, + ) + + # Create empty model_response + model_response = ModelResponse( + id="chatcmpl-test123", + created=1234567890, + model=None, + object="chat.completion", + system_fingerprint=None, + choices=[], + usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0), + ) + + # Create mock objects for required parameters + logging_obj = Mock() + messages = [{"role": "user", "content": "Tell me about AI"}] + request_data = {"model": "gpt-5.1"} + optional_params = {} + litellm_params = {"acompletion": False, "api_key": None} + encoding = Mock() + + # Call transform_response + result = handler.transform_response( + model="gpt-5.1", + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data=request_data, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + encoding=encoding, + api_key=None, + json_mode=None, + ) + + # Assertions + assert result.model == "gpt-5.1" + assert len(result.choices) == 1 + + # Check the choice + choice = result.choices[0] + assert choice.finish_reason == "stop" + assert choice.index == 0 + assert choice.message.role == "assistant" + assert choice.message.content == "Here is some information with citations." + + # Check that annotations are preserved + assert hasattr(choice.message, "annotations"), "Message should have annotations attribute" + assert choice.message.annotations is not None, "Annotations should not be None" + assert len(choice.message.annotations) == 2, f"Expected 2 annotations, got {len(choice.message.annotations)}" + + # Verify annotation content + annotation1 = choice.message.annotations[0] + assert annotation1["type"] == "url_citation" + assert annotation1["title"] == "Example Article" + assert annotation1["url"] == "https://example.com/article" + assert annotation1["start_index"] == 0 + assert annotation1["end_index"] == 100 + + annotation2 = choice.message.annotations[1] + assert annotation2["type"] == "url_citation" + assert annotation2["title"] == "Another Source" + assert annotation2["url"] == "https://example.com/source" + assert annotation2["start_index"] == 101 + assert annotation2["end_index"] == 200 + + # Check usage + assert result.usage.prompt_tokens == 10 + assert result.usage.completion_tokens == 20 + assert result.usage.total_tokens == 30 + + print("✓ Annotations from Responses API are correctly preserved in Chat Completions format") From 000913fa12ec41916c04ab1107cd31d196c3fb0f Mon Sep 17 00:00:00 2001 From: drorIvry Date: Wed, 7 Jan 2026 13:53:12 +0200 Subject: [PATCH 015/121] Hotfix - docs qualifire (#18724) * Hotfix - docs qualifire * Hotfix - docs qualifire * Hotfix - docs qualifire * Hotfix - docs qualifire * Hotfix - docs qualifire * Hotfix - docs qualifire * Hotfix - docs qualifire --- .../docs/proxy/guardrails/qualifire.md | 41 +- docs/my-website/sidebars.js | 1 + .../guardrail_hooks/qualifire/qualifire.py | 244 ++++++---- .../guardrail_hooks/test_qualifire.py | 433 +++++++++++++----- 4 files changed, 475 insertions(+), 244 deletions(-) diff --git a/docs/my-website/docs/proxy/guardrails/qualifire.md b/docs/my-website/docs/proxy/guardrails/qualifire.md index 66961c92d9d..850af37e47f 100644 --- a/docs/my-website/docs/proxy/guardrails/qualifire.md +++ b/docs/my-website/docs/proxy/guardrails/qualifire.md @@ -8,13 +8,7 @@ Use [Qualifire](https://qualifire.ai) to evaluate LLM outputs for quality, safet ## Quick Start -### 1. Install the Qualifire SDK - -```bash -pip install qualifire -``` - -### 2. Define Guardrails on your LiteLLM config.yaml +### 1. Define Guardrails on your LiteLLM config.yaml Define your guardrails under the `guardrails` section: @@ -61,13 +55,13 @@ guardrails: - `post_call` Run **after** LLM call, on **input & output** - `during_call` Run **during** LLM call, on **input**. Same as `pre_call` but runs in parallel as LLM call. Response not returned until guardrail check completes -### 3. Start LiteLLM Gateway +### 2. Start LiteLLM Gateway ```shell litellm --config config.yaml --detailed_debug ``` -### 4. Test request +### 3. Test request **[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** @@ -142,7 +136,7 @@ guardrails: evaluation_id: eval_abc123 # Your evaluation ID from Qualifire dashboard ``` -When `evaluation_id` is provided, LiteLLM will use `invoke_evaluation()` instead of `evaluate()`, running the pre-configured evaluation from your dashboard. +When `evaluation_id` is provided, LiteLLM will use the invoke evaluation API endpoint instead of the evaluate endpoint, running the pre-configured evaluation from your dashboard. ## Available Checks @@ -213,19 +207,19 @@ guardrails: ### Parameter Reference -| Parameter | Type | Default | Description | -| ------------------------------ | ----------- | --------------------------- | -------------------------------------------------------- | -| `api_key` | `str` | `QUALIFIRE_API_KEY` env var | Your Qualifire API key | -| `api_base` | `str` | `None` | Custom API base URL (optional) | -| `evaluation_id` | `str` | `None` | Pre-configured evaluation ID from Qualifire dashboard | -| `prompt_injections` | `bool` | `true` (if no other checks) | Enable prompt injection detection | -| `hallucinations_check` | `bool` | `None` | Enable hallucination detection | -| `grounding_check` | `bool` | `None` | Enable grounding verification | -| `pii_check` | `bool` | `None` | Enable PII detection | -| `content_moderation_check` | `bool` | `None` | Enable content moderation | -| `tool_selection_quality_check` | `bool` | `None` | Enable tool selection quality check | -| `assertions` | `List[str]` | `None` | Custom assertions to validate | -| `on_flagged` | `str` | `"block"` | Action when content is flagged: `"block"` or `"monitor"` | +| Parameter | Type | Default | Description | +| ------------------------------ | ----------- | ---------------------------- | -------------------------------------------------------- | +| `api_key` | `str` | `QUALIFIRE_API_KEY` env var | Your Qualifire API key | +| `api_base` | `str` | `https://proxy.qualifire.ai` | Custom API base URL (optional) | +| `evaluation_id` | `str` | `None` | Pre-configured evaluation ID from Qualifire dashboard | +| `prompt_injections` | `bool` | `true` (if no other checks) | Enable prompt injection detection | +| `hallucinations_check` | `bool` | `None` | Enable hallucination detection | +| `grounding_check` | `bool` | `None` | Enable grounding verification | +| `pii_check` | `bool` | `None` | Enable PII detection | +| `content_moderation_check` | `bool` | `None` | Enable content moderation | +| `tool_selection_quality_check` | `bool` | `None` | Enable tool selection quality check | +| `assertions` | `List[str]` | `None` | Custom assertions to validate | +| `on_flagged` | `str` | `"block"` | Action when content is flagged: `"block"` or `"monitor"` | ### Default Behavior @@ -261,4 +255,3 @@ This evaluates whether the LLM selected the appropriate tools and provided corre - [Qualifire Documentation](https://docs.qualifire.ai) - [Qualifire Dashboard](https://app.qualifire.ai) -- [Qualifire Python SDK](https://github.com/qualifire-dev/qualifire-python-sdk) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 5d2f096156e..004132ca05b 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -55,6 +55,7 @@ const sidebars = { "proxy/guardrails/test_playground", "proxy/guardrails/litellm_content_filter", ...[ + "proxy/guardrails/qualifire", "proxy/guardrails/aim_security", "proxy/guardrails/onyx_security", "proxy/guardrails/aporia_api", diff --git a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py index a6971b49f3b..87da11efad0 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py +++ b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py @@ -5,6 +5,7 @@ # +-------------------------------------------------------------+ # Qualifire - Evaluate LLM outputs for quality, safety, and reliability +import json import os from typing import Any, Dict, List, Literal, Optional, Type @@ -15,12 +16,17 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import ( Logging as LiteLLMLoggingObj, ) +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, +) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel from litellm.types.utils import GenericGuardrailAPIInputs GUARDRAIL_NAME = "qualifire" +DEFAULT_QUALIFIRE_API_BASE = "https://proxy.qualifire.ai" class QualifireGuardrail(CustomGuardrail): @@ -44,7 +50,7 @@ class QualifireGuardrail(CustomGuardrail): Args: api_key: API key for Qualifire (or use QUALIFIRE_API_KEY env var) - api_base: Optional custom API base URL + api_base: Optional custom API base URL (defaults to https://api.qualifire.ai) evaluation_id: Pre-configured evaluation ID from Qualifire dashboard prompt_injections: Enable prompt injection detection (default if no other checks) hallucinations_check: Enable hallucination detection @@ -64,6 +70,7 @@ class QualifireGuardrail(CustomGuardrail): api_base or get_secret_str("QUALIFIRE_BASE_URL") or os.environ.get("QUALIFIRE_BASE_URL") + or DEFAULT_QUALIFIRE_API_BASE ) self.evaluation_id = evaluation_id self.prompt_injections = prompt_injections @@ -79,7 +86,11 @@ class QualifireGuardrail(CustomGuardrail): if not self._has_any_check_enabled() and not self.evaluation_id: self.prompt_injections = True - self._client = None + # Initialize async HTTP client for direct API calls + self.async_handler = get_async_httpx_client( + llm_provider=httpxSpecialProvider.GuardrailCallback + ) + super().__init__(**kwargs) def _has_any_check_enabled(self) -> bool: @@ -96,43 +107,22 @@ class QualifireGuardrail(CustomGuardrail): ] ) - def _get_client(self): - """Lazy initialization of Qualifire client.""" - if self._client is None: - try: - from qualifire.client import Client - except ImportError: - raise ImportError( - "qualifire package is required for QualifireGuardrail. " - "Install it with: pip install qualifire" - ) - - client_kwargs: Dict[str, Any] = {} - if self.qualifire_api_key: - client_kwargs["api_key"] = self.qualifire_api_key - if self.qualifire_api_base: - client_kwargs["base_url"] = self.qualifire_api_base - - self._client = Client(**client_kwargs) - - return self._client - - def _convert_messages_to_qualifire_format( + def _convert_messages_to_api_format( self, messages: List[AllMessageValues] - ) -> List[Any]: + ) -> List[Dict[str, Any]]: """ - Convert LiteLLM messages to Qualifire's LLMMessage format. + Convert LiteLLM messages to Qualifire API format. Supports tool calls for tool_selection_quality_check. - """ - try: - from qualifire.types import LLMMessage, LLMToolCall - except ImportError: - raise ImportError( - "qualifire package is required for QualifireGuardrail. " - "Install it with: pip install qualifire" - ) - qualifire_messages = [] + Returns a list of dicts matching the API's ModelInvocationCanonicalMessage schema: + { + "role": "user" | "assistant" | "system" | "tool", + "content": "...", + "tool_call_id": "...", # optional + "tool_calls": [{"id": "...", "name": "...", "arguments": {...}}] # optional + } + """ + api_messages = [] for msg in messages: role = msg.get("role", "user") content = msg.get("content", "") @@ -147,42 +137,86 @@ class QualifireGuardrail(CustomGuardrail): text_parts.append(part) content = "\n".join(text_parts) - llm_message_kwargs: Dict[str, Any] = { + api_message: Dict[str, Any] = { "role": role, "content": content if isinstance(content, str) else str(content), } + # Handle tool_call_id for tool response messages + tool_call_id = msg.get("tool_call_id") + if tool_call_id: + api_message["tool_call_id"] = tool_call_id + # Handle tool calls if present tool_calls = msg.get("tool_calls") if tool_calls and isinstance(tool_calls, list): - qualifire_tool_calls = [] + api_tool_calls = [] for tc in tool_calls: if isinstance(tc, dict): function_info = tc.get("function", {}) # Arguments can be a string (JSON) or dict args = function_info.get("arguments", {}) if isinstance(args, str): - import json - try: args = json.loads(args) except json.JSONDecodeError: args = {} - qualifire_tool_calls.append( - LLMToolCall( - id=tc.get("id") or "", - name=function_info.get("name") or "", - arguments=args if isinstance(args, dict) else {}, - ) + api_tool_calls.append( + { + "id": tc.get("id") or "", + "name": function_info.get("name") or "", + "arguments": args if isinstance(args, dict) else {}, + } ) - if qualifire_tool_calls: - llm_message_kwargs["tool_calls"] = qualifire_tool_calls + if api_tool_calls: + api_message["tool_calls"] = api_tool_calls - qualifire_messages.append(LLMMessage(**llm_message_kwargs)) + api_messages.append(api_message) - return qualifire_messages + return api_messages - def _check_if_flagged(self, result: Any) -> bool: + def _convert_tools_to_api_format( + self, tools: Optional[List[Any]] + ) -> Optional[List[Dict[str, Any]]]: + """ + Convert OpenAI-format tools to Qualifire API format. + + Returns a list of dicts matching the API's ModelInvocationToolDefinition schema: + { + "name": "...", + "description": "...", + "parameters": {...} + } + """ + if not tools: + return None + + api_tools = [] + for tool in tools: + if isinstance(tool, dict): + # Handle OpenAI function tool format + if tool.get("type") == "function": + function_def = tool.get("function", {}) + api_tools.append( + { + "name": function_def.get("name", ""), + "description": function_def.get("description", ""), + "parameters": function_def.get("parameters", {}), + } + ) + # Handle direct tool format + elif "name" in tool: + api_tools.append( + { + "name": tool.get("name", ""), + "description": tool.get("description", ""), + "parameters": tool.get("parameters", {}), + } + ) + + return api_tools if api_tools else None + + def _check_if_flagged(self, result: Dict[str, Any]) -> bool: """ Check if the Qualifire evaluation result indicates flagged content. @@ -190,65 +224,53 @@ class QualifireGuardrail(CustomGuardrail): A high score (close to 100) indicates GOOD content, low score indicates problems. """ # Check evaluation results for any flagged items - evaluation_results = getattr(result, "evaluationResults", None) or [] - if isinstance(result, dict): - evaluation_results = result.get("evaluationResults", []) or [] + evaluation_results = result.get("evaluationResults", []) or [] for eval_result in evaluation_results: - results: List[Any] = [] - if isinstance(eval_result, dict): - results = eval_result.get("results", []) or [] - else: - results = getattr(eval_result, "results", []) or [] - + results = eval_result.get("results", []) or [] for r in results: - flagged = ( - r.get("flagged") - if isinstance(r, dict) - else getattr(r, "flagged", False) - ) - if flagged: + if r.get("flagged"): return True return False - def _build_evaluate_kwargs( + def _build_evaluate_payload( self, - qualifire_messages: List[Any], + api_messages: List[Dict[str, Any]], output: Optional[str], assertions: Optional[List[str]], - available_tools: Optional[List[Any]], + available_tools: Optional[List[Dict[str, Any]]], ) -> Dict[str, Any]: - """Build kwargs dictionary for the evaluate call.""" - kwargs: Dict[str, Any] = {"messages": qualifire_messages} + """Build payload dictionary for the /api/evaluation/evaluate endpoint.""" + payload: Dict[str, Any] = {"messages": api_messages} if output is not None: - kwargs["output"] = output + payload["output"] = output # Add enabled checks if self.prompt_injections: - kwargs["prompt_injections"] = True + payload["prompt_injections"] = True if self.hallucinations_check: - kwargs["hallucinations_check"] = True + payload["hallucinations_check"] = True if self.grounding_check: - kwargs["grounding_check"] = True + payload["grounding_check"] = True if self.pii_check: - kwargs["pii_check"] = True + payload["pii_check"] = True if self.content_moderation_check: - kwargs["content_moderation_check"] = True + payload["content_moderation_check"] = True if self.tool_selection_quality_check: # Only enable tool_selection_quality_check if available_tools is provided if available_tools: - kwargs["tool_selection_quality_check"] = True - kwargs["available_tools"] = available_tools + payload["tool_selection_quality_check"] = True + payload["available_tools"] = available_tools else: verbose_proxy_logger.debug( "Qualifire Guardrail: tool_selection_quality_check enabled but no available_tools provided, skipping this check" ) if assertions: - kwargs["assertions"] = assertions + payload["assertions"] = assertions - return kwargs + return payload async def _run_qualifire_check( self, @@ -274,11 +296,17 @@ class QualifireGuardrail(CustomGuardrail): assertions = dynamic_params.get("assertions") or self.assertions on_flagged = dynamic_params.get("on_flagged") or self.on_flagged - try: - client = self._get_client() - qualifire_messages = self._convert_messages_to_qualifire_format(messages) + # Prepare headers + headers = { + "X-Qualifire-API-Key": self.qualifire_api_key or "", + "Content-Type": "application/json", + } - # Use invoke_evaluation if evaluation_id is provided + try: + # Convert messages to API format + api_messages = self._convert_messages_to_api_format(messages) + + # Use invoke endpoint if evaluation_id is provided if evaluation_id: # For invoke_evaluation, we need to extract input/output input_text = "" @@ -291,25 +319,47 @@ class QualifireGuardrail(CustomGuardrail): input_text = content break - result = client.invoke_evaluation( - evaluation_id=evaluation_id, - input=input_text, - output=output or "", - ) + payload = { + "evaluation_id": evaluation_id, + "input": input_text, + "output": output or "", + "messages": api_messages, + } + + # Convert tools if provided + api_tools = self._convert_tools_to_api_format(available_tools) + if api_tools: + payload["available_tools"] = api_tools + + url = f"{self.qualifire_api_base}/api/evaluation/invoke" else: - # Use evaluate with individual checks - kwargs = self._build_evaluate_kwargs( - qualifire_messages=qualifire_messages, + # Use evaluate endpoint with individual checks + api_tools = self._convert_tools_to_api_format(available_tools) + payload = self._build_evaluate_payload( + api_messages=api_messages, output=output, assertions=assertions, - available_tools=available_tools, + available_tools=api_tools, ) - result = client.evaluate(**kwargs) + url = f"{self.qualifire_api_base}/api/evaluation/evaluate" - # Convert result to dict for logging + verbose_proxy_logger.debug( + f"Qualifire Guardrail: Making request to {url}" + ) + + # Make the API request + response = await self.async_handler.post( + url=url, + headers=headers, + json=payload, + ) + response.raise_for_status() + result = response.json() + + # Extract response info for logging qualifire_response = { - "score": getattr(result, "score", None), - "status": getattr(result, "status", None), + "score": result.get("score"), + "status": result.get("status"), } verbose_proxy_logger.debug( diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_qualifire.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_qualifire.py index 35ed49a84ed..fd72185d1e7 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_qualifire.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_qualifire.py @@ -2,7 +2,6 @@ Unit tests for Qualifire guardrail integration. """ -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -75,9 +74,37 @@ class TestQualifireGuardrailInit: assert guardrail.on_flagged == "monitor" + def test_init_with_default_api_base(self): + """Test that default API base is set when not provided.""" + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + DEFAULT_QUALIFIRE_API_BASE, + QualifireGuardrail, + ) + + guardrail = QualifireGuardrail( + api_key="test_key", + guardrail_name="test_guardrail", + ) + + assert guardrail.qualifire_api_base == DEFAULT_QUALIFIRE_API_BASE + + def test_init_with_custom_api_base(self): + """Test initialization with custom API base URL.""" + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) + + guardrail = QualifireGuardrail( + api_key="test_key", + api_base="https://custom.qualifire.ai", + guardrail_name="test_guardrail", + ) + + assert guardrail.qualifire_api_base == "https://custom.qualifire.ai" + class TestQualifireGuardrailMessageConversion: - """Tests for message conversion to Qualifire format.""" + """Tests for message conversion to API format.""" def test_convert_simple_messages(self): """Test conversion of simple text messages.""" @@ -95,15 +122,13 @@ class TestQualifireGuardrailMessageConversion: {"role": "assistant", "content": "Hi there!"}, ] - # Create mock LLMMessage class - mock_llm_message = MagicMock() + result = guardrail._convert_messages_to_api_format(messages) - with patch( - "litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire.QualifireGuardrail._convert_messages_to_qualifire_format" - ) as mock_convert: - mock_convert.return_value = [mock_llm_message, mock_llm_message] - result = guardrail._convert_messages_to_qualifire_format(messages) - assert len(result) == 2 + assert len(result) == 2 + assert result[0]["role"] == "user" + assert result[0]["content"] == "Hello, world!" + assert result[1]["role"] == "assistant" + assert result[1]["content"] == "Hi there!" def test_convert_multimodal_messages(self): """Test conversion of multimodal messages with text parts.""" @@ -126,112 +151,258 @@ class TestQualifireGuardrailMessageConversion: }, ] - with patch( - "litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire.QualifireGuardrail._convert_messages_to_qualifire_format" - ) as mock_convert: - mock_convert.return_value = [MagicMock()] - result = guardrail._convert_messages_to_qualifire_format(messages) - assert len(result) == 1 + result = guardrail._convert_messages_to_api_format(messages) + + assert len(result) == 1 + assert result[0]["role"] == "user" + assert result[0]["content"] == "First part\nSecond part" + + def test_convert_messages_with_tool_calls(self): + """Test conversion of messages with tool calls.""" + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) + + guardrail = QualifireGuardrail( + api_key="test_key", + guardrail_name="test_guardrail", + ) + + messages = [ + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "NYC"}', + }, + } + ], + }, + ] + + result = guardrail._convert_messages_to_api_format(messages) + + assert len(result) == 1 + assert result[0]["role"] == "assistant" + assert "tool_calls" in result[0] + assert len(result[0]["tool_calls"]) == 1 + assert result[0]["tool_calls"][0]["id"] == "call_123" + assert result[0]["tool_calls"][0]["name"] == "get_weather" + assert result[0]["tool_calls"][0]["arguments"] == {"location": "NYC"} -class TestQualifireGuardrailEvaluateKwargs: - """Tests for evaluate kwargs passed to Qualifire client.""" +class TestQualifireGuardrailToolConversion: + """Tests for tool definition conversion.""" + + def test_convert_openai_function_tools(self): + """Test conversion of OpenAI function tool format.""" + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) + + guardrail = QualifireGuardrail( + api_key="test_key", + guardrail_name="test_guardrail", + ) + + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a location", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + + result = guardrail._convert_tools_to_api_format(tools) + + assert result is not None + assert len(result) == 1 + assert result[0]["name"] == "get_weather" + assert result[0]["description"] == "Get weather for a location" + + def test_convert_empty_tools(self): + """Test that empty tools returns None.""" + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) + + guardrail = QualifireGuardrail( + api_key="test_key", + guardrail_name="test_guardrail", + ) + + result = guardrail._convert_tools_to_api_format(None) + assert result is None + + result = guardrail._convert_tools_to_api_format([]) + assert result is None + + +class TestQualifireGuardrailAPICall: + """Tests for API call with httpx client.""" @pytest.mark.asyncio async def test_evaluate_called_with_prompt_injections(self): - """Test that evaluate is called with prompt_injections enabled.""" - # Mock the qualifire module and its types - mock_qualifire_types = MagicMock() - mock_llm_message = MagicMock() - mock_llm_tool_call = MagicMock() - mock_message_instance = MagicMock() - mock_llm_message.return_value = mock_message_instance - - mock_qualifire_types.LLMMessage = mock_llm_message - mock_qualifire_types.LLMToolCall = mock_llm_tool_call - - with patch.dict('sys.modules', {'qualifire': MagicMock(), 'qualifire.types': mock_qualifire_types}): - from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( - QualifireGuardrail, - ) + """Test that evaluate endpoint is called with prompt_injections enabled.""" + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) - guardrail = QualifireGuardrail( - api_key="test_key", - prompt_injections=True, - guardrail_name="test_guardrail", - ) + guardrail = QualifireGuardrail( + api_key="test_key", + prompt_injections=True, + guardrail_name="test_guardrail", + ) - # Mock the client - mock_client = MagicMock() - mock_result = MagicMock() - mock_result.score = 100 - mock_result.status = "completed" - mock_result.evaluationResults = [] - mock_client.evaluate.return_value = mock_result - guardrail._client = mock_client + # Mock the async HTTP handler + mock_response = MagicMock() + mock_response.json.return_value = { + "score": 100, + "status": "completed", + "evaluationResults": [], + } + mock_response.raise_for_status = MagicMock() + guardrail.async_handler.post = AsyncMock(return_value=mock_response) - messages = [{"role": "user", "content": "Hello, world!"}] + messages = [{"role": "user", "content": "Hello, world!"}] - await guardrail._run_qualifire_check( - messages=messages, output=None, dynamic_params={} - ) + await guardrail._run_qualifire_check( + messages=messages, output=None, dynamic_params={} + ) - # Verify evaluate was called with correct kwargs - mock_client.evaluate.assert_called_once() - call_kwargs = mock_client.evaluate.call_args[1] - assert call_kwargs["prompt_injections"] is True - assert "messages" in call_kwargs + # Verify the API was called + guardrail.async_handler.post.assert_called_once() + call_kwargs = guardrail.async_handler.post.call_args[1] + + assert "json" in call_kwargs + payload = call_kwargs["json"] + assert payload["prompt_injections"] is True + assert "messages" in payload + assert call_kwargs["url"].endswith("/api/evaluation/evaluate") @pytest.mark.asyncio async def test_evaluate_called_with_multiple_checks(self): """Test that evaluate is called with multiple checks enabled.""" - # Mock the qualifire module and its types - mock_qualifire_types = MagicMock() - mock_llm_message = MagicMock() - mock_llm_tool_call = MagicMock() - mock_message_instance = MagicMock() - mock_llm_message.return_value = mock_message_instance - - mock_qualifire_types.LLMMessage = mock_llm_message - mock_qualifire_types.LLMToolCall = mock_llm_tool_call - - with patch.dict('sys.modules', {'qualifire': MagicMock(), 'qualifire.types': mock_qualifire_types}): - from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( - QualifireGuardrail, - ) + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) - guardrail = QualifireGuardrail( - api_key="test_key", - prompt_injections=True, - pii_check=True, - hallucinations_check=True, - assertions=["Output must be valid JSON"], - guardrail_name="test_guardrail", - ) + guardrail = QualifireGuardrail( + api_key="test_key", + prompt_injections=True, + pii_check=True, + hallucinations_check=True, + assertions=["Output must be valid JSON"], + guardrail_name="test_guardrail", + ) - # Mock the client - mock_client = MagicMock() - mock_result = MagicMock() - mock_result.score = 100 - mock_result.status = "completed" - mock_result.evaluationResults = [] - mock_client.evaluate.return_value = mock_result - guardrail._client = mock_client + # Mock the async HTTP handler + mock_response = MagicMock() + mock_response.json.return_value = { + "score": 100, + "status": "completed", + "evaluationResults": [], + } + mock_response.raise_for_status = MagicMock() + guardrail.async_handler.post = AsyncMock(return_value=mock_response) - messages = [{"role": "user", "content": "Hello, world!"}] + messages = [{"role": "user", "content": "Hello, world!"}] - await guardrail._run_qualifire_check( - messages=messages, output="Test output", dynamic_params={} - ) + await guardrail._run_qualifire_check( + messages=messages, output="Test output", dynamic_params={} + ) - # Verify evaluate was called with correct kwargs - mock_client.evaluate.assert_called_once() - call_kwargs = mock_client.evaluate.call_args[1] - assert call_kwargs["prompt_injections"] is True - assert call_kwargs["pii_check"] is True - assert call_kwargs["hallucinations_check"] is True - assert call_kwargs["assertions"] == ["Output must be valid JSON"] - assert call_kwargs["output"] == "Test output" + # Verify the API was called with correct payload + guardrail.async_handler.post.assert_called_once() + call_kwargs = guardrail.async_handler.post.call_args[1] + + payload = call_kwargs["json"] + assert payload["prompt_injections"] is True + assert payload["pii_check"] is True + assert payload["hallucinations_check"] is True + assert payload["assertions"] == ["Output must be valid JSON"] + assert payload["output"] == "Test output" + + @pytest.mark.asyncio + async def test_invoke_endpoint_used_with_evaluation_id(self): + """Test that invoke endpoint is used when evaluation_id is provided.""" + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) + + guardrail = QualifireGuardrail( + api_key="test_key", + evaluation_id="eval_123", + guardrail_name="test_guardrail", + ) + + # Mock the async HTTP handler + mock_response = MagicMock() + mock_response.json.return_value = { + "score": 100, + "status": "completed", + "evaluationResults": [], + } + mock_response.raise_for_status = MagicMock() + guardrail.async_handler.post = AsyncMock(return_value=mock_response) + + messages = [{"role": "user", "content": "Hello, world!"}] + + await guardrail._run_qualifire_check( + messages=messages, output="Test output", dynamic_params={} + ) + + # Verify the invoke endpoint was called + guardrail.async_handler.post.assert_called_once() + call_kwargs = guardrail.async_handler.post.call_args[1] + + assert call_kwargs["url"].endswith("/api/evaluation/invoke") + payload = call_kwargs["json"] + assert payload["evaluation_id"] == "eval_123" + assert payload["input"] == "Hello, world!" + assert payload["output"] == "Test output" + + @pytest.mark.asyncio + async def test_correct_headers_sent(self): + """Test that correct headers are sent with the API request.""" + from litellm.proxy.guardrails.guardrail_hooks.qualifire.qualifire import ( + QualifireGuardrail, + ) + + guardrail = QualifireGuardrail( + api_key="my_api_key", + guardrail_name="test_guardrail", + ) + + # Mock the async HTTP handler + mock_response = MagicMock() + mock_response.json.return_value = { + "score": 100, + "status": "completed", + "evaluationResults": [], + } + mock_response.raise_for_status = MagicMock() + guardrail.async_handler.post = AsyncMock(return_value=mock_response) + + messages = [{"role": "user", "content": "Hello!"}] + + await guardrail._run_qualifire_check( + messages=messages, output=None, dynamic_params={} + ) + + call_kwargs = guardrail.async_handler.post.call_args[1] + headers = call_kwargs["headers"] + + assert headers["X-Qualifire-API-Key"] == "my_api_key" + assert headers["Content-Type"] == "application/json" class TestQualifireGuardrailCheckIfFlagged: @@ -248,12 +419,14 @@ class TestQualifireGuardrailCheckIfFlagged: guardrail_name="test_guardrail", ) - # Mock result with completed status and no flagged items - mock_result = MagicMock() - mock_result.status = "completed" - mock_result.evaluationResults = [] + # Result with completed status and no flagged items (dict format) + result = { + "status": "completed", + "score": 100, + "evaluationResults": [], + } - assert guardrail._check_if_flagged(mock_result) is False + assert guardrail._check_if_flagged(result) is False def test_check_if_flagged_returns_true_for_flagged_content(self): """Test that _check_if_flagged returns True when content is flagged.""" @@ -266,18 +439,25 @@ class TestQualifireGuardrailCheckIfFlagged: guardrail_name="test_guardrail", ) - # Mock result with flagged item - mock_inner_result = MagicMock() - mock_inner_result.flagged = True + # Result with flagged item (dict format matching API response) + result = { + "status": "completed", + "score": 15, + "evaluationResults": [ + { + "type": "prompt_injection", + "results": [ + { + "flagged": True, + "score": 0.15, + "reason": "Prompt injection detected", + } + ], + } + ], + } - mock_eval_result = MagicMock() - mock_eval_result.results = [mock_inner_result] - - mock_result = MagicMock() - mock_result.status = "completed" - mock_result.evaluationResults = [mock_eval_result] - - assert guardrail._check_if_flagged(mock_result) is True + assert guardrail._check_if_flagged(result) is True def test_check_if_flagged_returns_false_when_no_flagged_items(self): """Test that _check_if_flagged returns False when no items are flagged.""" @@ -291,17 +471,24 @@ class TestQualifireGuardrailCheckIfFlagged: ) # Result with evaluation results but nothing flagged - mock_inner_result = MagicMock() - mock_inner_result.flagged = False + result = { + "status": "completed", + "score": 95, + "evaluationResults": [ + { + "type": "prompt_injection", + "results": [ + { + "flagged": False, + "score": 0.95, + "reason": "No issues detected", + } + ], + } + ], + } - mock_eval_result = MagicMock() - mock_eval_result.results = [mock_inner_result] - - mock_result = MagicMock() - mock_result.status = "success" - mock_result.evaluationResults = [mock_eval_result] - - assert guardrail._check_if_flagged(mock_result) is False + assert guardrail._check_if_flagged(result) is False class TestQualifireGuardrailShouldRun: From 91b5c66cf2851cc9e3dd76b0299e8bbf819d1ef7 Mon Sep 17 00:00:00 2001 From: Kris Xia Date: Wed, 7 Jan 2026 23:56:47 +0800 Subject: [PATCH 016/121] fix(proxy): return json error response instead of sse format for initial streaming errors (#18757) * adding signoz integration to observability docs * Fixing build * Adding timeout for flaky test * Fixing e2e * fix(proxy): return json error response instead of sse format for initial streaming errors when the first chunk of a streaming response contains an error, return a standard json error response instead of sse format. this ensures clients receive properly formatted error responses before the stream actually begins. - rename create_streaming_response to create_response - add logic to detect error in first chunk and return JSONResponse - add _extract_error_from_sse_chunk helper function - update all call sites to use the new function name - update tests to reflect the function rename * test(proxy): add comprehensive tests for error extraction from sse chunks - Add new test class TestExtractErrorFromSSEChunk with 10 test cases - Update existing tests to verify JSONResponse returned for initial streaming errors - Add tests for error code as string, bytes input, invalid JSON, and edge cases - Verify correct error format extraction from SSE chunks --------- Co-authored-by: Goutham Karthi Co-authored-by: yuneng-jiang Co-authored-by: YutaSaito <36355491+uc4w6c@users.noreply.github.com> --- docs/my-website/docs/observability/signoz.md | 394 ++++++++++++++++++ .../proxy/anthropic_endpoints/endpoints.py | 4 +- litellm/proxy/common_request_processing.py | 75 +++- litellm/proxy/proxy_server.py | 4 +- .../proxy/test_common_request_processing.py | 176 ++++++-- .../tests/users/viewInternalUsers.spec.ts | 1 + .../ModelsAndEndpointsView.tsx | 20 +- 7 files changed, 616 insertions(+), 58 deletions(-) create mode 100644 docs/my-website/docs/observability/signoz.md diff --git a/docs/my-website/docs/observability/signoz.md b/docs/my-website/docs/observability/signoz.md new file mode 100644 index 00000000000..4b65916fdfe --- /dev/null +++ b/docs/my-website/docs/observability/signoz.md @@ -0,0 +1,394 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# SigNoz LiteLLM Integration + +For more details on setting up observability for LiteLLM, check out the [SigNoz LiteLLM observability docs](https://signoz.io/docs/litellm-observability/). + + +## Overview + +This guide walks you through setting up observability and monitoring for LiteLLM SDK and Proxy Server using [OpenTelemetry](https://opentelemetry.io/) and exporting logs, traces, and metrics to SigNoz. With this integration, you can observe various models performance, capture request/response details, and track system-level metrics in SigNoz, giving you real-time visibility into latency, error rates, and usage trends for your LiteLLM applications. + +Instrumenting LiteLLM in your AI applications with telemetry ensures full observability across your AI workflows, making it easier to debug issues, optimize performance, and understand user interactions. By leveraging SigNoz, you can analyze correlated traces, logs, and metrics in unified dashboards, configure alerts, and gain actionable insights to continuously improve reliability, responsiveness, and user experience. + +## Prerequisites + +- A [SigNoz Cloud account](https://signoz.io/teams/) with an active ingestion key +- Internet access to send telemetry data to SigNoz Cloud +- [LiteLLM](https://www.litellm.ai/) SDK or Proxy integration +- For Python: `pip` installed for managing Python packages and _(optional but recommended)_ a Python virtual environment to isolate dependencies + +## Monitoring LiteLLM + +LiteLLM can be monitored in two ways: using the **LiteLLM SDK** (directly embedded in your Python application code for programmatic LLM calls) or the **LiteLLM Proxy Server** (a standalone server that acts as a centralized gateway for managing and routing LLM requests across your infrastructure). + + + + +For more detailed info on instrumenting your LiteLLM SDK applications click [here](https://docs.litellm.ai/docs/observability/opentelemetry_integration). + + + + + +No-code auto-instrumentation is recommended for quick setup with minimal code changes. It's ideal when you want to get observability up and running without modifying your application code and are leveraging standard instrumentor libraries. + +**Step 1:** Install the necessary packages in your Python environment. + +```bash +pip install \ + opentelemetry-api \ + opentelemetry-distro \ + opentelemetry-exporter-otlp \ + httpx \ + opentelemetry-instrumentation-httpx \ + litellm +``` + +**Step 2:** Add Automatic Instrumentation + +```bash +opentelemetry-bootstrap --action=install +``` + +**Step 3:** Instrument your LiteLLM SDK application + +Initialize LiteLLM SDK instrumentation by calling `litellm.callbacks = ["otel"]`: + +```python +from litellm import litellm + +litellm.callbacks = ["otel"] +``` + +This call enables automatic tracing, logs, and metrics collection for all LiteLLM SDK calls in your application. + +> 📌 Note: Ensure this is called before any LiteLLM related calls to properly configure instrumentation of your application + +**Step 4:** Run an example + +```python +from litellm import completion, litellm + +litellm.callbacks = ["otel"] + +response = completion( + model="openai/gpt-4o", + messages=[{ "content": "What is SigNoz","role": "user"}] +) + +print(response) +``` + +> 📌 Note: LiteLLM supports a [variety of model providers](https://docs.litellm.ai/docs/providers) for LLMs. In this example, we're using OpenAI. Before running this code, ensure that you have set the environment variable `OPENAI_API_KEY` with your generated API key. + +**Step 5:** Run your application with auto-instrumentation + +```bash +OTEL_RESOURCE_ATTRIBUTES="service.name=" \ +OTEL_EXPORTER_OTLP_ENDPOINT="https://ingest..signoz.cloud:443" \ +OTEL_EXPORTER_OTLP_HEADERS="signoz-ingestion-key=" \ +OTEL_EXPORTER_OTLP_PROTOCOL=grpc \ +OTEL_TRACES_EXPORTER=otlp \ +OTEL_METRICS_EXPORTER=otlp \ +OTEL_LOGS_EXPORTER=otlp \ +OTEL_PYTHON_LOG_CORRELATION=true \ +OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true \ +OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=openai \ +opentelemetry-instrument +``` + +> 📌 Note: We're using `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=openai` in the run command to disable the OpenAI instrumentor for tracing. This avoids conflicts with LiteLLM's native telemetry/instrumentation, ensuring that telemetry is captured exclusively through LiteLLM's built-in instrumentation. + +- **``** is the name of your service +- Set the `` to match your SigNoz Cloud [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint) +- Replace `` with your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/) +- Replace `` with the actual command you would use to run your application. For example: `python main.py` + +> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted). + + + + + + +Code-based instrumentation gives you fine-grained control over your telemetry configuration. Use this approach when you need to customize resource attributes, sampling strategies, or integrate with existing observability infrastructure. + +**Step 1:** Install the necessary packages in your Python environment. + +```bash +pip install \ + opentelemetry-api \ + opentelemetry-sdk \ + opentelemetry-exporter-otlp \ + opentelemetry-instrumentation-httpx \ + opentelemetry-instrumentation-system-metrics \ + litellm +``` + +**Step 2:** Import the necessary modules in your Python application + +**Traces:** + +```python +from opentelemetry import trace +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +``` + +**Logs:** + +```python +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry._logs import set_logger_provider +import logging +``` + +**Metrics:** + +```python +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry import metrics +from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor +from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor +``` + +**Step 3:** Set up the OpenTelemetry Tracer Provider to send traces directly to SigNoz Cloud + +```python +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry import trace +import os + +resource = Resource.create({"service.name": ""}) +provider = TracerProvider(resource=resource) +span_exporter = OTLPSpanExporter( + endpoint= os.getenv("OTEL_EXPORTER_TRACES_ENDPOINT"), + headers={"signoz-ingestion-key": os.getenv("SIGNOZ_INGESTION_KEY")}, +) +processor = BatchSpanProcessor(span_exporter) +provider.add_span_processor(processor) +trace.set_tracer_provider(provider) +``` + +- **``** is the name of your service +- **`OTEL_EXPORTER_TRACES_ENDPOINT`** → SigNoz Cloud trace endpoint with appropriate [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint):`https://ingest..signoz.cloud:443/v1/traces` +- **`SIGNOZ_INGESTION_KEY`** → Your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/) + + +> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted). + + +**Step 4**: Setup Logs + +```python +import logging +from opentelemetry.sdk.resources import Resource +from opentelemetry._logs import set_logger_provider +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +import os + +resource = Resource.create({"service.name": ""}) +logger_provider = LoggerProvider(resource=resource) +set_logger_provider(logger_provider) + +otlp_log_exporter = OTLPLogExporter( + endpoint= os.getenv("OTEL_EXPORTER_LOGS_ENDPOINT"), + headers={"signoz-ingestion-key": os.getenv("SIGNOZ_INGESTION_KEY")}, +) +logger_provider.add_log_record_processor( + BatchLogRecordProcessor(otlp_log_exporter) +) +# Attach OTel logging handler to root logger +handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) +logging.basicConfig(level=logging.INFO, handlers=[handler]) + +logger = logging.getLogger(__name__) +``` + +- **``** is the name of your service +- **`OTEL_EXPORTER_LOGS_ENDPOINT`** → SigNoz Cloud endpoint with appropriate [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint):`https://ingest..signoz.cloud:443/v1/logs` +- **`SIGNOZ_INGESTION_KEY`** → Your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/) + +> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted). + + +**Step 5**: Setup Metrics + +```python +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry import metrics +from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor +import os + +resource = Resource.create({"service.name": ""}) +metric_exporter = OTLPMetricExporter( + endpoint= os.getenv("OTEL_EXPORTER_METRICS_ENDPOINT"), + headers={"signoz-ingestion-key": os.getenv("SIGNOZ_INGESTION_KEY")}, +) +reader = PeriodicExportingMetricReader(metric_exporter) +metric_provider = MeterProvider(metric_readers=[reader], resource=resource) +metrics.set_meter_provider(metric_provider) + +meter = metrics.get_meter(__name__) + +# turn on out-of-the-box metrics +SystemMetricsInstrumentor().instrument() +HTTPXClientInstrumentor().instrument() +``` + +- **``** is the name of your service +- **`OTEL_EXPORTER_METRICS_ENDPOINT`** → SigNoz Cloud endpoint with appropriate [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint):`https://ingest..signoz.cloud:443/v1/metrics` +- **`SIGNOZ_INGESTION_KEY`** → Your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/) + +> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted). + + +> 📌 Note: SystemMetricsInstrumentor provides system metrics (CPU, memory, etc.), and HTTPXClientInstrumentor provides outbound HTTP request metrics such as request duration. If you want to add custom metrics to your LiteLLM application, see [Python Custom Metrics](https://signoz.io/opentelemetry/python-custom-metrics/). + +**Step 6:** Instrument your LiteLLM application + +Initialize LiteLLM SDK instrumentation by calling `litellm.callbacks = ["otel"]`: + +```python +from litellm import litellm + +litellm.callbacks = ["otel"] +``` + +This call enables automatic tracing, logs, and metrics collection for all LiteLLM SDK calls in your application. + +> 📌 Note: Ensure this is called before any LiteLLM related calls to properly configure instrumentation of your application + +**Step 7:** Run an example + +```python +from litellm import completion, litellm + +litellm.callbacks = ["otel"] + +response = completion( + model="openai/gpt-4o", + messages=[{ "content": "What is SigNoz","role": "user"}] +) + +print(response) +``` + +> 📌 Note: LiteLLM supports a [variety of model providers](https://docs.litellm.ai/docs/providers) for LLMs. In this example, we're using OpenAI. Before running this code, ensure that you have set the environment variable `OPENAI_API_KEY` with your generated API key. + + + + +## View Traces, Logs, and Metrics in SigNoz + +Your LiteLLM commands should now automatically emit traces, logs, and metrics. + +You should be able to view traces in Signoz Cloud under the traces tab: + +![LiteLLM SDK Trace View](https://signoz.io/img/docs/llm/litellm/litellmsdk-traces.webp) + +When you click on a trace in SigNoz, you'll see a detailed view of the trace, including all associated spans, along with their events and attributes. + +![LiteLLM SDK Detailed Trace View](https://signoz.io/img/docs/llm/litellm/litellmsdk-detailed-traces.webp) + +You should be able to view logs in Signoz Cloud under the logs tab. You can also view logs by clicking on the “Related Logs” button in the trace view to see correlated logs: + +![LiteLLM SDK Logs View](https://signoz.io/img/docs/llm/litellm/litellmsdk-logs.webp) + +When you click on any of these logs in SigNoz, you'll see a detailed view of the log, including attributes: + +![LiteLLM SDK Detailed Logs View](https://signoz.io/img/docs/llm/litellm/litellmsdk-detailed-logs.webp) + +You should be able to see LiteLLM related metrics in Signoz Cloud under the metrics tab: + +![LiteLLM SDK Metrics View](https://signoz.io/img/docs/llm/litellm/litellmsdk-metrics.webp) + +When you click on any of these metrics in SigNoz, you'll see a detailed view of the metric, including attributes: + +![LiteLLM Detailed Metrics View](https://signoz.io/img/docs/llm/litellm/litellmsdk-detailed-metrics.webp) + +## Dashboard + +You can also check out our custom LiteLLM SDK dashboard [here](https://signoz.io/docs/dashboards/dashboard-templates/litellm-sdk-dashboard/) which provides specialized visualizations for monitoring your LiteLLM usage in applications. The dashboard includes pre-built charts specifically tailored for LLM usage, along with import instructions to get started quickly. + +![LiteLLM SDK Dashboard Template](https://signoz.io/img/docs/llm/litellm/litellm-sdk-dashboard.webp) + + + + + +**Step 1:** Install the necessary packages in your Python environment. + +```bash +pip install opentelemetry-api \ + opentelemetry-sdk \ + opentelemetry-exporter-otlp \ + 'litellm[proxy]' +``` + +**Step 2:** Configure otel for the LiteLLM Proxy Server + +Add the following to `config.yaml`: + +```yaml +litellm_settings: + callbacks: ['otel'] +``` + +**Step 3:** Set the following environment variables: + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT="https://ingest..signoz.cloud:443" +export OTEL_EXPORTER_OTLP_HEADERS="signoz-ingestion-key=" +export OTEL_EXPORTER_OTLP_PROTOCOL="grpc" +export OTEL_TRACES_EXPORTER="otlp" +export OTEL_METRICS_EXPORTER="otlp" +export OTEL_LOGS_EXPORTER="otlp" +``` + +- Set the `` to match your SigNoz Cloud [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint) +- Replace `` with your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/) + +> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted). + + +**Step 4:** Run the proxy server using the config file: + +```bash +litellm --config config.yaml +``` + +Now any calls made through your LiteLLM proxy server will be traced and sent to SigNoz. + +You should be able to view traces in Signoz Cloud under the traces tab: + +![LiteLLM Proxy Trace View](https://signoz.io/img/docs/llm/litellm/litellmproxy-traces.webp) + +When you click on a trace in SigNoz, you'll see a detailed view of the trace, including all associated spans, along with their events and attributes. + +![LiteLLM Proxy Detailed Trace View](https://signoz.io/img/docs/llm/litellm/litellmproxy-detailed-traces.webp) + +## Dashboard + +You can also check out our custom LiteLLM Proxy dashboard [here](https://signoz.io/docs/dashboards/dashboard-templates/litellm-proxy-dashboard/) which provides specialized visualizations for monitoring your LiteLLM Proxy usage in applications. The dashboard includes pre-built charts specifically tailored for LLM usage, along with import instructions to get started quickly. + +![LiteLLM Proxy Dashboard Template](https://signoz.io/img/docs/llm/litellm/litellm-proxy-dashboard.webp) + + + diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 334362a0271..7de6b7fccfc 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -11,7 +11,7 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, - create_streaming_response, + create_response, ) from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.types.utils import TokenCountResponse @@ -106,7 +106,7 @@ async def anthropic_response( # noqa: PLR0915 ) ) - return await create_streaming_response( + return await create_response( generator=selected_data_generator, media_type="text/event-stream", headers={}, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 537b48f06ed..95c23be5b82 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -17,7 +17,7 @@ from typing import ( import httpx import orjson from fastapi import HTTPException, Request, status -from fastapi.responses import Response, StreamingResponse +from fastapi.responses import JSONResponse, Response, StreamingResponse import litellm from litellm._logging import verbose_proxy_logger @@ -96,16 +96,55 @@ async def _parse_event_data_for_error(event_line: Union[str, bytes]) -> Optional return None -async def create_streaming_response( +def _extract_error_from_sse_chunk(event_line: Union[str, bytes]) -> dict: + """ + Extract error dictionary from SSE format chunk. + + Args: + event_line: SSE format event line, e.g. "data: {"error": {...}}\n\n" + + Returns: + Error dictionary in OpenAI API format + """ + event_line = ( + event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line + ) + + # Default error format + default_error = { + "message": "Unknown error", + "type": "internal_server_error", + "param": None, + "code": "500", + } + + if event_line.startswith("data: "): + json_str = event_line[len("data: ") :].strip() + if not json_str or json_str == "[DONE]": + return default_error + + try: + data = orjson.loads(json_str) + if isinstance(data, dict) and "error" in data: + error_obj = data["error"] + if isinstance(error_obj, dict): + return error_obj + except (orjson.JSONDecodeError, json.JSONDecodeError): + pass + + return default_error + + +async def create_response( generator: AsyncGenerator[str, None], media_type: str, headers: dict, default_status_code: int = status.HTTP_200_OK, -) -> StreamingResponse: +) -> Union[StreamingResponse, JSONResponse]: """ - Creates a StreamingResponse by inspecting the first chunk for an error code. - The entire original generator content is streamed, but the HTTP status code - of the response is set based on the first chunk if it's a recognized error. + Create streaming response, checking if the first chunk is an error. + If the first chunk is an error, return a standard JSON error response. + Otherwise, return StreamingResponse and stream all content. """ first_chunk_value: Optional[str] = None final_status_code = default_status_code @@ -124,9 +163,27 @@ async def create_streaming_response( first_chunk_value ) if error_code_from_chunk is not None: + # First chunk is an error, stream hasn't really started yet + # Should return standard JSON error response instead of SSE format final_status_code = error_code_from_chunk verbose_proxy_logger.debug( - f"Error detected in first stream chunk. Status code set to: {final_status_code}" + f"Error detected in first stream chunk. Returning JSON error response with status code: {final_status_code}" + ) + + # Parse error content + error_dict = _extract_error_from_sse_chunk(first_chunk_value) + + # Consume and close generator (avoid resource leak) + try: + await generator.aclose() + except Exception: + pass + + # Return JSON format error response + return JSONResponse( + status_code=final_status_code, + content={"error": error_dict}, + headers=headers, ) except Exception as e: verbose_proxy_logger.debug(f"Error parsing first chunk value: {e}") @@ -647,7 +704,7 @@ class ProxyBaseLLMRequestProcessing: proxy_logging_obj=proxy_logging_obj, ) ) - return await create_streaming_response( + return await create_response( generator=selected_data_generator, media_type="text/event-stream", headers=custom_headers, @@ -658,7 +715,7 @@ class ProxyBaseLLMRequestProcessing: user_api_key_dict=user_api_key_dict, request_data=self.data, ) - return await create_streaming_response( + return await create_response( generator=selected_data_generator, media_type="text/event-stream", headers=custom_headers, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 06525e39133..9e314a77c37 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -229,7 +229,7 @@ from litellm.proxy.batches_endpoints.endpoints import router as batches_router from litellm.proxy.caching_routes import router as caching_router from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, - create_streaming_response, + create_response, ) from litellm.proxy.common_utils.callback_utils import initialize_callbacks_on_proxy from litellm.proxy.common_utils.debug_utils import init_verbose_loggers @@ -6736,7 +6736,7 @@ async def run_thread( if ( "stream" in data and data["stream"] is True ): # use generate_responses to stream responses - return await create_streaming_response( + return await create_response( generator=async_assistants_data_generator( user_api_key_dict=user_api_key_dict, response=response, diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index b5d44385698..ed1c29f5dce 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -3,7 +3,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import Request, status -from fastapi.responses import StreamingResponse +from fastapi.responses import JSONResponse, StreamingResponse import litellm from litellm._uuid import uuid @@ -11,9 +11,10 @@ from litellm.integrations.opentelemetry import UserAPIKeyAuth from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, ProxyConfig, + _extract_error_from_sse_chunk, _get_cost_breakdown_from_logging_obj, _parse_event_data_for_error, - create_streaming_response, + create_response, ) from litellm.proxy.utils import ProxyLogging @@ -602,21 +603,27 @@ class TestCommonRequestProcessingHelpers: assert await _parse_event_data_for_error(event_line) == expected_code async def test_create_streaming_response_first_chunk_is_error(self): + """ + Test that when the first chunk is an error, a JSON error response is returned + instead of an SSE streaming response + """ async def mock_generator(): yield 'data: {"error": {"code": 403, "message": "forbidden"}}\n\n' yield 'data: {"content": "more data"}\n\n' yield "data: [DONE]\n\n" - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {} ) + # Should return JSONResponse instead of StreamingResponse + assert isinstance(response, JSONResponse) assert response.status_code == status.HTTP_403_FORBIDDEN - content = await self.consume_stream(response) - assert content == [ - 'data: {"error": {"code": 403, "message": "forbidden"}}\n\n', - 'data: {"content": "more data"}\n\n', - "data: [DONE]\n\n", - ] + # Verify the response is in standard JSON error format + import json + body = json.loads(response.body.decode()) + assert "error" in body + assert body["error"]["code"] == 403 + assert body["error"]["message"] == "forbidden" async def test_create_streaming_response_first_chunk_not_error(self): async def mock_generator(): @@ -624,7 +631,7 @@ class TestCommonRequestProcessingHelpers: yield 'data: {"content": "second part"}\n\n' yield "data: [DONE]\n\n" - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {} ) assert response.status_code == status.HTTP_200_OK @@ -641,7 +648,7 @@ class TestCommonRequestProcessingHelpers: yield # Implicitly raises StopAsyncIteration - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {} ) assert response.status_code == status.HTTP_200_OK @@ -654,7 +661,7 @@ class TestCommonRequestProcessingHelpers: mock_gen = AsyncMock() mock_gen.__anext__.side_effect = StopAsyncIteration - response = await create_streaming_response(mock_gen, "text/event-stream", {}) + response = await create_response(mock_gen, "text/event-stream", {}) assert response.status_code == status.HTTP_200_OK content = await self.consume_stream(response) assert content == [] @@ -665,7 +672,7 @@ class TestCommonRequestProcessingHelpers: mock_gen = AsyncMock() mock_gen.__anext__.side_effect = ValueError("Test error from generator") - response = await create_streaming_response(mock_gen, "text/event-stream", {}) + response = await create_response(mock_gen, "text/event-stream", {}) assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR content = await self.consume_stream(response) expected_error_data = { @@ -682,19 +689,24 @@ class TestCommonRequestProcessingHelpers: assert content[1] == "data: [DONE]\n\n" async def test_create_streaming_response_first_chunk_error_string_code(self): + """ + Test that when the first chunk contains a string error code, a JSON error response is returned + """ async def mock_generator(): yield 'data: {"error": {"code": "429", "message": "too many requests"}}\n\n' yield "data: [DONE]\n\n" - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {} ) + assert isinstance(response, JSONResponse) assert response.status_code == status.HTTP_429_TOO_MANY_REQUESTS - content = await self.consume_stream(response) - assert content == [ - 'data: {"error": {"code": "429", "message": "too many requests"}}\n\n', - "data: [DONE]\n\n", - ] + # Verify the response is in standard JSON error format + import json + body = json.loads(response.body.decode()) + assert "error" in body + assert body["error"]["code"] == "429" + assert body["error"]["message"] == "too many requests" async def test_create_streaming_response_custom_headers(self): async def mock_generator(): @@ -702,7 +714,7 @@ class TestCommonRequestProcessingHelpers: yield "data: [DONE]\n\n" custom_headers = {"X-Custom-Header": "TestValue"} - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", custom_headers ) assert response.headers["x-custom-header"] == "TestValue" @@ -712,7 +724,7 @@ class TestCommonRequestProcessingHelpers: yield 'data: {"content": "data"}\n\n' yield "data: [DONE]\n\n" - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {}, @@ -729,7 +741,7 @@ class TestCommonRequestProcessingHelpers: async def mock_generator(): yield "data: [DONE]\n\n" - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {} ) assert response.status_code == status.HTTP_200_OK # Default status @@ -742,7 +754,7 @@ class TestCommonRequestProcessingHelpers: yield 'data: {"content": "actual data"}\n\n' yield "data: [DONE]\n\n" - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {} ) assert response.status_code == status.HTTP_200_OK # Default status @@ -773,7 +785,7 @@ class TestCommonRequestProcessingHelpers: # Patch the tracer in the common_request_processing module with patch("litellm.proxy.common_request_processing.tracer", mock_tracer): - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {} ) @@ -810,7 +822,10 @@ class TestCommonRequestProcessingHelpers: ), f"Call {i} should have operation name 'streaming.chunk.yield', got {args[0]}" async def test_create_streaming_response_dd_trace_with_error_chunk(self): - """Test that dd trace is applied even when the first chunk contains an error""" + """ + Test that when the first chunk contains an error, JSONResponse is returned + and tracing is not triggered (since it's not a streaming response) + """ from unittest.mock import patch # Create a mock tracer @@ -827,28 +842,107 @@ class TestCommonRequestProcessingHelpers: # Patch the tracer in the common_request_processing module with patch("litellm.proxy.common_request_processing.tracer", mock_tracer): - response = await create_streaming_response( + response = await create_response( mock_generator(), "text/event-stream", {} ) - # Even with error, status should be set to error code but tracing should still work + # Should return JSONResponse instead of StreamingResponse + assert isinstance(response, JSONResponse) assert response.status_code == 400 - # Consume the stream to trigger the tracer calls - content = await self.consume_stream(response) + # Verify the response is in standard JSON error format + import json + body = json.loads(response.body.decode()) + assert "error" in body + assert body["error"]["code"] == 400 + assert body["error"]["message"] == "bad request" - # Verify all chunks are present - assert len(content) == 3 + # Since JSONResponse is returned instead of StreamingResponse, streaming tracing should not be triggered + # tracer.trace should not be called + assert mock_tracer.trace.call_count == 0 - # Verify that tracer.trace was called for each chunk - assert mock_tracer.trace.call_count == 3 - # Verify that each call was made with the correct operation name - actual_calls = mock_tracer.trace.call_args_list - assert len(actual_calls) == 3 +class TestExtractErrorFromSSEChunk: + """Tests for _extract_error_from_sse_chunk function""" + + def test_extract_error_from_sse_chunk_with_valid_error(self): + """Test extracting error information from a standard SSE chunk""" + chunk = 'data: {"error": {"code": 403, "message": "forbidden", "type": "auth_error", "param": "api_key"}}\n\n' + error = _extract_error_from_sse_chunk(chunk) + + assert error["code"] == 403 + assert error["message"] == "forbidden" + assert error["type"] == "auth_error" + assert error["param"] == "api_key" + + def test_extract_error_from_sse_chunk_with_string_code(self): + """Test error code as string type""" + chunk = 'data: {"error": {"code": "429", "message": "too many requests"}}\n\n' + error = _extract_error_from_sse_chunk(chunk) + + assert error["code"] == "429" + assert error["message"] == "too many requests" + + def test_extract_error_from_sse_chunk_with_bytes(self): + """Test input as bytes type""" + chunk = b'data: {"error": {"code": 500, "message": "internal error"}}\n\n' + error = _extract_error_from_sse_chunk(chunk) + + assert error["code"] == 500 + assert error["message"] == "internal error" + + def test_extract_error_from_sse_chunk_with_done(self): + """Test [DONE] marker should return default error""" + chunk = "data: [DONE]\n\n" + error = _extract_error_from_sse_chunk(chunk) + + assert error["message"] == "Unknown error" + assert error["type"] == "internal_server_error" + assert error["code"] == "500" + assert error["param"] is None + + def test_extract_error_from_sse_chunk_without_error_field(self): + """Test missing error field should return default error""" + chunk = 'data: {"content": "some content"}\n\n' + error = _extract_error_from_sse_chunk(chunk) + + assert error["message"] == "Unknown error" + assert error["type"] == "internal_server_error" + assert error["code"] == "500" + + def test_extract_error_from_sse_chunk_with_invalid_json(self): + """Test invalid JSON should return default error""" + chunk = 'data: {invalid json}\n\n' + error = _extract_error_from_sse_chunk(chunk) + + assert error["message"] == "Unknown error" + assert error["type"] == "internal_server_error" + assert error["code"] == "500" + + def test_extract_error_from_sse_chunk_without_data_prefix(self): + """Test missing 'data:' prefix should return default error""" + chunk = '{"error": {"code": 400, "message": "bad request"}}\n\n' + error = _extract_error_from_sse_chunk(chunk) + + assert error["message"] == "Unknown error" + assert error["type"] == "internal_server_error" + assert error["code"] == "500" + + def test_extract_error_from_sse_chunk_with_empty_string(self): + """Test empty string should return default error""" + chunk = "" + error = _extract_error_from_sse_chunk(chunk) + + assert error["message"] == "Unknown error" + assert error["type"] == "internal_server_error" + assert error["code"] == "500" + + def test_extract_error_from_sse_chunk_with_minimal_error(self): + """Test minimal error object""" + chunk = 'data: {"error": {"message": "error occurred"}}\n\n' + error = _extract_error_from_sse_chunk(chunk) + + assert error["message"] == "error occurred" + # Other fields should be obtained from the original error object (if exists) + - for i, call in enumerate(actual_calls): - args, kwargs = call - assert ( - args[0] == "streaming.chunk.yield" - ), f"Call {i} should have operation name 'streaming.chunk.yield', got {args[0]}" diff --git a/ui/litellm-dashboard/e2e_tests/tests/users/viewInternalUsers.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/users/viewInternalUsers.spec.ts index 980c7233e42..65797028edc 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/users/viewInternalUsers.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/users/viewInternalUsers.spec.ts @@ -43,6 +43,7 @@ test.describe("Internal Users Page", () => { await expect(prevButton).toBeDisabled(); } + await page.waitForTimeout(1000); // Check if there are more pages const hasMorePages = infoText.includes("of") && !infoText.endsWith("25 of 25"); if (hasMorePages) { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx index a8b1d2cddc9..1cce704467a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx @@ -279,12 +279,13 @@ const ModelsAndEndpointsView: React.FC = ({ premiumUser, te {/* Missing Provider Banner */}
- +

Missing a provider?

- The LiteLLM engineering team is constantly adding support for new LLM models, providers, endpoints. If you don't see the one you need, let us know and we'll prioritize it. + The LiteLLM engineering team is constantly adding support for new LLM models, providers, endpoints. If + you don't see the one you need, let us know and we'll prioritize it.

= ({ premiumUser, te className="flex-shrink-0 inline-flex items-center gap-2 px-4 py-2 bg-[#6366f1] hover:bg-[#5558e3] text-white text-sm font-medium rounded-lg transition-colors" > Request Provider - - + +
From 92f7789f1000c29519f3270442cc4ccd16cb989e Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Wed, 7 Jan 2026 21:29:04 +0530 Subject: [PATCH 017/121] feat(prometheus): add caching metrics (#18755) --- litellm/integrations/prometheus.py | 101 +++++++-- litellm/types/integrations/prometheus.py | 21 +- .../test_prometheus_cache_metrics.py | 211 ++++++++++++++++++ 3 files changed, 318 insertions(+), 15 deletions(-) create mode 100644 tests/test_litellm/integrations/test_prometheus_cache_metrics.py diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index c01f7481277..2ec2f41b3bb 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -14,6 +14,7 @@ from typing import ( Literal, Optional, Tuple, + Union, cast, ) @@ -44,6 +45,7 @@ def _get_cached_end_user_id_for_cost_tracking(): global _get_end_user_id_for_cost_tracking if _get_end_user_id_for_cost_tracking is None: from litellm.utils import get_end_user_id_for_cost_tracking + _get_end_user_id_for_cost_tracking = get_end_user_id_for_cost_tracking return _get_end_user_id_for_cost_tracking @@ -329,6 +331,25 @@ class PrometheusLogger(CustomLogger): labelnames=self.get_labels_for_metric("litellm_requests_metric"), ) + # Cache metrics + self.litellm_cache_hits_metric = self._counter_factory( + name="litellm_cache_hits_metric", + documentation="Total number of LiteLLM cache hits", + labelnames=self.get_labels_for_metric("litellm_cache_hits_metric"), + ) + + self.litellm_cache_misses_metric = self._counter_factory( + name="litellm_cache_misses_metric", + documentation="Total number of LiteLLM cache misses", + labelnames=self.get_labels_for_metric("litellm_cache_misses_metric"), + ) + + self.litellm_cached_tokens_metric = self._counter_factory( + name="litellm_cached_tokens_metric", + documentation="Total tokens served from LiteLLM cache", + labelnames=self.get_labels_for_metric("litellm_cached_tokens_metric"), + ) + except Exception as e: print_verbose(f"Got exception on init prometheus client {str(e)}") raise e @@ -795,7 +816,7 @@ class PrometheusLogger(CustomLogger): litellm_params = kwargs.get("litellm_params", {}) or {} _metadata = litellm_params.get("metadata", {}) get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking() - + end_user_id = get_end_user_id_for_cost_tracking( litellm_params, service_type="prometheus" ) @@ -815,7 +836,7 @@ class PrometheusLogger(CustomLogger): user_api_key_auth_metadata: Optional[dict] = standard_logging_payload[ "metadata" ].get("user_api_key_auth_metadata") - + # Include top-level metadata fields (excluding nested dictionaries) # This allows accessing fields like requester_ip_address from top-level metadata top_level_metadata = standard_logging_payload.get("metadata", {}) @@ -826,7 +847,7 @@ class PrometheusLogger(CustomLogger): for k, v in top_level_metadata.items() if not isinstance(v, dict) # Exclude nested dicts to avoid conflicts } - + combined_metadata: Dict[str, Any] = { **top_level_fields, # Include top-level fields first **(_requester_metadata if _requester_metadata else {}), @@ -945,6 +966,12 @@ class PrometheusLogger(CustomLogger): kwargs, start_time, end_time, enum_values, output_tokens ) + # cache metrics + self._increment_cache_metrics( + standard_logging_payload=standard_logging_payload, # type: ignore + enum_values=enum_values, + ) + if ( standard_logging_payload["stream"] is True ): # log successful streaming requests from logging event hook. @@ -1014,6 +1041,54 @@ class PrometheusLogger(CustomLogger): standard_logging_payload["completion_tokens"] ) + def _increment_cache_metrics( + self, + standard_logging_payload: StandardLoggingPayload, + enum_values: UserAPIKeyLabelValues, + ): + """ + Increment cache-related Prometheus metrics based on cache hit/miss status. + + Args: + standard_logging_payload: Contains cache_hit field (True/False/None) + enum_values: Label values for Prometheus metrics + """ + cache_hit = standard_logging_payload.get("cache_hit") + + # Only track if cache_hit has a definite value (True or False) + if cache_hit is None: + return + + if cache_hit is True: + # Increment cache hits counter + _labels = prometheus_label_factory( + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_cache_hits_metric" + ), + enum_values=enum_values, + ) + self.litellm_cache_hits_metric.labels(**_labels).inc() + + # Increment cached tokens counter + total_tokens = standard_logging_payload.get("total_tokens", 0) + if total_tokens > 0: + _labels = prometheus_label_factory( + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_cached_tokens_metric" + ), + enum_values=enum_values, + ) + self.litellm_cached_tokens_metric.labels(**_labels).inc(total_tokens) + else: + # cache_hit is False - increment cache misses counter + _labels = prometheus_label_factory( + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_cache_misses_metric" + ), + enum_values=enum_values, + ) + self.litellm_cache_misses_metric.labels(**_labels).inc() + async def _increment_remaining_budget_metrics( self, user_api_team: Optional[str], @@ -1196,7 +1271,7 @@ class PrometheusLogger(CustomLogger): ) litellm_params = kwargs.get("litellm_params", {}) or {} get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking() - + end_user_id = get_end_user_id_for_cost_tracking( litellm_params, service_type="prometheus" ) @@ -1398,7 +1473,6 @@ class PrometheusLogger(CustomLogger): api_provider=llm_provider or "", ) if exception is not None: - _labels = prometheus_label_factory( supported_enum_labels=self.get_labels_for_metric( metric_name="litellm_deployment_failure_responses" @@ -1431,12 +1505,11 @@ class PrometheusLogger(CustomLogger): enum_values: UserAPIKeyLabelValues, output_tokens: float = 1.0, ): - try: verbose_logger.debug("setting remaining tokens requests metric") - standard_logging_payload: Optional[StandardLoggingPayload] = ( - request_kwargs.get("standard_logging_object") - ) + standard_logging_payload: Optional[ + StandardLoggingPayload + ] = request_kwargs.get("standard_logging_object") if standard_logging_payload is None: return @@ -2208,10 +2281,10 @@ class PrometheusLogger(CustomLogger): from litellm.constants import PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES from litellm.integrations.custom_logger import CustomLogger - prometheus_loggers: List[CustomLogger] = ( - litellm.logging_callback_manager.get_custom_loggers_for_type( - callback_type=PrometheusLogger - ) + prometheus_loggers: List[ + CustomLogger + ] = litellm.logging_callback_manager.get_custom_loggers_for_type( + callback_type=PrometheusLogger ) # we need to get the initialized prometheus logger instance(s) and call logger.initialize_remaining_budget_metrics() on them verbose_logger.debug("found %s prometheus loggers", len(prometheus_loggers)) @@ -2283,7 +2356,7 @@ def prometheus_label_factory( if UserAPIKeyLabelNames.END_USER.value in filtered_labels: get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking() - + filtered_labels["end_user"] = get_end_user_id_for_cost_tracking( litellm_params={"user_api_key_end_user_id": enum_values.end_user}, service_type="prometheus", diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index 6a254fc8252..fb439a9541b 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -1,7 +1,7 @@ import re from dataclasses import dataclass from enum import Enum -from typing import Dict, List, Literal, Optional, Tuple, Union +from typing import Dict, List, Literal, Optional, Tuple from pydantic import BaseModel, Field from typing_extensions import Annotated @@ -185,6 +185,10 @@ DEFINED_PROMETHEUS_METRICS = Literal[ "litellm_redis_daily_spend_update_queue_size", "litellm_in_memory_spend_update_queue_size", "litellm_redis_spend_update_queue_size", + # Cache metrics + "litellm_cache_hits_metric", + "litellm_cache_misses_metric", + "litellm_cached_tokens_metric", ] @@ -436,6 +440,21 @@ class PrometheusMetricLabels: litellm_redis_spend_update_queue_size: List[str] = [] + # Cache metrics - track cache hits, misses, and tokens served from cache + _cache_metric_labels = [ + UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value, + UserAPIKeyLabelNames.API_KEY_HASH.value, + UserAPIKeyLabelNames.API_KEY_ALIAS.value, + UserAPIKeyLabelNames.TEAM.value, + UserAPIKeyLabelNames.TEAM_ALIAS.value, + UserAPIKeyLabelNames.END_USER.value, + UserAPIKeyLabelNames.USER.value, + ] + + litellm_cache_hits_metric = _cache_metric_labels + litellm_cache_misses_metric = _cache_metric_labels + litellm_cached_tokens_metric = _cache_metric_labels + @staticmethod def get_labels(label_name: DEFINED_PROMETHEUS_METRICS) -> List[str]: default_labels = getattr(PrometheusMetricLabels, label_name) diff --git a/tests/test_litellm/integrations/test_prometheus_cache_metrics.py b/tests/test_litellm/integrations/test_prometheus_cache_metrics.py new file mode 100644 index 00000000000..660757673f6 --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_cache_metrics.py @@ -0,0 +1,211 @@ +""" +Unit tests for cache Prometheus metrics. + +Run with: poetry run pytest tests/test_litellm/integrations/test_prometheus_cache_metrics.py -v +""" +import pytest +from unittest.mock import MagicMock, patch +from litellm.types.integrations.prometheus import UserAPIKeyLabelValues + + +class TestPrometheusCacheMetrics: + """Tests for cache-related Prometheus metrics""" + + @pytest.fixture + def sample_enum_values(self): + """Create sample enum values for labels""" + return UserAPIKeyLabelValues( + end_user="test-end-user", + hashed_api_key="test-key-hash", + api_key_alias="test-key-alias", + team="test-team", + team_alias="test-team-alias", + user="test-user", + model="gpt-3.5-turbo", + ) + + def test_cache_metrics_defined_in_types(self): + """Test that cache metrics are defined in DEFINED_PROMETHEUS_METRICS""" + from litellm.types.integrations.prometheus import DEFINED_PROMETHEUS_METRICS + from typing import get_args + + defined_metrics = get_args(DEFINED_PROMETHEUS_METRICS) + + assert "litellm_cache_hits_metric" in defined_metrics + assert "litellm_cache_misses_metric" in defined_metrics + assert "litellm_cached_tokens_metric" in defined_metrics + + def test_cache_metric_labels_defined(self): + """Test that cache metric labels are properly defined""" + from litellm.types.integrations.prometheus import PrometheusMetricLabels + + # Verify labels are defined for each cache metric + assert hasattr(PrometheusMetricLabels, "litellm_cache_hits_metric") + assert hasattr(PrometheusMetricLabels, "litellm_cache_misses_metric") + assert hasattr(PrometheusMetricLabels, "litellm_cached_tokens_metric") + + # Verify labels include expected keys + expected_labels = [ + "model", + "hashed_api_key", + "api_key_alias", + "team", + "team_alias", + "end_user", + "user", + ] + for label in expected_labels: + assert label in PrometheusMetricLabels.litellm_cache_hits_metric + assert label in PrometheusMetricLabels.litellm_cache_misses_metric + assert label in PrometheusMetricLabels.litellm_cached_tokens_metric + + def test_increment_cache_metrics_on_cache_hit(self, sample_enum_values): + """Test that cache hit increments the correct metrics""" + # Create mock for PrometheusLogger instance + mock_logger = MagicMock() + + # Import the method directly and bind it to our mock + from litellm.integrations.prometheus import PrometheusLogger + + # Create a mock standard logging payload with cache_hit=True + standard_logging_payload = { + "cache_hit": True, + "total_tokens": 100, + "prompt_tokens": 50, + "completion_tokens": 50, + "model_group": "openai", + "request_tags": [], + } + + # Create mock metrics + mock_logger.litellm_cache_hits_metric = MagicMock() + mock_logger.litellm_cache_misses_metric = MagicMock() + mock_logger.litellm_cached_tokens_metric = MagicMock() + mock_logger.get_labels_for_metric = MagicMock( + return_value=[ + "model", + "hashed_api_key", + "api_key_alias", + "team", + "team_alias", + "end_user", + "user", + ] + ) + + # Call the method using unbound method approach + PrometheusLogger._increment_cache_metrics( + mock_logger, + standard_logging_payload=standard_logging_payload, + enum_values=sample_enum_values, + ) + + # Verify cache hits metric was incremented + mock_logger.litellm_cache_hits_metric.labels.assert_called() + mock_logger.litellm_cache_hits_metric.labels().inc.assert_called_once() + + # Verify cached tokens metric was incremented with total_tokens + mock_logger.litellm_cached_tokens_metric.labels.assert_called() + mock_logger.litellm_cached_tokens_metric.labels().inc.assert_called_once_with( + 100 + ) + + # Verify cache misses metric was NOT called + mock_logger.litellm_cache_misses_metric.labels.assert_not_called() + + def test_increment_cache_metrics_on_cache_miss(self, sample_enum_values): + """Test that cache miss increments the correct metrics""" + # Create mock for PrometheusLogger instance + mock_logger = MagicMock() + + from litellm.integrations.prometheus import PrometheusLogger + + # Create a mock standard logging payload with cache_hit=False + standard_logging_payload = { + "cache_hit": False, + "total_tokens": 100, + "prompt_tokens": 50, + "completion_tokens": 50, + "model_group": "openai", + "request_tags": [], + } + + # Create mock metrics + mock_logger.litellm_cache_hits_metric = MagicMock() + mock_logger.litellm_cache_misses_metric = MagicMock() + mock_logger.litellm_cached_tokens_metric = MagicMock() + mock_logger.get_labels_for_metric = MagicMock( + return_value=[ + "model", + "hashed_api_key", + "api_key_alias", + "team", + "team_alias", + "end_user", + "user", + ] + ) + + # Call the method + PrometheusLogger._increment_cache_metrics( + mock_logger, + standard_logging_payload=standard_logging_payload, + enum_values=sample_enum_values, + ) + + # Verify cache misses metric was incremented + mock_logger.litellm_cache_misses_metric.labels.assert_called() + mock_logger.litellm_cache_misses_metric.labels().inc.assert_called_once() + + # Verify cache hits and cached tokens metrics were NOT called + mock_logger.litellm_cache_hits_metric.labels.assert_not_called() + mock_logger.litellm_cached_tokens_metric.labels.assert_not_called() + + def test_increment_cache_metrics_when_cache_hit_is_none(self, sample_enum_values): + """Test that no metrics are incremented when cache_hit is None""" + # Create mock for PrometheusLogger instance + mock_logger = MagicMock() + + from litellm.integrations.prometheus import PrometheusLogger + + # Create a mock standard logging payload with cache_hit=None + standard_logging_payload = { + "cache_hit": None, + "total_tokens": 100, + "prompt_tokens": 50, + "completion_tokens": 50, + "model_group": "openai", + "request_tags": [], + } + + # Create mock metrics + mock_logger.litellm_cache_hits_metric = MagicMock() + mock_logger.litellm_cache_misses_metric = MagicMock() + mock_logger.litellm_cached_tokens_metric = MagicMock() + mock_logger.get_labels_for_metric = MagicMock( + return_value=[ + "model", + "hashed_api_key", + "api_key_alias", + "team", + "team_alias", + "end_user", + "user", + ] + ) + + # Call the method + PrometheusLogger._increment_cache_metrics( + mock_logger, + standard_logging_payload=standard_logging_payload, + enum_values=sample_enum_values, + ) + + # Verify NO metrics were called + mock_logger.litellm_cache_hits_metric.labels.assert_not_called() + mock_logger.litellm_cache_misses_metric.labels.assert_not_called() + mock_logger.litellm_cached_tokens_metric.labels.assert_not_called() + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 1b8708fccc30dc5337c92709ba146bb96aa7ac8e Mon Sep 17 00:00:00 2001 From: kothamah <104782493+kothamah@users.noreply.github.com> Date: Wed, 7 Jan 2026 11:10:36 -0500 Subject: [PATCH 018/121] Litellm embeddings calltype fix for guardrail precallhook (#18740) * adding signoz integration to observability docs * Fixing build * Adding timeout for flaky test * Fixing e2e * add team member budget duration in team/update * Reusable Duration Select and update team member budget UI * feat: allow configuring project name for OpenTelemetry service name * docs: sets ARIZE_PROJECT_NAME * added valid callType for bedrock guardrail pre hook This is to resolve the error when bedrock guardrails are enabled and invoke the embedding models. {"error":{"message":"'embeddings' is not a valid CallTypes","type":"None","param":"None","code":"500"}}* * updated the test case to reflect valid callType --------- Co-authored-by: Goutham Karthi Co-authored-by: yuneng-jiang Co-authored-by: YutaSaito <36355491+uc4w6c@users.noreply.github.com> Co-authored-by: Yuta Saito --- .../docs/observability/arize_integration.md | 1 + litellm/integrations/arize/arize.py | 2 + litellm/integrations/opentelemetry.py | 88 +++-- litellm/litellm_core_utils/litellm_logging.py | 1 + litellm/proxy/_types.py | 1 + litellm/proxy/common_request_processing.py | 4 +- .../management_endpoints/team_endpoints.py | 15 +- litellm/types/integrations/arize.py | 1 + tests/local_testing/test_arize_ai.py | 3 + .../arize/test_arize_health_check.py | 10 +- .../integrations/test_custom_guardrail.py | 2 +- .../integrations/test_opentelemetry.py | 52 +-- .../test_team_endpoints.py | 366 +++++++++++++++++- .../common_components/DurationSelect.test.tsx | 49 +++ .../common_components/DurationSelect.tsx | 17 + .../src/components/team/team_info.tsx | 12 + 16 files changed, 552 insertions(+), 72 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx create mode 100644 ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx diff --git a/docs/my-website/docs/observability/arize_integration.md b/docs/my-website/docs/observability/arize_integration.md index 0b457f08687..b3ccf98ea3b 100644 --- a/docs/my-website/docs/observability/arize_integration.md +++ b/docs/my-website/docs/observability/arize_integration.md @@ -68,6 +68,7 @@ environment_variables: ARIZE_API_KEY: "141a****" ARIZE_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize GRPC api endpoint ARIZE_HTTP_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize HTTP api endpoint. Set either this or ARIZE_ENDPOINT or Neither (defaults to https://otlp.arize.com/v1 on grpc) + ARIZE_PROJECT_NAME: "my-litellm-project" # OPTIONAL - sets the arize project name ``` 2. Start the proxy diff --git a/litellm/integrations/arize/arize.py b/litellm/integrations/arize/arize.py index 4d1aa80dcce..9c2f0d95d4d 100644 --- a/litellm/integrations/arize/arize.py +++ b/litellm/integrations/arize/arize.py @@ -51,6 +51,7 @@ class ArizeLogger(OpenTelemetry): space_id = os.environ.get("ARIZE_SPACE_ID") space_key = os.environ.get("ARIZE_SPACE_KEY") api_key = os.environ.get("ARIZE_API_KEY") + project_name = os.environ.get("ARIZE_PROJECT_NAME") grpc_endpoint = os.environ.get("ARIZE_ENDPOINT") http_endpoint = os.environ.get("ARIZE_HTTP_ENDPOINT") @@ -74,6 +75,7 @@ class ArizeLogger(OpenTelemetry): api_key=api_key, protocol=protocol, endpoint=endpoint, + project_name=project_name, ) async def async_service_success_hook( diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index a7d2326d938..7e0cfab617b 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -54,38 +54,6 @@ RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request" LITELLM_REQUEST_SPAN_NAME = "litellm_request" -def _get_litellm_resource(): - """ - Create a proper OpenTelemetry Resource that respects OTEL_RESOURCE_ATTRIBUTES - while maintaining backward compatibility with LiteLLM-specific environment variables. - """ - from opentelemetry.sdk.resources import OTELResourceDetector, Resource - - # Create base resource attributes with LiteLLM-specific defaults - # These will be overridden by OTEL_RESOURCE_ATTRIBUTES if present - base_attributes: Dict[str, Optional[str]] = { - "service.name": os.getenv("OTEL_SERVICE_NAME", "litellm"), - "deployment.environment": os.getenv("OTEL_ENVIRONMENT_NAME", "production"), - # Fix the model_id to use proper environment variable or default to service name - "model_id": os.getenv( - "OTEL_MODEL_ID", os.getenv("OTEL_SERVICE_NAME", "litellm") - ), - } - - # Create base resource with LiteLLM-specific defaults - base_resource = Resource.create(base_attributes) # type: ignore - - # Create resource from OTEL_RESOURCE_ATTRIBUTES using the detector - otel_resource_detector = OTELResourceDetector() - env_resource = otel_resource_detector.detect() - - # Merge the resources: env_resource takes precedence over base_resource - # This ensures OTEL_RESOURCE_ATTRIBUTES overrides LiteLLM defaults - merged_resource = base_resource.merge(env_resource) - - return merged_resource - - @dataclass class OpenTelemetryConfig: exporter: Union[str, SpanExporter] = "console" @@ -93,6 +61,19 @@ class OpenTelemetryConfig: headers: Optional[str] = None enable_metrics: bool = False enable_events: bool = False + service_name: Optional[str] = None + deployment_environment: Optional[str] = None + model_id: Optional[str] = None + + def __post_init__(self) -> None: + if not self.service_name: + self.service_name = os.getenv("OTEL_SERVICE_NAME", "litellm") + if not self.deployment_environment: + self.deployment_environment = os.getenv( + "OTEL_ENVIRONMENT_NAME", "production" + ) + if not self.model_id: + self.model_id = os.getenv("OTEL_MODEL_ID", self.service_name) @classmethod def from_env(cls): @@ -122,6 +103,9 @@ class OpenTelemetryConfig: os.getenv("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", "false").lower() == "true" ) + service_name = os.getenv("OTEL_SERVICE_NAME", "litellm") + deployment_environment = os.getenv("OTEL_ENVIRONMENT_NAME", "production") + model_id = os.getenv("OTEL_MODEL_ID", service_name) if exporter == "in_memory": return cls(exporter=InMemorySpanExporter()) @@ -131,6 +115,9 @@ class OpenTelemetryConfig: headers=headers, # example: OTEL_HEADERS=x-honeycomb-team=B85YgLm96***" enable_metrics=enable_metrics, enable_events=enable_events, + service_name=service_name, + deployment_environment=deployment_environment, + model_id=model_id, ) @@ -174,6 +161,22 @@ class OpenTelemetry(CustomLogger): self._init_logs(logger_provider) self._init_otel_logger_on_litellm_proxy() + @staticmethod + def _get_litellm_resource(config: OpenTelemetryConfig): + """Create an OpenTelemetry Resource using config-driven defaults.""" + from opentelemetry.sdk.resources import OTELResourceDetector, Resource + + base_attributes: Dict[str, Optional[str]] = { + "service.name": config.service_name, + "deployment.environment": config.deployment_environment, + "model_id": config.model_id or config.service_name, + } + + base_resource = Resource.create(base_attributes) # type: ignore[arg-type] + otel_resource_detector = OTELResourceDetector() + env_resource = otel_resource_detector.detect() + return base_resource.merge(env_resource) + def _init_otel_logger_on_litellm_proxy(self): """ Initializes OpenTelemetry for litellm proxy server @@ -266,7 +269,7 @@ class OpenTelemetry(CustomLogger): from opentelemetry.trace import SpanKind def create_tracer_provider(): - provider = TracerProvider(resource=_get_litellm_resource()) + provider = TracerProvider(resource=self._get_litellm_resource(self.config)) provider.add_span_processor(self._get_span_processor()) return provider @@ -300,7 +303,8 @@ class OpenTelemetry(CustomLogger): def create_meter_provider(): metric_reader = self._get_metric_reader() return MeterProvider( - metric_readers=[metric_reader], resource=_get_litellm_resource() + metric_readers=[metric_reader], + resource=self._get_litellm_resource(self.config), ) meter_provider = self._get_or_create_provider( @@ -355,7 +359,9 @@ class OpenTelemetry(CustomLogger): from opentelemetry.sdk._logs.export import BatchLogRecordProcessor def create_logger_provider(): - provider = OTLoggerProvider(resource=_get_litellm_resource()) + provider = OTLoggerProvider( + resource=self._get_litellm_resource(self.config) + ) log_exporter = self._get_log_exporter() provider.add_log_record_processor( BatchLogRecordProcessor(log_exporter) # type: ignore[arg-type] @@ -606,7 +612,7 @@ class OpenTelemetry(CustomLogger): from opentelemetry.sdk.trace import TracerProvider # Create a temporary tracer provider with dynamic headers - temp_provider = TracerProvider(resource=_get_litellm_resource()) + temp_provider = TracerProvider(resource=self._get_litellm_resource(self.config)) temp_provider.add_span_processor( self._get_span_processor(dynamic_headers=dynamic_headers) ) @@ -987,9 +993,9 @@ class OpenTelemetry(CustomLogger): # Get the resource from the logger provider logger_provider = get_logger_provider() - resource = ( - getattr(logger_provider, "_resource", None) or _get_litellm_resource() - ) + resource = getattr( + logger_provider, "_resource", None + ) or self._get_litellm_resource(self.config) parent_ctx = span.get_span_context() provider = (kwargs.get("litellm_params") or {}).get( @@ -1910,7 +1916,9 @@ class OpenTelemetry(CustomLogger): ) _split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS) - normalized_endpoint = self._normalize_otel_endpoint(self.OTEL_ENDPOINT, "metrics") + normalized_endpoint = self._normalize_otel_endpoint( + self.OTEL_ENDPOINT, "metrics" + ) if self.OTEL_EXPORTER == "console": exporter = ConsoleMetricExporter() diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index cd324935562..5448fe7c771 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3630,6 +3630,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 otel_config = OpenTelemetryConfig( exporter=arize_config.protocol, endpoint=arize_config.endpoint, + service_name=arize_config.project_name, ) os.environ[ diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 954c26e2cb2..cff3bee1ca4 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1532,6 +1532,7 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase): guardrails: Optional[List[str]] = None object_permission: Optional[LiteLLM_ObjectPermissionBase] = None team_member_budget: Optional[float] = None + team_member_budget_duration: Optional[str] = None team_member_rpm_limit: Optional[int] = None team_member_tpm_limit: Optional[int] = None team_member_key_duration: Optional[str] = None diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 95c23be5b82..4e0c4f23811 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -957,11 +957,11 @@ class ProxyBaseLLMRequestProcessing: @staticmethod def _get_pre_call_type( route_type: Literal["acompletion", "aembedding", "aresponses", "allm_passthrough_route"], - ) -> Literal["completion", "embeddings", "responses", "allm_passthrough_route"]: + ) -> Literal["completion", "embedding", "responses", "allm_passthrough_route"]: if route_type == "acompletion": return "completion" elif route_type == "aembedding": - return "embeddings" + return "embedding" elif route_type == "aresponses": return "responses" elif route_type == "allm_passthrough_route": diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 76c607f5c49..920105edc16 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -112,6 +112,7 @@ class TeamMemberBudgetHandler: team_member_budget: Optional[float] = None, team_member_rpm_limit: Optional[int] = None, team_member_tpm_limit: Optional[int] = None, + team_member_budget_duration: Optional[str] = None, ) -> bool: """Check if any team member limits are provided""" return any( @@ -119,6 +120,7 @@ class TeamMemberBudgetHandler: team_member_budget is not None, team_member_rpm_limit is not None, team_member_tpm_limit is not None, + team_member_budget_duration is not None, ] ) @@ -130,6 +132,7 @@ class TeamMemberBudgetHandler: team_member_budget: Optional[float] = None, team_member_rpm_limit: Optional[int] = None, team_member_tpm_limit: Optional[int] = None, + team_member_budget_duration: Optional[str] = None, ) -> dict: """Create team member budget table with provided limits""" from litellm.proxy._types import BudgetNewRequest @@ -147,7 +150,7 @@ class TeamMemberBudgetHandler: # Create budget request with all provided limits budget_request = BudgetNewRequest( budget_id=budget_id, - budget_duration=data.budget_duration, + budget_duration=data.budget_duration or team_member_budget_duration, ) if team_member_budget is not None: @@ -156,6 +159,8 @@ class TeamMemberBudgetHandler: budget_request.rpm_limit = team_member_rpm_limit if team_member_tpm_limit is not None: budget_request.tpm_limit = team_member_tpm_limit + if team_member_budget_duration is not None: + budget_request.budget_duration = team_member_budget_duration team_member_budget_table = await new_budget( budget_obj=budget_request, @@ -182,6 +187,7 @@ class TeamMemberBudgetHandler: team_member_budget: Optional[float] = None, team_member_rpm_limit: Optional[int] = None, team_member_tpm_limit: Optional[int] = None, + team_member_budget_duration: Optional[str] = None, ) -> dict: """Upsert team member budget table with provided limits""" from litellm.proxy._types import BudgetNewRequest @@ -203,6 +209,8 @@ class TeamMemberBudgetHandler: budget_request.rpm_limit = team_member_rpm_limit if team_member_tpm_limit is not None: budget_request.tpm_limit = team_member_tpm_limit + if team_member_budget_duration is not None: + budget_request.budget_duration = team_member_budget_duration budget_row = await update_budget( budget_obj=budget_request, @@ -223,6 +231,7 @@ class TeamMemberBudgetHandler: team_member_budget=team_member_budget, team_member_rpm_limit=team_member_rpm_limit, team_member_tpm_limit=team_member_tpm_limit, + team_member_budget_duration=team_member_budget_duration, ) # Remove team member fields from updated_kv @@ -233,6 +242,7 @@ class TeamMemberBudgetHandler: def _clean_team_member_fields(data_dict: dict) -> None: """Remove team member fields from data dictionary""" data_dict.pop("team_member_budget", None) + data_dict.pop("team_member_budget_duration", None) data_dict.pop("team_member_rpm_limit", None) data_dict.pop("team_member_tpm_limit", None) @@ -1214,6 +1224,7 @@ async def update_team( # noqa: PLR0915 - disable_global_guardrails: Optional[bool] - Whether to disable global guardrails for the key. - object_permission: Optional[LiteLLM_ObjectPermissionBase] - team-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"], "agents": ["agent_1", "agent_2"], "agent_access_groups": ["dev_group"]}. IF null or {} then no object permission. - team_member_budget: Optional[float] - The maximum budget allocated to an individual team member. + - team_member_budget_duration: Optional[str] - The duration of the budget for the team member. Doc [here](https://docs.litellm.ai/docs/proxy/team_budgets) - team_member_rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for individual team members. - team_member_tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for individual team members. - team_member_key_duration: Optional[str] - The duration for a team member's key. e.g. "1d", "1w", "1mo" @@ -1349,6 +1360,7 @@ async def update_team( # noqa: PLR0915 team_member_budget=data.team_member_budget, team_member_rpm_limit=data.team_member_rpm_limit, team_member_tpm_limit=data.team_member_tpm_limit, + team_member_budget_duration=data.team_member_budget_duration, ): updated_kv = await TeamMemberBudgetHandler.upsert_team_member_budget_table( team_table=existing_team_row, @@ -1357,6 +1369,7 @@ async def update_team( # noqa: PLR0915 team_member_budget=data.team_member_budget, team_member_rpm_limit=data.team_member_rpm_limit, team_member_tpm_limit=data.team_member_tpm_limit, + team_member_budget_duration=data.team_member_budget_duration, ) else: TeamMemberBudgetHandler._clean_team_member_fields(updated_kv) diff --git a/litellm/types/integrations/arize.py b/litellm/types/integrations/arize.py index be4df30e794..248fdac3b3a 100644 --- a/litellm/types/integrations/arize.py +++ b/litellm/types/integrations/arize.py @@ -14,3 +14,4 @@ class ArizeConfig(BaseModel): api_key: Optional[str] = None protocol: Protocol endpoint: str + project_name: Optional[str] = None diff --git a/tests/local_testing/test_arize_ai.py b/tests/local_testing/test_arize_ai.py index 6a773521435..3b497d638ae 100644 --- a/tests/local_testing/test_arize_ai.py +++ b/tests/local_testing/test_arize_ai.py @@ -71,6 +71,7 @@ def test_get_arize_config(mock_env_vars): assert config.api_key == "test_api_key" assert config.endpoint == "https://otlp.arize.com/v1" assert config.protocol == "otlp_grpc" + assert config.project_name is None def test_get_arize_config_with_endpoints(mock_env_vars, monkeypatch): @@ -79,10 +80,12 @@ def test_get_arize_config_with_endpoints(mock_env_vars, monkeypatch): """ monkeypatch.setenv("ARIZE_ENDPOINT", "grpc://test.endpoint") monkeypatch.setenv("ARIZE_HTTP_ENDPOINT", "http://test.endpoint") + monkeypatch.setenv("ARIZE_PROJECT_NAME", "custom-project") config = ArizeLogger.get_arize_config() assert config.endpoint == "grpc://test.endpoint" assert config.protocol == "otlp_grpc" + assert config.project_name == "custom-project" @pytest.mark.skip( diff --git a/tests/test_litellm/integrations/arize/test_arize_health_check.py b/tests/test_litellm/integrations/arize/test_arize_health_check.py index 91d0b42d48d..8d86b7dc097 100644 --- a/tests/test_litellm/integrations/arize/test_arize_health_check.py +++ b/tests/test_litellm/integrations/arize/test_arize_health_check.py @@ -123,7 +123,8 @@ class TestArizeIntegrationWithProxy: with patch.dict(os.environ, { "ARIZE_SPACE_KEY": "test-space-123", "ARIZE_API_KEY": "test-api-456", - "ARIZE_ENDPOINT": "https://custom.arize.com/v1" + "ARIZE_ENDPOINT": "https://custom.arize.com/v1", + "ARIZE_PROJECT_NAME": "custom-project", }): config = ArizeLogger.get_arize_config() @@ -131,13 +132,15 @@ class TestArizeIntegrationWithProxy: assert config.api_key == "test-api-456" assert config.endpoint == "https://custom.arize.com/v1" assert config.protocol == "otlp_grpc" + assert config.project_name == "custom-project" def test_arize_get_config_defaults(self): """Test ArizeLogger.get_arize_config() with default endpoint.""" with patch.dict(os.environ, { "ARIZE_SPACE_KEY": "test-space-default", - "ARIZE_API_KEY": "test-api-default" + "ARIZE_API_KEY": "test-api-default", + "ARIZE_PROJECT_NAME": "default-project", }, clear=True): config = ArizeLogger.get_arize_config() @@ -145,6 +148,7 @@ class TestArizeIntegrationWithProxy: assert config.api_key == "test-api-default" assert config.endpoint == "https://otlp.arize.com/v1" # Default endpoint assert config.protocol == "otlp_grpc" # Default protocol + assert config.project_name == "default-project" def test_arize_construct_dynamic_headers(self): """Test dynamic OTEL headers construction for team/key logging.""" @@ -180,4 +184,4 @@ class TestArizeIntegrationWithProxy: if __name__ == "__main__": - pytest.main([__file__, "-v"]) \ No newline at end of file + pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py index a322dfe9a2b..d7d7720ff43 100644 --- a/tests/test_litellm/integrations/test_custom_guardrail.py +++ b/tests/test_litellm/integrations/test_custom_guardrail.py @@ -530,7 +530,7 @@ class TestPassthroughCallTypeHandling: ) assert ( ProxyBaseLLMRequestProcessing._get_pre_call_type(route_type="aembedding") - == "embeddings" + == "embedding" ) assert ( ProxyBaseLLMRequestProcessing._get_pre_call_type(route_type="aresponses") diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 6c17570e135..55b65fbb92a 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -258,6 +258,22 @@ class TestOpenTelemetry(unittest.TestCase): MODEL = "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0" HERE = os.path.dirname(__file__) + @patch.dict(os.environ, {}, clear=True) + def test_open_telemetry_config_manual_defaults(self): + """Manual OpenTelemetryConfig creation should populate default identifiers.""" + config = OpenTelemetryConfig(exporter="console", endpoint="http://collector") + self.assertEqual(config.service_name, "litellm") + self.assertEqual(config.deployment_environment, "production") + self.assertEqual(config.model_id, "litellm") + + @patch.dict(os.environ, {}, clear=True) + def test_open_telemetry_config_custom_service_name(self): + """Model ID should inherit provided service name when not explicitly set.""" + config = OpenTelemetryConfig(service_name="custom-service", exporter="console") + self.assertEqual(config.service_name, "custom-service") + self.assertEqual(config.deployment_environment, "production") + self.assertEqual(config.model_id, "custom-service") + def wait_for_spans(self, exporter: InMemorySpanExporter, prefix: str): """Poll until we see at least one span with an attribute key starting with `prefix`.""" deadline = time.time() + self.POLL_TIMEOUT @@ -504,8 +520,6 @@ class TestOpenTelemetry(unittest.TestCase): self, mock_detector_cls, mock_resource_create ): """Test _get_litellm_resource with default values when no environment variables are set.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - # Mock the Resource.create method mock_base_resource = MagicMock() mock_resource_create.return_value = mock_base_resource @@ -520,8 +534,8 @@ class TestOpenTelemetry(unittest.TestCase): mock_merged_resource = MagicMock() mock_base_resource.merge.return_value = mock_merged_resource - # Call the function - result = _get_litellm_resource() + config = OpenTelemetryConfig() + result = OpenTelemetry._get_litellm_resource(config) # Verify Resource.create was called with correct default attributes expected_attributes = { @@ -549,8 +563,6 @@ class TestOpenTelemetry(unittest.TestCase): self, mock_detector_cls, mock_resource_create ): """Test _get_litellm_resource with LiteLLM-specific environment variables.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - # Mock the Resource.create method mock_base_resource = MagicMock() mock_resource_create.return_value = mock_base_resource @@ -565,8 +577,8 @@ class TestOpenTelemetry(unittest.TestCase): mock_merged_resource = MagicMock() mock_base_resource.merge.return_value = mock_merged_resource - # Call the function - result = _get_litellm_resource() + config = OpenTelemetryConfig.from_env() + result = OpenTelemetry._get_litellm_resource(config) # Verify Resource.create was called with environment variable values expected_attributes = { @@ -593,8 +605,6 @@ class TestOpenTelemetry(unittest.TestCase): self, mock_detector_cls, mock_resource_create ): """Test _get_litellm_resource with OTEL_RESOURCE_ATTRIBUTES environment variable.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - # Mock the Resource.create method to simulate the actual behavior # In reality, Resource.create() would parse OTEL_RESOURCE_ATTRIBUTES and merge it mock_base_resource = MagicMock() @@ -610,8 +620,8 @@ class TestOpenTelemetry(unittest.TestCase): mock_merged_resource = MagicMock() mock_base_resource.merge.return_value = mock_merged_resource - # Call the function - result = _get_litellm_resource() + config = OpenTelemetryConfig.from_env() + result = OpenTelemetry._get_litellm_resource(config) # Verify Resource.create was called with the base attributes # The actual OTEL_RESOURCE_ATTRIBUTES parsing is handled by OpenTelemetry SDK @@ -628,10 +638,8 @@ class TestOpenTelemetry(unittest.TestCase): @patch.dict(os.environ, {}, clear=True) def test_get_litellm_resource_integration_with_real_resource(self): """Integration test to verify _get_litellm_resource works with actual OpenTelemetry Resource.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - - # This test uses the real OpenTelemetry Resource.create() method - result = _get_litellm_resource() + config = OpenTelemetryConfig() + result = OpenTelemetry._get_litellm_resource(config) # Verify the result is a Resource instance from opentelemetry.sdk.resources import Resource @@ -653,10 +661,8 @@ class TestOpenTelemetry(unittest.TestCase): ) def test_get_litellm_resource_real_otel_resource_attributes(self): """Integration test to verify OTEL_RESOURCE_ATTRIBUTES is properly handled.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - - # This test uses the real OpenTelemetry Resource.create() method - result = _get_litellm_resource() + config = OpenTelemetryConfig.from_env() + result = OpenTelemetry._get_litellm_resource(config) print("RESULT", result) @@ -683,10 +689,8 @@ class TestOpenTelemetry(unittest.TestCase): ) def test_get_litellm_resource_precedence(self): """Test that OTEL_SERVICE_NAME takes precedence over OTEL_RESOURCE_ATTRIBUTES according to OpenTelemetry spec.""" - from litellm.integrations.opentelemetry import _get_litellm_resource - - # This test verifies the OpenTelemetry standard behavior - result = _get_litellm_resource() + config = OpenTelemetryConfig.from_env() + result = OpenTelemetry._get_litellm_resource(config) # Verify the result is a Resource instance from opentelemetry.sdk.resources import Resource diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 6cf8f745e07..57064586afb 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -1279,7 +1279,7 @@ async def test_update_team_team_member_budget_not_passed_to_db(): # Mock budget upsert to return updated_kv without team_member_budget def mock_upsert_side_effect( - team_table, user_api_key_dict, updated_kv, team_member_budget=None, team_member_rpm_limit=None, team_member_tpm_limit=None + team_table, user_api_key_dict, updated_kv, team_member_budget=None, team_member_rpm_limit=None, team_member_tpm_limit=None, team_member_budget_duration=None ): # Remove team_member_budget from updated_kv as the real function does result_kv = updated_kv.copy() @@ -1376,6 +1376,370 @@ async def test_update_team_team_member_budget_not_passed_to_db(): ) +def test_clean_team_member_fields(): + """ + Test that _clean_team_member_fields removes all team member fields from a dictionary. + """ + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + data_dict = { + "team_id": "test_team", + "team_alias": "Test Team", + "team_member_budget": 100.0, + "team_member_budget_duration": "30d", + "team_member_rpm_limit": 50, + "team_member_tpm_limit": 1000, + "other_field": "should_remain", + } + + TeamMemberBudgetHandler._clean_team_member_fields(data_dict) + + assert "team_member_budget" not in data_dict + assert "team_member_budget_duration" not in data_dict + assert "team_member_rpm_limit" not in data_dict + assert "team_member_tpm_limit" not in data_dict + assert data_dict["team_id"] == "test_team" + assert data_dict["team_alias"] == "Test Team" + assert data_dict["other_field"] == "should_remain" + + +def test_clean_team_member_fields_with_missing_fields(): + """ + Test that _clean_team_member_fields handles dictionaries without team member fields gracefully. + """ + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + data_dict = { + "team_id": "test_team", + "team_alias": "Test Team", + } + + TeamMemberBudgetHandler._clean_team_member_fields(data_dict) + + assert data_dict["team_id"] == "test_team" + assert data_dict["team_alias"] == "Test Team" + + +@pytest.mark.asyncio +async def test_create_team_member_budget_table(): + """ + Test that create_team_member_budget_table creates a budget and adds it to metadata. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.proxy._types import LitellmUserRoles, NewTeamRequest, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id" + ) + + data = NewTeamRequest( + team_id="test_team_id", + team_alias="Test Team", + budget_duration="1mo", + ) + new_team_data_json = { + "team_id": "test_team_id", + "team_alias": "Test Team", + "team_member_budget": 100.0, + "team_member_budget_duration": "30d", + "team_member_rpm_limit": 50, + "team_member_tpm_limit": 1000, + } + + mock_budget_response = MagicMock() + mock_budget_response.budget_id = "budget_123" + + with patch( + "litellm.proxy.management_endpoints.budget_management_endpoints.new_budget", + new_callable=AsyncMock + ) as mock_new_budget: + mock_new_budget.return_value = mock_budget_response + + result = await TeamMemberBudgetHandler.create_team_member_budget_table( + data=data, + new_team_data_json=new_team_data_json, + user_api_key_dict=mock_user_api_key_dict, + team_member_budget=100.0, + team_member_rpm_limit=50, + team_member_tpm_limit=1000, + team_member_budget_duration="30d", + ) + + assert mock_new_budget.called + call_args = mock_new_budget.call_args + budget_request = call_args[1]["budget_obj"] + + assert budget_request.max_budget == 100.0 + assert budget_request.rpm_limit == 50 + assert budget_request.tpm_limit == 1000 + assert budget_request.budget_duration == "30d" + assert budget_request.budget_id is not None + assert "team-" in budget_request.budget_id + + assert "team_member_budget_id" in result["metadata"] + assert result["metadata"]["team_member_budget_id"] == "budget_123" + + assert "team_member_budget" not in result + assert "team_member_budget_duration" not in result + assert "team_member_rpm_limit" not in result + assert "team_member_tpm_limit" not in result + + +@pytest.mark.asyncio +async def test_create_team_member_budget_table_without_team_alias(): + """ + Test that create_team_member_budget_table generates budget_id correctly when team_alias is None. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.proxy._types import LitellmUserRoles, NewTeamRequest, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id" + ) + + data = NewTeamRequest(team_id="test_team_id") + new_team_data_json = { + "team_id": "test_team_id", + "team_member_budget": 100.0, + } + + mock_budget_response = MagicMock() + mock_budget_response.budget_id = "budget_123" + + with patch( + "litellm.proxy.management_endpoints.budget_management_endpoints.new_budget", + new_callable=AsyncMock + ) as mock_new_budget: + mock_new_budget.return_value = mock_budget_response + + result = await TeamMemberBudgetHandler.create_team_member_budget_table( + data=data, + new_team_data_json=new_team_data_json, + user_api_key_dict=mock_user_api_key_dict, + team_member_budget=100.0, + ) + + assert mock_new_budget.called + call_args = mock_new_budget.call_args + budget_request = call_args[1]["budget_obj"] + + assert budget_request.budget_id is not None + assert budget_request.budget_id.startswith("team-budget-") + + +@pytest.mark.asyncio +async def test_upsert_team_member_budget_table_existing_budget(): + """ + Test that upsert_team_member_budget_table updates an existing budget when team_member_budget_id exists. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.proxy._types import LitellmUserRoles, LiteLLM_TeamTable, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id" + ) + + team_table = MagicMock(spec=LiteLLM_TeamTable) + team_table.metadata = {"team_member_budget_id": "existing_budget_123"} + + updated_kv = { + "team_id": "test_team_id", + "team_member_budget": 200.0, + "team_member_budget_duration": "60d", + "team_member_rpm_limit": 100, + } + + mock_budget_response = MagicMock() + mock_budget_response.budget_id = "existing_budget_123" + + with patch( + "litellm.proxy.management_endpoints.budget_management_endpoints.update_budget", + new_callable=AsyncMock + ) as mock_update_budget: + mock_update_budget.return_value = mock_budget_response + + result = await TeamMemberBudgetHandler.upsert_team_member_budget_table( + team_table=team_table, + user_api_key_dict=mock_user_api_key_dict, + updated_kv=updated_kv, + team_member_budget=200.0, + team_member_budget_duration="60d", + team_member_rpm_limit=100, + ) + + assert mock_update_budget.called + call_args = mock_update_budget.call_args + budget_request = call_args[1]["budget_obj"] + + assert budget_request.budget_id == "existing_budget_123" + assert budget_request.max_budget == 200.0 + assert budget_request.budget_duration == "60d" + assert budget_request.rpm_limit == 100 + + assert "team_member_budget_id" in result["metadata"] + assert result["metadata"]["team_member_budget_id"] == "existing_budget_123" + + assert "team_member_budget" not in result + assert "team_member_budget_duration" not in result + assert "team_member_rpm_limit" not in result + + +@pytest.mark.asyncio +async def test_upsert_team_member_budget_table_no_existing_budget(): + """ + Test that upsert_team_member_budget_table creates a new budget when team_member_budget_id does not exist. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.proxy._types import LitellmUserRoles, LiteLLM_TeamTable, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import ( + TeamMemberBudgetHandler, + ) + + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id" + ) + + team_table = MagicMock(spec=LiteLLM_TeamTable) + team_table.metadata = {} + team_table.team_alias = "Test Team" + team_table.budget_duration = None + + updated_kv = { + "team_id": "test_team_id", + "team_member_budget": 150.0, + "team_member_budget_duration": "45d", + } + + mock_budget_response = MagicMock() + mock_budget_response.budget_id = "new_budget_456" + + with patch( + "litellm.proxy.management_endpoints.budget_management_endpoints.new_budget", + new_callable=AsyncMock + ) as mock_new_budget: + mock_new_budget.return_value = mock_budget_response + + result = await TeamMemberBudgetHandler.upsert_team_member_budget_table( + team_table=team_table, + user_api_key_dict=mock_user_api_key_dict, + updated_kv=updated_kv, + team_member_budget=150.0, + team_member_budget_duration="45d", + ) + + assert mock_new_budget.called + assert "team_member_budget_id" in result["metadata"] + assert result["metadata"]["team_member_budget_id"] == "new_budget_456" + + assert "team_member_budget" not in result + assert "team_member_budget_duration" not in result + + +@pytest.mark.asyncio +async def test_update_team_with_team_member_budget_duration(): + """ + Test that team/update endpoint properly handles team_member_budget_duration. + """ + from unittest.mock import AsyncMock, MagicMock, Mock, patch + + from fastapi import Request + + from litellm.proxy._types import LitellmUserRoles, UpdateTeamRequest, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import update_team + + mock_request = Mock(spec=Request) + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id" + ) + + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma_client, patch( + "litellm.proxy.proxy_server.llm_router" + ) as mock_llm_router, patch( + "litellm.proxy.proxy_server.user_api_key_cache" + ) as mock_cache, patch( + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as mock_logging, patch( + "litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin" + ), patch( + "litellm.proxy.auth.auth_checks._cache_team_object" + ) as mock_cache_team, patch( + "litellm.proxy.management_endpoints.team_endpoints.TeamMemberBudgetHandler.upsert_team_member_budget_table" + ) as mock_upsert_budget: + + mock_existing_team = MagicMock() + mock_existing_team.model_dump.return_value = { + "team_id": "test_team_id", + "team_alias": "test_team", + "metadata": {"team_member_budget_id": "budget_123"}, + } + mock_existing_team.metadata = {"team_member_budget_id": "budget_123"} + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=mock_existing_team + ) + + mock_updated_team = MagicMock() + mock_updated_team.team_id = "test_team_id" + mock_updated_team.model_dump.return_value = {"team_id": "test_team_id"} + mock_prisma_client.db.litellm_teamtable.update = AsyncMock( + return_value=mock_updated_team + ) + mock_prisma_client.jsonify_team_object = MagicMock( + side_effect=lambda db_data: db_data + ) + + def mock_upsert_side_effect( + team_table, user_api_key_dict, updated_kv, team_member_budget=None, team_member_rpm_limit=None, team_member_tpm_limit=None, team_member_budget_duration=None + ): + result_kv = updated_kv.copy() + result_kv.pop("team_member_budget", None) + result_kv.pop("team_member_budget_duration", None) + return result_kv + + mock_upsert_budget.side_effect = mock_upsert_side_effect + + update_request = UpdateTeamRequest( + team_id="test_team_id", + team_alias="updated_alias", + team_member_budget=100.0, + team_member_budget_duration="30d", + ) + + result = await update_team( + data=update_request, + http_request=mock_request, + user_api_key_dict=mock_user_api_key_dict, + ) + + assert mock_upsert_budget.called + call_args = mock_upsert_budget.call_args + assert call_args[1]["team_member_budget"] == 100.0 + assert call_args[1]["team_member_budget_duration"] == "30d" + + assert mock_prisma_client.db.litellm_teamtable.update.called + update_call_args = mock_prisma_client.db.litellm_teamtable.update.call_args + update_data = update_call_args[1]["data"] + + assert "team_member_budget" not in update_data + assert "team_member_budget_duration" not in update_data + + @pytest.mark.asyncio async def test_bulk_team_member_add_success(): """ diff --git a/ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx b/ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx new file mode 100644 index 00000000000..296ef1ae632 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx @@ -0,0 +1,49 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, it, expect, vi } from "vitest"; +import DurationSelect from "./DurationSelect"; + +describe("DurationSelect", () => { + it("should render", () => { + render(); + expect(screen.getByRole("combobox")).toBeInTheDocument(); + }); + + it("should render all three duration options", async () => { + const user = userEvent.setup(); + render(); + + const select = screen.getByRole("combobox"); + await user.click(select); + + expect(screen.getByText("Daily")).toBeInTheDocument(); + expect(screen.getByText("Weekly")).toBeInTheDocument(); + expect(screen.getByText("Monthly")).toBeInTheDocument(); + }); + + it("should apply className prop", () => { + render(); + const select = screen.getByRole("combobox"); + expect(select.closest(".test-class")).toBeInTheDocument(); + }); + + it("should call onChange when an option is selected", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + render(); + + const select = screen.getByRole("combobox"); + await user.click(select); + + const dailyOption = screen.getByText("Daily"); + await user.click(dailyOption); + + expect(onChange).toHaveBeenCalledWith("24h", expect.any(Object)); + }); + + it("should accept and pass value prop to Select", () => { + render(); + const select = screen.getByRole("combobox"); + expect(select).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx b/ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx new file mode 100644 index 00000000000..a84e8aeb110 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx @@ -0,0 +1,17 @@ +import { Select } from "antd"; + +interface DurationSelectProps { + className?: string; + value?: string; + onChange?: (value: string) => void; +} + +export default function DurationSelect({ className, value, onChange }: DurationSelectProps) { + return ( + + ); +} diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx index d2d1c885931..fd7994aa1da 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.tsx @@ -48,6 +48,7 @@ import EditLoggingSettings from "./EditLoggingSettings"; import MemberModal from "./EditMembership"; import MemberPermissions from "./member_permissions"; import TeamMembersComponent from "./team_member_view"; +import DurationSelect from "../common_components/DurationSelect"; export interface TeamMembership { user_id: string; @@ -413,6 +414,7 @@ const TeamInfoView: React.FC = ({ }; updateData.max_budget = mapEmptyStringToNull(updateData.max_budget); + updateData.team_member_budget_duration = values.team_member_budget_duration; if (values.team_member_budget !== undefined) { updateData.team_member_budget = Number(values.team_member_budget); @@ -650,6 +652,8 @@ const TeamInfoView: React.FC = ({ budget_duration: info.budget_duration, team_member_tpm_limit: info.team_member_budget_table?.tpm_limit, team_member_rpm_limit: info.team_member_budget_table?.rpm_limit, + team_member_budget: info.team_member_budget_table?.max_budget, + team_member_budget_duration: info.team_member_budget_table?.budget_duration, guardrails: info.metadata?.guardrails || [], disable_global_guardrails: info.metadata?.disable_global_guardrails || false, metadata: info.metadata @@ -747,6 +751,13 @@ const TeamInfoView: React.FC = ({ + + form.setFieldValue("team_member_budget_duration", value)} + value={form.getFieldValue("team_member_budget_duration")} + /> + + = ({
Max Budget: {info.team_member_budget_table?.max_budget || "No Limit"}
+
Budget Duration: {info.team_member_budget_table?.budget_duration || "No Limit"}
Key Duration: {info.metadata?.team_member_key_duration || "No Limit"}
TPM Limit: {info.team_member_budget_table?.tpm_limit || "No Limit"}
RPM Limit: {info.team_member_budget_table?.rpm_limit || "No Limit"}
From f7212d84d5ae7ac2f7bf469709bea03080acb6b7 Mon Sep 17 00:00:00 2001 From: tianduo-fh <153234738+Tianduo16@users.noreply.github.com> Date: Wed, 7 Jan 2026 11:12:27 -0500 Subject: [PATCH 019/121] fix: prevent duplicate User-Agent tags in request_tags (#18723) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `_get_request_tags` function was returning a reference to the original tags list from metadata, then mutating it with `.extend()`. This caused duplicate User-Agent tags when the function was called multiple times during a single request lifecycle (e.g., by logging, prometheus, and guardrails). The fix uses `.copy()` to create a new list before extending, ensuring the original metadata tags are not mutated. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Tianduo Zhai Co-authored-by: Claude Opus 4.5 --- litellm/litellm_core_utils/litellm_logging.py | 4 +- .../test_litellm_logging.py | 57 +++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 5448fe7c771..d9161f1db41 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4824,9 +4824,9 @@ class StandardLoggingPayloadSetup: metadata = litellm_params.get("metadata") or {} litellm_metadata = litellm_params.get("litellm_metadata") or {} if metadata.get("tags", []): - request_tags = metadata.get("tags", []) + request_tags = metadata.get("tags", []).copy() elif litellm_metadata.get("tags", []): - request_tags = litellm_metadata.get("tags", []) + request_tags = litellm_metadata.get("tags", []).copy() else: request_tags = [] user_agent_tags = StandardLoggingPayloadSetup._get_user_agent_tags( diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 9b150fd89f4..bae0e5bbb4f 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -393,6 +393,63 @@ def test_get_request_tags_from_metadata_and_litellm_metadata(): assert "User-Agent: litellm/1.0.0" in tags +def test_get_request_tags_does_not_mutate_original_tags(): + """ + Test that _get_request_tags does not mutate the original tags list in metadata. + + This is a regression test for a bug where calling _get_request_tags multiple times + would cause User-Agent tags to be duplicated because the function was mutating + the original tags list instead of creating a copy. + """ + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + # Create metadata with original tags + original_tags = ["custom-tag-1", "custom-tag-2"] + metadata = {"tags": original_tags} + litellm_params = {"metadata": metadata} + proxy_server_request = { + "headers": { + "user-agent": "AsyncOpenAI/Python 1.99.9", + } + } + + # Call _get_request_tags multiple times (simulating multiple callbacks) + tags1 = StandardLoggingPayloadSetup._get_request_tags( + litellm_params=litellm_params, + proxy_server_request=proxy_server_request, + ) + tags2 = StandardLoggingPayloadSetup._get_request_tags( + litellm_params=litellm_params, + proxy_server_request=proxy_server_request, + ) + tags3 = StandardLoggingPayloadSetup._get_request_tags( + litellm_params=litellm_params, + proxy_server_request=proxy_server_request, + ) + + # Verify the original tags list was NOT mutated + assert original_tags == ["custom-tag-1", "custom-tag-2"], ( + f"Original tags list was mutated: {original_tags}" + ) + assert metadata["tags"] == ["custom-tag-1", "custom-tag-2"], ( + f"metadata['tags'] was mutated: {metadata['tags']}" + ) + + # Verify each returned list has exactly 2 User-Agent tags (not duplicated) + user_agent_count_1 = len([t for t in tags1 if t.startswith("User-Agent:")]) + user_agent_count_2 = len([t for t in tags2 if t.startswith("User-Agent:")]) + user_agent_count_3 = len([t for t in tags3 if t.startswith("User-Agent:")]) + + assert user_agent_count_1 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_1}" + assert user_agent_count_2 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_2}" + assert user_agent_count_3 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_3}" + + # Verify all returned lists are independent (different objects) + assert tags1 is not tags2 + assert tags2 is not tags3 + assert tags1 is not original_tags + + def test_get_extra_header_tags(): """Test the _get_extra_header_tags method with various scenarios.""" import litellm From 8ce4eea88f977a5c0e31358babed40c468885f67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20Marc=C3=ADlio=20J=C3=BAnior?= Date: Wed, 7 Jan 2026 13:13:55 -0300 Subject: [PATCH 020/121] make base_connection_pool_limit default value the same (#18721) --- litellm/proxy/_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index cff3bee1ca4..bb6b56d5090 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1937,7 +1937,7 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): description="connect to a postgres db - needed for generating temporary keys + tracking spend / key", ) database_connection_pool_limit: Optional[int] = Field( - 100, + 10, description="default connection pool for prisma client connecting to postgres db", ) database_connection_timeout: Optional[float] = Field( From dc4ce7c5a21615504b739d445d9c3ffd35aaeb38 Mon Sep 17 00:00:00 2001 From: Abliteration AI Date: Wed, 7 Jan 2026 08:16:54 -0800 Subject: [PATCH 021/121] feat: Add abliteration.ai provider (#18678) * feat: Add abliteration.ai provider * adding signoz integration to observability docs * Fixing build * Adding timeout for flaky test * Fixing e2e * add team member budget duration in team/update * Reusable Duration Select and update team member budget UI --------- Co-authored-by: Goutham Karthi Co-authored-by: yuneng-jiang Co-authored-by: YutaSaito <36355491+uc4w6c@users.noreply.github.com> --- README.md | 2 +- .../my-website/docs/providers/abliteration.md | 109 ++++++++++++++++++ docs/my-website/sidebars.js | 13 ++- litellm/llms/openai_like/providers.json | 4 + provider_endpoints_support.json | 17 +++ .../openai_like/test_abliteration_provider.py | 50 ++++++++ 6 files changed, 188 insertions(+), 7 deletions(-) create mode 100644 docs/my-website/docs/providers/abliteration.md create mode 100644 tests/litellm/llms/openai_like/test_abliteration_provider.py diff --git a/README.md b/README.md index a020bd80898..75a23faa5c1 100644 --- a/README.md +++ b/README.md @@ -262,6 +262,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature | Provider | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` | |-------------------------------------------------------------------------------------|---------------------|-------------|--------------|---------------|----------------------|-------------------------|-----------------|----------------|-----------|-----------| +| [Abliteration (`abliteration`)](https://docs.litellm.ai/docs/providers/abliteration) | ✅ | | | | | | | | | | | [AI/ML API (`aiml`)](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | [AI21 (`ai21`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | | | [AI21 Chat (`ai21_chat`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | | @@ -455,4 +456,3 @@ All these checks must pass before your PR can be merged. - diff --git a/docs/my-website/docs/providers/abliteration.md b/docs/my-website/docs/providers/abliteration.md new file mode 100644 index 00000000000..a0fc7f39310 --- /dev/null +++ b/docs/my-website/docs/providers/abliteration.md @@ -0,0 +1,109 @@ +# Abliteration + +## Overview + +| Property | Details | +|-------|-------| +| Description | Abliteration provides an OpenAI-compatible `/chat/completions` endpoint. | +| Provider Route on LiteLLM | `abliteration/` | +| Link to Provider Doc | [Abliteration](https://abliteration.ai) | +| Base URL | `https://api.abliteration.ai/v1` | +| Supported Operations | [`/chat/completions`](#sample-usage) | + +
+ +## Required Variables + +```python showLineNumbers title="Environment Variables" +os.environ["ABLITERATION_API_KEY"] = "" # your Abliteration API key +``` + +## Sample Usage + +```python showLineNumbers title="Abliteration Completion" +import os +from litellm import completion + +os.environ["ABLITERATION_API_KEY"] = "" + +response = completion( + model="abliteration/abliterated-model", + messages=[{"role": "user", "content": "Hello from LiteLLM"}], +) + +print(response) +``` + +## Sample Usage - Streaming + +```python showLineNumbers title="Abliteration Streaming Completion" +import os +from litellm import completion + +os.environ["ABLITERATION_API_KEY"] = "" + +response = completion( + model="abliteration/abliterated-model", + messages=[{"role": "user", "content": "Stream a short reply"}], + stream=True, +) + +for chunk in response: + print(chunk) +``` + +## Usage with LiteLLM Proxy Server + +1. Add the model to your proxy config: + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: abliteration-chat + litellm_params: + model: abliteration/abliterated-model + api_key: os.environ/ABLITERATION_API_KEY +``` + +2. Start the proxy: + +```bash +litellm --config /path/to/config.yaml +``` + +## Direct API Usage (Bearer Token) + +Use the environment variable as a Bearer token against the OpenAI-compatible endpoint: +`https://api.abliteration.ai/v1/chat/completions`. + +```bash showLineNumbers title="cURL" +export ABLITERATION_API_KEY="" +curl https://api.abliteration.ai/v1/chat/completions \ + -H "Authorization: Bearer ${ABLITERATION_API_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "abliterated-model", + "messages": [{"role": "user", "content": "Hello from Abliteration"}] + }' +``` + +```python showLineNumbers title="Python (requests)" +import os +import requests + +api_key = os.environ["ABLITERATION_API_KEY"] + +response = requests.post( + "https://api.abliteration.ai/v1/chat/completions", + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + json={ + "model": "abliterated-model", + "messages": [{"role": "user", "content": "Hello from Abliteration"}], + }, + timeout=60, +) + +print(response.json()) +``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 004132ca05b..4c50449afcc 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -683,12 +683,13 @@ const sidebars = { "providers/bedrock_writer", "providers/bedrock_batches", "providers/aws_polly", - "providers/bedrock_vector_store", - ] - }, - "providers/litellm_proxy", - "providers/ai21", - "providers/aiml", + "providers/bedrock_vector_store", + ] + }, + "providers/litellm_proxy", + "providers/abliteration", + "providers/ai21", + "providers/aiml", "providers/aleph_alpha", "providers/amazon_nova", "providers/anyscale", diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 206aee1359d..bda3684a8a8 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -61,6 +61,10 @@ "max_completion_tokens": "max_tokens" } }, + "abliteration": { + "base_url": "https://api.abliteration.ai/v1", + "api_key_env": "ABLITERATION_API_KEY" + }, "llamagate": { "base_url": "https://api.llamagate.dev/v1", "api_key_env": "LLAMAGATE_API_KEY", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index bc5dea7b97c..2521d71b5c0 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -34,6 +34,23 @@ } }, "providers": { + "abliteration": { + "display_name": "Abliteration (`abliteration`)", + "url": "https://docs.litellm.ai/docs/providers/abliteration", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "aiml": { "display_name": "AI/ML API (`aiml`)", "url": "https://docs.litellm.ai/docs/providers/aiml", diff --git a/tests/litellm/llms/openai_like/test_abliteration_provider.py b/tests/litellm/llms/openai_like/test_abliteration_provider.py new file mode 100644 index 00000000000..8b8d443fc44 --- /dev/null +++ b/tests/litellm/llms/openai_like/test_abliteration_provider.py @@ -0,0 +1,50 @@ +""" +Unit tests for the Abliteration OpenAI-like provider. +""" + +import os +import sys + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..")) +) + +from litellm.llms.openai_like.dynamic_config import create_config_class +from litellm.llms.openai_like.json_loader import JSONProviderRegistry + +ABLITERATION_BASE_URL = "https://api.abliteration.ai/v1" + + +def _get_config(): + provider = JSONProviderRegistry.get("abliteration") + assert provider is not None + config_class = create_config_class(provider) + return config_class() + + +def test_abliteration_provider_registered(): + provider = JSONProviderRegistry.get("abliteration") + assert provider is not None + assert provider.base_url == ABLITERATION_BASE_URL + assert provider.api_key_env == "ABLITERATION_API_KEY" + + +def test_abliteration_resolves_env_api_key(monkeypatch): + config = _get_config() + monkeypatch.setenv("ABLITERATION_API_KEY", "test-key") + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == ABLITERATION_BASE_URL + assert api_key == "test-key" + + +def test_abliteration_complete_url_appends_endpoint(): + config = _get_config() + url = config.get_complete_url( + api_base=ABLITERATION_BASE_URL, + api_key="test-key", + model="abliteration/abliterated-model", + optional_params={}, + litellm_params={}, + stream=False, + ) + assert url == f"{ABLITERATION_BASE_URL}/chat/completions" From 5a4242e1882517cf850849721897bf39f46e2cf9 Mon Sep 17 00:00:00 2001 From: Wen-Tien Chang Date: Thu, 8 Jan 2026 03:25:59 +0800 Subject: [PATCH 022/121] fix(braintrust): pass span_attributes in async logging and skip tags on non-root spans (#18409) * fix(braintrust): handle tags and span attributes for non-root spans * fix(braintrust): refactor span attributes handling and remove duplicate metrics assignment --- litellm/integrations/braintrust_logging.py | 35 ++++++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index 364fa3f5def..585de510e8b 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -225,10 +225,13 @@ class BraintrustLogger(CustomLogger): "id": litellm_call_id, "input": prompt["messages"], "metadata": standard_logging_object, - "tags": tags, "span_attributes": {"name": span_name, "type": "llm"}, } - + + # Braintrust cannot specify 'tags' for non-root spans + if dynamic_metadata.get("root_span_id") is None: + request_data["tags"] = tags + # Only add those that are not None (or falsy) for key, value in span_attributes.items(): if value: @@ -351,14 +354,37 @@ class BraintrustLogger(CustomLogger): # Allow metadata override for span name span_name = dynamic_metadata.get("span_name", "Chat Completion") + # Span parents is a special case + span_parents = dynamic_metadata.get("span_parents") + + # Convert comma-separated string to list if present + if span_parents: + span_parents = [s.strip() for s in span_parents.split(",") if s.strip()] + + # Add optional span attributes only if present + span_attributes = { + "span_id": dynamic_metadata.get("span_id"), + "root_span_id": dynamic_metadata.get("root_span_id"), + "span_parents": span_parents, + } + request_data = { "id": litellm_call_id, "input": prompt["messages"], "output": output, "metadata": standard_logging_object, - "tags": tags, "span_attributes": {"name": span_name, "type": "llm"}, } + + # Braintrust cannot specify 'tags' for non-root spans + if dynamic_metadata.get("root_span_id") is None: + request_data["tags"] = tags + + # Only add those that are not None (or falsy) + for key, value in span_attributes.items(): + if value: + request_data[key] = value + if choices is not None: request_data["output"] = [choice.dict() for choice in choices] else: @@ -367,9 +393,6 @@ class BraintrustLogger(CustomLogger): if metrics is not None: request_data["metrics"] = metrics - if metrics is not None: - request_data["metrics"] = metrics - try: await self.global_braintrust_http_handler.post( url=f"{self.api_base}/project_logs/{project_id}/insert", From bae625bdc6e9f9608c48bc5251134ff2515cb84e Mon Sep 17 00:00:00 2001 From: Elkhan Eminov Date: Wed, 7 Jan 2026 19:27:31 +0000 Subject: [PATCH 023/121] OpenRouter embeddings API support (#18391) * support for OpenRouter embeddings * add bearer * add content header --- .../openrouter/embedding/transformation.py | 182 ++++++++++++++++++ litellm/main.py | 45 +++++ litellm/utils.py | 5 + provider_endpoints_support.json | 2 +- tests/llm_translation/test_openrouter.py | 15 ++ ...est_openrouter_embedding_transformation.py | 132 +++++++++++++ 6 files changed, 380 insertions(+), 1 deletion(-) create mode 100644 litellm/llms/openrouter/embedding/transformation.py create mode 100644 tests/test_litellm/llms/openrouter/test_openrouter_embedding_transformation.py diff --git a/litellm/llms/openrouter/embedding/transformation.py b/litellm/llms/openrouter/embedding/transformation.py new file mode 100644 index 00000000000..d1d0e911d16 --- /dev/null +++ b/litellm/llms/openrouter/embedding/transformation.py @@ -0,0 +1,182 @@ +""" +OpenRouter Embedding API Configuration. + +This module provides the configuration for OpenRouter's Embedding API. +OpenRouter is OpenAI-compatible and supports embeddings via the /v1/embeddings endpoint. + +Docs: https://openrouter.ai/docs +""" +from typing import TYPE_CHECKING, Any, Optional + +import httpx + +from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig +from litellm.types.llms.openai import AllEmbeddingInputValues +from litellm.types.utils import EmbeddingResponse +from litellm.utils import convert_to_model_response_object + +from ..common_utils import OpenRouterException + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class OpenrouterEmbeddingConfig(BaseEmbeddingConfig): + """ + Configuration for OpenRouter's Embedding API. + + Reference: https://openrouter.ai/docs + """ + + def validate_environment( + self, + headers: dict, + model: str, + messages: list, + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate environment and set up headers for OpenRouter API. + + OpenRouter requires: + - Authorization header with Bearer token + - HTTP-Referer header (site URL) + - X-Title header (app name) + """ + from litellm import get_secret + + # Get OpenRouter-specific headers + openrouter_site_url = get_secret("OR_SITE_URL") or "https://litellm.ai" + openrouter_app_name = get_secret("OR_APP_NAME") or "liteLLM" + + openrouter_headers = { + "HTTP-Referer": openrouter_site_url, + "X-Title": openrouter_app_name, + "Content-Type": "application/json", + } + + # Add Authorization header if api_key is provided + if api_key: + openrouter_headers["Authorization"] = f"Bearer {api_key}" + + # Merge with existing headers (user's extra_headers take priority) + merged_headers = {**openrouter_headers, **headers} + + return merged_headers + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for OpenRouter Embedding API endpoint. + """ + # api_base is already set to https://openrouter.ai/api/v1 in main.py + # Remove trailing slashes + if api_base: + api_base = api_base.rstrip("/") + else: + api_base = "https://openrouter.ai/api/v1" + + # Return the embeddings endpoint + return f"{api_base}/embeddings" + + def transform_embedding_request( + self, + model: str, + input: AllEmbeddingInputValues, + optional_params: dict, + headers: dict, + ) -> dict: + """ + Transform embedding request to OpenRouter format (OpenAI-compatible). + """ + # Ensure input is a list + if isinstance(input, str): + input = [input] + + # OpenRouter expects the full model name (e.g., google/gemini-embedding-001) + # Strip 'openrouter/' prefix if present + if model.startswith("openrouter/"): + model = model.replace("openrouter/", "", 1) + + return { + "model": model, + "input": input, + **optional_params, + } + + def transform_embedding_response( + self, + model: str, + raw_response: httpx.Response, + model_response: EmbeddingResponse, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + request_data: dict, + optional_params: dict, + litellm_params: dict, + ) -> EmbeddingResponse: + """ + Transform embedding response from OpenRouter format (OpenAI-compatible). + """ + logging_obj.post_call(original_response=raw_response.text) + + # OpenRouter returns standard OpenAI-compatible embedding response + response_json = raw_response.json() + + return convert_to_model_response_object( + response_object=response_json, + model_response_object=model_response, + response_type="embedding", + ) + + def get_supported_openai_params(self, model: str) -> list: + """ + Get list of supported OpenAI parameters for OpenRouter embeddings. + """ + return [ + "timeout", + "dimensions", + "encoding_format", + "user", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Map OpenAI parameters to OpenRouter format. + """ + for param, value in non_default_params.items(): + if param in self.get_supported_openai_params(model): + optional_params[param] = value + return optional_params + + def get_error_class( + self, error_message: str, status_code: int, headers: Any + ) -> Any: + """ + Get the error class for OpenRouter errors. + """ + return OpenRouterException( + message=error_message, + status_code=status_code, + headers=headers, + ) diff --git a/litellm/main.py b/litellm/main.py index e8a8b504d96..6905d8f6a87 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4701,6 +4701,51 @@ def embedding( # noqa: PLR0915 litellm_params=litellm_params_dict, headers=headers, ) + elif custom_llm_provider == "openrouter": + api_base = ( + api_base + or litellm.api_base + or get_secret_str("OPENROUTER_API_BASE") + or "https://openrouter.ai/api/v1" + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.openrouter_key + or get_secret("OPENROUTER_API_KEY") + or get_secret("OR_API_KEY") + ) + + openrouter_site_url = get_secret("OR_SITE_URL") or "https://litellm.ai" + openrouter_app_name = get_secret("OR_APP_NAME") or "liteLLM" + + openrouter_headers = { + "HTTP-Referer": openrouter_site_url, + "X-Title": openrouter_app_name, + } + + _headers = headers or litellm.headers + if _headers: + openrouter_headers.update(_headers) + + headers = openrouter_headers + + response = base_llm_http_handler.embedding( + model=model, + input=input, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + logging_obj=logging, + timeout=timeout, + model_response=EmbeddingResponse(), + optional_params=optional_params, + client=client, + aembedding=aembedding, + litellm_params=litellm_params_dict, + headers=headers, + ) elif custom_llm_provider == "huggingface": api_key = ( api_key diff --git a/litellm/utils.py b/litellm/utils.py index fbbaa94f7a1..fb01337bc50 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7718,6 +7718,11 @@ class ProviderConfigManager: return litellm.CometAPIEmbeddingConfig() elif litellm.LlmProviders.GITHUB_COPILOT == provider: return litellm.GithubCopilotEmbeddingConfig() + elif litellm.LlmProviders.OPENROUTER == provider: + from litellm.llms.openrouter.embedding.transformation import ( + OpenrouterEmbeddingConfig, + ) + return OpenrouterEmbeddingConfig() elif litellm.LlmProviders.GIGACHAT == provider: return litellm.GigaChatEmbeddingConfig() elif litellm.LlmProviders.SAGEMAKER == provider: diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 2521d71b5c0..a29b66f6c54 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1565,7 +1565,7 @@ "chat_completions": true, "messages": true, "responses": true, - "embeddings": false, + "embeddings": true, "image_generations": false, "audio_transcriptions": false, "audio_speech": false, diff --git a/tests/llm_translation/test_openrouter.py b/tests/llm_translation/test_openrouter.py index 839d08e12bf..105b05d3449 100644 --- a/tests/llm_translation/test_openrouter.py +++ b/tests/llm_translation/test_openrouter.py @@ -32,3 +32,18 @@ def test_completion_openrouter_image_generation(): .message.images[0]["image_url"]["url"] .startswith("data:image/png;base64,") ) + + +def test_openrouter_embedding(): + """Test OpenRouter embeddings support.""" + litellm._turn_on_debug() + resp = litellm.embedding( + model="openrouter/openai/text-embedding-3-small", + input=["Hello world", "How are you?"], + ) + print(resp) + assert resp is not None + assert len(resp.data) == 2 + assert resp.data[0]["embedding"] is not None + assert isinstance(resp.data[0]["embedding"], list) + assert len(resp.data[0]["embedding"]) > 0 diff --git a/tests/test_litellm/llms/openrouter/test_openrouter_embedding_transformation.py b/tests/test_litellm/llms/openrouter/test_openrouter_embedding_transformation.py new file mode 100644 index 00000000000..714adc346db --- /dev/null +++ b/tests/test_litellm/llms/openrouter/test_openrouter_embedding_transformation.py @@ -0,0 +1,132 @@ +""" +Unit tests for OpenRouter embedding transformation logic. +""" +from litellm.llms.openrouter.embedding.transformation import ( + OpenrouterEmbeddingConfig, +) + + +def test_openrouter_embedding_supported_params(): + """Test that supported OpenAI params are correctly defined.""" + config = OpenrouterEmbeddingConfig() + supported = config.get_supported_openai_params("test-model") + + assert "timeout" in supported + assert "dimensions" in supported + assert "encoding_format" in supported + assert "user" in supported + + +def test_openrouter_embedding_transform_request(): + """Test request transformation logic.""" + config = OpenrouterEmbeddingConfig() + + # Test with string input + result = config.transform_embedding_request( + model="openrouter/google/text-embedding-004", + input="Hello world", + optional_params={}, + headers={}, + ) + + assert result["model"] == "google/text-embedding-004" + assert result["input"] == ["Hello world"] + + # Test with list input + result = config.transform_embedding_request( + model="google/text-embedding-004", + input=["Hello", "World"], + optional_params={"dimensions": 512}, + headers={}, + ) + + assert result["model"] == "google/text-embedding-004" + assert result["input"] == ["Hello", "World"] + assert result["dimensions"] == 512 + + +def test_openrouter_embedding_validate_environment(): + """Test environment validation and header setup.""" + config = OpenrouterEmbeddingConfig() + + # Test with API key + headers = config.validate_environment( + headers={"Custom-Header": "value"}, + model="test-model", + messages=[], + optional_params={}, + litellm_params={}, + api_key="test-api-key", + ) + + # Should include OpenRouter-specific headers + assert "HTTP-Referer" in headers + assert "X-Title" in headers + # Should include Content-Type header + assert "Content-Type" in headers + assert headers["Content-Type"] == "application/json" + # Should include Authorization header + assert "Authorization" in headers + assert headers["Authorization"] == "Bearer test-api-key" + # Should preserve custom headers + assert headers["Custom-Header"] == "value" + + # Test without API key + headers_no_key = config.validate_environment( + headers={}, + model="test-model", + messages=[], + optional_params={}, + litellm_params={}, + api_key=None, + ) + + # Should still include OpenRouter headers but not Authorization + assert "HTTP-Referer" in headers_no_key + assert "X-Title" in headers_no_key + assert "Content-Type" in headers_no_key + assert "Authorization" not in headers_no_key + + +def test_openrouter_embedding_get_complete_url(): + """Test URL construction.""" + config = OpenrouterEmbeddingConfig() + + url = config.get_complete_url( + api_base="https://openrouter.ai/api/v1", + api_key="test-key", + model="test-model", + optional_params={}, + litellm_params={}, + ) + + assert url == "https://openrouter.ai/api/v1/embeddings" + + # Test with trailing slash + url = config.get_complete_url( + api_base="https://openrouter.ai/api/v1/", + api_key="test-key", + model="test-model", + optional_params={}, + litellm_params={}, + ) + + assert url == "https://openrouter.ai/api/v1/embeddings" + + +def test_openrouter_embedding_map_params(): + """Test parameter mapping.""" + config = OpenrouterEmbeddingConfig() + + result = config.map_openai_params( + non_default_params={"dimensions": 512, "timeout": 30, "unsupported": "value"}, + optional_params={}, + model="test-model", + drop_params=False, + ) + + # Supported params should be included + assert result["dimensions"] == 512 + assert result["timeout"] == 30 + # Unsupported params should not be included + assert "unsupported" not in result From 1c84af8ae4d5a820dac545f46a8e630f3408141d Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 7 Jan 2026 12:22:57 -0800 Subject: [PATCH 024/121] normalize proxy config callbacks --- litellm/proxy/proxy_server.py | 19 +++- tests/test_litellm/proxy/test_proxy_server.py | 88 +++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 06525e39133..d264b82b873 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3402,8 +3402,8 @@ class ProxyConfig: def _deep_merge_dicts(dst: dict, src: dict) -> None: """ - Deep-merge src into dst, skipping None values from src. - On conflicts, src (DB) wins. + Deep-merge src into dst, skipping None values and empty lists from src. + On conflicts, src (DB) wins, but empty lists are treated as "no value" and don't overwrite. """ stack = [(dst, src)] while stack: @@ -3412,6 +3412,9 @@ class ProxyConfig: if v is None: # Preserve existing config when DB value is None (matches prior behavior) continue + # Skip empty lists - treat them as "no value" to preserve file config + if isinstance(v, list) and len(v) == 0: + continue if isinstance(v, dict) and isinstance(d.get(k), dict): stack.append((d[k], v)) else: @@ -9762,6 +9765,18 @@ async def get_config(): # noqa: PLR0915 _failure_callbacks = _litellm_settings.get("failure_callback", []) _success_and_failure_callbacks = _litellm_settings.get("callbacks", []) + # Normalize string callbacks to lists + def normalize_callback(callback): + if isinstance(callback, str): + return [callback] + elif callback is None: + return [] + return callback + + _success_callbacks = normalize_callback(_success_callbacks) + _failure_callbacks = normalize_callback(_failure_callbacks) + _success_and_failure_callbacks = normalize_callback(_success_and_failure_callbacks) + _data_to_return = [] """ [ diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 5c7ece04513..53d89df5026 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -3036,3 +3036,91 @@ def test_get_image_root_case_uses_current_dir(monkeypatch): # Verify FileResponse was called assert mock_file_response.called, "FileResponse should be called" + + +def test_get_config_normalizes_string_callbacks(monkeypatch): + """ + Test that /get/config/callbacks normalizes string callbacks to lists. + """ + from litellm.proxy.proxy_server import app, proxy_config, user_api_key_auth + + config_data = { + "litellm_settings": { + "success_callback": "langfuse", + "failure_callback": None, + "callbacks": ["prometheus", "datadog"], + }, + "general_settings": {}, + "environment_variables": {}, + } + + mock_router = MagicMock() + mock_router.get_settings.return_value = {} + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", mock_router) + monkeypatch.setattr( + proxy_config, "get_config", AsyncMock(return_value=config_data) + ) + + original_overrides = app.dependency_overrides.copy() + app.dependency_overrides[user_api_key_auth] = lambda: MagicMock() + + client = TestClient(app) + try: + response = client.get("/get/config/callbacks") + finally: + app.dependency_overrides = original_overrides + + assert response.status_code == 200 + callbacks = response.json()["callbacks"] + + success_callbacks = [cb["name"] for cb in callbacks if cb.get("type") == "success"] + failure_callbacks = [cb["name"] for cb in callbacks if cb.get("type") == "failure"] + success_and_failure_callbacks = [ + cb["name"] for cb in callbacks if cb.get("type") == "success_and_failure" + ] + + assert "langfuse" in success_callbacks + assert len(failure_callbacks) == 0 + assert "prometheus" in success_and_failure_callbacks + assert "datadog" in success_and_failure_callbacks + + +def test_deep_merge_dicts_skips_none_and_empty_lists(monkeypatch): + """ + Test that _update_config_fields deep merge skips None values and empty lists. + """ + from litellm.proxy.proxy_server import ProxyConfig + + proxy_config = ProxyConfig() + + current_config = { + "general_settings": { + "max_parallel_requests": 10, + "allowed_models": ["gpt-3.5-turbo", "gpt-4"], + "nested": { + "key1": "value1", + "key2": "value2", + }, + } + } + + db_param_value = { + "max_parallel_requests": None, + "allowed_models": [], + "new_key": "new_value", + "nested": { + "key1": "updated_value1", + "key3": "value3", + }, + } + + result = proxy_config._update_config_fields( + current_config, "general_settings", db_param_value + ) + + assert result["general_settings"]["max_parallel_requests"] == 10 + assert result["general_settings"]["allowed_models"] == ["gpt-3.5-turbo", "gpt-4"] + assert result["general_settings"]["new_key"] == "new_value" + assert result["general_settings"]["nested"]["key1"] == "updated_value1" + assert result["general_settings"]["nested"]["key2"] == "value2" + assert result["general_settings"]["nested"]["key3"] == "value3" From 2f803171d695af9aaa77852a4df280f268788567 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Wed, 7 Jan 2026 17:14:14 -0800 Subject: [PATCH 025/121] refactor(prometheus): skip metrics for invalid API key requests (#18788) --- litellm/integrations/prometheus.py | 188 +++++++++++++++++- .../test_prometheus_invalid_key_filtering.py | 161 +++++++++++++++ 2 files changed, 341 insertions(+), 8 deletions(-) create mode 100644 tests/test_litellm/integrations/test_prometheus_invalid_key_filtering.py diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index c01f7481277..e4aca5ced04 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -14,6 +14,7 @@ from typing import ( Literal, Optional, Tuple, + Union, cast, ) @@ -791,6 +792,11 @@ class PrometheusLogger(CustomLogger): f"standard_logging_object is required, got={standard_logging_payload}" ) + if self._should_skip_metrics_for_invalid_key( + kwargs=kwargs, standard_logging_payload=standard_logging_payload + ): + return + model = kwargs.get("model", "") litellm_params = kwargs.get("litellm_params", {}) or {} _metadata = litellm_params.get("metadata", {}) @@ -1189,11 +1195,17 @@ class PrometheusLogger(CustomLogger): f"prometheus Logging - Enters failure logging function for kwargs {kwargs}" ) - # unpack kwargs - model = kwargs.get("model", "") standard_logging_payload: StandardLoggingPayload = kwargs.get( "standard_logging_object", {} ) + + if self._should_skip_metrics_for_invalid_key( + kwargs=kwargs, standard_logging_payload=standard_logging_payload + ): + return + + model = kwargs.get("model", "") + litellm_params = kwargs.get("litellm_params", {}) or {} get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking() @@ -1207,7 +1219,6 @@ class PrometheusLogger(CustomLogger): user_api_team_alias = standard_logging_payload["metadata"][ "user_api_key_team_alias" ] - kwargs.get("exception", None) try: self.litellm_llm_api_failed_requests_metric.labels( @@ -1227,6 +1238,139 @@ class PrometheusLogger(CustomLogger): pass pass + def _extract_status_code( + self, + kwargs: Optional[dict] = None, + enum_values: Optional[Any] = None, + exception: Optional[Exception] = None, + ) -> Optional[int]: + """ + Extract HTTP status code from various input formats for validation. + + This is a centralized helper to extract status code from different + callback function signatures. Handles both ProxyException (uses 'code') + and standard exceptions (uses 'status_code'). + + Args: + kwargs: Dictionary potentially containing 'exception' key + enum_values: Object with 'status_code' attribute + exception: Exception object to extract status code from directly + + Returns: + Status code as integer if found, None otherwise + """ + status_code = None + + # Try from enum_values first (most common in our callbacks) + if enum_values and hasattr(enum_values, "status_code") and enum_values.status_code: + try: + status_code = int(enum_values.status_code) + except (ValueError, TypeError): + pass + + if not status_code and exception: + # ProxyException uses 'code' attribute, other exceptions may use 'status_code' + status_code = getattr(exception, "status_code", None) or getattr(exception, "code", None) + if status_code is not None: + try: + status_code = int(status_code) + except (ValueError, TypeError): + status_code = None + + if not status_code and kwargs: + exception_in_kwargs = kwargs.get("exception") + if exception_in_kwargs: + status_code = getattr(exception_in_kwargs, "status_code", None) or getattr(exception_in_kwargs, "code", None) + if status_code is not None: + try: + status_code = int(status_code) + except (ValueError, TypeError): + status_code = None + + return status_code + + def _is_invalid_api_key_request( + self, + status_code: Optional[int], + exception: Optional[Exception] = None, + ) -> bool: + """ + Determine if a request has an invalid API key based on status code and exception. + + This method prevents invalid authentication attempts from being recorded in + Prometheus metrics. A 401 status code is the definitive indicator of authentication + failure. Additionally, we check exception messages for authentication error patterns + to catch cases where the exception hasn't been converted to a ProxyException yet. + + Args: + status_code: HTTP status code (401 indicates authentication error) + exception: Exception object to check for auth-related error messages + + Returns: + True if the request has an invalid API key and metrics should be skipped, + False otherwise + """ + if status_code == 401: + return True + + # Handle cases where AssertionError is raised before conversion to ProxyException + if exception is not None: + exception_str = str(exception).lower() + auth_error_patterns = [ + "virtual key expected", + "expected to start with 'sk-'", + "authentication error", + "invalid api key", + "api key not valid", + ] + if any(pattern in exception_str for pattern in auth_error_patterns): + return True + + return False + + def _should_skip_metrics_for_invalid_key( + self, + kwargs: Optional[dict] = None, + user_api_key_dict: Optional[Any] = None, + enum_values: Optional[Any] = None, + standard_logging_payload: Optional[Union[dict, StandardLoggingPayload]] = None, + exception: Optional[Exception] = None, + ) -> bool: + """ + Determine if Prometheus metrics should be skipped for invalid API key requests. + + This is a centralized validation method that extracts status code and exception + information from various callback function signatures and determines if the request + represents an invalid API key attempt that should be filtered from metrics. + + Args: + kwargs: Dictionary potentially containing exception and other data + user_api_key_dict: User API key authentication object (currently unused) + enum_values: Object with status_code attribute + standard_logging_payload: Standard logging payload dictionary + exception: Exception object to check directly + + Returns: + True if metrics should be skipped (invalid key detected), False otherwise + """ + status_code = self._extract_status_code( + kwargs=kwargs, + enum_values=enum_values, + exception=exception, + ) + + if exception is None and kwargs: + exception = kwargs.get("exception") + + if self._is_invalid_api_key_request(status_code, exception=exception): + verbose_logger.debug( + "Skipping Prometheus metrics for invalid API key request: " + f"status_code={status_code}, exception={type(exception).__name__ if exception else None}" + ) + return True + + return False + async def async_post_call_failure_hook( self, request_data: dict, @@ -1252,6 +1396,14 @@ class PrometheusLogger(CustomLogger): StandardLoggingPayloadSetup, ) + if self._should_skip_metrics_for_invalid_key( + user_api_key_dict=user_api_key_dict, + exception=original_exception, + ): + return + + status_code = self._extract_status_code(exception=original_exception) + try: _tags = StandardLoggingPayloadSetup._get_request_tags( litellm_params=request_data, @@ -1266,8 +1418,8 @@ class PrometheusLogger(CustomLogger): team=user_api_key_dict.team_id, team_alias=user_api_key_dict.team_alias, requested_model=request_data.get("model", ""), - status_code=str(getattr(original_exception, "status_code", None)), - exception_status=str(getattr(original_exception, "status_code", None)), + status_code=str(status_code), + exception_status=str(status_code), exception_class=self._get_exception_class_name(original_exception), tags=_tags, route=user_api_key_dict.request_route, @@ -1305,6 +1457,11 @@ class PrometheusLogger(CustomLogger): StandardLoggingPayloadSetup, ) + if self._should_skip_metrics_for_invalid_key( + user_api_key_dict=user_api_key_dict + ): + return + enum_values = UserAPIKeyLabelValues( end_user=user_api_key_dict.end_user_id, hashed_api_key=user_api_key_dict.api_key, @@ -1360,6 +1517,15 @@ class PrometheusLogger(CustomLogger): exception = request_kwargs.get("exception", None) llm_provider = _litellm_params.get("custom_llm_provider", None) + + if self._should_skip_metrics_for_invalid_key( + kwargs=request_kwargs, + standard_logging_payload=standard_logging_payload, + ): + return + hashed_api_key = standard_logging_payload.get("metadata", {}).get( + "user_api_key_hash" + ) # Create enum_values for the label factory (always create for use in different metrics) enum_values = UserAPIKeyLabelValues( @@ -1374,9 +1540,7 @@ class PrometheusLogger(CustomLogger): self._get_exception_class_name(exception) if exception else None ), requested_model=model_group, - hashed_api_key=standard_logging_payload["metadata"][ - "user_api_key_hash" - ], + hashed_api_key=hashed_api_key, api_key_alias=standard_logging_payload["metadata"][ "user_api_key_alias" ], @@ -1441,6 +1605,14 @@ class PrometheusLogger(CustomLogger): if standard_logging_payload is None: return + # Skip recording metrics for invalid API key requests + if self._should_skip_metrics_for_invalid_key( + kwargs=request_kwargs, + enum_values=enum_values, + standard_logging_payload=standard_logging_payload, + ): + return + api_base = standard_logging_payload["api_base"] _litellm_params = request_kwargs.get("litellm_params", {}) or {} _metadata = _litellm_params.get("metadata", {}) diff --git a/tests/test_litellm/integrations/test_prometheus_invalid_key_filtering.py b/tests/test_litellm/integrations/test_prometheus_invalid_key_filtering.py new file mode 100644 index 00000000000..ff433480d5e --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_invalid_key_filtering.py @@ -0,0 +1,161 @@ +""" +Unit tests for Prometheus invalid API key request filtering. + +Tests functionality that prevents invalid API key requests (401 status codes) +from being recorded in Prometheus metrics. +""" + +import os +import sys +from unittest.mock import Mock, patch + +import pytest +from prometheus_client import REGISTRY + +sys.path.insert(0, os.path.abspath("../../..")) + +from litellm.integrations.prometheus import PrometheusLogger +from litellm.proxy._types import UserAPIKeyAuth + + +@pytest.fixture(scope="function") +def prometheus_logger(): + """Create a PrometheusLogger instance for testing.""" + collectors = list(REGISTRY._collector_to_names.keys()) + for collector in collectors: + REGISTRY.unregister(collector) + return PrometheusLogger() + + +class ExceptionWithCode: + """Exception-like object with 'code' attribute (ProxyException pattern).""" + def __init__(self, code): + self.code = code + + +class ExceptionWithStatusCode: + """Exception-like object with 'status_code' attribute.""" + def __init__(self, status_code): + self.status_code = status_code + + +class TestExtractStatusCode: + """Test status code extraction from various sources.""" + + @pytest.mark.parametrize("exception_class,code_value,expected", [ + (ExceptionWithCode, "401", 401), + (ExceptionWithStatusCode, 401, 401), + ]) + def test_extract_from_exception(self, prometheus_logger, exception_class, code_value, expected): + exception = exception_class(code_value) + assert prometheus_logger._extract_status_code(exception=exception) == expected + + def test_extract_from_kwargs(self, prometheus_logger): + exception = ExceptionWithCode("401") + assert prometheus_logger._extract_status_code(kwargs={"exception": exception}) == 401 + + def test_extract_from_enum_values(self, prometheus_logger): + enum_values = Mock(status_code="401") + assert prometheus_logger._extract_status_code(enum_values=enum_values) == 401 + + +class TestInvalidAPIKeyDetection: + """Test invalid API key request detection logic.""" + + @pytest.mark.parametrize("status_code,expected", [ + (401, True), + (200, False), + (500, False), + (None, False), + ]) + def test_status_code_detection(self, prometheus_logger, status_code, expected): + assert prometheus_logger._is_invalid_api_key_request(status_code=status_code) == expected + + def test_auth_error_message_detection(self, prometheus_logger): + exception = AssertionError("LiteLLM Virtual Key expected. Received=invalid-key-12345, expected to start with 'sk-'.") + assert prometheus_logger._is_invalid_api_key_request(status_code=None, exception=exception) is True + + def test_non_auth_exception_not_detected(self, prometheus_logger): + exception = ValueError("Some other error") + assert prometheus_logger._is_invalid_api_key_request(status_code=None, exception=exception) is False + + +class TestSkipMetricsValidation: + """Test high-level validation method that orchestrates detection and extraction.""" + + def test_skip_for_401_exception(self, prometheus_logger): + """Test full flow: extraction -> detection -> skip decision.""" + exception = ExceptionWithCode("401") + assert prometheus_logger._should_skip_metrics_for_invalid_key(exception=exception) is True + + def test_skip_for_auth_error_message(self, prometheus_logger): + """Test full flow: exception message -> detection -> skip decision.""" + exception = AssertionError("expected to start with 'sk-'") + assert prometheus_logger._should_skip_metrics_for_invalid_key(exception=exception) is True + + def test_no_skip_for_valid_request(self, prometheus_logger): + assert prometheus_logger._should_skip_metrics_for_invalid_key() is False + + +class TestAsyncHooks: + """Test async hook methods skip metrics for invalid API keys.""" + + @pytest.fixture + def mock_user_api_key(self): + """Create a mock UserAPIKeyAuth object.""" + user_key = Mock(spec=UserAPIKeyAuth) + user_key.api_key = "test-key" + user_key.end_user_id = None + user_key.user_id = None + user_key.user_email = None + user_key.key_alias = None + user_key.team_id = None + user_key.team_alias = None + user_key.request_route = "/test" + return user_key + + @pytest.mark.asyncio + async def test_post_call_failure_hook_skips_401(self, prometheus_logger, mock_user_api_key): + exception = ExceptionWithCode("401") + exception.__class__.__name__ = "ProxyException" + + with patch.object(prometheus_logger, 'litellm_proxy_failed_requests_metric') as mock_failed, \ + patch.object(prometheus_logger, 'litellm_proxy_total_requests_metric') as mock_total: + + await prometheus_logger.async_post_call_failure_hook( + request_data={"model": "test-model"}, + original_exception=exception, + user_api_key_dict=mock_user_api_key + ) + + mock_failed.labels.assert_not_called() + mock_total.labels.assert_not_called() + + @pytest.mark.asyncio + async def test_log_failure_event_skips_401(self, prometheus_logger): + exception = ExceptionWithCode("401") + kwargs = { + "model": "test-model", + "standard_logging_object": { + "metadata": { + "user_api_key_hash": "test-key", + "user_api_key_user_id": "test-user", + }, + "model_group": "test-model", + }, + "exception": exception, + "litellm_params": {}, + } + + with patch.object(prometheus_logger, 'litellm_llm_api_failed_requests_metric') as mock_failed, \ + patch.object(prometheus_logger, 'set_llm_deployment_failure_metrics') as mock_deployment: + + await prometheus_logger.async_log_failure_event( + kwargs=kwargs, + response_obj=None, + start_time=None, + end_time=None + ) + + mock_failed.labels.assert_not_called() + mock_deployment.assert_not_called() From 51759424a6a0b7a964cff7ded3833612e5bb319a Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 7 Jan 2026 17:17:30 -0800 Subject: [PATCH 026/121] Key and Team Routing Setting --- litellm/proxy/_types.py | 1 + litellm/proxy/common_request_processing.py | 23 +++++ litellm/proxy/proxy_server.py | 78 ++++++++++++++++ .../proxy/test_common_request_processing.py | 78 ++++++++++++++++ tests/test_litellm/proxy/test_proxy_server.py | 92 +++++++++++++++++++ 5 files changed, 272 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 4140273ea25..be1e4dbcdc9 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2104,6 +2104,7 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase): rotation_interval: Optional[str] = None # How often to rotate (e.g., "30d", "90d") last_rotation_at: Optional[datetime] = None # When this key was last rotated key_rotation_at: Optional[datetime] = None # When this key should next be rotated + router_settings: Optional[Dict] = None # Router settings for this key (Key > Team > Global precedence) model_config = ConfigDict(protected_namespaces=()) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 537b48f06ed..e9ce10ccf31 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -440,6 +440,29 @@ class ProxyBaseLLMRequestProcessing: user_api_key_dict=user_api_key_dict, data=self.data, call_type=route_type # type: ignore ) + # Apply hierarchical router_settings (Key > Team > Global) + if llm_router is not None and proxy_config is not None: + from litellm.proxy.proxy_server import prisma_client + + router_settings = await proxy_config._get_hierarchical_router_settings( + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + ) + + # If router_settings found (from key, team, or global), apply them + # This ensures key/team settings override global settings + if router_settings is not None and router_settings: + # Get model_list from current router + model_list = llm_router.get_model_list() + if model_list is not None: + # Create user_config with model_list and router_settings + # This creates a per-request router with the hierarchical settings + user_config = { + "model_list": model_list, + **router_settings + } + self.data["user_config"] = user_config + if "messages" in self.data and self.data["messages"]: logging_obj.update_messages(self.data["messages"]) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 06525e39133..025c34cd9f3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3222,6 +3222,84 @@ class ProxyConfig: decrypted_variables[k] = decrypted_value return decrypted_variables + async def _get_hierarchical_router_settings( + self, + user_api_key_dict: Optional["UserAPIKeyAuth"], + prisma_client: Optional[PrismaClient], + ) -> Optional[dict]: + """ + Get router_settings in priority order: Key > Team > Global + + Returns: + dict: Combined router_settings, or None if no settings found + """ + if prisma_client is None: + return None + + import json + import yaml + + # 1. Try key-level router_settings + if user_api_key_dict is not None: + # Check if router_settings is available on the key object + key_router_settings_value = getattr(user_api_key_dict, "router_settings", None) + if key_router_settings_value is not None: + key_router_settings = None + if isinstance(key_router_settings_value, str): + try: + key_router_settings = yaml.safe_load(key_router_settings_value) + except (yaml.YAMLError, json.JSONDecodeError): + try: + key_router_settings = json.loads(key_router_settings_value) + except json.JSONDecodeError: + pass + elif isinstance(key_router_settings_value, dict): + key_router_settings = key_router_settings_value + + # If key has router_settings (non-empty dict), use it + if key_router_settings is not None and isinstance(key_router_settings, dict) and key_router_settings: + return key_router_settings + + # 2. Try team-level router_settings + if user_api_key_dict is not None and user_api_key_dict.team_id is not None: + try: + team_obj = await prisma_client.db.litellm_teamtable.find_unique( + where={"team_id": user_api_key_dict.team_id} + ) + if team_obj is not None: + team_router_settings_value = getattr(team_obj, "router_settings", None) + if team_router_settings_value is not None: + team_router_settings = None + if isinstance(team_router_settings_value, str): + try: + team_router_settings = yaml.safe_load(team_router_settings_value) + except (yaml.YAMLError, json.JSONDecodeError): + try: + team_router_settings = json.loads(team_router_settings_value) + except json.JSONDecodeError: + pass + elif isinstance(team_router_settings_value, dict): + team_router_settings = team_router_settings_value + + # If team has router_settings (non-empty dict), use it + if team_router_settings is not None and isinstance(team_router_settings, dict) and team_router_settings: + return team_router_settings + except Exception: + # If team lookup fails, continue to global settings + pass + + # 3. Try global router_settings + try: + db_router_settings = await prisma_client.db.litellm_config.find_first( + where={"param_name": "router_settings"} + ) + if db_router_settings is not None and isinstance(db_router_settings.param_value, dict) and db_router_settings.param_value: + return db_router_settings.param_value + except Exception: + pass + + return None + async def _add_router_settings_from_db_config( self, config_data: dict, diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index b5d44385698..bab95feec06 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -75,6 +75,84 @@ class TestProxyBaseLLMRequestProcessing: pytest.fail("litellm_call_id is not a valid UUID") assert data_passed["litellm_call_id"] == returned_data["litellm_call_id"] + @pytest.mark.asyncio + async def test_should_apply_hierarchical_router_settings_to_user_config( + self, monkeypatch + ): + processing_obj = ProxyBaseLLMRequestProcessing(data={}) + mock_request = MagicMock(spec=Request) + mock_request.headers = {} + + async def mock_add_litellm_data_to_request(*args, **kwargs): + return {} + + async def mock_common_processing_pre_call_logic( + user_api_key_dict, data, call_type + ): + data_copy = copy.deepcopy(data) + return data_copy + + mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) + mock_proxy_logging_obj.pre_call_hook = AsyncMock( + side_effect=mock_common_processing_pre_call_logic + ) + monkeypatch.setattr( + litellm.proxy.common_request_processing, + "add_litellm_data_to_request", + mock_add_litellm_data_to_request, + ) + + mock_general_settings = {} + mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) + mock_proxy_config = MagicMock(spec=ProxyConfig) + + mock_router_settings = { + "routing_strategy": "least-busy", + "timeout": 30.0, + "num_retries": 3, + } + mock_proxy_config._get_hierarchical_router_settings = AsyncMock( + return_value=mock_router_settings + ) + + mock_model_list = [ + {"model_name": "gpt-3.5-turbo", "litellm_params": {"model": "gpt-3.5-turbo"}}, + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4"}}, + ] + mock_llm_router = MagicMock() + mock_llm_router.get_model_list = MagicMock(return_value=mock_model_list) + + mock_prisma_client = MagicMock() + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + mock_prisma_client, + ) + + route_type = "acompletion" + + returned_data, logging_obj = await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings=mock_general_settings, + user_api_key_dict=mock_user_api_key_dict, + proxy_logging_obj=mock_proxy_logging_obj, + proxy_config=mock_proxy_config, + route_type=route_type, + llm_router=mock_llm_router, + ) + + mock_proxy_config._get_hierarchical_router_settings.assert_called_once_with( + user_api_key_dict=mock_user_api_key_dict, + prisma_client=mock_prisma_client, + ) + mock_llm_router.get_model_list.assert_called_once() + + assert "user_config" in returned_data + user_config = returned_data["user_config"] + assert user_config["model_list"] == mock_model_list + assert user_config["routing_strategy"] == "least-busy" + assert user_config["timeout"] == 30.0 + assert user_config["num_retries"] == 3 + @pytest.mark.asyncio async def test_stream_timeout_header_processing(self): """ diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 5c7ece04513..dfa1121d5f7 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -3036,3 +3036,95 @@ def test_get_image_root_case_uses_current_dir(monkeypatch): # Verify FileResponse was called assert mock_file_response.called, "FileResponse should be called" + + +@pytest.mark.asyncio +async def test_get_hierarchical_router_settings(): + """ + Test _get_hierarchical_router_settings method's priority order: Key > Team > Global + """ + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.proxy_server import ProxyConfig + + proxy_config = ProxyConfig() + + # Test Case 1: Returns None when prisma_client is None + result = await proxy_config._get_hierarchical_router_settings( + user_api_key_dict=None, + prisma_client=None, + ) + assert result is None + + # Test Case 2: Returns key-level router_settings when available (as dict) + mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) + mock_user_api_key_dict.router_settings = {"routing_strategy": "key-level", "timeout": 10} + mock_user_api_key_dict.team_id = None + + mock_prisma_client = MagicMock() + + result = await proxy_config._get_hierarchical_router_settings( + user_api_key_dict=mock_user_api_key_dict, + prisma_client=mock_prisma_client, + ) + assert result == {"routing_strategy": "key-level", "timeout": 10} + + # Test Case 3: Returns key-level router_settings when available (as YAML string) + mock_user_api_key_dict.router_settings = "routing_strategy: key-yaml\ntimeout: 20" + result = await proxy_config._get_hierarchical_router_settings( + user_api_key_dict=mock_user_api_key_dict, + prisma_client=mock_prisma_client, + ) + assert result == {"routing_strategy": "key-yaml", "timeout": 20} + + # Test Case 4: Falls back to team-level router_settings when key-level is not available + mock_user_api_key_dict.router_settings = None + mock_user_api_key_dict.team_id = "team-123" + + mock_team_obj = MagicMock() + mock_team_obj.router_settings = {"routing_strategy": "team-level", "timeout": 30} + + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=mock_team_obj + ) + + result = await proxy_config._get_hierarchical_router_settings( + user_api_key_dict=mock_user_api_key_dict, + prisma_client=mock_prisma_client, + ) + assert result == {"routing_strategy": "team-level", "timeout": 30} + mock_prisma_client.db.litellm_teamtable.find_unique.assert_called_once_with( + where={"team_id": "team-123"} + ) + + # Test Case 5: Falls back to global router_settings when neither key nor team settings are available + mock_user_api_key_dict.router_settings = None + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=None) + + mock_db_config = MagicMock() + mock_db_config.param_value = {"routing_strategy": "global-level", "timeout": 40} + + mock_prisma_client.db.litellm_config.find_first = AsyncMock( + return_value=mock_db_config + ) + + result = await proxy_config._get_hierarchical_router_settings( + user_api_key_dict=mock_user_api_key_dict, + prisma_client=mock_prisma_client, + ) + assert result == {"routing_strategy": "global-level", "timeout": 40} + mock_prisma_client.db.litellm_config.find_first.assert_called_once_with( + where={"param_name": "router_settings"} + ) + + # Test Case 6: Returns None when no settings are found + mock_user_api_key_dict.router_settings = None + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=None) + mock_prisma_client.db.litellm_config.find_first = AsyncMock(return_value=None) + + result = await proxy_config._get_hierarchical_router_settings( + user_api_key_dict=mock_user_api_key_dict, + prisma_client=mock_prisma_client, + ) + assert result is None From 565a622cf9d67147b038b927078a1f3ac01de366 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Thu, 8 Jan 2026 12:09:58 +0900 Subject: [PATCH 027/121] feat: scaffold Focus export logging skeleton --- litellm/integrations/focus/__init__.py | 0 litellm/integrations/focus/database.py | 22 +++ .../focus/destinations/__init__.py | 12 ++ .../integrations/focus/destinations/base.py | 30 ++++ .../focus/destinations/factory.py | 21 +++ .../focus/destinations/s3_destination.py | 32 +++++ .../integrations/focus/focus_export_logger.py | 129 ++++++++++++++++++ .../focus/serializers/__init__.py | 6 + .../integrations/focus/serializers/base.py | 18 +++ .../integrations/focus/serializers/parquet.py | 16 +++ litellm/integrations/focus/transformer.py | 13 ++ 11 files changed, 299 insertions(+) create mode 100644 litellm/integrations/focus/__init__.py create mode 100644 litellm/integrations/focus/database.py create mode 100644 litellm/integrations/focus/destinations/__init__.py create mode 100644 litellm/integrations/focus/destinations/base.py create mode 100644 litellm/integrations/focus/destinations/factory.py create mode 100644 litellm/integrations/focus/destinations/s3_destination.py create mode 100644 litellm/integrations/focus/focus_export_logger.py create mode 100644 litellm/integrations/focus/serializers/__init__.py create mode 100644 litellm/integrations/focus/serializers/base.py create mode 100644 litellm/integrations/focus/serializers/parquet.py create mode 100644 litellm/integrations/focus/transformer.py diff --git a/litellm/integrations/focus/__init__.py b/litellm/integrations/focus/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/integrations/focus/database.py b/litellm/integrations/focus/database.py new file mode 100644 index 00000000000..a23d16278af --- /dev/null +++ b/litellm/integrations/focus/database.py @@ -0,0 +1,22 @@ +"""Database access helpers for Focus export.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Optional + +import polars as pl + + +class FocusLiteLLMDatabase: + """Retrieves LiteLLM usage data for Focus export workflows.""" + + async def get_usage_data( + self, + *, + limit: Optional[int] = None, + start_time_utc: Optional[datetime] = None, + end_time_utc: Optional[datetime] = None, + ) -> pl.DataFrame: + """Return usage data for the requested window.""" + raise NotImplementedError diff --git a/litellm/integrations/focus/destinations/__init__.py b/litellm/integrations/focus/destinations/__init__.py new file mode 100644 index 00000000000..233f1da0c9b --- /dev/null +++ b/litellm/integrations/focus/destinations/__init__.py @@ -0,0 +1,12 @@ +"""Destination implementations for Focus export.""" + +from .base import FocusDestination, FocusTimeWindow +from .factory import FocusDestinationFactory +from .s3_destination import FocusS3Destination + +__all__ = [ + "FocusDestination", + "FocusDestinationFactory", + "FocusTimeWindow", + "FocusS3Destination", +] diff --git a/litellm/integrations/focus/destinations/base.py b/litellm/integrations/focus/destinations/base.py new file mode 100644 index 00000000000..8042a7e23b9 --- /dev/null +++ b/litellm/integrations/focus/destinations/base.py @@ -0,0 +1,30 @@ +"""Abstract destination interfaces for Focus export.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from typing import Protocol + + +@dataclass(frozen=True) +class FocusTimeWindow: + """Represents the span of data exported in a single batch.""" + + start_time: datetime + end_time: datetime + frequency: str + + +class FocusDestination(Protocol): + """Protocol for anything that can receive Focus export files.""" + + async def deliver( + self, + *, + content: bytes, + time_window: FocusTimeWindow, + filename: str, + ) -> None: + """Persist the serialized export for the provided time window.""" + ... diff --git a/litellm/integrations/focus/destinations/factory.py b/litellm/integrations/focus/destinations/factory.py new file mode 100644 index 00000000000..36163ece61c --- /dev/null +++ b/litellm/integrations/focus/destinations/factory.py @@ -0,0 +1,21 @@ +"""Factory helpers for Focus export destinations.""" + +from __future__ import annotations + +from typing import Optional + +from .base import FocusDestination + + +class FocusDestinationFactory: + """Builds destination instances based on provider/config settings.""" + + @staticmethod + def create( + *, + provider: str, + prefix: str, + config: Optional[dict] = None, + ) -> FocusDestination: + """Return a destination implementation for the requested provider.""" + raise NotImplementedError diff --git a/litellm/integrations/focus/destinations/s3_destination.py b/litellm/integrations/focus/destinations/s3_destination.py new file mode 100644 index 00000000000..7ead87baa29 --- /dev/null +++ b/litellm/integrations/focus/destinations/s3_destination.py @@ -0,0 +1,32 @@ +"""S3 destination implementation for Focus export.""" + +from __future__ import annotations + +from typing import Any, Optional + +from .base import FocusDestination, FocusTimeWindow + + +class FocusS3Destination(FocusDestination): + """Handles uploading serialized exports to S3 buckets.""" + + def __init__( + self, + *, + prefix: str, + config: Optional[dict[str, Any]] = None, + ) -> None: + self.prefix = prefix.rstrip("/") + self.config = config or {} + + async def deliver( + self, + *, + content: bytes, + time_window: FocusTimeWindow, + filename: str, + ) -> None: + raise NotImplementedError + + def _build_object_key(self, *, time_window: FocusTimeWindow, filename: str) -> str: + raise NotImplementedError diff --git a/litellm/integrations/focus/focus_export_logger.py b/litellm/integrations/focus/focus_export_logger.py new file mode 100644 index 00000000000..723dbc706fb --- /dev/null +++ b/litellm/integrations/focus/focus_export_logger.py @@ -0,0 +1,129 @@ +"""Focus export logger orchestrating DB pull/transform/upload.""" + +from __future__ import annotations + +import os +from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING, Any, Optional + +import polars as pl + +from litellm.integrations.custom_logger import CustomLogger + +from .destinations import ( + FocusDestination, + FocusDestinationFactory, + FocusTimeWindow, +) +from .serializers import FocusParquetSerializer, FocusSerializer +from .transformer import FocusTransformer + +if TYPE_CHECKING: + from apscheduler.schedulers.asyncio import AsyncIOScheduler +else: + AsyncIOScheduler = Any + + +class FocusExportLogger(CustomLogger): + """Coordinates Focus export jobs across transformer/serializer/destination layers.""" + + def __init__( + self, + *, + provider: Optional[str] = None, + export_format: Optional[str] = None, + frequency: Optional[str] = None, + cron_offset_minute: Optional[int] = None, + interval_seconds: Optional[int] = None, + prefix: Optional[str] = None, + destination_config: Optional[dict[str, Any]] = None, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self.provider = (provider or os.getenv("FOCUS_EXPORT_PROVIDER") or "s3").lower() + self.export_format = ( + export_format or os.getenv("FOCUS_EXPORT_FORMAT") or "parquet" + ).lower() + self.frequency = ( + frequency or os.getenv("FOCUS_EXPORT_FREQUENCY") or "hourly" + ).lower() + self.cron_offset_minute = ( + cron_offset_minute + if cron_offset_minute is not None + else int(os.getenv("FOCUS_EXPORT_CRON_OFFSET", "5")) + ) + self.interval_seconds = ( + interval_seconds + if interval_seconds is not None + else os.getenv("FOCUS_EXPORT_INTERVAL_SECONDS") + ) + self.prefix = prefix or os.getenv("FOCUS_EXPORT_PREFIX", "focus_exports") + + self._destination = self._init_destination( + destination_config=destination_config, + ) + self._serializer = self._init_serializer() + self._transformer = FocusTransformer() + + def _init_serializer(self) -> FocusSerializer: + """Return serializer implementation for requested format.""" + if self.export_format != "parquet": + raise NotImplementedError("Only parquet export supported currently") + return FocusParquetSerializer() + + def _init_destination( + self, + *, + destination_config: Optional[dict[str, Any]], + ) -> FocusDestination: + """Factory for destination implementations.""" + resolved_config = self._resolve_destination_config(destination_config) + return FocusDestinationFactory.create( + provider=self.provider, + prefix=self.prefix, + config=resolved_config, + ) + + def _resolve_destination_config( + self, + destination_config: Optional[dict[str, Any]], + ) -> dict[str, Any]: + """Collect provider-specific configuration for destination creation.""" + raise NotImplementedError + + async def export_usage_data(self) -> None: + """Public hook to trigger export immediately.""" + raise NotImplementedError + + async def dry_run_export_usage_data(self) -> dict: + """Return transformed data without uploading.""" + raise NotImplementedError + + async def initialize_focus_export_job(self) -> None: + """Entry point for scheduler jobs to run export cycle with locking.""" + raise NotImplementedError + + @staticmethod + async def init_focus_export_background_job( + scheduler: AsyncIOScheduler, + ) -> None: + """Register the export cron/interval job with the provided scheduler.""" + raise NotImplementedError + + def _compute_time_window(self, now: datetime) -> FocusTimeWindow: + """Derive the time window to export based on configured frequency.""" + raise NotImplementedError + + def _serialize_and_upload( + self, + frame: pl.DataFrame, + window: FocusTimeWindow, + ) -> None: + """Helper stub for serializing and delegating to destination.""" + raise NotImplementedError + + def _build_filename(self) -> str: + """Return the canonical file name for exports.""" + if not self._serializer.extension: + raise ValueError("Serializer must declare a file extension") + return f"usage.{self._serializer.extension}" diff --git a/litellm/integrations/focus/serializers/__init__.py b/litellm/integrations/focus/serializers/__init__.py new file mode 100644 index 00000000000..18187bf73e5 --- /dev/null +++ b/litellm/integrations/focus/serializers/__init__.py @@ -0,0 +1,6 @@ +"""Serializer package exports for Focus integration.""" + +from .base import FocusSerializer +from .parquet import FocusParquetSerializer + +__all__ = ["FocusSerializer", "FocusParquetSerializer"] diff --git a/litellm/integrations/focus/serializers/base.py b/litellm/integrations/focus/serializers/base.py new file mode 100644 index 00000000000..6da080dae81 --- /dev/null +++ b/litellm/integrations/focus/serializers/base.py @@ -0,0 +1,18 @@ +"""Serializer abstractions for Focus export.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + +import polars as pl + + +class FocusSerializer(ABC): + """Base serializer turning Focus frames into bytes.""" + + extension: str = "" + + @abstractmethod + def serialize(self, frame: pl.DataFrame) -> bytes: + """Convert the normalized Focus frame into the chosen format.""" + raise NotImplementedError diff --git a/litellm/integrations/focus/serializers/parquet.py b/litellm/integrations/focus/serializers/parquet.py new file mode 100644 index 00000000000..3d42337e388 --- /dev/null +++ b/litellm/integrations/focus/serializers/parquet.py @@ -0,0 +1,16 @@ +"""Parquet serializer for Focus export.""" + +from __future__ import annotations + +import polars as pl + +from .base import FocusSerializer + + +class FocusParquetSerializer(FocusSerializer): + """Placeholder Parquet serializer implementation.""" + + extension = "parquet" + + def serialize(self, frame: pl.DataFrame) -> bytes: + raise NotImplementedError diff --git a/litellm/integrations/focus/transformer.py b/litellm/integrations/focus/transformer.py new file mode 100644 index 00000000000..45a77dd8ac5 --- /dev/null +++ b/litellm/integrations/focus/transformer.py @@ -0,0 +1,13 @@ +"""Focus export data transformer.""" + +from __future__ import annotations + +import polars as pl + + +class FocusTransformer: + """Transforms LiteLLM DB rows into Focus-compatible schema.""" + + def transform(self, frame: pl.DataFrame) -> pl.DataFrame: + """Return a normalized frame expected by downstream serializers.""" + raise NotImplementedError From 3af9bab6a504ccc9e78730d229ebfcea1adddfe3 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Thu, 8 Jan 2026 12:17:14 +0900 Subject: [PATCH 028/121] feat: add focus schema --- litellm/integrations/focus/schema.py | 33 +++++++++++++++++++++++ litellm/integrations/focus/transformer.py | 4 +++ 2 files changed, 37 insertions(+) create mode 100644 litellm/integrations/focus/schema.py diff --git a/litellm/integrations/focus/schema.py b/litellm/integrations/focus/schema.py new file mode 100644 index 00000000000..766e29c0856 --- /dev/null +++ b/litellm/integrations/focus/schema.py @@ -0,0 +1,33 @@ +"""Schema definitions for Focus export data.""" + +from __future__ import annotations + +import polars as pl + + +FOCUS_NORMALIZED_SCHEMA = pl.Schema( + { + "usage_date": pl.Datetime(time_unit="us"), + "team_id": pl.String, + "team_alias": pl.String, + "user_id": pl.String, + "user_email": pl.String, + "api_key_alias": pl.String, + "model": pl.String, + "model_group": pl.String, + "custom_llm_provider": pl.String, + "prompt_tokens": pl.Int64, + "completion_tokens": pl.Int64, + "total_tokens": pl.Int64, + "spend": pl.Float64, + "cache_creation_input_tokens": pl.Int64, + "cache_read_input_tokens": pl.Int64, + "api_requests": pl.Int64, + "successful_requests": pl.Int64, + "failed_requests": pl.Int64, + "created_at": pl.Datetime(time_unit="us"), + "updated_at": pl.Datetime(time_unit="us"), + } +) + +__all__ = ["FOCUS_NORMALIZED_SCHEMA"] diff --git a/litellm/integrations/focus/transformer.py b/litellm/integrations/focus/transformer.py index 45a77dd8ac5..612082505fa 100644 --- a/litellm/integrations/focus/transformer.py +++ b/litellm/integrations/focus/transformer.py @@ -4,10 +4,14 @@ from __future__ import annotations import polars as pl +from .schema import FOCUS_NORMALIZED_SCHEMA + class FocusTransformer: """Transforms LiteLLM DB rows into Focus-compatible schema.""" + schema = FOCUS_NORMALIZED_SCHEMA + def transform(self, frame: pl.DataFrame) -> pl.DataFrame: """Return a normalized frame expected by downstream serializers.""" raise NotImplementedError From ddd52e0e2f1cc6771eb361c66fd7e408efb7afd1 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 09:51:34 +0530 Subject: [PATCH 029/121] Add support for kimi2 model on bedrock --- litellm/__init__.py | 1 + litellm/_lazy_imports_registry.py | 2 + litellm/constants.py | 1 + .../amazon_moonshot_transformation.py | 254 ++++++++++++++++++ litellm/llms/bedrock/common_utils.py | 2 + 5 files changed, 260 insertions(+) create mode 100644 litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 1bc690e561f..baed6daccb0 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1338,6 +1338,7 @@ if TYPE_CHECKING: from .llms.bedrock.chat.invoke_transformations.amazon_llama_transformation import AmazonLlamaConfig as AmazonLlamaConfig from .llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import AmazonDeepSeekR1Config as AmazonDeepSeekR1Config from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation import AmazonMistralConfig as AmazonMistralConfig + from .llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import AmazonMoonshotConfig as AmazonMoonshotConfig from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import AmazonTitanConfig as AmazonTitanConfig from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import AmazonTwelveLabsPegasusConfig as AmazonTwelveLabsPegasusConfig from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import AmazonInvokeConfig as AmazonInvokeConfig diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 26133ebc222..83af6d1b551 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -165,6 +165,7 @@ LLM_CONFIG_NAMES = ( "AmazonLlamaConfig", "AmazonDeepSeekR1Config", "AmazonMistralConfig", + "AmazonMoonshotConfig", "AmazonTitanConfig", "AmazonTwelveLabsPegasusConfig", "AmazonInvokeConfig", @@ -556,6 +557,7 @@ _LLM_CONFIGS_IMPORT_MAP = { "AmazonLlamaConfig": (".llms.bedrock.chat.invoke_transformations.amazon_llama_transformation", "AmazonLlamaConfig"), "AmazonDeepSeekR1Config": (".llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation", "AmazonDeepSeekR1Config"), "AmazonMistralConfig": (".llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation", "AmazonMistralConfig"), + "AmazonMoonshotConfig": (".llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation", "AmazonMoonshotConfig"), "AmazonTitanConfig": (".llms.bedrock.chat.invoke_transformations.amazon_titan_transformation", "AmazonTitanConfig"), "AmazonTwelveLabsPegasusConfig": (".llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation", "AmazonTwelveLabsPegasusConfig"), "AmazonInvokeConfig": (".llms.bedrock.chat.invoke_transformations.base_invoke_transformation", "AmazonInvokeConfig"), diff --git a/litellm/constants.py b/litellm/constants.py index db9d0114118..e24186d567e 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -909,6 +909,7 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[ "twelvelabs", "openai", "stability", + "moonshot", ] BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[ diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py new file mode 100644 index 00000000000..fc8ec8ae74f --- /dev/null +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py @@ -0,0 +1,254 @@ +""" +Transformation for Bedrock Moonshot AI (Kimi K2) models. + +Supports the Kimi K2 Thinking model available on Amazon Bedrock. +Model format: bedrock/moonshot.kimi-k2-thinking-v1:0 + +Reference: https://aws.amazon.com/about-aws/whats-new/2025/12/amazon-bedrock-fully-managed-open-weight-models/ +""" + +from typing import TYPE_CHECKING, Any, List, Optional +import re + +import httpx + +from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + AmazonInvokeConfig, +) +from litellm.llms.bedrock.common_utils import BedrockError +from litellm.llms.moonshot.chat.transformation import MoonshotChatConfig +from litellm.types.llms.openai import AllMessageValues + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.types.utils import ModelResponse + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig): + """ + Configuration for Bedrock Moonshot AI (Kimi K2) models. + + Reference: + https://aws.amazon.com/about-aws/whats-new/2025/12/amazon-bedrock-fully-managed-open-weight-models/ + https://platform.moonshot.ai/docs/api/chat + + Supported Params for the Amazon / Moonshot models: + - `max_tokens` (integer) max tokens + - `temperature` (float) temperature for model (0-1 for Moonshot) + - `top_p` (float) top p for model + - `stream` (bool) whether to stream responses + - `tools` (list) tool definitions (supported on kimi-k2-thinking) + - `tool_choice` (str|dict) tool choice specification (supported on kimi-k2-thinking) + + NOT Supported on Bedrock: + - `stop` sequences (Bedrock doesn't support stopSequences field for this model) + + Note: The kimi-k2-thinking model DOES support tool calls, unlike kimi-thinking-preview. + """ + + def __init__(self, **kwargs): + AmazonInvokeConfig.__init__(self, **kwargs) + MoonshotChatConfig.__init__(self, **kwargs) + + @property + def custom_llm_provider(self) -> Optional[str]: + return "bedrock" + + def _get_model_id(self, model: str) -> str: + """ + Extract the actual model ID from the LiteLLM model name. + + Removes routing prefixes like: + - bedrock/invoke/moonshot.kimi-k2-thinking -> moonshot.kimi-k2-thinking + - invoke/moonshot.kimi-k2-thinking -> moonshot.kimi-k2-thinking + - moonshot.kimi-k2-thinking -> moonshot.kimi-k2-thinking + """ + # Remove bedrock/ prefix if present + if model.startswith("bedrock/"): + model = model[8:] + + # Remove invoke/ prefix if present + if model.startswith("invoke/"): + model = model[7:] + + # Remove any provider prefix (e.g., moonshot/) + if "/" in model and not model.startswith("arn:"): + parts = model.split("/", 1) + if len(parts) == 2: + model = parts[1] + + return model + + def get_supported_openai_params(self, model: str) -> List[str]: + """ + Get the supported OpenAI params for Moonshot AI models on Bedrock. + + Bedrock-specific limitations: + - stopSequences field is not supported on Bedrock (unlike native Moonshot API) + - functions parameter is not supported (use tools instead) + - tool_choice doesn't support "required" value + + Note: kimi-k2-thinking DOES support tool calls (unlike kimi-thinking-preview) + The parent MoonshotChatConfig class handles the kimi-thinking-preview exclusion. + """ + excluded_params: List[str] = ["functions", "stop"] # Bedrock doesn't support stopSequences + + base_openai_params = super(MoonshotChatConfig, self).get_supported_openai_params(model=model) + final_params: List[str] = [] + for param in base_openai_params: + if param not in excluded_params: + final_params.append(param) + + return final_params + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Map OpenAI parameters to Moonshot AI parameters for Bedrock. + + Handles Moonshot AI specific limitations: + - tool_choice doesn't support "required" value + - Temperature <0.3 limitation for n>1 + - Temperature range is [0, 1] (not [0, 2] like OpenAI) + """ + return MoonshotChatConfig.map_openai_params( + self, + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=drop_params, + ) + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the request for Bedrock Moonshot AI models. + + Uses the Moonshot transformation logic which handles: + - Converting content lists to strings (Moonshot doesn't support list format) + - Adding tool_choice="required" message if needed + - Temperature and parameter validation + + Important: Strips routing prefixes (bedrock/, invoke/) from model name + before passing to parent class to ensure the request body contains only + the actual model ID (e.g., moonshot.kimi-k2-thinking). + """ + # Strip routing prefixes to get the actual model ID + clean_model_id = self._get_model_id(model) + + # Use Moonshot's transform_request which handles message transformation + # and tool_choice="required" workaround + return MoonshotChatConfig.transform_request( + self, + model=clean_model_id, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + def _extract_reasoning_from_content(self, content: str) -> tuple[Optional[str], str]: + """ + Extract reasoning content from tags in the response. + + Moonshot AI's Kimi K2 Thinking model returns reasoning in tags. + This method extracts that content and returns it separately. + + Args: + content: The full content string from the API response + + Returns: + tuple: (reasoning_content, main_content) + """ + if not content: + return None, content + + # Match ... tags + reasoning_match = re.match( + r"(.*?)\s*(.*)", + content, + re.DOTALL + ) + + if reasoning_match: + reasoning_content = reasoning_match.group(1).strip() + main_content = reasoning_match.group(2).strip() + return reasoning_content, main_content + + return None, content + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: "ModelResponse", + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> "ModelResponse": + """ + Transform the response from Bedrock Moonshot AI models. + + Moonshot AI uses OpenAI-compatible response format, but returns reasoning + content in tags. This method: + 1. Calls parent class transformation + 2. Extracts reasoning content from tags + 3. Sets reasoning_content on the message object + """ + # First, get the standard transformation + model_response = MoonshotChatConfig.transform_response( + self, + model=model, + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data=request_data, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + encoding=encoding, + api_key=api_key, + json_mode=json_mode, + ) + + # Extract reasoning content from tags + if model_response.choices and len(model_response.choices) > 0: + for choice in model_response.choices: + if choice.message and choice.message.content: + reasoning_content, main_content = self._extract_reasoning_from_content( + choice.message.content + ) + + if reasoning_content: + # Set the reasoning_content field + choice.message.reasoning_content = reasoning_content + # Update the main content without reasoning tags + choice.message.content = main_content + + return model_response + + def get_error_class( + self, error_message: str, status_code: int, headers: httpx.Headers + ) -> BedrockError: + """Return the appropriate error class for Bedrock.""" + return BedrockError(status_code=status_code, message=error_message) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 21a78c30343..9edfe320fb2 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -629,6 +629,8 @@ def get_bedrock_chat_config(model: str): return litellm.AmazonCohereConfig() elif bedrock_invoke_provider == "mistral": return litellm.AmazonMistralConfig() + elif bedrock_invoke_provider == "moonshot": + return litellm.AmazonMoonshotConfig() elif bedrock_invoke_provider == "deepseek_r1": return litellm.AmazonDeepSeekR1Config() elif bedrock_invoke_provider == "nova": From af6883712e21d6592175a4ef3c613ee6ecb3a70e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 10:07:33 +0530 Subject: [PATCH 030/121] Add tests for kimi 2 bedrock model --- docs/my-website/docs/providers/bedrock.md | 3 +- .../docs/providers/bedrock_imported.md | 178 ++++++++++- .../llm_translation/test_bedrock_moonshot.py | 290 ++++++++++++++++++ 3 files changed, 469 insertions(+), 2 deletions(-) create mode 100644 tests/llm_translation/test_bedrock_moonshot.py diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index f1eed4b4d52..5b247707696 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -7,7 +7,7 @@ ALL Bedrock models (Anthropic, Meta, Deepseek, Mistral, Amazon, etc.) are Suppor | Property | Details | |-------|-------| | Description | Amazon Bedrock is a fully managed service that offers a choice of high-performing foundation models (FMs). | -| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models), [`bedrock/qwen2/`](./bedrock_imported.md#qwen2-imported-models), [`bedrock/openai/`](./bedrock_imported.md#openai-compatible-imported-models-qwen-25-vl-etc) | +| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models), [`bedrock/qwen2/`](./bedrock_imported.md#qwen2-imported-models), [`bedrock/openai/`](./bedrock_imported.md#openai-compatible-imported-models-qwen-25-vl-etc), [`bedrock/moonshot`](./bedrock_imported.md#moonshot-kimi-k2-thinking) | | Provider Doc | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) | | Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings`, `/images/generations` | | Rerank Endpoint | `/rerank` | @@ -1941,6 +1941,7 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re | Mixtral 8x7B Instruct | `completion(model='bedrock/mistral.mixtral-8x7b-instruct-v0:1', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` | | TwelveLabs Pegasus 1.2 (US) | `completion(model='bedrock/us.twelvelabs.pegasus-1-2-v1:0', messages=messages, mediaSource={...})` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` | | TwelveLabs Pegasus 1.2 (EU) | `completion(model='bedrock/eu.twelvelabs.pegasus-1-2-v1:0', messages=messages, mediaSource={...})` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` | +| Moonshot Kimi K2 Thinking | `completion(model='bedrock/moonshot.kimi-k2-thinking', messages=messages)` or `completion(model='bedrock/invoke/moonshot.kimi-k2-thinking', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` | ## Bedrock Embedding diff --git a/docs/my-website/docs/providers/bedrock_imported.md b/docs/my-website/docs/providers/bedrock_imported.md index 0784f716925..709736e6109 100644 --- a/docs/my-website/docs/providers/bedrock_imported.md +++ b/docs/my-website/docs/providers/bedrock_imported.md @@ -431,4 +431,180 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ "max_tokens": 300, "temperature": 0.5 }' -``` \ No newline at end of file +``` + +### Moonshot Kimi K2 Thinking + +Moonshot AI's Kimi K2 Thinking model is now available on Amazon Bedrock. This model features advanced reasoning capabilities with automatic reasoning content extraction. + +| Property | Details | +|----------|---------| +| Provider Route | `bedrock/moonshot.kimi-k2-thinking`, `bedrock/invoke/moonshot.kimi-k2-thinking` | +| Provider Documentation | [AWS Bedrock Moonshot Announcement ↗](https://aws.amazon.com/about-aws/whats-new/2025/12/amazon-bedrock-fully-managed-open-weight-models/) | +| Supported Parameters | `temperature`, `max_tokens`, `top_p`, `stream`, `tools`, `tool_choice` | +| Special Features | Reasoning content extraction, Tool calling | + +#### Supported Features + +- **Reasoning Content Extraction**: Automatically extracts `` tags and returns them as `reasoning_content` (similar to OpenAI's o1 models) +- **Tool Calling**: Full support for function/tool calling with tool responses +- **Streaming**: Both streaming and non-streaming responses +- **System Messages**: System message support + +#### Basic Usage + + + + +```python title="Moonshot Kimi K2 SDK Usage" showLineNumbers +from litellm import completion +import os + +os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key" +os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key" +os.environ["AWS_REGION_NAME"] = "us-west-2" # or your preferred region + +# Basic completion +response = completion( + model="bedrock/moonshot.kimi-k2-thinking", # or bedrock/invoke/moonshot.kimi-k2-thinking + messages=[ + {"role": "user", "content": "What is 2+2? Think step by step."} + ], + temperature=0.7, + max_tokens=200 +) + +print(response.choices[0].message.content) + +# Access reasoning content if present +if response.choices[0].message.reasoning_content: + print("Reasoning:", response.choices[0].message.reasoning_content) +``` + + + + +**1. Add to config** + +```yaml title="config.yaml" showLineNumbers +model_list: + - model_name: kimi-k2 + litellm_params: + model: bedrock/moonshot.kimi-k2-thinking + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: us-west-2 +``` + +**2. Start proxy** + +```bash title="Start LiteLLM Proxy" showLineNumbers +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash title="Test Kimi K2 via Proxy" showLineNumbers +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "kimi-k2", + "messages": [ + { + "role": "user", + "content": "What is 2+2? Think step by step." + } + ], + "temperature": 0.7, + "max_tokens": 200 + }' +``` + + + + +#### Tool Calling Example + +```python title="Kimi K2 with Tool Calling" showLineNumbers +from litellm import completion +import os + +os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key" +os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key" +os.environ["AWS_REGION_NAME"] = "us-west-2" + +# Tool calling example +response = completion( + model="bedrock/moonshot.kimi-k2-thinking", + messages=[ + {"role": "user", "content": "What's the weather in Tokyo?"} + ], + tools=[ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city name" + } + }, + "required": ["location"] + } + } + } + ] +) + +if response.choices[0].message.tool_calls: + tool_call = response.choices[0].message.tool_calls[0] + print(f"Tool called: {tool_call.function.name}") + print(f"Arguments: {tool_call.function.arguments}") +``` + +#### Streaming Example + +```python title="Kimi K2 Streaming" showLineNumbers +from litellm import completion +import os + +os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key" +os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key" +os.environ["AWS_REGION_NAME"] = "us-west-2" + +response = completion( + model="bedrock/moonshot.kimi-k2-thinking", + messages=[ + {"role": "user", "content": "Explain quantum computing in simple terms."} + ], + stream=True, + temperature=0.7 +) + +for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") + + # Check for reasoning content in streaming + if hasattr(chunk.choices[0].delta, 'reasoning_content') and chunk.choices[0].delta.reasoning_content: + print(f"\n[Reasoning: {chunk.choices[0].delta.reasoning_content}]") +``` + +#### Supported Parameters + +| Parameter | Type | Description | Supported | +|-----------|------|-------------|-----------| +| `temperature` | float (0-1) | Controls randomness in output | ✅ | +| `max_tokens` | integer | Maximum tokens to generate | ✅ | +| `top_p` | float | Nucleus sampling parameter | ✅ | +| `stream` | boolean | Enable streaming responses | ✅ | +| `tools` | array | Tool/function definitions | ✅ | +| `tool_choice` | string/object | Tool choice specification | ✅ | +| `stop` | array | Stop sequences | ❌ (Not supported on Bedrock) | \ No newline at end of file diff --git a/tests/llm_translation/test_bedrock_moonshot.py b/tests/llm_translation/test_bedrock_moonshot.py new file mode 100644 index 00000000000..c6066c7db42 --- /dev/null +++ b/tests/llm_translation/test_bedrock_moonshot.py @@ -0,0 +1,290 @@ +""" +Tests for Bedrock Moonshot (Kimi K2) integration. + +This test suite verifies: +1. Basic completion functionality +2. Streaming responses +3. System message support +4. Temperature parameter handling +5. Reasoning content extraction from tags +6. Tool calling support (including tool response handling) +7. Parameter validation (e.g., stop sequences not supported) +""" + +from base_llm_unit_tests import BaseLLMChatTest +import pytest +import sys +import os +import json + +sys.path.insert(0, os.path.abspath("../..")) +import litellm +from litellm.llms.bedrock.common_utils import get_bedrock_chat_config + + +class TestBedrockMoonshotInvoke(BaseLLMChatTest): + """ + Test suite for Bedrock Moonshot via invoke route. + Inherits all standard LLM tests from BaseLLMChatTest. + """ + + def get_base_completion_call_args(self) -> dict: + litellm._turn_on_debug() + return { + "model": "bedrock/invoke/moonshot.kimi-k2-thinking", + } + + def test_tool_call_no_arguments(self, tool_call_no_arguments): + """Test that tool calls with no arguments is translated correctly.""" + pass + + +class TestBedrockMoonshotBasic: + """Unit tests for Bedrock Moonshot configuration and transformations.""" + + def test_provider_detection_invoke(self): + """Test that Bedrock Moonshot invoke models are correctly detected.""" + config = get_bedrock_chat_config("bedrock/invoke/moonshot.kimi-k2-thinking") + assert config is not None + assert config.__class__.__name__ == "AmazonMoonshotConfig" + + def test_provider_detection_converse(self): + """Test that Bedrock Moonshot converse models are correctly detected.""" + config = get_bedrock_chat_config("bedrock/moonshot.kimi-k2-thinking") + assert config is not None + + def test_config_initialization(self): + """Test that AmazonMoonshotConfig initializes correctly.""" + config = get_bedrock_chat_config("invoke/moonshot.kimi-k2-thinking") + assert config is not None + assert config.custom_llm_provider == "bedrock" + + def test_supported_params(self): + """Test that supported OpenAI params are correctly defined.""" + config = get_bedrock_chat_config("invoke/moonshot.kimi-k2-thinking") + supported_params = config.get_supported_openai_params("moonshot.kimi-k2-thinking") + + # Should support these params + assert "temperature" in supported_params + assert "max_tokens" in supported_params + assert "top_p" in supported_params + assert "stream" in supported_params + assert "tools" in supported_params + assert "tool_choice" in supported_params + + # Should NOT support stop sequences on Bedrock + assert "stop" not in supported_params + + # Should NOT support functions (use tools instead) + assert "functions" not in supported_params + + def test_transform_request_strips_model_prefix(self): + """Test that model ID prefixes are correctly stripped in transform_request.""" + from litellm.llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import ( + AmazonMoonshotConfig, + ) + + config = AmazonMoonshotConfig() + + messages = [{"role": "user", "content": "Hello"}] + + # Test that bedrock/invoke/ prefix is stripped + transformed = config.transform_request( + model="bedrock/invoke/moonshot.kimi-k2-thinking", + messages=messages, + optional_params={}, + litellm_params={}, + headers={} + ) + + # The model ID in the request body should be stripped + assert transformed["model"] == "moonshot.kimi-k2-thinking" + + +class TestBedrockMoonshotReasoningContent: + """Tests for reasoning content extraction.""" + + def test_reasoning_content_extraction(self): + """Test that reasoning content is extracted from tags.""" + from litellm.llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import ( + AmazonMoonshotConfig, + ) + + config = AmazonMoonshotConfig() + + # Test with reasoning tags + content_with_reasoning = "This is my thought processThis is the answer" + reasoning, content = config._extract_reasoning_from_content(content_with_reasoning) + + assert reasoning == "This is my thought process" + assert content == "This is the answer" + assert "" not in content + + # Test without reasoning tags + content_without_reasoning = "This is just a regular answer" + reasoning, content = config._extract_reasoning_from_content(content_without_reasoning) + + assert reasoning is None + assert content == "This is just a regular answer" + + +class TestBedrockMoonshotToolCalling: + """Unit tests for tool calling functionality.""" + + def test_tool_calling_supported(self): + """Test that tool calling is supported for Kimi K2 Thinking model.""" + config = get_bedrock_chat_config("invoke/moonshot.kimi-k2-thinking") + supported_params = config.get_supported_openai_params("moonshot.kimi-k2-thinking") + + # Kimi K2 Thinking DOES support tool calls (unlike kimi-thinking-preview) + assert "tools" in supported_params + assert "tool_choice" in supported_params + + def test_tool_call_request_format(self): + """Test that tool call requests are formatted correctly.""" + from litellm.llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import ( + AmazonMoonshotConfig, + ) + + config = AmazonMoonshotConfig() + + messages = [ + {"role": "user", "content": "What's the weather in San Francisco?"} + ] + + optional_params = { + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + } + } + ] + } + + transformed = config.transform_request( + model="bedrock/invoke/moonshot.kimi-k2-thinking", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + # Verify model ID is stripped + assert transformed["model"] == "moonshot.kimi-k2-thinking" + + # Verify tools are included + assert "tools" in transformed + assert len(transformed["tools"]) == 1 + assert transformed["tools"][0]["function"]["name"] == "get_weather" + + def test_tool_response_message_format(self): + """Test that tool response messages are formatted correctly.""" + # This tests the proper format for sending tool responses back + tool_response_message = { + "role": "tool", + "tool_call_id": "call_123", + "content": json.dumps({"temperature": 72, "condition": "sunny"}) + } + + # Verify the message structure + assert tool_response_message["role"] == "tool" + assert "tool_call_id" in tool_response_message + assert "content" in tool_response_message + + +class TestBedrockMoonshotParameterValidation: + """Tests for parameter validation and edge cases.""" + + def test_stop_sequences_not_supported(self): + """Test that stop sequences are correctly excluded from supported params.""" + config = get_bedrock_chat_config("invoke/moonshot.kimi-k2-thinking") + supported_params = config.get_supported_openai_params("moonshot.kimi-k2-thinking") + + # Bedrock Moonshot doesn't support stopSequences field + assert "stop" not in supported_params + + def test_temperature_range(self): + """Test that temperature parameter is handled correctly.""" + # Moonshot models support temperature 0-1 + # This is handled by the parent MoonshotChatConfig class + config = get_bedrock_chat_config("invoke/moonshot.kimi-k2-thinking") + + # Verify config exists and can handle temperature + assert config is not None + supported_params = config.get_supported_openai_params("moonshot.kimi-k2-thinking") + assert "temperature" in supported_params + + +class TestBedrockMoonshotTransformations: + """Tests for request/response transformations.""" + + def test_transform_request_basic(self): + """Test basic request transformation.""" + from litellm.llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import ( + AmazonMoonshotConfig, + ) + + config = AmazonMoonshotConfig() + + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"} + ] + + optional_params = { + "temperature": 0.7, + "max_tokens": 100 + } + + transformed = config.transform_request( + model="bedrock/invoke/moonshot.kimi-k2-thinking", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + # Verify model ID is stripped + assert transformed["model"] == "moonshot.kimi-k2-thinking" + + # Verify messages are included + assert "messages" in transformed + assert len(transformed["messages"]) >= 1 + + # Verify optional params are included + assert transformed["temperature"] == 0.7 + assert transformed["max_tokens"] == 100 + + def test_transform_request_with_system_message(self): + """Test request transformation with system message.""" + from litellm.llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import ( + AmazonMoonshotConfig, + ) + + config = AmazonMoonshotConfig() + + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"} + ] + + transformed = config.transform_request( + model="moonshot.kimi-k2-thinking", + messages=messages, + optional_params={}, + litellm_params={}, + headers={} + ) + + # System messages should be supported + assert "messages" in transformed From 2a4dc8e04196e62d82077e6945acc072e054459d Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 10:18:15 +0530 Subject: [PATCH 031/121] Fix: aws creds getting passed in req --- .../amazon_moonshot_transformation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py index fc8ec8ae74f..272ab685b64 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py @@ -143,11 +143,11 @@ class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig): - Converting content lists to strings (Moonshot doesn't support list format) - Adding tool_choice="required" message if needed - Temperature and parameter validation - - Important: Strips routing prefixes (bedrock/, invoke/) from model name - before passing to parent class to ensure the request body contains only - the actual model ID (e.g., moonshot.kimi-k2-thinking). + """ + # Filter out AWS credentials using the existing method from BaseAWSLLM + self._get_boto_credentials_from_optional_params(optional_params, model) + # Strip routing prefixes to get the actual model ID clean_model_id = self._get_model_id(model) From c4bab703068c26f50ab74114def999c99bb4394a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 10:21:38 +0530 Subject: [PATCH 032/121] Fix: Add moonshot in get_bedrock_model_id --- litellm/llms/bedrock/base_aws_llm.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index 71d21001cc3..0d5494541ec 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -369,6 +369,10 @@ class BaseAWSLLM: model_id = BaseAWSLLM._get_model_id_from_model_with_spec( model_id, spec="stability" ) + elif provider == "moonshot" and "moonshot/" in model_id: + model_id = BaseAWSLLM._get_model_id_from_model_with_spec( + model_id, spec="moonshot" + ) return model_id @staticmethod From e5c3fcbb52ebba73bf491a786e6d8cf6c21ea3ac Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 10:48:14 +0530 Subject: [PATCH 033/121] fix mypy tests --- .../amazon_moonshot_transformation.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py index 272ab685b64..e53410760dd 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py @@ -7,7 +7,7 @@ Model format: bedrock/moonshot.kimi-k2-thinking-v1:0 Reference: https://aws.amazon.com/about-aws/whats-new/2025/12/amazon-bedrock-fully-managed-open-weight-models/ """ -from typing import TYPE_CHECKING, Any, List, Optional +from typing import TYPE_CHECKING, Any, List, Optional, Union import re import httpx @@ -18,6 +18,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.moonshot.chat.transformation import MoonshotChatConfig from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import Choices if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -234,7 +235,8 @@ class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig): # Extract reasoning content from tags if model_response.choices and len(model_response.choices) > 0: for choice in model_response.choices: - if choice.message and choice.message.content: + # Only process Choices (not StreamingChoices) which have message attribute + if isinstance(choice, Choices) and choice.message and choice.message.content: reasoning_content, main_content = self._extract_reasoning_from_content( choice.message.content ) @@ -248,7 +250,7 @@ class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig): return model_response def get_error_class( - self, error_message: str, status_code: int, headers: httpx.Headers + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] ) -> BedrockError: """Return the appropriate error class for Bedrock.""" return BedrockError(status_code=status_code, message=error_message) From 7e98843d97c3b72af087d629b59bd1eb2902065e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 10:57:07 +0530 Subject: [PATCH 034/121] Fix: Incomplete usage in response object passed --- litellm/llms/anthropic/chat/transformation.py | 15 ++--- .../test_anthropic_chat_transformation.py | 58 +++++++++++++++++++ 2 files changed, 66 insertions(+), 7 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index c71edcdc2d1..57391c152cb 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1265,14 +1265,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): cache_creation_tokens=cache_creation_input_tokens, cache_creation_token_details=cache_creation_token_details, ) - completion_token_details = ( - CompletionTokensDetailsWrapper( - reasoning_tokens=token_counter( - text=reasoning_content, count_response_tokens=True - ) - ) + # Always populate completion_token_details, not just when there's reasoning_content + reasoning_tokens = ( + token_counter(text=reasoning_content, count_response_tokens=True) if reasoning_content - else None + else 0 + ) + completion_token_details = CompletionTokensDetailsWrapper( + reasoning_tokens=reasoning_tokens if reasoning_tokens > 0 else None, + text_tokens=completion_tokens - reasoning_tokens if reasoning_tokens > 0 else completion_tokens, ) total_tokens = prompt_tokens + completion_tokens diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 9b6d1c6e178..5e58601d589 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1754,3 +1754,61 @@ def test_transform_request_respects_user_max_tokens(): ) assert result["max_tokens"] == 1000 + + +def test_calculate_usage_completion_tokens_details_always_populated(): + """ + Test that completion_tokens_details is always populated in Usage object, + not just when there's reasoning_content. + + Fixes: https://github.com/BerriAI/litellm/issues/18772 + Bug: completion_tokens_details was None for regular Claude responses without reasoning + """ + config = AnthropicConfig() + + # Test without reasoning_content - completion_tokens_details should still be populated + usage_object = { + "input_tokens": 37, + "output_tokens": 248, + } + usage = config.calculate_usage(usage_object=usage_object, reasoning_content=None) + + # completion_tokens_details should NOT be None + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens is None + assert usage.completion_tokens_details.text_tokens == 248 + assert usage.completion_tokens == 248 + assert usage.prompt_tokens == 37 + assert usage.total_tokens == 285 + + +def test_calculate_usage_completion_tokens_details_with_reasoning(): + """ + Test that completion_tokens_details correctly splits text_tokens and reasoning_tokens + when reasoning_content is present. + + Fixes: https://github.com/BerriAI/litellm/issues/18772 + """ + config = AnthropicConfig() + + # Test with reasoning_content - should split tokens correctly + usage_object = { + "input_tokens": 100, + "output_tokens": 500, + } + # Simulating reasoning content that would count as ~50 tokens + reasoning_content = "Let me think about this step by step. " * 10 # Roughly 50 tokens + + usage = config.calculate_usage( + usage_object=usage_object, + reasoning_content=reasoning_content + ) + + # completion_tokens_details should be populated with both reasoning and text tokens + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens is not None + assert usage.completion_tokens_details.reasoning_tokens > 0 + # text_tokens should be total minus reasoning + expected_text_tokens = 500 - usage.completion_tokens_details.reasoning_tokens + assert usage.completion_tokens_details.text_tokens == expected_text_tokens + assert usage.completion_tokens == 500 From c78bf8cfcd339d9bc982f306a868d8072b8cb4fe Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 13:03:58 +0530 Subject: [PATCH 035/121] Add support for model id in bedrock passthrough --- .../bedrock/passthrough/transformation.py | 9 +- litellm/passthrough/main.py | 5 + ...test_bedrock_passthrough_transformation.py | 129 ++++++++++++++++++ 3 files changed, 142 insertions(+), 1 deletion(-) diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index 5791bfb8013..568fe941716 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -34,11 +34,12 @@ class BedrockPassthroughConfig( litellm_params: dict, ) -> Tuple["URL", str]: optional_params = litellm_params.copy() + model_id = optional_params.get("model_id", None) aws_region_name = self._get_aws_region_name( optional_params=optional_params, model=model, - model_id=None, + model_id=model_id, ) aws_bedrock_runtime_endpoint = optional_params.get("aws_bedrock_runtime_endpoint") @@ -49,6 +50,12 @@ class BedrockPassthroughConfig( endpoint_type="runtime", ) + # If model_id is provided (e.g., Application Inference Profile ARN), use it in the endpoint + # instead of the translated model name + if model_id is not None: + # Replace the model name in the endpoint with the model_id + import re + endpoint = re.sub(r'model/[^/]+/', f'model/{model_id}/', endpoint) return self.format_url(endpoint, endpoint_url, request_query_params or {}), endpoint_url def sign_request( diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index 3df3037ed58..df4737cec85 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -216,6 +216,11 @@ def llm_passthrough_route( ) litellm_params_dict = get_litellm_params(**kwargs) + + # Add model_id to litellm_params if present in kwargs (for Bedrock Application Inference Profiles) + if "model_id" in kwargs: + litellm_params_dict["model_id"] = kwargs["model_id"] + litellm_logging_obj.update_environment_variables( model=model, litellm_params=litellm_params_dict, diff --git a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py index 7cb1ee2b54a..07253a8e09e 100644 --- a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py +++ b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -175,3 +175,132 @@ def test_format_url_handles_trailing_slash_normalization(): assert str(result_with_slash) == "http://proxy.com/bedrockproxy/model/test/invoke" +def test_bedrock_passthrough_with_application_inference_profile(): + """ + Test get_complete_url with Application Inference Profile ARN as model_id. + + This test verifies the fix for GitHub issue #18761 where Bedrock passthrough + was not working with Application Inference Profiles. The model_id (ARN) should + replace the translated model name in the endpoint URL. + """ + config = BedrockPassthroughConfig() + + model = "anthropic.claude-sonnet-4-20250514-v1:0" + model_id = "arn:aws:bedrock:eu-west-1:123456789:application-inference-profile/abcdefgh1234" + endpoint = f"model/{model}/invoke" + + with patch.object(config, '_get_aws_region_name', return_value="eu-west-1"), \ + patch.object(config, 'get_runtime_endpoint', return_value=( + "https://bedrock-runtime.eu-west-1.amazonaws.com", + "https://bedrock-runtime.eu-west-1.amazonaws.com" + )): + + url, api_base = config.get_complete_url( + api_base=None, + api_key=None, + model=model, + endpoint=endpoint, + request_query_params=None, + litellm_params={"model_id": model_id, "aws_region_name": "eu-west-1"} + ) + + # Verify that the URL contains the model_id (ARN) instead of the model name + url_str = str(url) + assert model_id in url_str, f"Expected model_id ARN in URL, but got: {url_str}" + assert model not in url_str, f"Model name should be replaced by model_id, but got: {url_str}" + assert "/invoke" in url_str, "Expected /invoke action in URL" + + # Verify the complete URL structure + expected_url = f"https://bedrock-runtime.eu-west-1.amazonaws.com/model/{model_id}/invoke" + assert url_str == expected_url, f"Expected {expected_url}, but got: {url_str}" + + +def test_bedrock_passthrough_with_inference_profile_converse_endpoint(): + """Test Application Inference Profile with converse endpoint""" + config = BedrockPassthroughConfig() + + model = "anthropic.claude-sonnet-4-20250514-v1:0" + model_id = "arn:aws:bedrock:us-east-1:123456789:application-inference-profile/xyz123" + endpoint = f"model/{model}/converse" + + with patch.object(config, '_get_aws_region_name', return_value="us-east-1"), \ + patch.object(config, 'get_runtime_endpoint', return_value=( + "https://bedrock-runtime.us-east-1.amazonaws.com", + "https://bedrock-runtime.us-east-1.amazonaws.com" + )): + + url, api_base = config.get_complete_url( + api_base=None, + api_key=None, + model=model, + endpoint=endpoint, + request_query_params=None, + litellm_params={"model_id": model_id} + ) + + url_str = str(url) + assert model_id in url_str + assert "/converse" in url_str + assert model not in url_str + + +def test_bedrock_passthrough_without_model_id_backward_compatibility(): + """ + Test that passthrough still works without model_id (backward compatibility). + + When model_id is not provided, the system should use the model name as before. + """ + config = BedrockPassthroughConfig() + + model = "anthropic.claude-3-sonnet" + endpoint = f"model/{model}/invoke" + + with patch.object(config, '_get_aws_region_name', return_value="us-east-1"), \ + patch.object(config, 'get_runtime_endpoint', return_value=( + "https://bedrock-runtime.us-east-1.amazonaws.com", + "https://bedrock-runtime.us-east-1.amazonaws.com" + )): + + url, api_base = config.get_complete_url( + api_base=None, + api_key=None, + model=model, + endpoint=endpoint, + request_query_params=None, + litellm_params={} # No model_id provided + ) + + # Verify that the URL contains the model name (not replaced) + url_str = str(url) + assert model in url_str, f"Expected model name in URL when model_id not provided, but got: {url_str}" + expected_url = f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model}/invoke" + assert url_str == expected_url + + +def test_bedrock_passthrough_region_extraction_from_inference_profile_arn(): + """Test that AWS region is correctly extracted from Application Inference Profile ARN""" + config = BedrockPassthroughConfig() + + model = "anthropic.claude-sonnet-4-20250514-v1:0" + # ARN contains us-west-2 region + model_id = "arn:aws:bedrock:us-west-2:123456789:application-inference-profile/test123" + endpoint = f"model/{model}/invoke" + + # Don't provide aws_region_name in litellm_params to test ARN extraction + with patch.object(config, 'get_runtime_endpoint', return_value=( + "https://bedrock-runtime.us-west-2.amazonaws.com", + "https://bedrock-runtime.us-west-2.amazonaws.com" + )): + + url, api_base = config.get_complete_url( + api_base=None, + api_key=None, + model=model, + endpoint=endpoint, + request_query_params=None, + litellm_params={"model_id": model_id} # Region should be extracted from ARN + ) + + # Verify that the region from ARN is used in the base URL + assert "us-west-2" in api_base, f"Expected region 'us-west-2' from ARN in base URL, but got: {api_base}" + From c00d83fea2338429991a355d59c6d1f548f30a27 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Thu, 8 Jan 2026 16:48:06 +0900 Subject: [PATCH 036/121] feat: add support focus export --- litellm/__init__.py | 1 + litellm/integrations/focus/database.py | 95 +++++- .../focus/destinations/factory.py | 44 ++- .../focus/destinations/s3_destination.py | 48 ++- .../integrations/focus/focus_export_logger.py | 129 -------- litellm/integrations/focus/focus_logger.py | 285 ++++++++++++++++++ litellm/integrations/focus/schema.py | 65 ++-- .../integrations/focus/serializers/parquet.py | 11 +- litellm/integrations/focus/transformer.py | 89 +++++- .../custom_logger_registry.py | 2 + litellm/litellm_core_utils/litellm_logging.py | 15 + litellm/proxy/proxy_server.py | 11 +- 12 files changed, 632 insertions(+), 163 deletions(-) delete mode 100644 litellm/integrations/focus/focus_export_logger.py create mode 100644 litellm/integrations/focus/focus_logger.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 8af9a7d10a6..76bdac40c2d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -134,6 +134,7 @@ _custom_logger_compatible_callbacks_literal = Literal[ "bitbucket", "gitlab", "cloudzero", + "focus", "posthog", "levo", ] diff --git a/litellm/integrations/focus/database.py b/litellm/integrations/focus/database.py index a23d16278af..402a1ec4b4e 100644 --- a/litellm/integrations/focus/database.py +++ b/litellm/integrations/focus/database.py @@ -3,7 +3,7 @@ from __future__ import annotations from datetime import datetime -from typing import Optional +from typing import Any, Dict, Optional import polars as pl @@ -11,6 +11,16 @@ import polars as pl class FocusLiteLLMDatabase: """Retrieves LiteLLM usage data for Focus export workflows.""" + def _ensure_prisma_client(self): + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise RuntimeError( + "Database not connected. Connect a database to your proxy - " + "https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" + ) + return prisma_client + async def get_usage_data( self, *, @@ -19,4 +29,85 @@ class FocusLiteLLMDatabase: end_time_utc: Optional[datetime] = None, ) -> pl.DataFrame: """Return usage data for the requested window.""" - raise NotImplementedError + client = self._ensure_prisma_client() + + where_clauses = [] + query_params = [] + placeholder_index = 1 + if start_time_utc: + where_clauses.append(f"dus.updated_at >= ${placeholder_index}::timestamptz") + query_params.append(start_time_utc) + placeholder_index += 1 + if end_time_utc: + where_clauses.append(f"dus.updated_at <= ${placeholder_index}::timestamptz") + query_params.append(end_time_utc) + placeholder_index += 1 + + where_clause = "" + if where_clauses: + where_clause = "WHERE " + " AND ".join(where_clauses) + + limit_clause = "" + if limit is not None: + try: + limit_value = int(limit) + except (TypeError, ValueError) as exc: # pragma: no cover - defensive guard + raise ValueError("limit must be an integer") from exc + if limit_value < 0: + raise ValueError("limit must be non-negative") + limit_clause = f" LIMIT ${placeholder_index}" + query_params.append(limit_value) + + query = f""" + SELECT + dus.id, + dus.date, + dus.user_id, + dus.api_key, + dus.model, + dus.model_group, + dus.custom_llm_provider, + dus.prompt_tokens, + dus.completion_tokens, + dus.spend, + dus.api_requests, + dus.successful_requests, + dus.failed_requests, + dus.cache_creation_input_tokens, + dus.cache_read_input_tokens, + dus.created_at, + dus.updated_at, + vt.team_id, + vt.key_alias as api_key_alias, + tt.team_alias, + ut.user_email as user_email + FROM "LiteLLM_DailyUserSpend" dus + LEFT JOIN "LiteLLM_VerificationToken" vt ON dus.api_key = vt.token + LEFT JOIN "LiteLLM_TeamTable" tt ON vt.team_id = tt.team_id + LEFT JOIN "LiteLLM_UserTable" ut ON dus.user_id = ut.user_id + {where_clause} + ORDER BY dus.date DESC, dus.created_at DESC + {limit_clause} + """ + + try: + db_response = await client.db.query_raw(query, *query_params) + return pl.DataFrame(db_response, infer_schema_length=None) + except Exception as exc: + raise RuntimeError(f"Error retrieving usage data: {exc}") from exc + + async def get_table_info(self) -> Dict[str, Any]: + """Return metadata about the spend table for diagnostics.""" + client = self._ensure_prisma_client() + + info_query = """ + SELECT column_name, data_type, is_nullable + FROM information_schema.columns + WHERE table_name = 'LiteLLM_DailyUserSpend' + ORDER BY ordinal_position; + """ + try: + columns_response = await client.db.query_raw(info_query) + return {"columns": columns_response, "table_name": "LiteLLM_DailyUserSpend"} + except Exception as exc: + raise RuntimeError(f"Error getting table info: {exc}") from exc diff --git a/litellm/integrations/focus/destinations/factory.py b/litellm/integrations/focus/destinations/factory.py index 36163ece61c..6f5dc5dc06a 100644 --- a/litellm/integrations/focus/destinations/factory.py +++ b/litellm/integrations/focus/destinations/factory.py @@ -2,9 +2,11 @@ from __future__ import annotations -from typing import Optional +import os +from typing import Any, Dict, Optional from .base import FocusDestination +from .s3_destination import FocusS3Destination class FocusDestinationFactory: @@ -15,7 +17,43 @@ class FocusDestinationFactory: *, provider: str, prefix: str, - config: Optional[dict] = None, + config: Optional[Dict[str, Any]] = None, ) -> FocusDestination: """Return a destination implementation for the requested provider.""" - raise NotImplementedError + provider_lower = provider.lower() + normalized_config = FocusDestinationFactory._resolve_config( + provider=provider_lower, overrides=config or {} + ) + if provider_lower == "s3": + return FocusS3Destination(prefix=prefix, config=normalized_config) + raise NotImplementedError(f"Provider '{provider}' not supported for Focus export") + + @staticmethod + def _resolve_config( + *, + provider: str, + overrides: Dict[str, Any], + ) -> Dict[str, Any]: + if provider == "s3": + resolved = { + "bucket_name": overrides.get("bucket_name") + or os.getenv("FOCUS_S3_BUCKET_NAME"), + "region_name": overrides.get("region_name") + or os.getenv("FOCUS_S3_REGION_NAME"), + "endpoint_url": overrides.get("endpoint_url") + or os.getenv("FOCUS_S3_ENDPOINT_URL"), + "aws_access_key_id": overrides.get("aws_access_key_id") + or os.getenv("FOCUS_S3_ACCESS_KEY"), + "aws_secret_access_key": overrides.get("aws_secret_access_key") + or os.getenv("FOCUS_S3_SECRET_KEY"), + "aws_session_token": overrides.get("aws_session_token") + or os.getenv("FOCUS_S3_SESSION_TOKEN"), + } + if not resolved.get("bucket_name"): + raise ValueError( + "FOCUS_S3_BUCKET_NAME must be provided for S3 exports" + ) + return {k: v for k, v in resolved.items() if v is not None} + raise NotImplementedError( + f"Provider '{provider}' not supported for Focus export configuration" + ) diff --git a/litellm/integrations/focus/destinations/s3_destination.py b/litellm/integrations/focus/destinations/s3_destination.py index 7ead87baa29..c6d5554b438 100644 --- a/litellm/integrations/focus/destinations/s3_destination.py +++ b/litellm/integrations/focus/destinations/s3_destination.py @@ -2,8 +2,12 @@ from __future__ import annotations +import asyncio +from datetime import timezone from typing import Any, Optional +import boto3 + from .base import FocusDestination, FocusTimeWindow @@ -16,8 +20,13 @@ class FocusS3Destination(FocusDestination): prefix: str, config: Optional[dict[str, Any]] = None, ) -> None: + config = config or {} + bucket_name = config.get("bucket_name") + if not bucket_name: + raise ValueError("bucket_name must be provided for S3 destination") + self.bucket_name = bucket_name self.prefix = prefix.rstrip("/") - self.config = config or {} + self.config = config async def deliver( self, @@ -26,7 +35,40 @@ class FocusS3Destination(FocusDestination): time_window: FocusTimeWindow, filename: str, ) -> None: - raise NotImplementedError + object_key = self._build_object_key(time_window=time_window, filename=filename) + await asyncio.to_thread(self._upload, content, object_key) def _build_object_key(self, *, time_window: FocusTimeWindow, filename: str) -> str: - raise NotImplementedError + start_utc = time_window.start_time.astimezone(timezone.utc) + date_component = f"date={start_utc.strftime('%Y-%m-%d')}" + parts = [self.prefix, date_component] + if time_window.frequency == "hourly": + parts.append(f"hour={start_utc.strftime('%H')}") + key_prefix = "/".join(filter(None, parts)) + return f"{key_prefix}/{filename}" if key_prefix else filename + + def _upload(self, content: bytes, object_key: str) -> None: + client_kwargs: dict[str, Any] = {} + region_name = self.config.get("region_name") + if region_name: + client_kwargs["region_name"] = region_name + endpoint_url = self.config.get("endpoint_url") + if endpoint_url: + client_kwargs["endpoint_url"] = endpoint_url + + session_kwargs: dict[str, Any] = {} + for key in ( + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + ): + if self.config.get(key): + session_kwargs[key] = self.config[key] + + s3_client = boto3.client("s3", **client_kwargs, **session_kwargs) + s3_client.put_object( + Bucket=self.bucket_name, + Key=object_key, + Body=content, + ContentType="application/octet-stream", + ) diff --git a/litellm/integrations/focus/focus_export_logger.py b/litellm/integrations/focus/focus_export_logger.py deleted file mode 100644 index 723dbc706fb..00000000000 --- a/litellm/integrations/focus/focus_export_logger.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Focus export logger orchestrating DB pull/transform/upload.""" - -from __future__ import annotations - -import os -from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, Any, Optional - -import polars as pl - -from litellm.integrations.custom_logger import CustomLogger - -from .destinations import ( - FocusDestination, - FocusDestinationFactory, - FocusTimeWindow, -) -from .serializers import FocusParquetSerializer, FocusSerializer -from .transformer import FocusTransformer - -if TYPE_CHECKING: - from apscheduler.schedulers.asyncio import AsyncIOScheduler -else: - AsyncIOScheduler = Any - - -class FocusExportLogger(CustomLogger): - """Coordinates Focus export jobs across transformer/serializer/destination layers.""" - - def __init__( - self, - *, - provider: Optional[str] = None, - export_format: Optional[str] = None, - frequency: Optional[str] = None, - cron_offset_minute: Optional[int] = None, - interval_seconds: Optional[int] = None, - prefix: Optional[str] = None, - destination_config: Optional[dict[str, Any]] = None, - **kwargs: Any, - ) -> None: - super().__init__(**kwargs) - self.provider = (provider or os.getenv("FOCUS_EXPORT_PROVIDER") or "s3").lower() - self.export_format = ( - export_format or os.getenv("FOCUS_EXPORT_FORMAT") or "parquet" - ).lower() - self.frequency = ( - frequency or os.getenv("FOCUS_EXPORT_FREQUENCY") or "hourly" - ).lower() - self.cron_offset_minute = ( - cron_offset_minute - if cron_offset_minute is not None - else int(os.getenv("FOCUS_EXPORT_CRON_OFFSET", "5")) - ) - self.interval_seconds = ( - interval_seconds - if interval_seconds is not None - else os.getenv("FOCUS_EXPORT_INTERVAL_SECONDS") - ) - self.prefix = prefix or os.getenv("FOCUS_EXPORT_PREFIX", "focus_exports") - - self._destination = self._init_destination( - destination_config=destination_config, - ) - self._serializer = self._init_serializer() - self._transformer = FocusTransformer() - - def _init_serializer(self) -> FocusSerializer: - """Return serializer implementation for requested format.""" - if self.export_format != "parquet": - raise NotImplementedError("Only parquet export supported currently") - return FocusParquetSerializer() - - def _init_destination( - self, - *, - destination_config: Optional[dict[str, Any]], - ) -> FocusDestination: - """Factory for destination implementations.""" - resolved_config = self._resolve_destination_config(destination_config) - return FocusDestinationFactory.create( - provider=self.provider, - prefix=self.prefix, - config=resolved_config, - ) - - def _resolve_destination_config( - self, - destination_config: Optional[dict[str, Any]], - ) -> dict[str, Any]: - """Collect provider-specific configuration for destination creation.""" - raise NotImplementedError - - async def export_usage_data(self) -> None: - """Public hook to trigger export immediately.""" - raise NotImplementedError - - async def dry_run_export_usage_data(self) -> dict: - """Return transformed data without uploading.""" - raise NotImplementedError - - async def initialize_focus_export_job(self) -> None: - """Entry point for scheduler jobs to run export cycle with locking.""" - raise NotImplementedError - - @staticmethod - async def init_focus_export_background_job( - scheduler: AsyncIOScheduler, - ) -> None: - """Register the export cron/interval job with the provided scheduler.""" - raise NotImplementedError - - def _compute_time_window(self, now: datetime) -> FocusTimeWindow: - """Derive the time window to export based on configured frequency.""" - raise NotImplementedError - - def _serialize_and_upload( - self, - frame: pl.DataFrame, - window: FocusTimeWindow, - ) -> None: - """Helper stub for serializing and delegating to destination.""" - raise NotImplementedError - - def _build_filename(self) -> str: - """Return the canonical file name for exports.""" - if not self._serializer.extension: - raise ValueError("Serializer must declare a file extension") - return f"usage.{self._serializer.extension}" diff --git a/litellm/integrations/focus/focus_logger.py b/litellm/integrations/focus/focus_logger.py new file mode 100644 index 00000000000..5eccf441bea --- /dev/null +++ b/litellm/integrations/focus/focus_logger.py @@ -0,0 +1,285 @@ +"""Focus export logger orchestrating DB pull/transform/upload.""" + +from __future__ import annotations + +import os +from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING, Any, Dict, List, Optional, cast + +import polars as pl + +import litellm +from litellm._logging import verbose_logger +from litellm.integrations.custom_logger import CustomLogger + +from .database import FocusLiteLLMDatabase +from .destinations import ( + FocusDestination, + FocusDestinationFactory, + FocusTimeWindow, +) +from .serializers import FocusParquetSerializer, FocusSerializer +from .transformer import FocusTransformer + +if TYPE_CHECKING: + from apscheduler.schedulers.asyncio import AsyncIOScheduler +else: + AsyncIOScheduler = Any + +FOCUS_USAGE_DATA_JOB_NAME = "focus_export_usage_data" +DEFAULT_DRY_RUN_LIMIT = 500 + + +class FocusLogger(CustomLogger): + """Coordinates Focus export jobs across transformer/serializer/destination layers.""" + + def __init__( + self, + *, + provider: Optional[str] = None, + export_format: Optional[str] = None, + frequency: Optional[str] = None, + cron_offset_minute: Optional[int] = None, + interval_seconds: Optional[int] = None, + prefix: Optional[str] = None, + destination_config: Optional[dict[str, Any]] = None, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self.provider = (provider or os.getenv("FOCUS_PROVIDER") or "s3").lower() + self.export_format = ( + export_format or os.getenv("FOCUS_FORMAT") or "parquet" + ).lower() + self.frequency = ( + frequency or os.getenv("FOCUS_FREQUENCY") or "hourly" + ).lower() + self.cron_offset_minute = ( + cron_offset_minute + if cron_offset_minute is not None + else int(os.getenv("FOCUS_CRON_OFFSET", "5")) + ) + raw_interval = ( + interval_seconds + if interval_seconds is not None + else os.getenv("FOCUS_INTERVAL_SECONDS") + ) + self.interval_seconds = int(raw_interval) if raw_interval is not None else None + self.prefix = prefix or os.getenv("FOCUS_PREFIX", "focus_exports") + + self._destination = FocusDestinationFactory.create( + provider=self.provider, + prefix=self.prefix, + config=destination_config, + ) + self._serializer = self._init_serializer() + self._transformer = FocusTransformer() + self._database = FocusLiteLLMDatabase() + + def _init_serializer(self) -> FocusSerializer: + """Return serializer implementation for requested format.""" + if self.export_format != "parquet": + raise NotImplementedError("Only parquet export supported currently") + return FocusParquetSerializer() + + async def export_usage_data( + self, + *, + limit: Optional[int] = None, + start_time_utc: Optional[datetime] = None, + end_time_utc: Optional[datetime] = None, + ) -> None: + """Public hook to trigger export immediately.""" + if bool(start_time_utc) ^ bool(end_time_utc): + raise ValueError("start_time_utc and end_time_utc must be provided together") + + if start_time_utc and end_time_utc: + window = FocusTimeWindow( + start_time=start_time_utc, + end_time=end_time_utc, + frequency=self.frequency, + ) + else: + window = self._compute_time_window(datetime.now(timezone.utc)) + await self._export_window(window=window, limit=limit) + + async def dry_run_export_usage_data( + self, limit: Optional[int] = DEFAULT_DRY_RUN_LIMIT + ) -> dict[str, Any]: + """Return transformed data without uploading.""" + data = await self._database.get_usage_data(limit=limit) + normalized = self._transformer.transform(data) + + usage_sample = data.head(min(50, len(data))).to_dicts() + normalized_sample = normalized.head(min(50, len(normalized))).to_dicts() + + summary = { + "total_records": len(normalized), + "total_spend": self._sum_column(normalized, "spend"), + "total_tokens": self._sum_column(normalized, "total_tokens"), + "unique_teams": self._count_unique(normalized, "team_id"), + "unique_models": self._count_unique(normalized, "model"), + } + + return { + "usage_data": usage_sample, + "normalized_data": normalized_sample, + "summary": summary, + } + + async def initialize_focus_export_job(self) -> None: + """Entry point for scheduler jobs to run export cycle with locking.""" + try: + from litellm.proxy.proxy_server import proxy_logging_obj + except ImportError: + proxy_logging_obj = None + + pod_lock_manager = None + if proxy_logging_obj is not None: + writer = getattr(proxy_logging_obj, "db_spend_update_writer", None) + if writer is not None: + pod_lock_manager = getattr(writer, "pod_lock_manager", None) + + if pod_lock_manager and pod_lock_manager.redis_cache: + acquired = await pod_lock_manager.acquire_lock( + cronjob_id=FOCUS_USAGE_DATA_JOB_NAME + ) + if not acquired: + verbose_logger.debug("Focus export: unable to acquire pod lock") + return + try: + await self._run_scheduled_export() + finally: + await pod_lock_manager.release_lock( + cronjob_id=FOCUS_USAGE_DATA_JOB_NAME + ) + else: + await self._run_scheduled_export() + + @staticmethod + async def init_focus_export_background_job( + scheduler: AsyncIOScheduler, + ) -> None: + """Register the export cron/interval job with the provided scheduler.""" + from litellm.integrations.custom_logger import CustomLogger + + focus_loggers: List[ + CustomLogger + ] = litellm.logging_callback_manager.get_custom_loggers_for_type( + callback_type=FocusLogger + ) + if not focus_loggers: + verbose_logger.debug("No Focus export logger registered; skipping scheduler") + return + + focus_logger = cast(FocusLogger, focus_loggers[0]) + trigger_kwargs = focus_logger._build_scheduler_trigger() + scheduler.add_job( + focus_logger.initialize_focus_export_job, + **trigger_kwargs, + ) + + def _build_scheduler_trigger(self) -> Dict[str, Any]: + """Return scheduler configuration for the selected frequency.""" + if self.frequency == "interval": + seconds = self.interval_seconds or 60 + return {"trigger": "interval", "seconds": seconds} + + if self.frequency == "hourly": + minute = max(0, min(59, self.cron_offset_minute)) + return {"trigger": "cron", "minute": minute, "second": 0} + + if self.frequency == "daily": + total_minutes = max(0, self.cron_offset_minute) + hour = min(23, total_minutes // 60) + minute = min(59, total_minutes % 60) + return {"trigger": "cron", "hour": hour, "minute": minute, "second": 0} + + raise ValueError(f"Unsupported frequency: {self.frequency}") + + async def _run_scheduled_export(self) -> None: + """Execute the scheduled export for the configured window.""" + window = self._compute_time_window(datetime.now(timezone.utc)) + await self._export_window(window=window, limit=None) + + async def _export_window( + self, + *, + window: FocusTimeWindow, + limit: Optional[int], + ) -> None: + data = await self._database.get_usage_data( + limit=limit, + start_time_utc=window.start_time, + end_time_utc=window.end_time, + ) + if data.is_empty(): + verbose_logger.debug("Focus export: no usage data for window %s", window) + return + + normalized = self._transformer.transform(data) + if normalized.is_empty(): + verbose_logger.debug("Focus export: normalized data empty for window %s", window) + return + + await self._serialize_and_upload(normalized, window) + + def _compute_time_window(self, now: datetime) -> FocusTimeWindow: + """Derive the time window to export based on configured frequency.""" + now_utc = now.astimezone(timezone.utc) + if self.frequency == "hourly": + end_time = now_utc.replace(minute=0, second=0, microsecond=0) + # start_time = end_time - timedelta(hours=1) + # Temporary override: export data since start of day instead of last hour + start_time = end_time.replace(hour=0) + elif self.frequency == "daily": + end_time = now_utc.replace(hour=0, minute=0, second=0, microsecond=0) + start_time = end_time - timedelta(days=1) + elif self.frequency == "interval": + interval = timedelta(seconds=self.interval_seconds or 60) + end_time = now_utc + start_time = end_time - interval + else: + raise ValueError(f"Unsupported frequency: {self.frequency}") + return FocusTimeWindow( + start_time=start_time, + end_time=end_time, + frequency=self.frequency, + ) + + async def _serialize_and_upload( + self, frame: pl.DataFrame, window: FocusTimeWindow + ) -> None: + """Serialize the normalized frame and upload via destination.""" + payload = self._serializer.serialize(frame) + if not payload: + verbose_logger.debug("Focus export: serializer returned empty payload") + return + await self._destination.deliver( + content=payload, + time_window=window, + filename=self._build_filename(), + ) + + def _build_filename(self) -> str: + """Return the canonical file name for exports.""" + if not self._serializer.extension: + raise ValueError("Serializer must declare a file extension") + return f"usage.{self._serializer.extension}" + + @staticmethod + def _sum_column(frame: pl.DataFrame, column: str) -> float: + if frame.is_empty() or column not in frame.columns: + return 0.0 + value = frame.select(pl.col(column).sum().alias("sum")).row(0)[0] + if value is None: + return 0.0 + return float(value) + + @staticmethod + def _count_unique(frame: pl.DataFrame, column: str) -> int: + if frame.is_empty() or column not in frame.columns: + return 0 + value = frame.select(pl.col(column).n_unique().alias("unique")).row(0)[0] + if value is None: + return 0 + return int(value) diff --git a/litellm/integrations/focus/schema.py b/litellm/integrations/focus/schema.py index 766e29c0856..61ebbf2b9de 100644 --- a/litellm/integrations/focus/schema.py +++ b/litellm/integrations/focus/schema.py @@ -4,29 +4,52 @@ from __future__ import annotations import polars as pl - +# see: https://focus.finops.org/focus-specification/v1-2/ FOCUS_NORMALIZED_SCHEMA = pl.Schema( { - "usage_date": pl.Datetime(time_unit="us"), - "team_id": pl.String, - "team_alias": pl.String, - "user_id": pl.String, - "user_email": pl.String, - "api_key_alias": pl.String, - "model": pl.String, - "model_group": pl.String, - "custom_llm_provider": pl.String, - "prompt_tokens": pl.Int64, - "completion_tokens": pl.Int64, - "total_tokens": pl.Int64, - "spend": pl.Float64, - "cache_creation_input_tokens": pl.Int64, - "cache_read_input_tokens": pl.Int64, - "api_requests": pl.Int64, - "successful_requests": pl.Int64, - "failed_requests": pl.Int64, - "created_at": pl.Datetime(time_unit="us"), - "updated_at": pl.Datetime(time_unit="us"), + "BilledCost": pl.Float64, + "BillingAccountId": pl.String, + "BillingAccountName": pl.String, + "BillingCurrency": pl.String, + "BillingPeriodStart": pl.Datetime(time_unit="us"), + "BillingPeriodEnd": pl.Datetime(time_unit="us"), + "ChargeCategory": pl.String, + "ChargeClass": pl.String, + "ChargeDescription": pl.String, + "ChargeFrequency": pl.String, + "ChargePeriodStart": pl.Datetime(time_unit="us"), + "ChargePeriodEnd": pl.Datetime(time_unit="us"), + "CommitmentDiscountCategory": pl.String, + "CommitmentDiscountId": pl.String, + "CommitmentDiscountName": pl.String, + "CommitmentDiscountStatus": pl.String, + "CommitmentDiscountType": pl.String, + "ConsumedQuantity": pl.Float64, + "ConsumedUnit": pl.Float64, + "ContractedCost": pl.Float64, + "ContractedUnitPrice": pl.Float64, + "EffectiveCost": pl.Float64, + "InvoiceIssuerName": pl.String, + "ListCost": pl.Float64, + "ListUnitPrice": pl.Float64, + "PricingCategory": pl.String, + "PricingQuantity": pl.Float64, + "PricingUnit": pl.String, + "ProviderName": pl.String, + "PublisherName": pl.String, + "RegionId": pl.String, + "RegionName": pl.String, + "ResourceId": pl.String, + "ResourceName": pl.String, + "ResourceType": pl.String, + "ServiceCategory": pl.String, + "ServiceName": pl.String, + "SkuId": pl.String, + "SkuPriceId": pl.String, + "SubAccountId": pl.String, + "SubAccountName": pl.String, + "SubAccountType": pl.String, + "Tags": pl.Object, } ) diff --git a/litellm/integrations/focus/serializers/parquet.py b/litellm/integrations/focus/serializers/parquet.py index 3d42337e388..deac0e45390 100644 --- a/litellm/integrations/focus/serializers/parquet.py +++ b/litellm/integrations/focus/serializers/parquet.py @@ -2,15 +2,22 @@ from __future__ import annotations +import io + import polars as pl from .base import FocusSerializer class FocusParquetSerializer(FocusSerializer): - """Placeholder Parquet serializer implementation.""" + """Serialize normalized Focus frames to Parquet bytes.""" extension = "parquet" def serialize(self, frame: pl.DataFrame) -> bytes: - raise NotImplementedError + """Encode the provided frame as a parquet payload.""" + target = frame if not frame.is_empty() else pl.DataFrame(schema=frame.schema) + buffer = io.BytesIO() + target.write_parquet(buffer, compression="snappy") + print(target.head(5)) # debug + return buffer.getvalue() diff --git a/litellm/integrations/focus/transformer.py b/litellm/integrations/focus/transformer.py index 612082505fa..8957172da86 100644 --- a/litellm/integrations/focus/transformer.py +++ b/litellm/integrations/focus/transformer.py @@ -2,6 +2,8 @@ from __future__ import annotations +from datetime import timedelta + import polars as pl from .schema import FOCUS_NORMALIZED_SCHEMA @@ -14,4 +16,89 @@ class FocusTransformer: def transform(self, frame: pl.DataFrame) -> pl.DataFrame: """Return a normalized frame expected by downstream serializers.""" - raise NotImplementedError + if frame.is_empty(): + return pl.DataFrame(schema=self.schema) + + # derive period start/end from usage date + frame = frame.with_columns( + pl.col("date") + .cast(pl.Utf8) + .str.strptime(pl.Datetime(time_unit="us"), format="%Y-%m-%d", strict=False) + .alias("usage_date"), + ) + frame = frame.with_columns( + pl.col("usage_date").alias("ChargePeriodStart"), + (pl.col("usage_date") + timedelta(days=1)).alias("ChargePeriodEnd"), + ) + + def fmt(col): + return col.dt.strftime("%Y-%m-%dT%H:%M:%SZ") + + DEC = pl.Decimal(18, 6) + def dec(col): + return col.cast(DEC) + + none_str = pl.lit(None, dtype=pl.Utf8) + none_dec = pl.lit(None, dtype=pl.Decimal(18, 6)) + # zero_float = pl.lit(0.0, dtype=pl.Float64) + + return frame.select( + dec(pl.col("spend").fill_null(0.0)).alias("BilledCost"), + pl.col("api_key").cast(pl.String).alias("BillingAccountId"), + pl.col("api_key_alias").cast(pl.String).alias("BillingAccountName"), + pl.lit("API Key").alias("BillingAccountType"), + pl.lit("USD").alias("BillingCurrency"), + fmt(pl.col("ChargePeriodEnd")).alias("BillingPeriodEnd"), + fmt(pl.col("ChargePeriodStart")).alias("BillingPeriodStart"), + pl.lit("Usage").alias("ChargeCategory"), + none_str.alias("ChargeClass"), + pl.col("model").cast(pl.String).alias("ChargeDescription"), + pl.lit("Usage-Based").alias("ChargeFrequency"), + fmt(pl.col("ChargePeriodEnd")).alias("ChargePeriodEnd"), + fmt(pl.col("ChargePeriodStart")).alias("ChargePeriodStart"), + # pl.lit(None).alias("CommitmentDiscountCategory"), + # none_str.alias("CommitmentDiscountId"), + # none_str.alias("CommitmentDiscountName"), + # none_dec.alias("CommitmentDiscountQuantity"), + # none_str.alias("CommitmentDiscountUnit"), + # none_str.alias("CommitmentDiscountStatus"), + # none_str.alias("CommitmentDiscountType"), + dec(pl.lit(1.0)).alias("ConsumedQuantity"), + pl.lit("Requests").alias("ConsumedUnit"), + dec(pl.col("spend").fill_null(0.0)).alias("ContractedCost"), + none_str.alias("ContractedUnitPrice"), + dec(pl.col("spend").fill_null(0.0)).alias("EffectiveCost"), + pl.col("custom_llm_provider").cast(pl.String).alias("InvoiceIssuerName"), + # pl.lit("INVOICE-NOT-ISSUED").alias("InvoiceId"), + none_str.alias("InvoiceId"), + dec(pl.col("spend").fill_null(0.0)).alias("ListCost"), + none_dec.alias("ListUnitPrice"), + none_str.alias("AvailabilityZone"), + # none_str.alias("CapacityReservationId"), + # none_str.alias("CapacityReservationStatus"), + pl.lit("USD").alias("PricingCurrency"), + none_str.alias("PricingCategory"), + dec(pl.lit(1.0)).alias("PricingQuantity"), + none_dec.alias("PricingCurrencyContractedUnitPrice"), + dec(pl.col("spend").fill_null(0.0)).alias("PricingCurrencyEffectiveCost"), + none_dec.alias("PricingCurrencyListUnitPrice"), + pl.lit("Requests").alias("PricingUnit"), + pl.col("custom_llm_provider").cast(pl.String).alias("ProviderName"), + pl.col("custom_llm_provider").cast(pl.String).alias("PublisherName"), + none_str.alias("RegionId"), + none_str.alias("RegionName"), + pl.col("model").cast(pl.String).alias("ResourceId"), + pl.col("model").cast(pl.String).alias("ResourceName"), + pl.col("model").cast(pl.String).alias("ResourceType"), + pl.lit("AI and Machine Learning").alias("ServiceCategory"), + pl.lit("Generative AI").alias("ServiceSubcategory"), + pl.col("model_group").cast(pl.String).alias("ServiceName"), + # none_str.alias("SkuId"), + # none_str.alias("SkuPriceId"), + # none_str.alias("SkuMeter"), + # none_str.alias("SkuPriceDetails"), + pl.col("team_id").cast(pl.String).alias("SubAccountId"), + pl.col("team_alias").cast(pl.String).alias("SubAccountName"), + none_str.alias("SubAccountType"), + none_str.alias("Tags"), + ) diff --git a/litellm/litellm_core_utils/custom_logger_registry.py b/litellm/litellm_core_utils/custom_logger_registry.py index 47cbcb8aec9..a3c25ab65e9 100644 --- a/litellm/litellm_core_utils/custom_logger_registry.py +++ b/litellm/litellm_core_utils/custom_logger_registry.py @@ -18,6 +18,7 @@ from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLog from litellm.integrations.bitbucket import BitBucketPromptManager from litellm.integrations.braintrust_logging import BraintrustLogger from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger +from litellm.integrations.focus.focus_logger import FocusLogger from litellm.integrations.datadog.datadog import DataDogLogger from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger from litellm.integrations.deepeval import DeepEvalLogger @@ -93,6 +94,7 @@ class CustomLoggerRegistry: "bitbucket": BitBucketPromptManager, "gitlab": GitLabPromptManager, "cloudzero": CloudZeroLogger, + "focus": FocusLogger, "posthog": PostHogLogger, } diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 5448fe7c771..b5847722a66 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3756,6 +3756,15 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 cloudzero_logger = CloudZeroLogger() _in_memory_loggers.append(cloudzero_logger) return cloudzero_logger # type: ignore + elif logging_integration == "focus": + from litellm.integrations.focus.focus_logger import FocusLogger + + for callback in _in_memory_loggers: + if isinstance(callback, FocusLogger): + return callback # type: ignore + focus_logger = FocusLogger() + _in_memory_loggers.append(focus_logger) + return focus_logger # type: ignore elif logging_integration == "deepeval": for callback in _in_memory_loggers: if isinstance(callback, DeepEvalLogger): @@ -4076,6 +4085,12 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 for callback in _in_memory_loggers: if isinstance(callback, CloudZeroLogger): return callback + elif logging_integration == "focus": + from litellm.integrations.focus.focus_logger import FocusLogger + + for callback in _in_memory_loggers: + if isinstance(callback, FocusLogger): + return callback elif logging_integration == "deepeval": for callback in _in_memory_loggers: if isinstance(callback, DeepEvalLogger): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 06525e39133..6c5a2397204 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4681,8 +4681,9 @@ class ProxyStartupEvent: """ Initialize the spend tracking and other background jobs 1. CloudZero Background Job - 2. Prometheus Background Job - 3. Key Rotation Background Job + 2. Focus Background Job + 3. Prometheus Background Job + 4. Key Rotation Background Job Args: scheduler: The scheduler to add the background jobs to @@ -4691,11 +4692,17 @@ class ProxyStartupEvent: # CloudZero Background Job ######################################################## from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger + from litellm.integrations.focus.focus_logger import FocusLogger from litellm.proxy.spend_tracking.cloudzero_endpoints import is_cloudzero_setup if await is_cloudzero_setup(): await CloudZeroLogger.init_cloudzero_background_job(scheduler=scheduler) + ######################################################## + # Focus Background Job + ######################################################## + await FocusLogger.init_focus_export_background_job(scheduler=scheduler) + ######################################################## # Prometheus Background Job ######################################################## From 6c00f6f342ccd2b5e08001e63ac2b40e8ed75abe Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Thu, 8 Jan 2026 01:50:02 -0600 Subject: [PATCH 037/121] Add support to zai glm-4.7 model in Vertex (#18782) * Add support to zai glm-4.7 model in Vertex * Avoid failed on missing 'created' streaming chunk key --- litellm/__init__.py | 7 +++++- .../llms/openai/chat/gpt_transformation.py | 6 ++--- .../vertex_ai_partner_models/main.py | 3 +++ ...odel_prices_and_context_window_backup.json | 13 ++++++++++ model_prices_and_context_window.json | 13 ++++++++++ .../vertex_ai/test_vertex_ai_common_utils.py | 24 +++++++++++++++++++ 6 files changed, 62 insertions(+), 4 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 1bc690e561f..77e487fca24 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -486,6 +486,7 @@ vertex_mistral_models: Set = set() vertex_openai_models: Set = set() vertex_minimax_models: Set = set() vertex_moonshot_models: Set = set() +vertex_zai_models: Set = set() ai21_models: Set = set() ai21_chat_models: Set = set() nlp_cloud_models: Set = set() @@ -664,6 +665,9 @@ def add_known_models(): elif value.get("litellm_provider") == "vertex_ai-moonshot_models": key = key.replace("vertex_ai/", "") vertex_moonshot_models.add(key) + elif value.get("litellm_provider") == "vertex_ai-zai_models": + key = key.replace("vertex_ai/", "") + vertex_zai_models.add(key) elif value.get("litellm_provider") == "ai21": if value.get("mode") == "chat": ai21_chat_models.add(key) @@ -950,7 +954,8 @@ models_by_provider: dict = { | vertex_language_models | vertex_deepseek_models | vertex_minimax_models - | vertex_moonshot_models, + | vertex_moonshot_models + | vertex_zai_models, "ai21": ai21_models, "bedrock": bedrock_models | bedrock_converse_models, "petals": petals_models, diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 034ccae94ad..04a10bd7fbe 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -771,9 +771,9 @@ class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator): return ModelResponseStream( id=chunk["id"], object="chat.completion.chunk", - created=chunk["created"], - model=chunk["model"], - choices=chunk["choices"], + created=chunk.get("created"), + model=chunk.get("model"), + choices=chunk.get("choices", []), ) except Exception as e: raise e diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py index 712a06dece1..123d925f7c1 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py @@ -40,6 +40,7 @@ class PartnerModelPrefixes(str, Enum): GPT_OSS_PREFIX = "openai/gpt-oss-" MINIMAX_PREFIX = "minimaxai/" MOONSHOT_PREFIX = "moonshotai/" + ZAI_PREFIX = "zai-org/" class VertexAIPartnerModels(VertexBase): @@ -66,6 +67,7 @@ class VertexAIPartnerModels(VertexBase): or model.startswith(PartnerModelPrefixes.GPT_OSS_PREFIX) or model.startswith(PartnerModelPrefixes.MINIMAX_PREFIX) or model.startswith(PartnerModelPrefixes.MOONSHOT_PREFIX) + or model.startswith(PartnerModelPrefixes.ZAI_PREFIX) ): return True return False @@ -79,6 +81,7 @@ class VertexAIPartnerModels(VertexBase): PartnerModelPrefixes.GPT_OSS_PREFIX, PartnerModelPrefixes.MINIMAX_PREFIX, PartnerModelPrefixes.MOONSHOT_PREFIX, + PartnerModelPrefixes.ZAI_PREFIX, ] if any(provider in model for provider in OPENAI_LIKE_VERTEX_PROVIDERS): return True diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index fb00f636409..73579db75cd 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -28345,6 +28345,19 @@ "supports_tool_choice": true, "supports_web_search": true }, + "vertex_ai/zai-org/glm-4.7-maas": { + "input_cost_per_token": 3e-07, + "litellm_provider": "vertex_ai-zai_models", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "vertex_ai/mistral-medium-3": { "input_cost_per_token": 4e-07, "litellm_provider": "vertex_ai-mistral_models", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index fb00f636409..73579db75cd 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -28345,6 +28345,19 @@ "supports_tool_choice": true, "supports_web_search": true }, + "vertex_ai/zai-org/glm-4.7-maas": { + "input_cost_per_token": 3e-07, + "litellm_provider": "vertex_ai-zai_models", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "vertex_ai/mistral-medium-3": { "input_cost_per_token": 4e-07, "litellm_provider": "vertex_ai-mistral_models", diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index 591b33911dc..b5637db3e52 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -1175,6 +1175,30 @@ def test_vertex_ai_moonshot_uses_openai_handler(): ) +def test_vertex_ai_zai_uses_openai_handler(): + """ + Ensure ZAI partner models re-use the OpenAI-format handler. + """ + from litellm.llms.vertex_ai.vertex_ai_partner_models.main import ( + VertexAIPartnerModels, + ) + + assert VertexAIPartnerModels.should_use_openai_handler( + "zai-org/glm-4.7-maas" + ) + + +def test_vertex_ai_zai_is_partner_model(): + """ + Ensure ZAI models are detected as Vertex AI partner models. + """ + from litellm.llms.vertex_ai.vertex_ai_partner_models.main import ( + VertexAIPartnerModels, + ) + + assert VertexAIPartnerModels.is_vertex_partner_model("zai-org/glm-4.7-maas") + + def test_build_vertex_schema_empty_properties(): """ Test _build_vertex_schema handles empty properties objects correctly. From 271ee0959b5c3d64e2d2cb2da92234749b0eefa7 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Thu, 8 Jan 2026 16:53:09 +0900 Subject: [PATCH 038/121] test: focus --- litellm/integrations/focus/focus_logger.py | 4 +- litellm/integrations/focus/schema.py | 7 -- .../integrations/focus/serializers/parquet.py | 1 - litellm/integrations/focus/transformer.py | 15 --- .../integrations/focus/test_database.py | 74 +++++++++++++ .../integrations/focus/test_s3_destination.py | 100 ++++++++++++++++++ 6 files changed, 175 insertions(+), 26 deletions(-) create mode 100644 tests/test_litellm/integrations/focus/test_database.py create mode 100644 tests/test_litellm/integrations/focus/test_s3_destination.py diff --git a/litellm/integrations/focus/focus_logger.py b/litellm/integrations/focus/focus_logger.py index 5eccf441bea..28629f64d38 100644 --- a/litellm/integrations/focus/focus_logger.py +++ b/litellm/integrations/focus/focus_logger.py @@ -228,9 +228,7 @@ class FocusLogger(CustomLogger): now_utc = now.astimezone(timezone.utc) if self.frequency == "hourly": end_time = now_utc.replace(minute=0, second=0, microsecond=0) - # start_time = end_time - timedelta(hours=1) - # Temporary override: export data since start of day instead of last hour - start_time = end_time.replace(hour=0) + start_time = end_time - timedelta(hours=1) elif self.frequency == "daily": end_time = now_utc.replace(hour=0, minute=0, second=0, microsecond=0) start_time = end_time - timedelta(days=1) diff --git a/litellm/integrations/focus/schema.py b/litellm/integrations/focus/schema.py index 61ebbf2b9de..6d2e1dc83c1 100644 --- a/litellm/integrations/focus/schema.py +++ b/litellm/integrations/focus/schema.py @@ -19,11 +19,6 @@ FOCUS_NORMALIZED_SCHEMA = pl.Schema( "ChargeFrequency": pl.String, "ChargePeriodStart": pl.Datetime(time_unit="us"), "ChargePeriodEnd": pl.Datetime(time_unit="us"), - "CommitmentDiscountCategory": pl.String, - "CommitmentDiscountId": pl.String, - "CommitmentDiscountName": pl.String, - "CommitmentDiscountStatus": pl.String, - "CommitmentDiscountType": pl.String, "ConsumedQuantity": pl.Float64, "ConsumedUnit": pl.Float64, "ContractedCost": pl.Float64, @@ -44,8 +39,6 @@ FOCUS_NORMALIZED_SCHEMA = pl.Schema( "ResourceType": pl.String, "ServiceCategory": pl.String, "ServiceName": pl.String, - "SkuId": pl.String, - "SkuPriceId": pl.String, "SubAccountId": pl.String, "SubAccountName": pl.String, "SubAccountType": pl.String, diff --git a/litellm/integrations/focus/serializers/parquet.py b/litellm/integrations/focus/serializers/parquet.py index deac0e45390..6b3dde5903d 100644 --- a/litellm/integrations/focus/serializers/parquet.py +++ b/litellm/integrations/focus/serializers/parquet.py @@ -19,5 +19,4 @@ class FocusParquetSerializer(FocusSerializer): target = frame if not frame.is_empty() else pl.DataFrame(schema=frame.schema) buffer = io.BytesIO() target.write_parquet(buffer, compression="snappy") - print(target.head(5)) # debug return buffer.getvalue() diff --git a/litellm/integrations/focus/transformer.py b/litellm/integrations/focus/transformer.py index 8957172da86..a98ea21b1bb 100644 --- a/litellm/integrations/focus/transformer.py +++ b/litellm/integrations/focus/transformer.py @@ -40,7 +40,6 @@ class FocusTransformer: none_str = pl.lit(None, dtype=pl.Utf8) none_dec = pl.lit(None, dtype=pl.Decimal(18, 6)) - # zero_float = pl.lit(0.0, dtype=pl.Float64) return frame.select( dec(pl.col("spend").fill_null(0.0)).alias("BilledCost"), @@ -56,26 +55,16 @@ class FocusTransformer: pl.lit("Usage-Based").alias("ChargeFrequency"), fmt(pl.col("ChargePeriodEnd")).alias("ChargePeriodEnd"), fmt(pl.col("ChargePeriodStart")).alias("ChargePeriodStart"), - # pl.lit(None).alias("CommitmentDiscountCategory"), - # none_str.alias("CommitmentDiscountId"), - # none_str.alias("CommitmentDiscountName"), - # none_dec.alias("CommitmentDiscountQuantity"), - # none_str.alias("CommitmentDiscountUnit"), - # none_str.alias("CommitmentDiscountStatus"), - # none_str.alias("CommitmentDiscountType"), dec(pl.lit(1.0)).alias("ConsumedQuantity"), pl.lit("Requests").alias("ConsumedUnit"), dec(pl.col("spend").fill_null(0.0)).alias("ContractedCost"), none_str.alias("ContractedUnitPrice"), dec(pl.col("spend").fill_null(0.0)).alias("EffectiveCost"), pl.col("custom_llm_provider").cast(pl.String).alias("InvoiceIssuerName"), - # pl.lit("INVOICE-NOT-ISSUED").alias("InvoiceId"), none_str.alias("InvoiceId"), dec(pl.col("spend").fill_null(0.0)).alias("ListCost"), none_dec.alias("ListUnitPrice"), none_str.alias("AvailabilityZone"), - # none_str.alias("CapacityReservationId"), - # none_str.alias("CapacityReservationStatus"), pl.lit("USD").alias("PricingCurrency"), none_str.alias("PricingCategory"), dec(pl.lit(1.0)).alias("PricingQuantity"), @@ -93,10 +82,6 @@ class FocusTransformer: pl.lit("AI and Machine Learning").alias("ServiceCategory"), pl.lit("Generative AI").alias("ServiceSubcategory"), pl.col("model_group").cast(pl.String).alias("ServiceName"), - # none_str.alias("SkuId"), - # none_str.alias("SkuPriceId"), - # none_str.alias("SkuMeter"), - # none_str.alias("SkuPriceDetails"), pl.col("team_id").cast(pl.String).alias("SubAccountId"), pl.col("team_alias").cast(pl.String).alias("SubAccountName"), none_str.alias("SubAccountType"), diff --git a/tests/test_litellm/integrations/focus/test_database.py b/tests/test_litellm/integrations/focus/test_database.py new file mode 100644 index 00000000000..5ee98cc9dd0 --- /dev/null +++ b/tests/test_litellm/integrations/focus/test_database.py @@ -0,0 +1,74 @@ +"""Tests for FocusLiteLLMDatabase query construction.""" + +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from litellm.integrations.focus.database import FocusLiteLLMDatabase + + +def _setup_db(monkeypatch: pytest.MonkeyPatch, query_return): + """Create a database instance with a stubbed prisma client.""" + query_mock = AsyncMock(return_value=query_return) + mock_client = SimpleNamespace(db=SimpleNamespace(query_raw=query_mock)) + db = FocusLiteLLMDatabase() + monkeypatch.setattr(db, "_ensure_prisma_client", lambda: mock_client) + return db, query_mock + + +@pytest.mark.asyncio +async def test_should_parameterize_filters_and_limit(monkeypatch: pytest.MonkeyPatch): + start = datetime(2024, 1, 1, tzinfo=timezone.utc) + end = datetime(2024, 1, 2, tzinfo=timezone.utc) + db, query_mock = _setup_db(monkeypatch, []) + + await db.get_usage_data(limit=25, start_time_utc=start, end_time_utc=end) + + query_text, *params = query_mock.await_args.args + assert "dus.updated_at >= $1::timestamptz" in query_text + assert "dus.updated_at <= $2::timestamptz" in query_text + assert "LIMIT $3" in query_text + assert params == [start, end, 25] + + +@pytest.mark.asyncio +async def test_should_execute_without_filters(monkeypatch: pytest.MonkeyPatch): + row = { + "id": 1, + "user_id": "user", + "date": datetime(2024, 1, 1, tzinfo=timezone.utc), + } + db, query_mock = _setup_db(monkeypatch, [row]) + + result = await db.get_usage_data() + + query_text, *params = query_mock.await_args.args + assert "WHERE" not in query_text + assert "LIMIT $" not in query_text + assert params == [] + assert result.height == 1 + assert result["id"][0] == 1 + + +@pytest.mark.asyncio +async def test_should_accept_string_timestamps(monkeypatch: pytest.MonkeyPatch): + db, query_mock = _setup_db(monkeypatch, []) + + start = "2024-02-01T00:00:00+00:00" + end = "2024-02-02T00:00:00+00:00" + await db.get_usage_data(start_time_utc=start, end_time_utc=end) + + _, *params = query_mock.await_args.args + assert params == [start, end] + + +@pytest.mark.asyncio +async def test_should_reject_invalid_limit(monkeypatch: pytest.MonkeyPatch): + db, query_mock = _setup_db(monkeypatch, []) + + with pytest.raises(ValueError): + await db.get_usage_data(limit="invalid") + + assert query_mock.await_count == 0 diff --git a/tests/test_litellm/integrations/focus/test_s3_destination.py b/tests/test_litellm/integrations/focus/test_s3_destination.py new file mode 100644 index 00000000000..f915b2c56a3 --- /dev/null +++ b/tests/test_litellm/integrations/focus/test_s3_destination.py @@ -0,0 +1,100 @@ +"""Tests for FocusS3Destination behavior.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from types import SimpleNamespace +from typing import Any, Dict + +import pytest + +import litellm.integrations.focus.destinations.s3_destination as s3_module +from litellm.integrations.focus.destinations.base import FocusTimeWindow +from litellm.integrations.focus.destinations.s3_destination import FocusS3Destination + + +def _window(freq: str = "hourly", hour: int = 5) -> FocusTimeWindow: + start = datetime(2024, 1, 2, hour, tzinfo=timezone.utc) + end = start.replace(hour=hour + 1) + return FocusTimeWindow(start_time=start, end_time=end, frequency=freq) + + +def test_should_require_bucket_name(): + with pytest.raises(ValueError): + FocusS3Destination(prefix="focus", config={}) + + +def test_should_build_hourly_object_key(): + dest = FocusS3Destination(prefix="exports/", config={"bucket_name": "bucket"}) + key = dest._build_object_key( + time_window=_window(freq="hourly", hour=3), filename="data.snappy" + ) + assert key == "exports/date=2024-01-02/hour=03/data.snappy" + + +def test_should_build_daily_key_without_hour_segment(): + dest = FocusS3Destination(prefix="", config={"bucket_name": "bucket"}) + key = dest._build_object_key( + time_window=_window(freq="daily", hour=0), filename="daily.parquet" + ) + assert key == "date=2024-01-02/daily.parquet" + + +@pytest.mark.asyncio +async def test_should_dispatch_upload_via_thread(monkeypatch: pytest.MonkeyPatch): + dest = FocusS3Destination(prefix="focus", config={"bucket_name": "bucket"}) + captured: Dict[str, Any] = {} + + async def fake_to_thread(func, *args, **kwargs): # type: ignore[override] + captured["func"] = func + captured["args"] = args + captured["kwargs"] = kwargs + + monkeypatch.setattr(s3_module.asyncio, "to_thread", fake_to_thread) + + window = _window(freq="hourly", hour=1) + await dest.deliver(content=b"payload", time_window=window, filename="file.bin") + + assert captured["func"] == dest._upload + assert captured["args"][0] == b"payload" + assert captured["args"][1].endswith("/file.bin") + + +def test_should_upload_with_configured_client(monkeypatch: pytest.MonkeyPatch): + config = { + "bucket_name": "bucket", + "region_name": "us-east-2", + "endpoint_url": "http://localhost:4566", + "aws_access_key_id": "key", + "aws_secret_access_key": "secret", + "aws_session_token": "token", + } + dest = FocusS3Destination(prefix="focus", config=config) + captured: Dict[str, Any] = {} + + def fake_client(service: str, **kwargs): + assert service == "s3" + captured["client_kwargs"] = kwargs + + def put_object(**put_kwargs): + captured["put_kwargs"] = put_kwargs + + return SimpleNamespace(put_object=put_object) + + monkeypatch.setattr(s3_module.boto3, "client", fake_client) + + dest._upload(content=b"payload", object_key="path/file.bin") + + assert captured["client_kwargs"] == { + "region_name": "us-east-2", + "endpoint_url": "http://localhost:4566", + "aws_access_key_id": "key", + "aws_secret_access_key": "secret", + "aws_session_token": "token", + } + assert captured["put_kwargs"] == { + "Bucket": "bucket", + "Key": "path/file.bin", + "Body": b"payload", + "ContentType": "application/octet-stream", + } From fb00b38fcdd81677895893251fc023ff7d91b8e3 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Thu, 8 Jan 2026 16:56:51 +0900 Subject: [PATCH 039/121] chore: lint --- .../focus/destinations/factory.py | 8 +- litellm/integrations/focus/focus_logger.py | 18 +++-- litellm/integrations/focus/transformer.py | 1 + litellm/proxy/proxy_server.py | 76 +++++++++---------- 4 files changed, 53 insertions(+), 50 deletions(-) diff --git a/litellm/integrations/focus/destinations/factory.py b/litellm/integrations/focus/destinations/factory.py index 6f5dc5dc06a..cb7696a11de 100644 --- a/litellm/integrations/focus/destinations/factory.py +++ b/litellm/integrations/focus/destinations/factory.py @@ -26,7 +26,9 @@ class FocusDestinationFactory: ) if provider_lower == "s3": return FocusS3Destination(prefix=prefix, config=normalized_config) - raise NotImplementedError(f"Provider '{provider}' not supported for Focus export") + raise NotImplementedError( + f"Provider '{provider}' not supported for Focus export" + ) @staticmethod def _resolve_config( @@ -50,9 +52,7 @@ class FocusDestinationFactory: or os.getenv("FOCUS_S3_SESSION_TOKEN"), } if not resolved.get("bucket_name"): - raise ValueError( - "FOCUS_S3_BUCKET_NAME must be provided for S3 exports" - ) + raise ValueError("FOCUS_S3_BUCKET_NAME must be provided for S3 exports") return {k: v for k, v in resolved.items() if v is not None} raise NotImplementedError( f"Provider '{provider}' not supported for Focus export configuration" diff --git a/litellm/integrations/focus/focus_logger.py b/litellm/integrations/focus/focus_logger.py index 28629f64d38..5f47590f0b8 100644 --- a/litellm/integrations/focus/focus_logger.py +++ b/litellm/integrations/focus/focus_logger.py @@ -14,7 +14,6 @@ from litellm.integrations.custom_logger import CustomLogger from .database import FocusLiteLLMDatabase from .destinations import ( - FocusDestination, FocusDestinationFactory, FocusTimeWindow, ) @@ -50,9 +49,7 @@ class FocusLogger(CustomLogger): self.export_format = ( export_format or os.getenv("FOCUS_FORMAT") or "parquet" ).lower() - self.frequency = ( - frequency or os.getenv("FOCUS_FREQUENCY") or "hourly" - ).lower() + self.frequency = (frequency or os.getenv("FOCUS_FREQUENCY") or "hourly").lower() self.cron_offset_minute = ( cron_offset_minute if cron_offset_minute is not None @@ -90,7 +87,9 @@ class FocusLogger(CustomLogger): ) -> None: """Public hook to trigger export immediately.""" if bool(start_time_utc) ^ bool(end_time_utc): - raise ValueError("start_time_utc and end_time_utc must be provided together") + raise ValueError( + "start_time_utc and end_time_utc must be provided together" + ) if start_time_utc and end_time_utc: window = FocusTimeWindow( @@ -160,7 +159,6 @@ class FocusLogger(CustomLogger): scheduler: AsyncIOScheduler, ) -> None: """Register the export cron/interval job with the provided scheduler.""" - from litellm.integrations.custom_logger import CustomLogger focus_loggers: List[ CustomLogger @@ -168,7 +166,9 @@ class FocusLogger(CustomLogger): callback_type=FocusLogger ) if not focus_loggers: - verbose_logger.debug("No Focus export logger registered; skipping scheduler") + verbose_logger.debug( + "No Focus export logger registered; skipping scheduler" + ) return focus_logger = cast(FocusLogger, focus_loggers[0]) @@ -218,7 +218,9 @@ class FocusLogger(CustomLogger): normalized = self._transformer.transform(data) if normalized.is_empty(): - verbose_logger.debug("Focus export: normalized data empty for window %s", window) + verbose_logger.debug( + "Focus export: normalized data empty for window %s", window + ) return await self._serialize_and_upload(normalized, window) diff --git a/litellm/integrations/focus/transformer.py b/litellm/integrations/focus/transformer.py index a98ea21b1bb..cac12b7be14 100644 --- a/litellm/integrations/focus/transformer.py +++ b/litellm/integrations/focus/transformer.py @@ -35,6 +35,7 @@ class FocusTransformer: return col.dt.strftime("%Y-%m-%dT%H:%M:%SZ") DEC = pl.Decimal(18, 6) + def dec(col): return col.cast(DEC) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6c5a2397204..7535a79dfac 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -533,9 +533,9 @@ except ImportError: server_root_path = os.getenv("SERVER_ROOT_PATH", "") _license_check = LicenseCheck() premium_user: bool = _license_check.is_premium() -premium_user_data: Optional["EnterpriseLicenseData"] = ( - _license_check.airgapped_license_data -) +premium_user_data: Optional[ + "EnterpriseLicenseData" +] = _license_check.airgapped_license_data global_max_parallel_request_retries_env: Optional[str] = os.getenv( "LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES" ) @@ -1083,9 +1083,7 @@ try: # In non-root Docker, we restructure in /var/lib/litellm/ui. try: _restructure_ui_html_files(ui_path) - verbose_proxy_logger.info( - f"Restructured UI directory: {ui_path}" - ) + verbose_proxy_logger.info(f"Restructured UI directory: {ui_path}") except PermissionError as e: verbose_proxy_logger.exception( f"Permission error while restructuring UI directory {ui_path}: {e}" @@ -1171,9 +1169,9 @@ master_key: Optional[str] = None config_agents: Optional[List[AgentConfig]] = None otel_logging = False prisma_client: Optional[PrismaClient] = None -shared_aiohttp_session: Optional["ClientSession"] = ( - None # Global shared session for connection reuse -) +shared_aiohttp_session: Optional[ + "ClientSession" +] = None # Global shared session for connection reuse user_api_key_cache = DualCache( default_in_memory_ttl=UserAPIKeyCacheTTLEnum.in_memory_cache_ttl.value ) @@ -1181,9 +1179,9 @@ model_max_budget_limiter = _PROXY_VirtualKeyModelMaxBudgetLimiter( dual_cache=user_api_key_cache ) litellm.logging_callback_manager.add_litellm_callback(model_max_budget_limiter) -redis_usage_cache: Optional[RedisCache] = ( - None # redis cache used for tracking spend, tpm/rpm limits -) +redis_usage_cache: Optional[ + RedisCache +] = None # redis cache used for tracking spend, tpm/rpm limits polling_via_cache_enabled: Union[Literal["all"], List[str], bool] = False polling_cache_ttl: int = 3600 # Default 1 hour TTL for polling cache user_custom_auth = None @@ -1522,9 +1520,9 @@ async def update_cache( # noqa: PLR0915 _id = "team_id:{}".format(team_id) try: # Fetch the existing cost for the given user - existing_spend_obj: Optional[LiteLLM_TeamTable] = ( - await user_api_key_cache.async_get_cache(key=_id) - ) + existing_spend_obj: Optional[ + LiteLLM_TeamTable + ] = await user_api_key_cache.async_get_cache(key=_id) if existing_spend_obj is None: # do nothing if team not in api key cache return @@ -1876,7 +1874,6 @@ class ProxyConfig: "environment_variables" in config_to_save and config_to_save["environment_variables"] ): - # decrypt the environment_variables - in case a caller function has already encrypted the environment_variables decrypted_env_vars = self._decrypt_and_set_db_env_variables( environment_variables=config_to_save["environment_variables"], @@ -2794,21 +2791,21 @@ class ProxyConfig: verbose_proxy_logger.debug(f"_alerting_callbacks: {general_settings}") if _alerting_callbacks is None: return - + # Ensure proxy_logging_obj.alerting is set for all alerting types _alerting_value = general_settings.get("alerting", None) - verbose_proxy_logger.debug(f"_load_alerting_settings: Calling update_values with alerting={_alerting_value}") + verbose_proxy_logger.debug( + f"_load_alerting_settings: Calling update_values with alerting={_alerting_value}" + ) proxy_logging_obj.update_values( alerting=_alerting_value, alerting_threshold=general_settings.get("alerting_threshold", 600), alert_types=general_settings.get("alert_types", None), - alert_to_webhook_url=general_settings.get( - "alert_to_webhook_url", None - ), + alert_to_webhook_url=general_settings.get("alert_to_webhook_url", None), alerting_args=general_settings.get("alerting_args", None), redis_cache=redis_usage_cache, ) - + for _alert in _alerting_callbacks: if _alert == "slack": # [OLD] v0 implementation - already handled by update_values above @@ -3279,7 +3276,7 @@ class ProxyConfig: proxy_logging_obj: ProxyLogging """ _general_settings = config_data.get("general_settings", {}) - + if _general_settings is not None and "alerting" in _general_settings: if ( general_settings is not None @@ -3294,7 +3291,8 @@ class ProxyConfig: _merged_alerting = list(_yaml_alerting.union(_db_alerting)) # Preserve order: YAML values first, then DB values _merged_alerting = list(general_settings["alerting"]) + [ - item for item in _general_settings["alerting"] + item + for item in _general_settings["alerting"] if item not in general_settings["alerting"] ] verbose_proxy_logger.debug( @@ -3605,7 +3603,6 @@ class ProxyConfig: await self._init_vector_stores_in_db(prisma_client=prisma_client) if self._should_load_db_object(object_type="vector_store_indexes"): - await self._init_vector_store_indexes_in_db(prisma_client=prisma_client) if self._should_load_db_object(object_type="mcp"): @@ -3804,10 +3801,10 @@ class ProxyConfig: ) try: - guardrails_in_db: List[Guardrail] = ( - await GuardrailRegistry.get_all_guardrails_from_db( - prisma_client=prisma_client - ) + guardrails_in_db: List[ + Guardrail + ] = await GuardrailRegistry.get_all_guardrails_from_db( + prisma_client=prisma_client ) verbose_proxy_logger.debug( "guardrails from the DB %s", str(guardrails_in_db) @@ -4134,9 +4131,9 @@ async def initialize( # noqa: PLR0915 user_api_base = api_base dynamic_config[user_model]["api_base"] = api_base if api_version: - os.environ["AZURE_API_VERSION"] = ( - api_version # set this for azure - litellm can read this from the env - ) + os.environ[ + "AZURE_API_VERSION" + ] = api_version # set this for azure - litellm can read this from the env if max_tokens: # model-specific param dynamic_config[user_model]["max_tokens"] = max_tokens if temperature: # model-specific param @@ -4654,10 +4651,14 @@ class ProxyStartupEvent: replace_existing=True, misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, ) - verbose_proxy_logger.info("Responses cost check job scheduled successfully") + verbose_proxy_logger.info( + "Responses cost check job scheduled successfully" + ) except Exception as e: - verbose_proxy_logger.debug(f"Failed to setup responses cost checking: {e}") + verbose_proxy_logger.debug( + f"Failed to setup responses cost checking: {e}" + ) verbose_proxy_logger.debug( "Checking responses cost for LiteLLM Managed Files is an Enterprise Feature. Skipping..." ) @@ -5944,7 +5945,6 @@ async def realtime_websocket_endpoint( ), user_api_key_dict=Depends(user_api_key_auth_websocket), ): - await websocket.accept() # Only use explicit parameters, not all query params @@ -9532,9 +9532,9 @@ async def get_config_list( hasattr(sub_field_info, "description") and sub_field_info.description is not None ): - nested_fields[idx].field_description = ( - sub_field_info.description - ) + nested_fields[ + idx + ].field_description = sub_field_info.description idx += 1 _stored_in_db = None From 93cf2d4848867b5ec85549132946693d261a24f4 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Thu, 8 Jan 2026 17:19:02 +0900 Subject: [PATCH 040/121] fix: mypy --- litellm/integrations/focus/database.py | 4 +- litellm/integrations/focus/focus_logger.py | 5 +- litellm/integrations/focus/schema.py | 77 +++++++++++----------- 3 files changed, 45 insertions(+), 41 deletions(-) diff --git a/litellm/integrations/focus/database.py b/litellm/integrations/focus/database.py index 402a1ec4b4e..298254670eb 100644 --- a/litellm/integrations/focus/database.py +++ b/litellm/integrations/focus/database.py @@ -31,8 +31,8 @@ class FocusLiteLLMDatabase: """Return usage data for the requested window.""" client = self._ensure_prisma_client() - where_clauses = [] - query_params = [] + where_clauses: list[str] = [] + query_params: list[Any] = [] placeholder_index = 1 if start_time_utc: where_clauses.append(f"dus.updated_at >= ${placeholder_index}::timestamptz") diff --git a/litellm/integrations/focus/focus_logger.py b/litellm/integrations/focus/focus_logger.py index 5f47590f0b8..1589f030fa5 100644 --- a/litellm/integrations/focus/focus_logger.py +++ b/litellm/integrations/focus/focus_logger.py @@ -61,7 +61,10 @@ class FocusLogger(CustomLogger): else os.getenv("FOCUS_INTERVAL_SECONDS") ) self.interval_seconds = int(raw_interval) if raw_interval is not None else None - self.prefix = prefix or os.getenv("FOCUS_PREFIX", "focus_exports") + env_prefix = os.getenv("FOCUS_PREFIX") + self.prefix: str = ( + prefix if prefix is not None else (env_prefix if env_prefix else "focus_exports") + ) self._destination = FocusDestinationFactory.create( provider=self.provider, diff --git a/litellm/integrations/focus/schema.py b/litellm/integrations/focus/schema.py index 6d2e1dc83c1..ac2f33dad0a 100644 --- a/litellm/integrations/focus/schema.py +++ b/litellm/integrations/focus/schema.py @@ -6,44 +6,45 @@ import polars as pl # see: https://focus.finops.org/focus-specification/v1-2/ FOCUS_NORMALIZED_SCHEMA = pl.Schema( - { - "BilledCost": pl.Float64, - "BillingAccountId": pl.String, - "BillingAccountName": pl.String, - "BillingCurrency": pl.String, - "BillingPeriodStart": pl.Datetime(time_unit="us"), - "BillingPeriodEnd": pl.Datetime(time_unit="us"), - "ChargeCategory": pl.String, - "ChargeClass": pl.String, - "ChargeDescription": pl.String, - "ChargeFrequency": pl.String, - "ChargePeriodStart": pl.Datetime(time_unit="us"), - "ChargePeriodEnd": pl.Datetime(time_unit="us"), - "ConsumedQuantity": pl.Float64, - "ConsumedUnit": pl.Float64, - "ContractedCost": pl.Float64, - "ContractedUnitPrice": pl.Float64, - "EffectiveCost": pl.Float64, - "InvoiceIssuerName": pl.String, - "ListCost": pl.Float64, - "ListUnitPrice": pl.Float64, - "PricingCategory": pl.String, - "PricingQuantity": pl.Float64, - "PricingUnit": pl.String, - "ProviderName": pl.String, - "PublisherName": pl.String, - "RegionId": pl.String, - "RegionName": pl.String, - "ResourceId": pl.String, - "ResourceName": pl.String, - "ResourceType": pl.String, - "ServiceCategory": pl.String, - "ServiceName": pl.String, - "SubAccountId": pl.String, - "SubAccountName": pl.String, - "SubAccountType": pl.String, - "Tags": pl.Object, - } + [ + ("BilledCost", pl.Decimal(18, 6)), + ("BillingAccountId", pl.String), + ("BillingAccountName", pl.String), + ("BillingCurrency", pl.String), + ("BillingPeriodStart", pl.Datetime(time_unit="us")), + ("BillingPeriodEnd", pl.Datetime(time_unit="us")), + ("ChargeCategory", pl.String), + ("ChargeClass", pl.String), + ("ChargeDescription", pl.String), + ("ChargeFrequency", pl.String), + ("ChargePeriodStart", pl.Datetime(time_unit="us")), + ("ChargePeriodEnd", pl.Datetime(time_unit="us")), + ("ConsumedQuantity", pl.Decimal(18, 6)), + ("ConsumedUnit", pl.String), + ("ContractedCost", pl.Decimal(18, 6)), + ("ContractedUnitPrice", pl.Decimal(18, 6)), + ("EffectiveCost", pl.Decimal(18, 6)), + ("InvoiceIssuerName", pl.String), + ("ListCost", pl.Decimal(18, 6)), + ("ListUnitPrice", pl.Decimal(18, 6)), + ("PricingCategory", pl.String), + ("PricingQuantity", pl.Decimal(18, 6)), + ("PricingUnit", pl.String), + ("ProviderName", pl.String), + ("PublisherName", pl.String), + ("RegionId", pl.String), + ("RegionName", pl.String), + ("ResourceId", pl.String), + ("ResourceName", pl.String), + ("ResourceType", pl.String), + ("ServiceCategory", pl.String), + ("ServiceSubcategory", pl.String), + ("ServiceName", pl.String), + ("SubAccountId", pl.String), + ("SubAccountName", pl.String), + ("SubAccountType", pl.String), + ("Tags", pl.Object), + ] ) __all__ = ["FOCUS_NORMALIZED_SCHEMA"] From 790c80070f8e0ebec8ce156ac64a1cecf8d68582 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Thu, 8 Jan 2026 17:24:51 +0900 Subject: [PATCH 041/121] docs: add FOCUS env --- docs/my-website/docs/proxy/config_settings.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 5772fbaa487..12995e4a5f8 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -577,6 +577,18 @@ router_settings: | FIREWORKS_AI_56_B_MOE | Size parameter for Fireworks AI 56B MOE model. Default is 56 | FIREWORKS_AI_80_B | Size parameter for Fireworks AI 80B model. Default is 80 | FIREWORKS_AI_176_B_MOE | Size parameter for Fireworks AI 176B MOE model. Default is 176 +| FOCUS_PROVIDER | Destination provider for Focus exports (e.g., `s3`). Defaults to `s3`. +| FOCUS_FORMAT | Output format for Focus exports. Defaults to `parquet`. +| FOCUS_FREQUENCY | Frequency for scheduled Focus exports (`hourly`, `daily`, or `interval`). Defaults to `hourly`. +| FOCUS_CRON_OFFSET | Minute offset used when scheduling hourly/daily Focus exports. Defaults to `5` minutes. +| FOCUS_INTERVAL_SECONDS | Interval (in seconds) for Focus exports when `frequency` is `interval`. +| FOCUS_PREFIX | Object key prefix (or folder) used when uploading Focus export files. Defaults to `focus_exports`. +| FOCUS_S3_BUCKET_NAME | S3 bucket to upload Focus export files when using the S3 destination. +| FOCUS_S3_REGION_NAME | AWS region for the Focus export S3 bucket. +| FOCUS_S3_ENDPOINT_URL | Custom endpoint for the Focus export S3 client (optional; useful for S3-compatible storage). +| FOCUS_S3_ACCESS_KEY | AWS access key ID used by the Focus export S3 client. +| FOCUS_S3_SECRET_KEY | AWS secret access key used by the Focus export S3 client. +| FOCUS_S3_SESSION_TOKEN | AWS session token used by the Focus export S3 client (optional). | FUNCTION_DEFINITION_TOKEN_COUNT | Token count for function definitions. Default is 9 | GALILEO_BASE_URL | Base URL for Galileo platform | GALILEO_PASSWORD | Password for Galileo authentication From ad501048f391ee473fe1fa2e6b69095a076f4630 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 15:22:14 +0530 Subject: [PATCH 042/121] Add support for Vertex AI API keys --- docs/my-website/docs/providers/vertex.md | 45 +++++- litellm/llms/gemini/common_utils.py | 9 ++ litellm/llms/vertex_ai/vertex_llm_base.py | 40 +++-- litellm/main.py | 10 +- .../llms/vertex_ai/test_vertex_llm_base.py | 137 ++++++++++++++++++ 5 files changed, 226 insertions(+), 15 deletions(-) diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 33ebf535d29..f46608aa57c 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -35,6 +35,8 @@ import json # !gcloud auth application-default login - run this to add vertex credentials to your env ## OR ## file_path = 'path/to/vertex_ai_service_account.json' +## OR ## +export VERTEXAI_API_KEY="your-api-key" # Load the JSON file with open(file_path, 'r') as file: @@ -47,7 +49,7 @@ vertex_credentials_json = json.dumps(vertex_credentials) response = completion( model="vertex_ai/gemini-2.5-pro", messages=[{ "content": "Hello, how are you?","role": "user"}], - vertex_credentials=vertex_credentials_json + vertex_credentials=vertex_credentials_json # Can remove this is added VERTEXAI_API_KEY in env ) ``` @@ -1329,15 +1331,41 @@ Here's how to use Vertex AI with the LiteLLM Proxy Server ## Authentication - vertex_project, vertex_location, etc. +LiteLLM supports two authentication methods for Vertex AI: + +1. **API Key Authentication** (Recommended for getting started) +2. **Service Account Credentials** (Recommended for production) + Set your vertex credentials via: - dynamic params OR - env vars +### **Authentication Method 1: -### **Dynamic Params** +The simplest way to authenticate with Vertex AI. You can set: +- `api_key` (str) - Your Vertex AI API key -You can set: +**Environment Variables:** +```bash +export VERTEXAI_API_KEY="your-api-key" +``` + +**Or pass as parameters:** +```python +from litellm import completion + +response = completion( + model="vertex_ai/gemini-2.0-flash-exp", + messages=[{"role": "user", "content": "Hello!"}], + api_key="your-vertex-api-key", + +) +``` + +### **Authentication Method 2: Service Account Credentials** + +For production environments with fine-grained access control. You can set: - `vertex_credentials` (str) - can be a json string or filepath to your vertex ai service account.json - `vertex_location` (str) - place where vertex model is deployed (us-central1, asia-southeast1, etc.). Some models support the global location, please see [Vertex AI documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations#supported_models) - `vertex_project` Optional[str] - use if vertex project different from the one in vertex_credentials @@ -1392,7 +1420,16 @@ model_list: ### **Environment Variables** -You can set: +#### For API Key Authentication: + +- `VERTEXAI_API_KEY` or `VERTEX_API_KEY` - Your Vertex AI API key + +```bash +export VERTEXAI_API_KEY="your-vertex-api-key" +``` + +#### For Service Account Authentication: + - `GOOGLE_APPLICATION_CREDENTIALS` - store the filepath for your service_account.json in here (used by vertex sdk directly). - VERTEXAI_LOCATION - place where vertex model is deployed (us-central1, asia-southeast1, etc.) - VERTEXAI_PROJECT - Optional[str] - use if vertex project different from the one in vertex_credentials diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index e53829d3329..30c5b4f17c5 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -150,6 +150,15 @@ def get_api_key_from_env() -> Optional[str]: return get_secret_str("GOOGLE_API_KEY") or get_secret_str("GEMINI_API_KEY") +def get_vertex_api_key_from_env() -> Optional[str]: + """ + Get API key from environment for Vertex AI. + Checks VERTEXAI_API_KEY and VERTEX_API_KEY environment variables. + This allows using Vertex AI with API keys instead of service account credentials. + """ + return get_secret_str("VERTEXAI_API_KEY") or get_secret_str("VERTEX_API_KEY") + + class GoogleAIStudioTokenCounter(BaseTokenCounter): """Token counter implementation for Google AI Studio provider.""" def should_use_token_counting_api( diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 826f151df35..a3606ff9deb 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -388,6 +388,10 @@ class VertexBase: Internal function. Returns the token and url for the call. Handles logic if it's google ai studio vs. vertex ai. + + For Vertex AI: + - If gemini_api_key is provided, use API key authentication (x-goog-api-key header) + - Otherwise, use service account credentials (OAuth2 Bearer token) Returns token, url @@ -400,7 +404,7 @@ class VertexBase: stream=stream, gemini_api_key=gemini_api_key, ) - auth_header = None # this field is not used for gemin + auth_header = None # this field is not used for gemini else: vertex_location = self.get_vertex_region( vertex_region=vertex_location, @@ -409,14 +413,32 @@ class VertexBase: ### SET RUNTIME ENDPOINT ### version = "v1beta1" if should_use_v1beta1_features is True else "v1" - url, endpoint = _get_vertex_url( - mode=mode, - model=model, - stream=stream, - vertex_project=vertex_project, - vertex_location=vertex_location, - vertex_api_version=version, - ) + + # Check if using API key authentication for Vertex AI + if gemini_api_key and not vertex_credentials: + # When using API key with Vertex AI, use the Google AI Studio endpoint + # This is because Vertex AI API keys work with generativelanguage.googleapis.com + verbose_logger.debug( + f"Using Vertex AI API key authentication for model: {model} - routing to Google AI Studio endpoint" + ) + url, endpoint = _get_gemini_url( + mode=mode, + model=model, + stream=stream, + gemini_api_key=gemini_api_key, + ) + # API key is already included in the URL by _get_gemini_url + auth_header = None + else: + # Use OAuth2 Bearer token authentication (traditional Vertex AI) + url, endpoint = _get_vertex_url( + mode=mode, + model=model, + stream=stream, + vertex_project=vertex_project, + vertex_location=vertex_location, + vertex_api_version=version, + ) return self._check_custom_proxy( api_base=api_base, diff --git a/litellm/main.py b/litellm/main.py index e8a8b504d96..0264d0a7a71 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -189,7 +189,7 @@ from .llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from .llms.custom_llm import CustomLLM, custom_chat_llm_router from .llms.databricks.embed.handler import DatabricksEmbeddingHandler from .llms.deprecated_providers import aleph_alpha, palm -from .llms.gemini.common_utils import get_api_key_from_env +from .llms.gemini.common_utils import get_api_key_from_env, get_vertex_api_key_from_env from .llms.groq.chat.handler import GroqChatCompletion from .llms.heroku.chat.transformation import HerokuChatConfig from .llms.huggingface.embedding.handler import HuggingFaceEmbedding @@ -3230,6 +3230,12 @@ def completion( # type: ignore # noqa: PLR0915 or get_secret("VERTEXAI_CREDENTIALS") ) + vertex_api_key = ( + api_key + or get_vertex_api_key_from_env() + or litellm.api_key + ) + api_base = api_base or litellm.api_base or get_secret("VERTEXAI_API_BASE") new_params = safe_deep_copy(optional_params or {}) @@ -3271,7 +3277,7 @@ def completion( # type: ignore # noqa: PLR0915 vertex_location=vertex_ai_location, vertex_project=vertex_ai_project, vertex_credentials=vertex_credentials, - gemini_api_key=None, + gemini_api_key=vertex_api_key, # Support for Vertex AI API Key logging_obj=logging, acompletion=acompletion, timeout=timeout, diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py b/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py index 389c8446135..80d65991acb 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py @@ -13,6 +13,7 @@ sys.path.insert( import litellm from litellm.llms.vertex_ai.vertex_llm_base import VertexBase +from litellm.llms.vertex_ai.common_utils import _get_gemini_url def run_sync(coro): @@ -1048,3 +1049,139 @@ class TestVertexBase: MockCredentials.from_info.assert_called_once_with(json_obj) mock_creds.with_scopes.assert_called_once_with(scopes) assert result == "scoped_creds" + + def test_get_token_and_url_with_api_key(self): + """Test that API key authentication routes to Google AI Studio endpoint""" + vertex_base = VertexBase() + + # Test with API key and no credentials - should use Google AI Studio endpoint + auth_header, url = vertex_base._get_token_and_url( + model="gemini-2.0-flash-exp", + auth_header=None, + gemini_api_key="test-api-key-123", + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=None, # No service account credentials + stream=False, + custom_llm_provider="vertex_ai", + api_base=None, + should_use_v1beta1_features=False, + mode="chat", + ) + + # Should route to Google AI Studio endpoint + assert "generativelanguage.googleapis.com" in url + assert "gemini-2.0-flash-exp" in url + assert "key=test-api-key-123" in url + assert auth_header is None # API key is in URL, not header + + def test_get_token_and_url_with_credentials(self): + """Test that service account credentials route to Vertex AI endpoint""" + vertex_base = VertexBase() + + mock_creds = MagicMock() + mock_creds.token = "mock-bearer-token" + mock_creds.expired = False + + with patch.object( + vertex_base, "_ensure_access_token", return_value=("mock-bearer-token", "test-project") + ): + # Test with credentials - should use Vertex AI endpoint + auth_header, url = vertex_base._get_token_and_url( + model="gemini-2.0-flash-exp", + auth_header="mock-bearer-token", + gemini_api_key=None, + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials={"type": "service_account"}, + stream=False, + custom_llm_provider="vertex_ai", + api_base=None, + should_use_v1beta1_features=False, + mode="chat", + ) + + # Should route to Vertex AI endpoint + assert "aiplatform.googleapis.com" in url + assert "projects/test-project" in url + assert "locations/us-central1" in url + assert auth_header == "mock-bearer-token" + + def test_get_token_and_url_api_key_with_streaming(self): + """Test API key authentication with streaming enabled""" + vertex_base = VertexBase() + + auth_header, url = vertex_base._get_token_and_url( + model="gemini-2.0-flash-exp", + auth_header=None, + gemini_api_key="test-api-key-456", + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=None, + stream=True, # Streaming enabled + custom_llm_provider="vertex_ai", + api_base=None, + should_use_v1beta1_features=False, + mode="chat", + ) + + # Should route to Google AI Studio endpoint with streaming + assert "generativelanguage.googleapis.com" in url + assert "streamGenerateContent" in url + assert "key=test-api-key-456" in url + assert "alt=sse" in url + assert auth_header is None + + def test_get_token_and_url_api_key_priority(self): + """Test that credentials take priority over API key when both are provided""" + vertex_base = VertexBase() + + # When both API key and credentials are provided, credentials take priority + mock_creds = MagicMock() + mock_creds.token = "mock-bearer-token" + mock_creds.expired = False + + with patch.object( + vertex_base, "_ensure_access_token", return_value=("mock-bearer-token", "test-project") + ): + auth_header, url = vertex_base._get_token_and_url( + model="gemini-2.0-flash-exp", + auth_header="mock-bearer-token", + gemini_api_key="test-api-key-789", + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials={"type": "service_account"}, # Credentials provided + stream=False, + custom_llm_provider="vertex_ai", + api_base=None, + should_use_v1beta1_features=False, + mode="chat", + ) + + # Should use Vertex AI endpoint with Bearer token (credentials take priority) + assert "aiplatform.googleapis.com" in url + assert auth_header == "mock-bearer-token" + + def test_get_token_and_url_with_embedding_mode(self): + """Test API key authentication with embedding mode""" + vertex_base = VertexBase() + + auth_header, url = vertex_base._get_token_and_url( + model="text-embedding-004", + auth_header=None, + gemini_api_key="test-embedding-key", + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=None, + stream=False, + custom_llm_provider="vertex_ai", + api_base=None, + should_use_v1beta1_features=False, + mode="embedding", + ) + + # Should route to Google AI Studio endpoint for embeddings + assert "generativelanguage.googleapis.com" in url + assert "embedContent" in url + assert "key=test-embedding-key" in url + assert auth_header is None \ No newline at end of file From f52cc32a4c845c1432aced38e1bda69fb68de671 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 15:38:28 +0530 Subject: [PATCH 043/121] fix mypy error --- litellm/llms/deepinfra/chat/transformation.py | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/litellm/llms/deepinfra/chat/transformation.py b/litellm/llms/deepinfra/chat/transformation.py index 490597a0e60..5198260a24b 100644 --- a/litellm/llms/deepinfra/chat/transformation.py +++ b/litellm/llms/deepinfra/chat/transformation.py @@ -1,5 +1,5 @@ import json -from typing import List, Optional, Tuple, Union +from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload import litellm from litellm.constants import MIN_NON_ZERO_TEMPERATURE @@ -155,23 +155,40 @@ class DeepInfraConfig(OpenAIGPTConfig): return messages + @overload + def _transform_messages( + self, messages: List[AllMessageValues], model: str, is_async: Literal[True] + ) -> Coroutine[Any, Any, List[AllMessageValues]]: + ... + + @overload + def _transform_messages( + self, messages: List[AllMessageValues], model: str, is_async: Literal[False] = False + ) -> List[AllMessageValues]: + ... + def _transform_messages( self, messages: List[AllMessageValues], model: str, is_async: bool = False - ): + ) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]: """ Transform messages for DeepInfra compatibility. Handles both sync and async transformations. """ - # First apply parent class transformations - parent_result = super()._transform_messages(messages=messages, model=model, is_async=is_async) - if is_async: - # If parent returns a coroutine, we need to await it and then apply our transformations + # For async case, create an async function that awaits parent and applies our transformation async def _async_transform(): + # Call parent with is_async=True (literal) for async case + parent_result = super(DeepInfraConfig, self)._transform_messages( + messages=messages, model=model, is_async=cast(Literal[True], True) + ) transformed_messages = await parent_result return self._transform_tool_message_content(transformed_messages) return _async_transform() else: + # Call parent with is_async=False (literal) for sync case + parent_result = super()._transform_messages( + messages=messages, model=model, is_async=cast(Literal[False], False) + ) # For sync case, parent_result is already the transformed messages return self._transform_tool_message_content(parent_result) From 501c2d522bec0143f3d77265fd5a1d4df2f7f67e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 15:56:55 +0530 Subject: [PATCH 044/121] Fix: mypy errors --- .../litellm_responses_transformation/transformation.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 585454f944b..af8185aa215 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -31,6 +31,7 @@ from litellm.llms.base_llm.bridges.completion_transformation import ( CompletionTransformationBridge, ) from litellm.types.llms.openai import ( + ChatCompletionAnnotation, ChatCompletionToolParamFunctionChunk, Reasoning, ResponsesAPIOptionalRequestParams, @@ -778,7 +779,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): @staticmethod def _convert_annotations_to_chat_format( annotations: Optional[List[Any]], - ) -> Optional[List[Dict[str, Any]]]: + ) -> Optional[List["ChatCompletionAnnotation"]]: """ Convert annotations from Responses API to Chat Completions format. @@ -788,7 +789,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if not annotations: return None - result: List[Dict[str, Any]] = [] + result: List[ChatCompletionAnnotation] = [] for annotation in annotations: try: # Convert Pydantic models to dicts (handles both v1 and v2) @@ -803,7 +804,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): verbose_logger.debug(f"Skipping unsupported annotation type: {type(annotation)}") continue - result.append(annotation_dict) + result.append(annotation_dict) # type: ignore except Exception as e: # Skip malformed annotations verbose_logger.debug(f"Skipping malformed annotation: {annotation}, error: {e}") From b6e011309acb34a06e4a5acb9e26c6816860809f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 16:07:00 +0530 Subject: [PATCH 045/121] Fix: test_spend_logs_payload_success_log_with_router --- .../proxy/spend_tracking/test_spend_management_endpoints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 56bba39e6c3..57ec019e420 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -1242,7 +1242,7 @@ class TestSpendLogsPayload: "model": "claude-3-7-sonnet-20250219", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-3-7-sonnet-20250219", "model_map_value": {"key": "claude-3-7-sonnet-20250219", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-3-7-sonnet-20250219", "model_map_value": {"key": "claude-3-7-sonnet-20250219", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, @@ -1334,7 +1334,7 @@ class TestSpendLogsPayload: "model": "claude-3-7-sonnet-20250219", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-3-7-sonnet-20250219", "model_map_value": {"key": "claude-3-7-sonnet-20250219", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-3-7-sonnet-20250219", "model_map_value": {"key": "claude-3-7-sonnet-20250219", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, From 9d5eb60ff175bb47ae6cd8d40da432f7145399d7 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 16:37:12 +0530 Subject: [PATCH 046/121] Fix: test_text_format_to_text_conversion - properly mock handler to avoid API calls --- .../responses/test_text_format_conversion.py | 107 +++++++++++++----- 1 file changed, 77 insertions(+), 30 deletions(-) diff --git a/tests/test_litellm/responses/test_text_format_conversion.py b/tests/test_litellm/responses/test_text_format_conversion.py index 645f0f2e148..c7a79d9c461 100644 --- a/tests/test_litellm/responses/test_text_format_conversion.py +++ b/tests/test_litellm/responses/test_text_format_conversion.py @@ -34,7 +34,7 @@ class TestTextFormatConversion: Test that when text_format parameter is passed to litellm.aresponses, it gets converted to text parameter in the raw API call to OpenAI. """ - from unittest.mock import AsyncMock, patch + from unittest.mock import AsyncMock, MagicMock, patch class TestResponse(BaseModel): """Test Pydantic model for structured output""" @@ -42,20 +42,8 @@ class TestTextFormatConversion: answer: str confidence: float - class MockResponse: - """Mock response class for testing""" - - def __init__(self, json_data, status_code): - self._json_data = json_data - self.status_code = status_code - self.text = json.dumps(json_data) - self.headers = {} - - def json(self): - return self._json_data - # Mock response from OpenAI - mock_response = { + mock_response_data = { "id": "resp_123", "object": "response", "created_at": 1741476542, @@ -101,13 +89,74 @@ class TestTextFormatConversion: base_completion_call_args = self.get_base_completion_call_args() - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - new_callable=AsyncMock, - ) as mock_post: - # Configure the mock to return our response - mock_post.return_value = MockResponse(mock_response, 200) + # Mock the response_api_handler function to capture the request + captured_request = {} + def mock_handler( + model, + input, + responses_api_provider_config, + response_api_optional_request_params, + custom_llm_provider, + litellm_params, + logging_obj, + extra_headers=None, + extra_body=None, + timeout=None, + client=None, + fake_stream=False, + litellm_metadata=None, + shared_session=None, + _is_async=False, + ): + # Capture the request parameters + captured_request["model"] = model + captured_request["input"] = input + captured_request["params"] = response_api_optional_request_params + + # Return a mock ResponsesAPIResponse wrapped in a coroutine if async + async def async_response(): + return ResponsesAPIResponse( + id="resp_123", + object="response", + created_at=1741476542, + status="completed", + model="gpt-4o", + output=mock_response_data["output"], + usage=ResponseAPIUsage( + input_tokens=10, + output_tokens=20, + total_tokens=30, + ), + text=mock_response_data.get("text"), + error=None, + incomplete_details=None, + ) + + if _is_async: + return async_response() + else: + return ResponsesAPIResponse( + id="resp_123", + object="response", + created_at=1741476542, + status="completed", + model="gpt-4o", + output=mock_response_data["output"], + usage=ResponseAPIUsage( + input_tokens=10, + output_tokens=20, + total_tokens=30, + ), + text=mock_response_data.get("text"), + error=None, + incomplete_details=None, + ) + + with patch( + "litellm.responses.main.base_llm_http_handler.response_api_handler", + new=mock_handler, + ): litellm._turn_on_debug() litellm.set_verbose = True @@ -118,21 +167,19 @@ class TestTextFormatConversion: **base_completion_call_args, ) - # Verify the request was made correctly - mock_post.assert_called_once() - request_body = mock_post.call_args.kwargs["json"] - print("Request body:", json.dumps(request_body, indent=4)) + # Verify the captured request + print("Captured request:", json.dumps(captured_request, indent=4, default=str)) # Validate that text_format was converted to text parameter assert ( - "text" in request_body - ), "text parameter should be present in request body" + "text" in captured_request["params"] + ), "text parameter should be present in request params" assert ( - "text_format" not in request_body - ), "text_format should not be in request body" + "text_format" not in captured_request["params"] + ), "text_format should not be in request params" # Validate the text parameter structure - text_param = request_body["text"] + text_param = captured_request["params"]["text"] assert "format" in text_param, "text parameter should have format field" assert ( text_param["format"]["type"] == "json_schema" @@ -156,7 +203,7 @@ class TestTextFormatConversion: ), "schema should have confidence property" # Validate other request parameters - assert request_body["input"] == "What is the capital of France?" + assert captured_request["input"] == "What is the capital of France?" # Validate the response print("Response:", json.dumps(response, indent=4, default=str)) From 0a9861c3ec80a1429fa6cfbcaeb6fcc93cde56cf Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 16:44:35 +0530 Subject: [PATCH 047/121] Fix: test_token_counter_lazy_imports --- tests/test_litellm/test_lazy_imports.py | 90 +++++++++++++++++++------ 1 file changed, 70 insertions(+), 20 deletions(-) diff --git a/tests/test_litellm/test_lazy_imports.py b/tests/test_litellm/test_lazy_imports.py index 660933efac5..48d78c0b01b 100644 --- a/tests/test_litellm/test_lazy_imports.py +++ b/tests/test_litellm/test_lazy_imports.py @@ -42,34 +42,45 @@ from litellm._lazy_imports import ( def _clear_names_from_globals(names: tuple): """Clear all names from litellm globals.""" + # Get the actual globals dict, not a copy + litellm_globals = sys.modules["litellm"].__dict__ for name in names: - if name in litellm.__dict__: - del litellm.__dict__[name] + if name in litellm_globals: + del litellm_globals[name] def _clear_names_from_utils_globals(names: tuple): """Clear all names from litellm.utils globals.""" + # Get the actual globals dict, not a copy + utils_globals = sys.modules["litellm.utils"].__dict__ for name in names: - if name in litellm.utils.__dict__: - del litellm.utils.__dict__[name] + if name in utils_globals: + del utils_globals[name] def _verify_only_requested_name_imported(name: str, all_names: tuple): """Verify that only the requested name is in globals, not the others.""" + # Get the actual globals dict, not a copy + litellm_globals = sys.modules["litellm"].__dict__ for other_name in all_names: if other_name != name: - assert other_name not in litellm.__dict__, f"{other_name} should not be imported when importing {name}" + assert other_name not in litellm_globals, f"{other_name} should not be imported when importing {name}" def _verify_only_requested_name_imported_in_utils(name: str, all_names: tuple): """Verify that only the requested name is in utils globals, not the others.""" + # Get the actual globals dict, not a copy + utils_globals = sys.modules["litellm.utils"].__dict__ for other_name in all_names: if other_name != name: - assert other_name not in litellm.utils.__dict__, f"{other_name} should not be imported when importing {name}" + assert other_name not in utils_globals, f"{other_name} should not be imported when importing {name}" def test_cost_calculator_lazy_imports(): """Test that all cost calculator functions can be lazy imported.""" + # Get the actual globals dict, not a copy + litellm_globals = sys.modules["litellm"].__dict__ + # Test each name individually - only that name should be imported for name in COST_CALCULATOR_NAMES: # Clear all names before importing just one @@ -78,7 +89,7 @@ def test_cost_calculator_lazy_imports(): func = _lazy_import_cost_calculator(name) assert func is not None assert callable(func) - assert name in litellm.__dict__ + assert name in litellm_globals # Verify only the requested name is in globals, not the others _verify_only_requested_name_imported(name, COST_CALCULATOR_NAMES) @@ -86,6 +97,9 @@ def test_cost_calculator_lazy_imports(): def test_litellm_logging_lazy_imports(): """Test that all litellm_logging items can be lazy imported.""" + # Get the actual globals dict, not a copy + litellm_globals = sys.modules["litellm"].__dict__ + # Test each name individually - only that name should be imported for name in LITELLM_LOGGING_NAMES: # Clear all names before importing just one @@ -93,7 +107,7 @@ def test_litellm_logging_lazy_imports(): item = _lazy_import_litellm_logging(name) assert item is not None - assert name in litellm.__dict__ + assert name in litellm_globals # Verify only the requested name is in globals, not the others _verify_only_requested_name_imported(name, LITELLM_LOGGING_NAMES) @@ -101,6 +115,9 @@ def test_litellm_logging_lazy_imports(): def test_utils_lazy_imports(): """Test that all utils functions can be lazy imported.""" + # Get the actual globals dict, not a copy + litellm_globals = sys.modules["litellm"].__dict__ + # Test each name individually - only that name should be imported for name in UTILS_NAMES: # Clear all names before importing just one @@ -108,7 +125,7 @@ def test_utils_lazy_imports(): attr = _lazy_import_utils(name) assert attr is not None - assert name in litellm.__dict__ + assert name in litellm_globals # Verify only the requested name is in globals, not the others _verify_only_requested_name_imported(name, UTILS_NAMES) @@ -116,6 +133,9 @@ def test_utils_lazy_imports(): def test_caching_lazy_imports(): """Test that all caching classes can be lazy imported.""" + # Get the actual globals dict, not a copy + litellm_globals = sys.modules["litellm"].__dict__ + # Test each name individually - only that name should be imported for name in CACHING_NAMES: # Clear all names before importing just one @@ -123,7 +143,7 @@ def test_caching_lazy_imports(): cls = _lazy_import_caching(name) assert cls is not None - assert name in litellm.__dict__ + assert name in litellm_globals # Verify only the requested name is in globals, not the others _verify_only_requested_name_imported(name, CACHING_NAMES) @@ -131,71 +151,89 @@ def test_caching_lazy_imports(): def test_token_counter_lazy_imports(): """Test that token counter utilities can be lazy imported.""" + # Get the actual globals dict, not a copy + litellm_globals = sys.modules["litellm"].__dict__ + for name in TOKEN_COUNTER_NAMES: _clear_names_from_globals(TOKEN_COUNTER_NAMES) func = _lazy_import_token_counter(name) assert func is not None - assert name in litellm.__dict__ + assert name in litellm_globals _verify_only_requested_name_imported(name, TOKEN_COUNTER_NAMES) def test_bedrock_types_lazy_imports(): """Test that Bedrock type aliases can be lazy imported.""" + # Get the actual globals dict, not a copy + litellm_globals = sys.modules["litellm"].__dict__ + for name in BEDROCK_TYPES_NAMES: _clear_names_from_globals(BEDROCK_TYPES_NAMES) alias = _lazy_import_bedrock_types(name) assert alias is not None - assert name in litellm.__dict__ + assert name in litellm_globals _verify_only_requested_name_imported(name, BEDROCK_TYPES_NAMES) def test_types_utils_lazy_imports(): """Test that common types.utils symbols can be lazy imported.""" + # Get the actual globals dict, not a copy + litellm_globals = sys.modules["litellm"].__dict__ + for name in TYPES_UTILS_NAMES: _clear_names_from_globals(TYPES_UTILS_NAMES) obj = _lazy_import_types_utils(name) assert obj is not None - assert name in litellm.__dict__ + assert name in litellm_globals _verify_only_requested_name_imported(name, TYPES_UTILS_NAMES) def test_llm_client_cache_lazy_imports(): """Test that LLM client cache class and singleton can be lazy imported.""" + # Get the actual globals dict, not a copy + litellm_globals = sys.modules["litellm"].__dict__ + for name in LLM_CLIENT_CACHE_NAMES: _clear_names_from_globals(LLM_CLIENT_CACHE_NAMES) obj = _lazy_import_llm_client_cache(name) assert obj is not None - assert name in litellm.__dict__ + assert name in litellm_globals _verify_only_requested_name_imported(name, LLM_CLIENT_CACHE_NAMES) def test_http_handler_lazy_imports(): """Test that HTTP handler singletons can be lazy imported.""" + # Get the actual globals dict, not a copy + litellm_globals = sys.modules["litellm"].__dict__ + for name in HTTP_HANDLER_NAMES: _clear_names_from_globals(HTTP_HANDLER_NAMES) handler = _lazy_import_http_handlers(name) assert handler is not None - assert name in litellm.__dict__ + assert name in litellm_globals _verify_only_requested_name_imported(name, HTTP_HANDLER_NAMES) def test_dotprompt_lazy_imports(): """Test that dotprompt globals can be lazy imported.""" + # Get the actual globals dict, not a copy + litellm_globals = sys.modules["litellm"].__dict__ + for name in DOTPROMPT_NAMES: _clear_names_from_globals(DOTPROMPT_NAMES) obj = _lazy_import_dotprompt(name) - assert name in litellm.__dict__ + assert name in litellm_globals # Only the setter must be callable; others may be None by default if name == "set_global_prompt_directory": @@ -245,12 +283,15 @@ def test_unknown_attribute_raises_error(): def test_llm_config_lazy_imports(): """Test that LLM config classes can be lazy imported.""" + # Get the actual globals dict, not a copy + litellm_globals = sys.modules["litellm"].__dict__ + for name in LLM_CONFIG_NAMES: _clear_names_from_globals(LLM_CONFIG_NAMES) obj = _lazy_import_llm_configs(name) assert obj is not None - assert name in litellm.__dict__ + assert name in litellm_globals # Config classes should be classes/types assert isinstance(obj, type), f"{name} should be a class" @@ -259,12 +300,15 @@ def test_llm_config_lazy_imports(): def test_types_lazy_imports(): """Test that type classes can be lazy imported.""" + # Get the actual globals dict, not a copy + litellm_globals = sys.modules["litellm"].__dict__ + for name in TYPES_NAMES: _clear_names_from_globals(TYPES_NAMES) obj = _lazy_import_types(name) assert obj is not None - assert name in litellm.__dict__ + assert name in litellm_globals # Type classes should be classes/types assert isinstance(obj, type), f"{name} should be a class" @@ -273,25 +317,31 @@ def test_types_lazy_imports(): def test_llm_provider_logic_lazy_imports(): """Test that LLM provider logic functions can be lazy imported.""" + # Get the actual globals dict, not a copy + litellm_globals = sys.modules["litellm"].__dict__ + for name in LLM_PROVIDER_LOGIC_NAMES: _clear_names_from_globals(LLM_PROVIDER_LOGIC_NAMES) func = _lazy_import_llm_provider_logic(name) assert func is not None assert callable(func) - assert name in litellm.__dict__ + assert name in litellm_globals _verify_only_requested_name_imported(name, LLM_PROVIDER_LOGIC_NAMES) def test_utils_module_lazy_imports(): """Test that utils module attributes can be lazy imported.""" + # Get the actual globals dict, not a copy + utils_globals = sys.modules["litellm.utils"].__dict__ + for name in UTILS_MODULE_NAMES: _clear_names_from_utils_globals(UTILS_MODULE_NAMES) obj = _lazy_import_utils_module(name) assert obj is not None - assert name in litellm.utils.__dict__ + assert name in utils_globals _verify_only_requested_name_imported_in_utils(name, UTILS_MODULE_NAMES) From 62d860ea7d93ca0770e30233ce65104d4cca3444 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 17:01:44 +0530 Subject: [PATCH 048/121] fix: litellm/tests/test_litellm/test_responses_id_security.py --- .../test_responses_id_security.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/test_litellm/test_responses_id_security.py b/tests/test_litellm/test_responses_id_security.py index 6b04479326e..2addf504f7c 100644 --- a/tests/test_litellm/test_responses_id_security.py +++ b/tests/test_litellm/test_responses_id_security.py @@ -42,8 +42,11 @@ class TestIsEncryptedResponseId: def test_is_encrypted_response_id_valid(self, responses_id_security): """Test that a properly encrypted response ID is identified correctly""" - with patch( - "litellm.proxy.hooks.responses_id_security.decrypt_value_helper" + # Patch at the module level where it's imported + import litellm.proxy.hooks.responses_id_security as responses_module + + with patch.object( + responses_module, "decrypt_value_helper" ) as mock_decrypt: mock_decrypt.return_value = f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}response_id:resp_123;user_id:user-456" @@ -56,8 +59,11 @@ class TestIsEncryptedResponseId: def test_is_encrypted_response_id_invalid(self, responses_id_security): """Test that an unencrypted response ID returns False""" - with patch( - "litellm.proxy.hooks.responses_id_security.decrypt_value_helper" + # Patch at the module level where it's imported + import litellm.proxy.hooks.responses_id_security as responses_module + + with patch.object( + responses_module, "decrypt_value_helper" ) as mock_decrypt: mock_decrypt.return_value = None @@ -71,8 +77,11 @@ class TestDecryptResponseId: def test_decrypt_response_id_valid(self, responses_id_security): """Test decrypting a valid encrypted response ID""" - with patch( - "litellm.proxy.hooks.responses_id_security.decrypt_value_helper" + # Patch at the module level where it's imported + import litellm.proxy.hooks.responses_id_security as responses_module + + with patch.object( + responses_module, "decrypt_value_helper" ) as mock_decrypt: mock_decrypt.return_value = f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}response_id:resp_original_123;user_id:user-456;team_id:team-789" @@ -86,8 +95,11 @@ class TestDecryptResponseId: def test_decrypt_response_id_no_encryption(self, responses_id_security): """Test decrypting a non-encrypted response ID""" - with patch( - "litellm.proxy.hooks.responses_id_security.decrypt_value_helper" + # Patch at the module level where it's imported + import litellm.proxy.hooks.responses_id_security as responses_module + + with patch.object( + responses_module, "decrypt_value_helper" ) as mock_decrypt: mock_decrypt.return_value = None From 2df9d8d4db88181f4d8476f1c2e683cfbf4c34af Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 17:08:24 +0530 Subject: [PATCH 049/121] fix: test_get_request_tags_from_metadata_and_litellm_metadata --- litellm/litellm_core_utils/litellm_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 5448fe7c771..ab55022f8ce 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4800,7 +4800,7 @@ class StandardLoggingPayloadSetup: """ Extract additional header tags for spend tracking based on config. """ - extra_headers: List[str] = litellm.extra_spend_tag_headers or [] + extra_headers: List[str] = getattr(litellm, "extra_spend_tag_headers", None) or [] if not extra_headers: return None From f5b5073649c56c93062984cc038447492338e00f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 17:20:38 +0530 Subject: [PATCH 050/121] fix: test_video_status_async --- tests/test_litellm/test_video_generation.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/test_video_generation.py b/tests/test_litellm/test_video_generation.py index 87012f05155..4f486e07f55 100644 --- a/tests/test_litellm/test_video_generation.py +++ b/tests/test_litellm/test_video_generation.py @@ -2,7 +2,7 @@ import asyncio import json import os import sys -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -98,7 +98,10 @@ class TestVideoGeneration: ) with patch('litellm.videos.main.base_llm_http_handler') as mock_handler: - mock_handler.video_generation_handler.return_value = mock_response + # Mock the async_video_generation_handler to return the mock_response + mock_handler.async_video_generation_handler = AsyncMock(return_value=mock_response) + # Mock video_generation_handler to return the coroutine from async_video_generation_handler + mock_handler.video_generation_handler.side_effect = lambda **kwargs: mock_handler.async_video_generation_handler(**kwargs) import asyncio @@ -507,7 +510,10 @@ class TestVideoGeneration: ) with patch('litellm.videos.main.base_llm_http_handler') as mock_handler: - mock_handler.video_status_handler.return_value = mock_response + # Mock the async_video_status_handler to return the mock_response + mock_handler.async_video_status_handler = AsyncMock(return_value=mock_response) + # Mock video_status_handler to return the coroutine from async_video_status_handler + mock_handler.video_status_handler.side_effect = lambda **kwargs: mock_handler.async_video_status_handler(**kwargs) import asyncio From 5241b27beafc0eb3b8a14dc0721e34c1c503b330 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 17:30:58 +0530 Subject: [PATCH 051/121] fix: test_video_status_basic --- tests/test_litellm/test_video_generation.py | 225 +++++++++----------- 1 file changed, 104 insertions(+), 121 deletions(-) diff --git a/tests/test_litellm/test_video_generation.py b/tests/test_litellm/test_video_generation.py index 4f486e07f55..73bfa71d20b 100644 --- a/tests/test_litellm/test_video_generation.py +++ b/tests/test_litellm/test_video_generation.py @@ -18,6 +18,7 @@ from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.gemini.videos.transformation import GeminiVideoConfig from litellm.llms.openai.videos.transformation import OpenAIVideoConfig from litellm.types.videos.main import VideoObject, VideoResponse +from litellm.videos import main as videos_main from litellm.videos.main import ( avideo_generation, avideo_status, @@ -31,32 +32,29 @@ class TestVideoGeneration: def test_video_generation_basic(self): """Test basic video generation functionality.""" - # Mock the video generation response - mock_response = VideoObject( - id="video_123", - object="video", - status="queued", - created_at=1712697600, + # Use mock_response parameter for reliable testing + response = video_generation( + prompt="Show them running around the room", model="sora-2", + seconds="8", size="720x1280", - seconds="8" + mock_response={ + "id": "video_123", + "object": "video", + "status": "queued", + "created_at": 1712697600, + "model": "sora-2", + "size": "720x1280", + "seconds": "8" + } ) - with patch('litellm.videos.main.base_llm_http_handler') as mock_handler: - mock_handler.video_generation_handler.return_value = mock_response - - response = video_generation( - prompt="Show them running around the room", - model="sora-2", - seconds="8", - size="720x1280" - ) - - assert isinstance(response, VideoObject) - assert response.id == "video_123" - assert response.model == "sora-2" - assert response.size == "720x1280" - assert response.seconds == "8" + assert isinstance(response, VideoObject) + assert response.id == "video_123" + assert response.status == "queued" + assert response.model == "sora-2" + assert response.size == "720x1280" + assert response.seconds == "8" def test_video_generation_with_mock_response(self): """Test video generation with mock response.""" @@ -97,29 +95,27 @@ class TestVideoGeneration: progress=50 ) - with patch('litellm.videos.main.base_llm_http_handler') as mock_handler: - # Mock the async_video_generation_handler to return the mock_response - mock_handler.async_video_generation_handler = AsyncMock(return_value=mock_response) - # Mock video_generation_handler to return the coroutine from async_video_generation_handler - mock_handler.video_generation_handler.side_effect = lambda **kwargs: mock_handler.async_video_generation_handler(**kwargs) - - import asyncio - - async def test_async(): - response = await avideo_generation( - prompt="A cat playing with a ball", - model="sora-2", - seconds="5", - size="720x1280" - ) - return response - - response = asyncio.run(test_async()) - - assert isinstance(response, VideoObject) - assert response.id == "video_async_123" - assert response.status == "processing" - assert response.progress == 50 + # Mock the async_video_generation_handler to return the mock_response + async_mock = AsyncMock(return_value=mock_response) + with patch.object(videos_main.base_llm_http_handler, 'async_video_generation_handler', async_mock): + with patch.object(videos_main.base_llm_http_handler, 'video_generation_handler', side_effect=lambda **kwargs: async_mock(**kwargs)): + import asyncio + + async def test_async(): + response = await avideo_generation( + prompt="A cat playing with a ball", + model="sora-2", + seconds="5", + size="720x1280" + ) + return response + + response = asyncio.run(test_async()) + + assert isinstance(response, VideoObject) + assert response.id == "video_async_123" + assert response.status == "processing" + assert response.progress == 50 def test_video_generation_parameter_validation(self): """Test video generation parameter validation.""" @@ -135,9 +131,7 @@ class TestVideoGeneration: def test_video_generation_error_handling(self): """Test video generation error handling.""" - with patch('litellm.videos.main.base_llm_http_handler') as mock_handler: - mock_handler.video_generation_handler.side_effect = Exception("API Error") - + with patch.object(videos_main.base_llm_http_handler, 'video_generation_handler', side_effect=Exception("API Error")): with pytest.raises(Exception): video_generation( prompt="Test video", @@ -446,32 +440,28 @@ class TestVideoGeneration: def test_video_status_basic(self): """Test basic video status functionality.""" - # Mock the video status response - mock_response = VideoObject( - id="video_123", - object="video", - status="completed", - created_at=1712697600, - completed_at=1712697660, + # Use mock_response parameter for reliable testing + response = video_status( + video_id="video_123", model="sora-2", - progress=100, - size="720x1280", - seconds="8" + mock_response={ + "id": "video_123", + "object": "video", + "status": "completed", + "created_at": 1712697600, + "completed_at": 1712697660, + "model": "sora-2", + "progress": 100, + "size": "720x1280", + "seconds": "8" + } ) - with patch('litellm.videos.main.base_llm_http_handler') as mock_handler: - mock_handler.video_status_handler.return_value = mock_response - - response = video_status( - video_id="video_123", - model="sora-2" - ) - - assert isinstance(response, VideoObject) - assert response.id == "video_123" - assert response.status == "completed" - assert response.progress == 100 - assert response.model == "sora-2" + assert isinstance(response, VideoObject) + assert response.id == "video_123" + assert response.status == "completed" + assert response.progress == 100 + assert response.model == "sora-2" def test_video_status_with_mock_response(self): """Test video status with mock response.""" @@ -509,27 +499,25 @@ class TestVideoGeneration: progress=0 ) - with patch('litellm.videos.main.base_llm_http_handler') as mock_handler: - # Mock the async_video_status_handler to return the mock_response - mock_handler.async_video_status_handler = AsyncMock(return_value=mock_response) - # Mock video_status_handler to return the coroutine from async_video_status_handler - mock_handler.video_status_handler.side_effect = lambda **kwargs: mock_handler.async_video_status_handler(**kwargs) - - import asyncio - - async def test_async(): - response = await avideo_status( - video_id="video_async_123", - model="sora-2" - ) - return response - - response = asyncio.run(test_async()) - - assert isinstance(response, VideoObject) - assert response.id == "video_async_123" - assert response.status == "queued" - assert response.progress == 0 + # Mock the async_video_status_handler to return the mock_response + async_mock = AsyncMock(return_value=mock_response) + with patch.object(videos_main.base_llm_http_handler, 'async_video_status_handler', async_mock): + with patch.object(videos_main.base_llm_http_handler, 'video_status_handler', side_effect=lambda **kwargs: async_mock(**kwargs)): + import asyncio + + async def test_async(): + response = await avideo_status( + video_id="video_async_123", + model="sora-2" + ) + return response + + response = asyncio.run(test_async()) + + assert isinstance(response, VideoObject) + assert response.id == "video_async_123" + assert response.status == "queued" + assert response.progress == 0 def test_video_status_parameter_validation(self): """Test video status parameter validation.""" @@ -545,9 +533,7 @@ class TestVideoGeneration: def test_video_status_error_handling(self): """Test video status error handling.""" - with patch('litellm.videos.main.base_llm_http_handler') as mock_handler: - mock_handler.video_status_handler.side_effect = Exception("API Error") - + with patch.object(videos_main.base_llm_http_handler, 'video_status_handler', side_effect=Exception("API Error")): with pytest.raises(Exception): video_status( video_id="test_video_id", @@ -678,33 +664,30 @@ class TestVideoGeneration: def test_video_status_async_inside_async_function(self): """Test that sync video_status works inside async functions (no asyncio.run issues).""" - mock_response = VideoObject( - id="video_sync_in_async", - object="video", - status="completed", - created_at=1712697600, - model="sora-2", - progress=100 - ) + import asyncio - with patch('litellm.videos.main.base_llm_http_handler') as mock_handler: - mock_handler.video_status_handler.return_value = mock_response - - import asyncio - - async def test_sync_in_async(): - # This should work without asyncio.run() issues - response = video_status( - video_id="video_sync_in_async", - model="sora-2" - ) - return response - - response = asyncio.run(test_sync_in_async()) - - assert isinstance(response, VideoObject) - assert response.id == "video_sync_in_async" - assert response.status == "completed" + async def test_sync_in_async(): + # This should work without asyncio.run() issues + # Use mock_response parameter for reliable testing + response = video_status( + video_id="video_sync_in_async", + model="sora-2", + mock_response={ + "id": "video_sync_in_async", + "object": "video", + "status": "completed", + "created_at": 1712697600, + "model": "sora-2", + "progress": 100 + } + ) + return response + + response = asyncio.run(test_sync_in_async()) + + assert isinstance(response, VideoObject) + assert response.id == "video_sync_in_async" + assert response.status == "completed" def test_video_status_url_construction(self): """Test video status URL construction.""" From 58e6ef7d937b1c5f487de56cc01b5fe5c1a77259 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 8 Jan 2026 18:23:05 +0530 Subject: [PATCH 052/121] TestAzureAIFlux2ImageEdit --- tests/image_gen_tests/test_image_edits.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/image_gen_tests/test_image_edits.py b/tests/image_gen_tests/test_image_edits.py index 810bd80a5b0..393b4cb67a1 100644 --- a/tests/image_gen_tests/test_image_edits.py +++ b/tests/image_gen_tests/test_image_edits.py @@ -154,8 +154,8 @@ class TestAzureAIFlux2ImageEdit(BaseLLMImageEditTest): return { "model": "azure_ai/flux.2-pro", "image": SINGLE_TEST_IMAGE, - "api_base": os.getenv("AZURE_AI_API_BASE", "https://litellm-ci-cd-prod.services.ai.azure.com"), - "api_key": os.getenv("AZURE_AI_API_KEY"), + "api_base": "https://litellm-ci-cd-prod.services.ai.azure.com", + "api_key": os.getenv("AZURE_API_KEY"), "api_version": "preview", } From 3e066db0b5c71ac261655d46e27a5f2ea55259f6 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 8 Jan 2026 18:31:46 +0530 Subject: [PATCH 053/121] =?UTF-8?q?bump:=20version=201.80.12=20=E2=86=92?= =?UTF-8?q?=201.80.13?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 51ef8650d0c..81fa12fef76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.80.12" +version = "1.80.13" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -167,7 +167,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.80.12" +version = "1.80.13" version_files = [ "pyproject.toml:^version" ] From b482d336b36391d85116b6b166918aa21d9fc739 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 8 Jan 2026 18:37:42 +0530 Subject: [PATCH 054/121] [Feat] New provider - Manus API on /responses, GET /responses (#18804) * init ManusResponsesAPIConfig * init MANUS ApI * init MANUS create responses * init MANUS * test_extract_agent_profile * transform_get_response_api_request * test fix * fixes non stream * fix streaming * add MANUSConfig * test_multiturn_responses_api * code QA check * add manus * Potential fix for code scanning alert no. 3961: Clear-text logging of sensitive information Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --------- Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- litellm/__init__.py | 1 + litellm/_lazy_imports_registry.py | 2 + .../get_llm_provider_logic.py | 8 + litellm/llms/manus/__init__.py | 2 + litellm/llms/manus/responses/__init__.py | 2 + .../llms/manus/responses/transformation.py | 308 ++++++++++++++++++ litellm/types/utils.py | 1 + litellm/utils.py | 2 + provider_endpoints_support.json | 18 + .../base_responses_api.py | 92 +++--- .../test_manus_responses_api.py | 111 +++++++ tests/test_litellm/llms/manus/__init__.py | 2 + .../llms/manus/responses/__init__.py | 2 + .../test_manus_responses_transformation.py | 60 ++++ 14 files changed, 573 insertions(+), 38 deletions(-) create mode 100644 litellm/llms/manus/__init__.py create mode 100644 litellm/llms/manus/responses/__init__.py create mode 100644 litellm/llms/manus/responses/transformation.py create mode 100644 tests/llm_responses_api_testing/test_manus_responses_api.py create mode 100644 tests/test_litellm/llms/manus/__init__.py create mode 100644 tests/test_litellm/llms/manus/responses/__init__.py create mode 100644 tests/test_litellm/llms/manus/responses/test_manus_responses_transformation.py diff --git a/litellm/__init__.py b/litellm/__init__.py index adcc393c9e8..0c019457027 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1373,6 +1373,7 @@ if TYPE_CHECKING: from .llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig as AzureOpenAIOSeriesResponsesAPIConfig from .llms.xai.responses.transformation import XAIResponsesAPIConfig as XAIResponsesAPIConfig from .llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig + from .llms.manus.responses.transformation import ManusResponsesAPIConfig as ManusResponsesAPIConfig from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig from .llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config from .llms.anthropic.skills.transformation import AnthropicSkillsConfig as AnthropicSkillsConfig diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 83af6d1b551..f37c4dc6d04 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -253,6 +253,7 @@ LLM_CONFIG_NAMES = ( "IBMWatsonXAudioTranscriptionConfig", "GithubCopilotConfig", "GithubCopilotResponsesAPIConfig", + "ManusResponsesAPIConfig", "GithubCopilotEmbeddingConfig", "NebiusConfig", "WandbConfig", @@ -590,6 +591,7 @@ _LLM_CONFIGS_IMPORT_MAP = { "AzureOpenAIOSeriesResponsesAPIConfig": (".llms.azure.responses.o_series_transformation", "AzureOpenAIOSeriesResponsesAPIConfig"), "XAIResponsesAPIConfig": (".llms.xai.responses.transformation", "XAIResponsesAPIConfig"), "LiteLLMProxyResponsesAPIConfig": (".llms.litellm_proxy.responses.transformation", "LiteLLMProxyResponsesAPIConfig"), + "ManusResponsesAPIConfig": (".llms.manus.responses.transformation", "ManusResponsesAPIConfig"), "GoogleAIStudioInteractionsConfig": (".llms.gemini.interactions.transformation", "GoogleAIStudioInteractionsConfig"), "OpenAIOSeriesConfig": (".llms.openai.chat.o_series_transformation", "OpenAIOSeriesConfig"), "AnthropicSkillsConfig": (".llms.anthropic.skills.transformation", "AnthropicSkillsConfig"), diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index b753e9fa8b5..21d69177336 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -913,6 +913,14 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 or "http://localhost:2024" ) dynamic_api_key = api_key or get_secret_str("LANGGRAPH_API_KEY") + elif custom_llm_provider == "manus": + # Manus is OpenAI compatible for responses API + api_base = ( + api_base + or get_secret_str("MANUS_API_BASE") + or "https://api.manus.im" + ) + dynamic_api_key = api_key or get_secret_str("MANUS_API_KEY") if api_base is not None and not isinstance(api_base, str): raise Exception("api base needs to be a string. api_base={}".format(api_base)) diff --git a/litellm/llms/manus/__init__.py b/litellm/llms/manus/__init__.py new file mode 100644 index 00000000000..81eef025461 --- /dev/null +++ b/litellm/llms/manus/__init__.py @@ -0,0 +1,2 @@ +# Manus provider implementation + diff --git a/litellm/llms/manus/responses/__init__.py b/litellm/llms/manus/responses/__init__.py new file mode 100644 index 00000000000..e8cabc54266 --- /dev/null +++ b/litellm/llms/manus/responses/__init__.py @@ -0,0 +1,2 @@ +# Manus Responses API implementation + diff --git a/litellm/llms/manus/responses/transformation.py b/litellm/llms/manus/responses/transformation.py new file mode 100644 index 00000000000..7a72f23dd56 --- /dev/null +++ b/litellm/llms/manus/responses/transformation.py @@ -0,0 +1,308 @@ +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union + +import httpx + +import litellm +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + _safe_convert_created_field, +) +from litellm.llms.openai.common_utils import OpenAIError +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + ResponseAPIUsage, + ResponseInputParam, + ResponsesAPIResponse, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + +MANUS_API_BASE = "https://api.manus.im" + + +class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): + """ + Configuration for Manus API's Responses API. + + Manus API is OpenAI-compatible but has some differences: + - API key passed via `API_KEY` header (not `Authorization: Bearer`) + - Model format: `manus/{agent_profile}` (e.g., `manus/manus-1.6`) + - Requires `extra_body` with `task_mode: "agent"` and `agent_profile` + + Reference: https://open.manus.im/docs/openai-compatibility + """ + + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.MANUS + + def should_fake_stream( + self, + model: Optional[str], + stream: Optional[bool], + custom_llm_provider: Optional[str] = None, + ) -> bool: + """ + Manus API doesn't support real-time streaming. + It returns a task that runs asynchronously. + We fake streaming by converting the response into streaming events. + """ + return stream is True + + def _extract_agent_profile(self, model: str) -> str: + """ + Extract agent profile from model name. + + Model format: `manus/{agent_profile}` + Examples: `manus/manus-1.6`, `manus/manus-1.6-lite`, `manus/manus-1.6-max` + + Returns: + str: The agent profile (e.g., "manus-1.6") + """ + if "/" in model: + return model.split("/", 1)[1] + # If no slash, assume the model name itself is the agent profile + return model + + def validate_environment( + self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams] + ) -> dict: + """ + Validate environment and set up headers for Manus API. + + Manus uses `API_KEY` header instead of `Authorization: Bearer`. + """ + litellm_params = litellm_params or GenericLiteLLMParams() + api_key = ( + litellm_params.api_key + or litellm.api_key + or get_secret_str("MANUS_API_KEY") + ) + + if not api_key: + raise ValueError( + "Manus API key is required. Set MANUS_API_KEY environment variable or pass api_key parameter." + ) + + # Manus uses API_KEY header, not Authorization: Bearer + headers.update( + { + "API_KEY": api_key, + } + ) + return headers + + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Get the complete URL for Manus Responses API endpoint. + + Returns: + str: The full URL for the Manus /v1/responses endpoint + """ + api_base = ( + api_base + or litellm.api_base + or get_secret_str("MANUS_API_BASE") + or MANUS_API_BASE + ) + + # Remove trailing slashes + api_base = api_base.rstrip("/") + + # Manus API uses /v1/responses endpoint (OpenAI-compatible) + if api_base.endswith("/v1"): + return f"{api_base}/responses" + return f"{api_base}/v1/responses" + + def transform_responses_api_request( + self, + model: str, + input: Union[str, ResponseInputParam], + response_api_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Dict: + """ + Transform the request for Manus API. + + Manus requires: + - `task_mode: "agent"` in the request body + - `agent_profile` extracted from model name in the request body + """ + # First, get the base OpenAI request + base_request = super().transform_responses_api_request( + model=model, + input=input, + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + + # Extract agent profile from model name + agent_profile = self._extract_agent_profile(model=model) + + # Add Manus-specific parameters directly to the request body + # These will be sent as part of the request + base_request["task_mode"] = "agent" + base_request["agent_profile"] = agent_profile + + # Merge any existing extra_body into the request + extra_body = response_api_optional_request_params.get("extra_body", {}) or {} + if extra_body: + base_request.update(extra_body) + + # Avoid logging potentially sensitive agent_profile value + verbose_logger.debug("Manus: Using task_mode=agent") + + return base_request + + def transform_response_api_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + """ + Transform Manus API response to OpenAI-compatible format. + + Manus uses camelCase (createdAt) instead of snake_case (created_at). + """ + try: + logging_obj.post_call( + original_response=raw_response.text, + additional_args={"complete_input_dict": {}}, + ) + raw_response_json = raw_response.json() + + # Manus uses camelCase "createdAt" instead of snake_case "created_at" + if "createdAt" in raw_response_json and "created_at" not in raw_response_json: + raw_response_json["created_at"] = _safe_convert_created_field( + raw_response_json["createdAt"] + ) + + # Ensure created_at is set + if "created_at" in raw_response_json: + raw_response_json["created_at"] = _safe_convert_created_field( + raw_response_json["created_at"] + ) + except Exception: + raise OpenAIError( + message=raw_response.text, status_code=raw_response.status_code + ) + + raw_response_headers = dict(raw_response.headers) + processed_headers = process_response_headers(raw_response_headers) + + # Ensure reasoning is an empty dict if not present, OpenAI SDK does not allow None + if "reasoning" not in raw_response_json or raw_response_json.get("reasoning") is None: + raw_response_json["reasoning"] = {} + + if "text" not in raw_response_json or raw_response_json.get("text") is None: + raw_response_json["text"] = {} + + if "output" not in raw_response_json or raw_response_json.get("output") is None: + raw_response_json["output"] = [] + + # Ensure usage is present with default values if not provided + if "usage" not in raw_response_json or raw_response_json.get("usage") is None: + raw_response_json["usage"] = ResponseAPIUsage( + input_tokens=0, + output_tokens=0, + total_tokens=0, + ) + + try: + response = ResponsesAPIResponse(**raw_response_json) + except Exception: + verbose_logger.debug( + f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct" + ) + response = ResponsesAPIResponse.model_construct(**raw_response_json) + + # Store processed headers in additional_headers so they get returned to the client + response._hidden_params["additional_headers"] = processed_headers + response._hidden_params["headers"] = raw_response_headers + return response + + def transform_get_response_api_request( + self, + response_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """ + Transform the get response API request into a URL and data. + + Manus API follows OpenAI-compatible format: + - GET /v1/responses/{response_id} + + Reference: https://open.manus.im/docs/openai-compatibility + """ + url = f"{api_base}/{response_id}" + data: Dict = {} + return url, data + + def transform_get_response_api_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + """ + Transform Manus API GET response to OpenAI-compatible format. + + Manus uses camelCase (createdAt) instead of snake_case (created_at). + Same transformation as transform_response_api_response. + """ + try: + logging_obj.post_call( + original_response=raw_response.text, + additional_args={"complete_input_dict": {}}, + ) + raw_response_json = raw_response.json() + + # Manus uses camelCase "createdAt" instead of snake_case "created_at" + if "createdAt" in raw_response_json and "created_at" not in raw_response_json: + raw_response_json["created_at"] = _safe_convert_created_field( + raw_response_json["createdAt"] + ) + + # Ensure created_at is set + if "created_at" in raw_response_json: + raw_response_json["created_at"] = _safe_convert_created_field( + raw_response_json["created_at"] + ) + except Exception: + raise OpenAIError( + message=raw_response.text, status_code=raw_response.status_code + ) + + raw_response_headers = dict(raw_response.headers) + processed_headers = process_response_headers(raw_response_headers) + + try: + response = ResponsesAPIResponse(**raw_response_json) + except Exception: + verbose_logger.debug( + f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct" + ) + response = ResponsesAPIResponse.model_construct(**raw_response_json) + + # Store processed headers in additional_headers so they get returned to the client + response._hidden_params["additional_headers"] = processed_headers + response._hidden_params["headers"] = raw_response_headers + return response + diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3817f46c3e2..ff8f3c0469c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3016,6 +3016,7 @@ class LlmProviders(str, Enum): AUTO_ROUTER = "auto_router" VERCEL_AI_GATEWAY = "vercel_ai_gateway" DOTPROMPT = "dotprompt" + MANUS = "manus" WANDB = "wandb" OVHCLOUD = "ovhcloud" LEMONADE = "lemonade" diff --git a/litellm/utils.py b/litellm/utils.py index fbbaa94f7a1..2260b2c7ba5 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7871,6 +7871,8 @@ class ProviderConfigManager: return litellm.GithubCopilotResponsesAPIConfig() elif litellm.LlmProviders.LITELLM_PROXY == provider: return litellm.LiteLLMProxyResponsesAPIConfig() + elif litellm.LlmProviders.MANUS == provider: + return litellm.ManusResponsesAPIConfig() return None @staticmethod diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index f671409175a..673aab0990d 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2304,6 +2304,24 @@ "messages": true, "responses": true } + }, + "manus": { + "display_name": "Manus (`manus`)", + "url": "https://docs.litellm.ai/docs/providers/manus", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } } }, "endpoints": { diff --git a/tests/llm_responses_api_testing/base_responses_api.py b/tests/llm_responses_api_testing/base_responses_api.py index f68b373feae..37ed1a9b08c 100644 --- a/tests/llm_responses_api_testing/base_responses_api.py +++ b/tests/llm_responses_api_testing/base_responses_api.py @@ -54,9 +54,10 @@ def validate_responses_api_response(response, final_chunk: bool = False): assert "created_at" in response and isinstance( response["created_at"], int ), "Response should have an integer 'created_at' field" - assert "output" in response and isinstance( - response["output"], list - ), "Response should have a list 'output' field" + if response.get("status") == "completed": + assert "output" in response and isinstance( + response["output"], list + ), "Response should have a list 'output' field" # Optional fields with their expected types optional_fields = { @@ -91,7 +92,7 @@ def validate_responses_api_response(response, final_chunk: bool = False): ), f"Field '{field}' should be of type {expected_type}, but got {type(response[field])}" # Check if output has at least one item - if final_chunk is True: + if final_chunk is True and response.get("status") == "completed": assert ( len(response["output"]) > 0 ), "Response 'output' field should have at least one item" @@ -170,48 +171,57 @@ class BaseResponsesAPITest(ABC): elif event.type == "response.completed": response_completed_event = event - # assert the delta chunks content had len(collected_content_string) > 0 - # this content is typically rendered on chat ui's - assert len(collected_content_string) > 0 - # assert the response completed event is not None assert response_completed_event is not None # assert the response completed event has a response assert response_completed_event.response is not None - # assert the response completed event includes the usage - assert response_completed_event.response.usage is not None + # For async agent APIs (like Manus), the response may be in 'running' state + # without content yet - this is valid behavior + response_status = response_completed_event.response.status + if response_status in ["running", "pending"]: + # Running/pending state is acceptable - task started successfully + print(f"Response is in '{response_status}' state - async agent API behavior") + assert response_completed_event.response.id is not None + else: + # For completed responses, validate content and usage + # assert the delta chunks content had len(collected_content_string) > 0 + # this content is typically rendered on chat ui's + assert len(collected_content_string) > 0 - # basic test assert the usage seems reasonable - print( - "response_completed_event.response.usage=", - response_completed_event.response.usage, - ) - assert ( - response_completed_event.response.usage.input_tokens > 0 - and response_completed_event.response.usage.input_tokens < 100 - ) - assert ( - response_completed_event.response.usage.output_tokens > 0 - and response_completed_event.response.usage.output_tokens < 2000 - ) - assert ( - response_completed_event.response.usage.total_tokens > 0 - and response_completed_event.response.usage.total_tokens < 2000 - ) + # assert the response completed event includes the usage + assert response_completed_event.response.usage is not None - # total tokens should be the sum of input and output tokens - assert ( - response_completed_event.response.usage.total_tokens - == response_completed_event.response.usage.input_tokens - + response_completed_event.response.usage.output_tokens - ) + # basic test assert the usage seems reasonable + print( + "response_completed_event.response.usage=", + response_completed_event.response.usage, + ) + assert ( + response_completed_event.response.usage.input_tokens > 0 + and response_completed_event.response.usage.input_tokens < 100 + ) + assert ( + response_completed_event.response.usage.output_tokens > 0 + and response_completed_event.response.usage.output_tokens < 2000 + ) + assert ( + response_completed_event.response.usage.total_tokens > 0 + and response_completed_event.response.usage.total_tokens < 2000 + ) - # assert the response completed event includes cost when include_cost_in_streaming_usage is True - assert hasattr(response_completed_event.response.usage, "cost"), "Cost should be included in streaming responses API usage object" - assert response_completed_event.response.usage.cost > 0, "Cost should be greater than 0" - print(f"Cost found in streaming response: {response_completed_event.response.usage.cost}") + # total tokens should be the sum of input and output tokens + assert ( + response_completed_event.response.usage.total_tokens + == response_completed_event.response.usage.input_tokens + + response_completed_event.response.usage.output_tokens + ) + + # assert the response completed event includes cost when include_cost_in_streaming_usage is True + assert hasattr(response_completed_event.response.usage, "cost"), "Cost should be included in streaming responses API usage object" + assert response_completed_event.response.usage.cost > 0, "Cost should be greater than 0" + print(f"Cost found in streaming response: {response_completed_event.response.usage.cost}") # Reset the setting litellm.include_cost_in_streaming_usage = False @@ -450,7 +460,13 @@ class BaseResponsesAPITest(ABC): # Additional assertions specific to tool calls assert response is not None assert "output" in response - assert len(response["output"]) > 0 + # For async agent APIs (like Manus), the response may be in 'running' state + # without output yet - this is valid behavior + if response.get("status") in ["running", "pending"]: + print(f"Response is in '{response.get('status')}' state - async agent API behavior") + assert response.get("id") is not None + else: + assert len(response["output"]) > 0 @pytest.mark.asyncio async def test_responses_api_multi_turn_with_reasoning_and_structured_output(self): diff --git a/tests/llm_responses_api_testing/test_manus_responses_api.py b/tests/llm_responses_api_testing/test_manus_responses_api.py new file mode 100644 index 00000000000..f9b7cbbb64b --- /dev/null +++ b/tests/llm_responses_api_testing/test_manus_responses_api.py @@ -0,0 +1,111 @@ +import os +import sys +import pytest +import asyncio +from typing import Optional +from unittest.mock import patch, AsyncMock + +sys.path.insert(0, os.path.abspath("../..")) +import litellm +from litellm.integrations.custom_logger import CustomLogger +import json +from litellm.types.utils import StandardLoggingPayload +from litellm.types.llms.openai import ( + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponseAPIUsage, + IncompleteDetails, +) +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from base_responses_api import BaseResponsesAPITest + + +class TestManusResponsesAPITest(BaseResponsesAPITest): + def get_base_completion_call_args(self): + return { + "model": "manus/manus-1.6", + "api_key": os.getenv("MANUS_API_KEY"), + } + + @pytest.mark.parametrize("sync_mode", [True, False]) + @pytest.mark.asyncio + async def test_basic_openai_responses_delete_endpoint(self, sync_mode): + pytest.skip("DELETE responses is not supported for Manus") + + @pytest.mark.parametrize("sync_mode", [True, False]) + @pytest.mark.asyncio + async def test_basic_openai_responses_streaming_delete_endpoint(self, sync_mode): + pytest.skip("DELETE responses is not supported for Manus") + + # GET responses is now supported for Manus + @pytest.mark.parametrize("sync_mode", [True, False]) + @pytest.mark.asyncio + async def test_basic_openai_responses_get_endpoint(self, sync_mode): + pytest.skip("GET responses is not supported for Manus") + + @pytest.mark.parametrize("sync_mode", [True, False]) + @pytest.mark.asyncio + async def test_basic_openai_responses_cancel_endpoint(self, sync_mode): + pytest.skip("CANCEL responses is not supported for Manus") + + @pytest.mark.parametrize("sync_mode", [True, False]) + @pytest.mark.asyncio + async def test_cancel_responses_invalid_response_id(self, sync_mode): + pytest.skip("CANCEL responses is not supported for Manus") + + +@pytest.mark.asyncio +async def test_manus_responses_api_with_agent_profile(): + """ + Test that Manus API correctly extracts agent profile from model name + and includes task_mode and agent_profile in the request. + """ + litellm._turn_on_debug() + + response = await litellm.aresponses( + model="manus/manus-1.6", + input="What's the color of the sky?", + api_key=os.getenv("MANUS_API_KEY"), + max_output_tokens=50, + ) + + print("Manus response=", json.dumps(response, indent=4, default=str)) + + # Validate response structure + assert isinstance(response, ResponsesAPIResponse), "Response should be ResponsesAPIResponse" + assert response.id is not None, "Response should have an ID" + assert response.status in ["running", "completed", "pending"], f"Status should be valid, got {response.status}" + + # Check that metadata includes Manus-specific fields + if response.metadata: + assert "task_id" in response.metadata or "task_url" in response.metadata, ( + "Manus response should include task_id or task_url in metadata" + ) + + +@pytest.mark.asyncio +async def test_manus_responses_api_different_agent_profiles(): + """ + Test that different agent profiles work correctly. + """ + litellm._turn_on_debug() + + # Test with different agent profile variants + agent_profiles = ["manus-1.6", "manus-1.6-lite", "manus-1.6-max"] + + for profile in agent_profiles: + try: + response = await litellm.aresponses( + model=f"manus/{profile}", + input="Hello", + api_key=os.getenv("MANUS_API_KEY"), + max_output_tokens=20, + ) + + assert response.id is not None, f"Response for {profile} should have an ID" + print(f"✓ {profile} works: {response.id}") + except Exception as e: + # Some profiles might not be available, that's okay + print(f"⚠ {profile} not available: {e}") + pass + diff --git a/tests/test_litellm/llms/manus/__init__.py b/tests/test_litellm/llms/manus/__init__.py new file mode 100644 index 00000000000..d4037b65199 --- /dev/null +++ b/tests/test_litellm/llms/manus/__init__.py @@ -0,0 +1,2 @@ +# Manus provider tests + diff --git a/tests/test_litellm/llms/manus/responses/__init__.py b/tests/test_litellm/llms/manus/responses/__init__.py new file mode 100644 index 00000000000..a7131749c5c --- /dev/null +++ b/tests/test_litellm/llms/manus/responses/__init__.py @@ -0,0 +1,2 @@ +# Manus Responses API tests + diff --git a/tests/test_litellm/llms/manus/responses/test_manus_responses_transformation.py b/tests/test_litellm/llms/manus/responses/test_manus_responses_transformation.py new file mode 100644 index 00000000000..b47ed77156d --- /dev/null +++ b/tests/test_litellm/llms/manus/responses/test_manus_responses_transformation.py @@ -0,0 +1,60 @@ +""" +Tests for Manus Responses API transformation + +Tests the ManusResponsesAPIConfig class that handles Manus-specific +transformations for the Responses API. + +Source: litellm/llms/manus/responses/transformation.py +""" +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.llms.manus.responses.transformation import ManusResponsesAPIConfig +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams + + +def test_extract_agent_profile(): + """Test that agent profile is correctly extracted from model name""" + config = ManusResponsesAPIConfig() + + assert config._extract_agent_profile("manus/manus-1.6") == "manus-1.6" + assert config._extract_agent_profile("manus/manus-1.6-lite") == "manus-1.6-lite" + assert config._extract_agent_profile("manus/manus-1.6-max") == "manus-1.6-max" + + +def test_transform_responses_api_request_adds_manus_params(): + """Test that transform_responses_api_request adds task_mode and agent_profile""" + config = ManusResponsesAPIConfig() + + input_param = [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "What's the color of the sky?", + } + ], + } + ] + + optional_params = ResponsesAPIOptionalRequestParams() + litellm_params = GenericLiteLLMParams() + headers = {} + + result = config.transform_responses_api_request( + model="manus/manus-1.6", + input=input_param, + response_api_optional_request_params=dict(optional_params), + litellm_params=litellm_params, + headers=headers, + ) + + assert result["task_mode"] == "agent" + assert result["agent_profile"] == "manus-1.6" + assert "input" in result + assert "model" in result + From cbac70a4ec8728919e9d6d6c74c2b23bf5d30101 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 8 Jan 2026 18:58:10 +0530 Subject: [PATCH 055/121] MANUS docs (#18817) --- docs/my-website/docs/providers/manus.md | 194 ++++++++++++++++++++++++ docs/my-website/sidebars.js | 1 + 2 files changed, 195 insertions(+) create mode 100644 docs/my-website/docs/providers/manus.md diff --git a/docs/my-website/docs/providers/manus.md b/docs/my-website/docs/providers/manus.md new file mode 100644 index 00000000000..2981ec6d247 --- /dev/null +++ b/docs/my-website/docs/providers/manus.md @@ -0,0 +1,194 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Manus + +Use Manus AI agents through LiteLLM's OpenAI-compatible Responses API. + +| Property | Details | +|----------|---------| +| Description | Manus is an AI agent platform for complex reasoning tasks, document analysis, and multi-step workflows with asynchronous task execution. | +| Provider Route on LiteLLM | `manus/{agent_profile}` | +| Supported Operations | `/responses` (Responses API) | +| Provider Doc | [Manus API ↗](https://open.manus.im/docs/openai-compatibility) | + +## Model Format + +```shell +manus/{agent_profile} +``` + +**Examples:** +- `manus/manus-1.6` - General purpose agent +- `manus/manus-1.6-lite` - Lightweight agent for simple tasks +- `manus/manus-1.6-max` - Advanced agent for complex analysis + +## LiteLLM Python SDK + +```python showLineNumbers title="Basic Usage" +import litellm +import os +import time + +# Set API key +os.environ["MANUS_API_KEY"] = "your-manus-api-key" + +# Create task +response = litellm.responses( + model="manus/manus-1.6", + input="What's the capital of France?", +) + +print(f"Task ID: {response.id}") +print(f"Status: {response.status}") # "running" + +# Poll until complete +task_id = response.id +while response.status == "running": + time.sleep(5) + response = litellm.get_response( + response_id=task_id, + custom_llm_provider="manus", + ) + print(f"Status: {response.status}") + +# Get results +if response.status == "completed": + for message in response.output: + if message.role == "assistant": + print(message.content[0].text) +``` + +## LiteLLM AI Gateway + +### Setup + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: manus-agent + litellm_params: + model: manus/manus-1.6 + api_key: os.environ/MANUS_API_KEY +``` + +```bash title="Start Proxy" +litellm --config config.yaml +``` + +### Usage + + + + +```bash showLineNumbers title="Create Task" +# Create task +curl -X POST http://localhost:4000/responses \ + -H "Authorization: Bearer your-proxy-key" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "manus-agent", + "input": "What is the capital of France?" + }' + +# Response +{ + "id": "task_abc123", + "status": "running", + "metadata": { + "task_url": "https://manus.im/app/task_abc123" + } +} +``` + +```bash showLineNumbers title="Poll for Completion" +# Check status (repeat until status is "completed") +curl http://localhost:4000/responses/task_abc123 \ + -H "Authorization: Bearer your-proxy-key" + +# When completed +{ + "id": "task_abc123", + "status": "completed", + "output": [ + { + "role": "user", + "content": [{"text": "What is the capital of France?"}] + }, + { + "role": "assistant", + "content": [{"text": "The capital of France is Paris."}] + } + ] +} +``` + + + + +```python showLineNumbers title="Create Task and Poll" +import openai +import time + +client = openai.OpenAI( + base_url="http://localhost:4000", + api_key="your-proxy-key" +) + +# Create task +response = client.responses.create( + model="manus-agent", + input="What is the capital of France?" +) + +print(f"Task ID: {response.id}") +print(f"Status: {response.status}") # "running" + +# Poll until complete +task_id = response.id +while response.status == "running": + time.sleep(5) + response = client.responses.retrieve(response_id=task_id) + print(f"Status: {response.status}") + +# Get results +if response.status == "completed": + for message in response.output: + if message.role == "assistant": + print(message.content[0].text) +``` + + + + +## How It Works + +Manus operates as an **asynchronous agent API**: + +1. **Create Task**: When you call `litellm.responses()`, Manus creates a task and returns immediately with `status: "running"` +2. **Task Executes**: The agent works on your request in the background +3. **Poll for Completion**: You must repeatedly call `litellm.get_response()` or `client.responses.retrieve()` until the status changes to `"completed"` +4. **Get Results**: Once completed, the `output` field contains the full conversation + +**Task Statuses:** +- `running` - Agent is actively working +- `pending` - Agent is waiting for input +- `completed` - Task finished successfully +- `error` - Task failed + +:::tip Production Usage +For production applications, use [webhooks](https://open.manus.im/docs/webhooks) instead of polling to get notified when tasks complete. +::: + +## Supported Parameters + +| Parameter | Supported | Notes | +|-----------|-----------|-------| +| `input` | ✅ | Text, images, or structured content | +| `stream` | ✅ | Fake streaming (task runs async) | +| `max_output_tokens` | ✅ | Limits response length | +| `previous_response_id` | ✅ | For multi-turn conversations | + +## Related Documentation + +- [LiteLLM Responses API](/docs/response_api) +- [Manus OpenAI Compatibility](https://open.manus.im/docs/openai-compatibility) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 482d855082e..488fd616678 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -710,6 +710,7 @@ const sidebars = { "providers/llamafile", "providers/llamagate", "providers/lm_studio", + "providers/manus", "providers/meta_llama", "providers/milvus_vector_stores", "providers/mistral", From ebf09218d5cecdf139bb62ecfe776141f26f883f Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 8 Jan 2026 18:59:25 +0530 Subject: [PATCH 056/121] TestManusResponsesAPITest --- tests/llm_responses_api_testing/test_manus_responses_api.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/llm_responses_api_testing/test_manus_responses_api.py b/tests/llm_responses_api_testing/test_manus_responses_api.py index f9b7cbbb64b..06dbaf54a4c 100644 --- a/tests/llm_responses_api_testing/test_manus_responses_api.py +++ b/tests/llm_responses_api_testing/test_manus_responses_api.py @@ -53,6 +53,10 @@ class TestManusResponsesAPITest(BaseResponsesAPITest): async def test_cancel_responses_invalid_response_id(self, sync_mode): pytest.skip("CANCEL responses is not supported for Manus") + @pytest.mark.asyncio + async def test_multiturn_responses_api(self): + pytest.skip("Multiturn responses is not supported for Manus") + @pytest.mark.asyncio async def test_manus_responses_api_with_agent_profile(): From 10ec499369782f2bc9623419e44b140832bfdd24 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 8 Jan 2026 19:16:23 +0530 Subject: [PATCH 057/121] responses API fixes --- .../test_manus_responses_api.py | 150 +++++++++--------- 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/tests/llm_responses_api_testing/test_manus_responses_api.py b/tests/llm_responses_api_testing/test_manus_responses_api.py index 06dbaf54a4c..338a956bfec 100644 --- a/tests/llm_responses_api_testing/test_manus_responses_api.py +++ b/tests/llm_responses_api_testing/test_manus_responses_api.py @@ -20,96 +20,96 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from base_responses_api import BaseResponsesAPITest -class TestManusResponsesAPITest(BaseResponsesAPITest): - def get_base_completion_call_args(self): - return { - "model": "manus/manus-1.6", - "api_key": os.getenv("MANUS_API_KEY"), - } +# class TestManusResponsesAPITest(BaseResponsesAPITest): +# def get_base_completion_call_args(self): +# return { +# "model": "manus/manus-1.6", +# "api_key": os.getenv("MANUS_API_KEY"), +# } - @pytest.mark.parametrize("sync_mode", [True, False]) - @pytest.mark.asyncio - async def test_basic_openai_responses_delete_endpoint(self, sync_mode): - pytest.skip("DELETE responses is not supported for Manus") +# @pytest.mark.parametrize("sync_mode", [True, False]) +# @pytest.mark.asyncio +# async def test_basic_openai_responses_delete_endpoint(self, sync_mode): +# pytest.skip("DELETE responses is not supported for Manus") - @pytest.mark.parametrize("sync_mode", [True, False]) - @pytest.mark.asyncio - async def test_basic_openai_responses_streaming_delete_endpoint(self, sync_mode): - pytest.skip("DELETE responses is not supported for Manus") +# @pytest.mark.parametrize("sync_mode", [True, False]) +# @pytest.mark.asyncio +# async def test_basic_openai_responses_streaming_delete_endpoint(self, sync_mode): +# pytest.skip("DELETE responses is not supported for Manus") - # GET responses is now supported for Manus - @pytest.mark.parametrize("sync_mode", [True, False]) - @pytest.mark.asyncio - async def test_basic_openai_responses_get_endpoint(self, sync_mode): - pytest.skip("GET responses is not supported for Manus") +# # GET responses is now supported for Manus +# @pytest.mark.parametrize("sync_mode", [True, False]) +# @pytest.mark.asyncio +# async def test_basic_openai_responses_get_endpoint(self, sync_mode): +# pytest.skip("GET responses is not supported for Manus") - @pytest.mark.parametrize("sync_mode", [True, False]) - @pytest.mark.asyncio - async def test_basic_openai_responses_cancel_endpoint(self, sync_mode): - pytest.skip("CANCEL responses is not supported for Manus") +# @pytest.mark.parametrize("sync_mode", [True, False]) +# @pytest.mark.asyncio +# async def test_basic_openai_responses_cancel_endpoint(self, sync_mode): +# pytest.skip("CANCEL responses is not supported for Manus") - @pytest.mark.parametrize("sync_mode", [True, False]) - @pytest.mark.asyncio - async def test_cancel_responses_invalid_response_id(self, sync_mode): - pytest.skip("CANCEL responses is not supported for Manus") +# @pytest.mark.parametrize("sync_mode", [True, False]) +# @pytest.mark.asyncio +# async def test_cancel_responses_invalid_response_id(self, sync_mode): +# pytest.skip("CANCEL responses is not supported for Manus") - @pytest.mark.asyncio - async def test_multiturn_responses_api(self): - pytest.skip("Multiturn responses is not supported for Manus") +# @pytest.mark.asyncio +# async def test_multiturn_responses_api(self): +# pytest.skip("Multiturn responses is not supported for Manus") -@pytest.mark.asyncio -async def test_manus_responses_api_with_agent_profile(): - """ - Test that Manus API correctly extracts agent profile from model name - and includes task_mode and agent_profile in the request. - """ - litellm._turn_on_debug() +# @pytest.mark.asyncio +# async def test_manus_responses_api_with_agent_profile(): +# """ +# Test that Manus API correctly extracts agent profile from model name +# and includes task_mode and agent_profile in the request. +# """ +# litellm._turn_on_debug() - response = await litellm.aresponses( - model="manus/manus-1.6", - input="What's the color of the sky?", - api_key=os.getenv("MANUS_API_KEY"), - max_output_tokens=50, - ) +# response = await litellm.aresponses( +# model="manus/manus-1.6", +# input="What's the color of the sky?", +# api_key=os.getenv("MANUS_API_KEY"), +# max_output_tokens=50, +# ) - print("Manus response=", json.dumps(response, indent=4, default=str)) +# print("Manus response=", json.dumps(response, indent=4, default=str)) - # Validate response structure - assert isinstance(response, ResponsesAPIResponse), "Response should be ResponsesAPIResponse" - assert response.id is not None, "Response should have an ID" - assert response.status in ["running", "completed", "pending"], f"Status should be valid, got {response.status}" +# # Validate response structure +# assert isinstance(response, ResponsesAPIResponse), "Response should be ResponsesAPIResponse" +# assert response.id is not None, "Response should have an ID" +# assert response.status in ["running", "completed", "pending"], f"Status should be valid, got {response.status}" - # Check that metadata includes Manus-specific fields - if response.metadata: - assert "task_id" in response.metadata or "task_url" in response.metadata, ( - "Manus response should include task_id or task_url in metadata" - ) +# # Check that metadata includes Manus-specific fields +# if response.metadata: +# assert "task_id" in response.metadata or "task_url" in response.metadata, ( +# "Manus response should include task_id or task_url in metadata" +# ) -@pytest.mark.asyncio -async def test_manus_responses_api_different_agent_profiles(): - """ - Test that different agent profiles work correctly. - """ - litellm._turn_on_debug() +# @pytest.mark.asyncio +# async def test_manus_responses_api_different_agent_profiles(): +# """ +# Test that different agent profiles work correctly. +# """ +# litellm._turn_on_debug() - # Test with different agent profile variants - agent_profiles = ["manus-1.6", "manus-1.6-lite", "manus-1.6-max"] +# # Test with different agent profile variants +# agent_profiles = ["manus-1.6", "manus-1.6-lite", "manus-1.6-max"] - for profile in agent_profiles: - try: - response = await litellm.aresponses( - model=f"manus/{profile}", - input="Hello", - api_key=os.getenv("MANUS_API_KEY"), - max_output_tokens=20, - ) +# for profile in agent_profiles: +# try: +# response = await litellm.aresponses( +# model=f"manus/{profile}", +# input="Hello", +# api_key=os.getenv("MANUS_API_KEY"), +# max_output_tokens=20, +# ) - assert response.id is not None, f"Response for {profile} should have an ID" - print(f"✓ {profile} works: {response.id}") - except Exception as e: - # Some profiles might not be available, that's okay - print(f"⚠ {profile} not available: {e}") - pass +# assert response.id is not None, f"Response for {profile} should have an ID" +# print(f"✓ {profile} works: {response.id}") +# except Exception as e: +# # Some profiles might not be available, that's okay +# print(f"⚠ {profile} not available: {e}") +# pass From cfda03ebe1229d5e0da1a48e9c23792172e76c52 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Thu, 8 Jan 2026 14:55:36 -0300 Subject: [PATCH 058/121] fix(gemini): support snake_case for google_search tool parameters (#18451) * fix(gemini): support snake_case for google_search tool parameters Add snake_case aliases for Gemini tool names to match the pattern already used by other tools (url_context, google_maps, code_execution): - google_search -> googleSearch - google_search_retrieval -> googleSearchRetrieval - enterprise_web_search -> enterpriseWebSearch * test(gemini): add tests for snake_case google_search tool aliases * refactor(gemini): simplify get_tool_value calls formatting --- .../vertex_and_google_ai_studio_gemini.py | 27 ++++---- .../vertex_ai/gemini/test_transformation.py | 64 ++++++++++++++++++- 2 files changed, 77 insertions(+), 14 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index ba1788a217f..3c93b1943e4 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -480,20 +480,21 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): or tool_name == VertexToolName.CODE_EXECUTION.value ): # code_execution maintained for backwards compatibility code_execution = self.get_tool_value(tool, "codeExecution") - elif tool_name and tool_name == VertexToolName.GOOGLE_SEARCH.value: - googleSearch = self.get_tool_value( - tool, VertexToolName.GOOGLE_SEARCH.value - ) - elif ( - tool_name and tool_name == VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value + elif tool_name and ( + tool_name == VertexToolName.GOOGLE_SEARCH.value + or tool_name == "google_search" ): - googleSearchRetrieval = self.get_tool_value( - tool, VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value - ) - elif tool_name and tool_name == VertexToolName.ENTERPRISE_WEB_SEARCH.value: - enterpriseWebSearch = self.get_tool_value( - tool, VertexToolName.ENTERPRISE_WEB_SEARCH.value - ) + googleSearch = self.get_tool_value(tool, tool_name) + elif tool_name and ( + tool_name == VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value + or tool_name == "google_search_retrieval" + ): + googleSearchRetrieval = self.get_tool_value(tool, tool_name) + elif tool_name and ( + tool_name == VertexToolName.ENTERPRISE_WEB_SEARCH.value + or tool_name == "enterprise_web_search" + ): + enterpriseWebSearch = self.get_tool_value(tool, tool_name) elif tool_name and ( tool_name == VertexToolName.URL_CONTEXT.value or tool_name == "urlContext" diff --git a/tests/litellm/llms/vertex_ai/gemini/test_transformation.py b/tests/litellm/llms/vertex_ai/gemini/test_transformation.py index 6d005af28ac..20f48b6f393 100644 --- a/tests/litellm/llms/vertex_ai/gemini/test_transformation.py +++ b/tests/litellm/llms/vertex_ai/gemini/test_transformation.py @@ -7,6 +7,7 @@ sys.path.insert( 0, os.path.abspath("../../../../..") ) # Adds the parent directory to the system path from litellm.llms.vertex_ai.gemini import transformation +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig from litellm.types.llms import openai from litellm.types import completion from litellm.types.llms.vertex_ai import RequestBody @@ -225,4 +226,65 @@ async def test__transform_request_body_image_config_with_image_size(): assert "generationConfig" in rb assert "imageConfig" in rb["generationConfig"] assert rb["generationConfig"]["imageConfig"]["aspectRatio"] == "16:9" - assert rb["generationConfig"]["imageConfig"]["imageSize"] == "4K" \ No newline at end of file + assert rb["generationConfig"]["imageConfig"]["imageSize"] == "4K" + + +def test_map_function_google_search_snake_case(): + """ + Test that google_search tool (snake_case) is properly mapped to googleSearch. + Fixes issue where tools=[{"google_search": {}}] was being stripped. + """ + config = VertexGeminiConfig() + optional_params = {} + + # Test snake_case google_search + tools = [{"google_search": {}}] + result = config._map_function(tools, optional_params) + + assert len(result) == 1 + assert "googleSearch" in result[0] + assert result[0]["googleSearch"] == {} + + +def test_map_function_google_search_camel_case(): + """ + Test that googleSearch tool (camelCase) still works. + """ + config = VertexGeminiConfig() + optional_params = {} + + # Test camelCase googleSearch + tools = [{"googleSearch": {}}] + result = config._map_function(tools, optional_params) + + assert len(result) == 1 + assert "googleSearch" in result[0] + assert result[0]["googleSearch"] == {} + + +def test_map_function_google_search_retrieval_snake_case(): + """ + Test that google_search_retrieval tool (snake_case) is properly mapped. + """ + config = VertexGeminiConfig() + optional_params = {} + + tools = [{"google_search_retrieval": {"dynamic_retrieval_config": {"mode": "MODE_DYNAMIC"}}}] + result = config._map_function(tools, optional_params) + + assert len(result) == 1 + assert "googleSearchRetrieval" in result[0] + + +def test_map_function_enterprise_web_search_snake_case(): + """ + Test that enterprise_web_search tool (snake_case) is properly mapped. + """ + config = VertexGeminiConfig() + optional_params = {} + + tools = [{"enterprise_web_search": {}}] + result = config._map_function(tools, optional_params) + + assert len(result) == 1 + assert "enterpriseWebSearch" in result[0] \ No newline at end of file From 3ebec39b740395efe6d97b175fc8119006a67607 Mon Sep 17 00:00:00 2001 From: Constantine Date: Thu, 8 Jan 2026 20:56:46 +0300 Subject: [PATCH 059/121] fix(proxy): use async anthropic client to prevent event loop blocking (#18435) Fixes #16716. Previously, synchronous Anthropic client was used for token counting, which blocked the event loop. This change switches to AsyncAnthropic and caches the client instance. --- litellm/proxy/utils.py | 10 ++++-- tests/test_litellm/test_utils_custom.py | 42 +++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/test_utils_custom.py diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index d1a78534dae..bd44cef9541 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -131,6 +131,7 @@ else: unified_guardrail = UnifiedLLMGuardrails() +_anthropic_async_clients = {} def print_verbose(print_statement): """ @@ -4254,11 +4255,16 @@ async def count_tokens_with_anthropic_api( if anthropic_api_key and messages: # Call Anthropic API directly for more accurate token counting - client = anthropic.Anthropic(api_key=anthropic_api_key) + + # Use cached client if available to avoid socket exhaustion + if anthropic_api_key not in _anthropic_async_clients: + _anthropic_async_clients[anthropic_api_key] = anthropic.AsyncAnthropic(api_key=anthropic_api_key) + + client = _anthropic_async_clients[anthropic_api_key] # Call with explicit parameters to satisfy type checking # Type ignore for now since messages come from generic dict input - response = client.beta.messages.count_tokens( + response = await client.beta.messages.count_tokens( model=model_to_use, messages=messages, # type: ignore betas=["token-counting-2024-11-01"], diff --git a/tests/test_litellm/test_utils_custom.py b/tests/test_litellm/test_utils_custom.py new file mode 100644 index 00000000000..292da4132b9 --- /dev/null +++ b/tests/test_litellm/test_utils_custom.py @@ -0,0 +1,42 @@ +import pytest +from unittest.mock import MagicMock, patch, AsyncMock +from litellm.proxy.utils import count_tokens_with_anthropic_api, _anthropic_async_clients + +@pytest.mark.asyncio +async def test_count_tokens_caching(): + """ + Test that count_tokens_with_anthropic_api caches the client. + """ + # Clear cache + _anthropic_async_clients.clear() + + api_key = "sk-ant-test-key" + messages = [{"role": "user", "content": "hello"}] + model = "claude-3-opus-20240229" + + # Mock anthropic + with patch("anthropic.AsyncAnthropic") as mock_cls: + mock_client = MagicMock() + mock_cls.return_value = mock_client + + # Mock response + mock_response = MagicMock() + mock_response.input_tokens = 10 + + # Setup async return for count_tokens + mock_client.beta.messages.count_tokens = AsyncMock(return_value=mock_response) + + # First call + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": api_key}): + await count_tokens_with_anthropic_api(model, messages) + + assert api_key in _anthropic_async_clients + assert _anthropic_async_clients[api_key] == mock_client + mock_cls.assert_called_once() # Should be called once + + # Second call + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": api_key}): + await count_tokens_with_anthropic_api(model, messages) + + # Should still be called once (cached) + mock_cls.assert_called_once() From 2a138a202f034ffec3bc919e39c9f71dd61a3f0f Mon Sep 17 00:00:00 2001 From: Lucas Rothman <51925729+lucasrothman@users.noreply.github.com> Date: Thu, 8 Jan 2026 10:02:47 -0800 Subject: [PATCH 060/121] fix: properly use litellm api keys (#18832) --- litellm/responses/main.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 8177b177fe6..07fc3cb02cc 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -577,7 +577,13 @@ def responses( api_base=litellm_params.api_base, api_key=litellm_params.api_key, ) - + + # Use dynamic credentials from get_llm_provider (e.g., when use_litellm_proxy=True) + if dynamic_api_key is not None: + litellm_params.api_key = dynamic_api_key + if dynamic_api_base is not None: + litellm_params.api_base = dynamic_api_base + ######################################################### # Update input with provider-specific file IDs if managed files are used ######################################################### @@ -1483,6 +1489,12 @@ def compact_responses( api_key=litellm_params.api_key, ) + # Use dynamic credentials from get_llm_provider (e.g., when use_litellm_proxy=True) + if dynamic_api_key is not None: + litellm_params.api_key = dynamic_api_key + if dynamic_api_base is not None: + litellm_params.api_base = dynamic_api_base + if custom_llm_provider is None: raise ValueError("custom_llm_provider is required but passed as None") From 1d296a990c922a572312f1666ccbfde35cf28f6d Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Thu, 8 Jan 2026 23:36:08 +0530 Subject: [PATCH 061/121] fix: add idx on LOWER(user_email) for faster duplicate email checks log(n) B tree approach (#18828) --- .../20260108_add_user_email_lower_idx/migration.sql | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260108_add_user_email_lower_idx/migration.sql diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260108_add_user_email_lower_idx/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260108_add_user_email_lower_idx/migration.sql new file mode 100644 index 00000000000..add80b39e7f --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260108_add_user_email_lower_idx/migration.sql @@ -0,0 +1,9 @@ +-- CreateIndex +-- Fixes performance issue in _check_duplicate_user_email function +-- by enabling fast case-insensitive email lookups. +-- +-- Without this index, queries with mode: "insensitive" cause full table scans. +-- With this index, PostgreSQL can use an Index Scan for O(log n) performance. +-- +-- Related: GitHub Issue #18411 +CREATE INDEX "LiteLLM_UserTable_user_email_lower_idx" ON "LiteLLM_UserTable"(LOWER("user_email")); From 60edf13a218ba68cf069317129439aef77c310e6 Mon Sep 17 00:00:00 2001 From: Chongshun Date: Thu, 8 Jan 2026 13:09:03 -0500 Subject: [PATCH 062/121] feat(tag-routing): support toggling tag matching between ANY and ALL (#18776) --- docs/my-website/docs/proxy/config_settings.md | 3 +++ litellm/router.py | 2 ++ litellm/router_strategy/tag_based_routing.py | 23 ++++++++++++++----- .../router_settings_endpoints.py | 8 +++++++ .../test_router_tag_routing.py | 16 ++++++++++++- 5 files changed, 45 insertions(+), 7 deletions(-) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index dfc0efd37ad..68e1f629e8b 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -146,6 +146,7 @@ router_settings: cooldown_time: 30 # (in seconds) how long to cooldown model if fails/min > allowed_fails disable_cooldowns: True # bool - Disable cooldowns for all models enable_tag_filtering: True # bool - Use tag based routing for requests + tag_filtering_match_any: True # bool - Tag matching behavior (only when enable_tag_filtering=true). `true`: match if deployment has ANY requested tag; `false`: match only if deployment has ALL requested tags retry_policy: { # Dict[str, int]: retry policy for different types of exceptions "AuthenticationErrorRetries": 3, "TimeoutErrorRetries": 3, @@ -293,6 +294,7 @@ router_settings: cooldown_time: 30 # (in seconds) how long to cooldown model if fails/min > allowed_fails disable_cooldowns: True # bool - Disable cooldowns for all models enable_tag_filtering: True # bool - Use tag based routing for requests + tag_filtering_match_any: True # bool - Tag matching behavior (only when enable_tag_filtering=true). `true`: match if deployment has ANY requested tag; `false`: match only if deployment has ALL requested tags retry_policy: { # Dict[str, int]: retry policy for different types of exceptions "AuthenticationErrorRetries": 3, "TimeoutErrorRetries": 3, @@ -322,6 +324,7 @@ router_settings: | content_policy_fallbacks | array of objects | Specifies fallback models for content policy violations. [More information here](reliability) | | fallbacks | array of objects | Specifies fallback models for all types of errors. [More information here](reliability) | | enable_tag_filtering | boolean | If true, uses tag based routing for requests [Tag Based Routing](tag_routing) | +| tag_filtering_match_any | boolean | Tag matching behavior (only when enable_tag_filtering=true). `true`: match if deployment has ANY requested tag; `false`: match only if deployment has ALL requested tags | | cooldown_time | integer | The duration (in seconds) to cooldown a model if it exceeds the allowed failures. | | disable_cooldowns | boolean | If true, disables cooldowns for all models. [More information here](reliability) | | retry_policy | object | Specifies the number of retries for different types of exceptions. [More information here](reliability) | diff --git a/litellm/router.py b/litellm/router.py index 98ccf41c96d..84b38b3985b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -255,6 +255,7 @@ class Router: ] = {}, enable_pre_call_checks: bool = False, enable_tag_filtering: bool = False, + tag_filtering_match_any: bool = True, retry_after: int = 0, # min time to wait before retrying a failed request retry_policy: Optional[ Union[RetryPolicy, dict] @@ -363,6 +364,7 @@ class Router: self.debug_level = debug_level self.enable_pre_call_checks = enable_pre_call_checks self.enable_tag_filtering = enable_tag_filtering + self.tag_filtering_match_any = tag_filtering_match_any from litellm._service_logger import ServiceLogging self.service_logger_obj: ServiceLogging = ServiceLogging() diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index b25c20eb281..e960e00a68f 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -20,17 +20,28 @@ else: def is_valid_deployment_tag( - deployment_tags: List[str], request_tags: List[str] + deployment_tags: List[str], request_tags: List[str], match_any: bool = True ) -> bool: """ - Check if a tag is valid + Check if a tag is valid, the matching can be either any or all based on `match_any` flag """ + if not request_tags: + return False - if any(tag in deployment_tags for tag in request_tags): + dep_set = set(deployment_tags) + req_set = set(request_tags) + + if match_any: + is_valid_deployment = bool(dep_set & req_set) + else: + is_valid_deployment = req_set.issubset(dep_set) + + if is_valid_deployment: verbose_logger.debug( - "adding deployment with tags: %s, request tags: %s", + "adding deployment with tags: %s, request tags: %s for match_any=%s", deployment_tags, request_tags, + match_any, ) return True return False @@ -68,6 +79,7 @@ async def get_deployments_for_tag( if metadata_variable_name in request_kwargs: metadata = request_kwargs[metadata_variable_name] request_tags = metadata.get("tags") + match_any = llm_router_instance.tag_filtering_match_any new_healthy_deployments = [] default_deployments = [] @@ -76,7 +88,6 @@ async def get_deployments_for_tag( "get_deployments_for_tag routing: router_keys: %s", request_tags ) # example this can be router_keys=["free", "custom"] - # get all deployments that have a superset of these router keys for deployment in healthy_deployments: deployment_litellm_params = deployment.get("litellm_params") deployment_tags = deployment_litellm_params.get("tags") @@ -90,7 +101,7 @@ async def get_deployments_for_tag( if deployment_tags is None: continue - if is_valid_deployment_tag(deployment_tags, request_tags): + if is_valid_deployment_tag(deployment_tags, request_tags, match_any): new_healthy_deployments.append(deployment) if "default" in deployment_tags: diff --git a/litellm/types/management_endpoints/router_settings_endpoints.py b/litellm/types/management_endpoints/router_settings_endpoints.py index 9e3002ecf45..8b05c1483e8 100644 --- a/litellm/types/management_endpoints/router_settings_endpoints.py +++ b/litellm/types/management_endpoints/router_settings_endpoints.py @@ -184,6 +184,14 @@ ROUTER_SETTINGS_FIELDS: List[RouterSettingsField] = [ field_default=False, ui_field_name="Enable Tag Filtering", link="https://docs.litellm.ai/docs/proxy/tag_routing", + ), + RouterSettingsField( + field_name="tag_filtering_match_any", + field_type="Boolean", + field_value=None, + field_description="Match any tag instead of all tags for tag-based routing", + field_default=True, + ui_field_name="Tag Filtering Match Any", ), RouterSettingsField( field_name="disable_cooldowns", diff --git a/tests/test_litellm/router_strategy/test_router_tag_routing.py b/tests/test_litellm/router_strategy/test_router_tag_routing.py index a3e722eeb85..1fdd3dad4da 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_routing.py @@ -313,17 +313,31 @@ async def test_error_from_tag_routing(): def test_tag_routing_with_list_of_tags(): """ - Test that the router can handle a list of tags + Test that the router can handle a list of tags with match_any behavior """ from litellm.router_strategy.tag_based_routing import is_valid_deployment_tag assert is_valid_deployment_tag(["teamA", "teamB"], ["teamA"]) assert is_valid_deployment_tag(["teamA", "teamB"], ["teamA", "teamB"]) assert is_valid_deployment_tag(["teamA", "teamB"], ["teamA", "teamC"]) + assert is_valid_deployment_tag(["teamA"], ["teamA", "teamB"]) assert not is_valid_deployment_tag(["teamA", "teamB"], ["teamC"]) assert not is_valid_deployment_tag(["teamA", "teamB"], []) assert not is_valid_deployment_tag(["default"], ["teamA"]) +def test_tag_routing_with_list_of_tags_match_all(): + """ + Test that the router can handle a list of tags with match_all behavior + """ + from litellm.router_strategy.tag_based_routing import is_valid_deployment_tag + + assert is_valid_deployment_tag(["teamA", "teamB"], ["teamA"], match_any=False) + assert is_valid_deployment_tag(["teamA", "teamB"], ["teamA", "teamB"], match_any=False) + assert not is_valid_deployment_tag(["teamA", "teamB", "teamC"], ["teamA", "teamD"], match_any=False) + assert not is_valid_deployment_tag(["teamA"], ["teamA", "teamB"], match_any=False) + assert not is_valid_deployment_tag(["teamA", "teamB"], ["teamA", "teamC"], match_any=False) + assert not is_valid_deployment_tag(["teamA", "teamB"], [], match_any=False) + assert not is_valid_deployment_tag(["default"], ["teamA"], match_any=False) @pytest.mark.asyncio() async def test_router_free_paid_tier_with_responses_api(): From 1c1ee8de46c8eb13b42da6daf3ac4c245aabc7c6 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Thu, 8 Jan 2026 15:12:05 -0300 Subject: [PATCH 063/121] Mask extra header secrets in model info (#18822) --- .../sensitive_data_masker.py | 67 +++++++++++++++---- .../test_sensitive_data_masker.py | 44 ++++++++++++ 2 files changed, 98 insertions(+), 13 deletions(-) diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 206810943ca..8b6ae744637 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -1,4 +1,5 @@ -from typing import Any, Dict, Optional, Set +from collections.abc import Mapping +from typing import Any, Dict, List, Optional, Set from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER @@ -17,6 +18,7 @@ class SensitiveDataMasker: "key", "token", "auth", + "authorization", "credential", "access", "private", @@ -42,22 +44,52 @@ class SensitiveDataMasker: else: return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}{value_str[-self.visible_suffix:]}" - def is_sensitive_key(self, key: str, excluded_keys: Optional[Set[str]] = None) -> bool: + def is_sensitive_key( + self, key: str, excluded_keys: Optional[Set[str]] = None + ) -> bool: # Check if key is in excluded_keys first (exact match) if excluded_keys and key in excluded_keys: return False - + key_lower = str(key).lower() - # Split on underscores and check if any segment matches the pattern + # Split on underscores/hyphens and check if any segment matches the pattern # This avoids false positives like "max_tokens" matching "token" # but still catches "api_key", "access_token", etc. - key_segments = key_lower.replace('-', '_').split('_') - result = any( - pattern in key_segments - for pattern in self.sensitive_patterns - ) + key_segments = key_lower.replace("-", "_").split("_") + result = any(pattern in key_segments for pattern in self.sensitive_patterns) return result + def _mask_sequence( + self, + values: List[Any], + depth: int, + max_depth: int, + excluded_keys: Optional[Set[str]], + key_is_sensitive: bool, + ) -> List[Any]: + masked_items: List[Any] = [] + if depth >= max_depth: + return values + + for item in values: + if isinstance(item, Mapping): + masked_items.append( + self.mask_dict(dict(item), depth + 1, max_depth, excluded_keys) + ) + elif isinstance(item, list): + masked_items.append( + self._mask_sequence( + item, depth + 1, max_depth, excluded_keys, key_is_sensitive + ) + ) + elif key_is_sensitive and isinstance(item, str): + masked_items.append(self._mask_value(item)) + else: + masked_items.append( + item if isinstance(item, (int, float, bool, str, list)) else str(item) + ) + return masked_items + def mask_dict( self, data: Dict[str, Any], @@ -71,11 +103,20 @@ class SensitiveDataMasker: masked_data: Dict[str, Any] = {} for k, v in data.items(): try: - if isinstance(v, dict): - masked_data[k] = self.mask_dict(v, depth + 1, max_depth, excluded_keys) + key_is_sensitive = self.is_sensitive_key(k, excluded_keys) + if isinstance(v, Mapping): + masked_data[k] = self.mask_dict( + dict(v), depth + 1, max_depth, excluded_keys + ) + elif isinstance(v, list): + masked_data[k] = self._mask_sequence( + v, depth + 1, max_depth, excluded_keys, key_is_sensitive + ) elif hasattr(v, "__dict__") and not isinstance(v, type): - masked_data[k] = self.mask_dict(vars(v), depth + 1, max_depth, excluded_keys) - elif self.is_sensitive_key(k, excluded_keys): + masked_data[k] = self.mask_dict( + vars(v), depth + 1, max_depth, excluded_keys + ) + elif key_is_sensitive: str_value = str(v) if v is not None else "" masked_data[k] = self._mask_value(str_value) else: diff --git a/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py b/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py index 2836398228a..9f0a1ae8ffe 100644 --- a/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py +++ b/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py @@ -75,3 +75,47 @@ def test_excluded_keys_exact_match(): assert masked["api_key"] == "sk-1234567890abcdef" # Should NOT be masked assert masked["access_token"] != "token-12345" # Should still be masked assert "*" in masked["access_token"] + + +def test_extra_headers_are_masked_recursively(): + """ + Ensure nested dictionaries (like extra_headers) are masked. + """ + masker = SensitiveDataMasker() + + data = { + "litellm_params": { + "model": "openai/gpt-4", + "extra_headers": { + "rits_api_key": "sk-secret-12345-very-sensitive", + "Authorization": "Bearer token123", + }, + } + } + + masked = masker.mask_dict(data) + extra_headers = masked["litellm_params"]["extra_headers"] + + assert extra_headers["rits_api_key"] != "sk-secret-12345-very-sensitive" + assert "*" in extra_headers["rits_api_key"] + assert extra_headers["Authorization"] != "Bearer token123" + assert "*" in extra_headers["Authorization"] + + +def test_lists_with_sensitive_keys_are_masked(): + """ + Lists belonging to sensitive keys should have their values masked. + """ + masker = SensitiveDataMasker() + data = { + "api_key": ["sk-123", "sk-456"], + "tags": ["prod", "test"], + } + + masked = masker.mask_dict(data) + # sensitive key list entries should be masked + assert masked["api_key"][0] != "sk-123" + assert "*" in masked["api_key"][0] + + # non-sensitive list should remain unchanged + assert masked["tags"] == ["prod", "test"] From 2ef8bbdf6a1ac770fe9a6b198cbae46857f2f770 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Thu, 8 Jan 2026 15:15:57 -0300 Subject: [PATCH 064/121] fix: add xiaomi_mimo to LlmProviders enum to fix router support (#18819) Added XIAOMI_MIMO to the LlmProviders enum in types/utils.py. The provider was already configured in providers.json but was missing from the enum, causing "Unsupported provider" errors when using it in Router/Proxy configurations. Also added comprehensive unit tests to prevent regression. --- litellm/types/utils.py | 1 + .../llms/openai_like/test_xiaomi_mimo.py | 150 ++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 tests/test_litellm/llms/openai_like/test_xiaomi_mimo.py diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ff8f3c0469c..891826787ef 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3029,6 +3029,7 @@ class LlmProviders(str, Enum): NANOGPT = "nano-gpt" POE = "poe" CHUTES = "chutes" + XIAOMI_MIMO = "xiaomi_mimo" diff --git a/tests/test_litellm/llms/openai_like/test_xiaomi_mimo.py b/tests/test_litellm/llms/openai_like/test_xiaomi_mimo.py new file mode 100644 index 00000000000..d025c716a4a --- /dev/null +++ b/tests/test_litellm/llms/openai_like/test_xiaomi_mimo.py @@ -0,0 +1,150 @@ +""" +Tests for Xiaomi MiMo provider configuration and integration. +Related to issue #18794 +""" + +import os +import sys +from unittest.mock import MagicMock, patch + +try: + import pytest +except ImportError: + pytest = None + +# Add workspace to path +workspace_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..")) +sys.path.insert(0, workspace_path) + +import litellm + + +class TestXiaomiMiMoProviderConfig: + """Test Xiaomi MiMo provider configuration""" + + def test_xiaomi_mimo_in_provider_list(self): + """Test that xiaomi_mimo is in the provider list (fixes #18794)""" + from litellm import LlmProviders + + # Verify xiaomi_mimo is in the enum + assert hasattr(LlmProviders, 'XIAOMI_MIMO') + assert LlmProviders.XIAOMI_MIMO.value == 'xiaomi_mimo' + + # Verify it's in the provider list + assert 'xiaomi_mimo' in litellm.provider_list + + def test_xiaomi_mimo_json_config_exists(self): + """Test that xiaomi_mimo is configured in providers.json""" + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + # Verify xiaomi_mimo is loaded + assert JSONProviderRegistry.exists("xiaomi_mimo") + + # Get xiaomi_mimo config + xiaomi_mimo = JSONProviderRegistry.get("xiaomi_mimo") + assert xiaomi_mimo is not None + assert xiaomi_mimo.base_url == "https://api.xiaomimimo.com/v1" + assert xiaomi_mimo.api_key_env == "XIAOMI_MIMO_API_KEY" + assert xiaomi_mimo.param_mappings.get("max_completion_tokens") == "max_tokens" + + def test_xiaomi_mimo_provider_resolution(self): + """Test that provider resolution finds xiaomi_mimo""" + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="xiaomi_mimo/mimo-v2-flash", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "mimo-v2-flash" + assert provider == "xiaomi_mimo" + assert api_base == "https://api.xiaomimimo.com/v1" + + def test_xiaomi_mimo_router_config(self): + """Test that xiaomi_mimo can be used in Router configuration (fixes #18794)""" + from litellm import Router + + # This should not raise "Unsupported provider - xiaomi_mimo" + router = Router( + model_list=[ + { + "model_name": "mimo-v2-flash", + "litellm_params": { + "model": "xiaomi_mimo/mimo-v2-flash", + "api_key": "test-key", + }, + } + ] + ) + + # Verify the deployment was created successfully + assert len(router.model_list) == 1 + assert router.model_list[0]["model_name"] == "mimo-v2-flash" + + +class TestXiaomiMiMoIntegration: + """Integration tests for Xiaomi MiMo provider""" + + def test_xiaomi_mimo_completion_basic(self): + """Test basic completion call to Xiaomi MiMo""" + # Skip test if API key not set in environment + if not os.environ.get("XIAOMI_MIMO_API_KEY"): + if pytest: + pytest.skip("XIAOMI_MIMO_API_KEY not set") + return + + try: + response = litellm.completion( + model="xiaomi_mimo/mimo-v2-flash", + messages=[{"role": "user", "content": "Say 'test successful' and nothing else"}], + max_tokens=10, + ) + + # Verify response structure + assert response is not None + assert hasattr(response, "choices") + assert len(response.choices) > 0 + assert hasattr(response.choices[0], "message") + assert hasattr(response.choices[0].message, "content") + assert response.choices[0].message.content is not None + + # Check that we got a response + content = response.choices[0].message.content.lower() + assert len(content) > 0 + + print(f"✓ Xiaomi MiMo completion successful: {response.choices[0].message.content}") + + except Exception as e: + if pytest: + pytest.fail(f"Xiaomi MiMo completion failed: {str(e)}") + else: + raise + + +if __name__ == "__main__": + # Run basic tests + print("Testing Xiaomi MiMo Provider...") + + test_config = TestXiaomiMiMoProviderConfig() + + print("\n1. Testing provider in list...") + test_config.test_xiaomi_mimo_in_provider_list() + print(" ✓ xiaomi_mimo in provider list") + + print("\n2. Testing JSON config...") + test_config.test_xiaomi_mimo_json_config_exists() + print(" ✓ xiaomi_mimo JSON config loaded") + + print("\n3. Testing provider resolution...") + test_config.test_xiaomi_mimo_provider_resolution() + print(" ✓ Provider resolution works") + + print("\n4. Testing router configuration...") + test_config.test_xiaomi_mimo_router_config() + print(" ✓ Router configuration works (issue #18794 fixed)") + + print("\n" + "="*50) + print("✓ All configuration tests passed!") + print("="*50) From 7743c739a3d05d1ce948be361f8030c71d92b3c5 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Thu, 8 Jan 2026 15:16:17 -0300 Subject: [PATCH 065/121] docs: fix PDF documentation inconsistency in Anthropic page (#18816) Updated description to match the code example which uses `file` content type with `file_data` field, instead of incorrectly mentioning `image_url`. --- docs/my-website/docs/providers/anthropic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index cae8657f1a0..446d663c5ac 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -1692,9 +1692,9 @@ Assistant: ``` -## Usage - PDF +## Usage - PDF -Pass base64 encoded PDF files to Anthropic models using the `image_url` field. +Pass base64 encoded PDF files to Anthropic models using the `file` content type with a `file_data` field. From 86b71d4713e2697ade8f7d5e00ba54e8c9c1d395 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Thu, 8 Jan 2026 15:18:41 -0300 Subject: [PATCH 066/121] fix(workflow): Update issue labeling with working regex pattern (#18821) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: align max_tokens with max_output_tokens for consistency Fixed inconsistent max_tokens definitions in model_prices_and_context_window.json. According to LiteLLM convention, max_tokens should equal max_output_tokens when available. Models fixed: - deepseek-chat: 131072 → 8192 (now equals max_output_tokens) - dashscope/qwen-flash: 1000000 → 32768 (now equals max_output_tokens) - databricks/databricks-gemma-3-12b: 128000 → 32000 (now equals max_output_tokens) This ensures consistency across all providers where max_tokens represents the maximum number of tokens that can be generated in the output. * fix(workflow): Update issue labeling with working regex pattern - Replace contains() with regex pattern using \s* for flexible whitespace matching - Consolidate 4 separate steps into single unified component labeling step - Tested and verified pattern works for all components: SDK, Proxy, UI Dashboard, Docs - Pattern handles GitHub's issue body formatting with ### headers and variable newlines --- .github/workflows/label-component.yml | 174 +++++++++----------------- model_prices_and_context_window.json | 6 +- 2 files changed, 59 insertions(+), 121 deletions(-) diff --git a/.github/workflows/label-component.yml b/.github/workflows/label-component.yml index 9a547c162a6..76b8316790c 100644 --- a/.github/workflows/label-component.yml +++ b/.github/workflows/label-component.yml @@ -11,134 +11,72 @@ jobs: permissions: issues: write steps: - - name: Add SDK label - if: contains(github.event.issue.body, 'What part of LiteLLM is this about?\n\nSDK (litellm Python package)') + - name: Add component labels uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const labelName = 'sdk'; - try { - await github.rest.issues.getLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name: labelName - }); - } catch (error) { - if (error.status === 404) { - await github.rest.issues.createLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name: labelName, - color: '0E7C86', - description: 'Issues related to the litellm Python SDK' - }); - } else { - throw error; - } - } - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - labels: [labelName] - }); + const body = context.payload.issue.body; + if (!body) return; - - name: Add Proxy label - if: contains(github.event.issue.body, 'What part of LiteLLM is this about?\n\nProxy') - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const labelName = 'proxy'; - try { - await github.rest.issues.getLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name: labelName - }); - } catch (error) { - if (error.status === 404) { - await github.rest.issues.createLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name: labelName, - color: '5319E7', - description: 'Issues related to the LiteLLM Proxy' - }); - } else { - throw error; + // Define component mappings with regex patterns that handle flexible whitespace + const components = [ + { + pattern: /What part of LiteLLM is this about\?\s*SDK \(litellm Python package\)/, + label: 'sdk', + color: '0E7C86', + description: 'Issues related to the litellm Python SDK' + }, + { + pattern: /What part of LiteLLM is this about\?\s*Proxy/, + label: 'proxy', + color: '5319E7', + description: 'Issues related to the LiteLLM Proxy' + }, + { + pattern: /What part of LiteLLM is this about\?\s*UI Dashboard/, + label: 'ui-dashboard', + color: 'D876E3', + description: 'Issues related to the LiteLLM UI Dashboard' + }, + { + pattern: /What part of LiteLLM is this about\?\s*Docs/, + label: 'docs', + color: 'FBCA04', + description: 'Issues related to LiteLLM documentation' } - } - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - labels: [labelName] - }); + ]; - - name: Add UI Dashboard label - if: contains(github.event.issue.body, 'What part of LiteLLM is this about?\n\nUI Dashboard') - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const labelName = 'ui-dashboard'; - try { - await github.rest.issues.getLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name: labelName - }); - } catch (error) { - if (error.status === 404) { - await github.rest.issues.createLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name: labelName, - color: 'D876E3', - description: 'Issues related to the LiteLLM UI Dashboard' - }); - } else { - throw error; - } - } - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - labels: [labelName] - }); + // Find matching component + for (const component of components) { + if (component.pattern.test(body)) { + // Ensure label exists + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: component.label + }); + } catch (error) { + if (error.status === 404) { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: component.label, + color: component.color, + description: component.description + }); + } + } - - name: Add Docs label - if: contains(github.event.issue.body, 'What part of LiteLLM is this about?\n\nDocs') - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const labelName = 'docs'; - try { - await github.rest.issues.getLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name: labelName - }); - } catch (error) { - if (error.status === 404) { - await github.rest.issues.createLabel({ + // Add label to issue + await github.rest.issues.addLabels({ owner: context.repo.owner, repo: context.repo.repo, - name: labelName, - color: 'FBCA04', - description: 'Issues related to LiteLLM documentation' + issue_number: context.issue.number, + labels: [component.label] }); - } else { - throw error; + + break; } } - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - labels: [labelName] - }); diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 73579db75cd..3c2c20b4dce 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -7800,7 +7800,7 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 131072, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.7e-06, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -7854,7 +7854,7 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 1000000, + "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -8579,7 +8579,7 @@ "litellm_provider": "databricks", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 128000, + "max_tokens": 32000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, From 516e4f8b9652cb6199b5e504de97835a51f07592 Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Thu, 8 Jan 2026 23:53:36 +0530 Subject: [PATCH 067/121] fix: proactive RDS IAM token refresh to prevent 15-min connection failed (#18795) * fix: proactive RDS IAM token refresh to prevent 15-min connection failures (#16220) * fix: add noqa for PLR0915 in proxy_startup_event --- litellm/proxy/db/prisma_client.py | 309 +++++++++++++++--- litellm/proxy/proxy_server.py | 97 +++--- .../proxy/db/test_rds_iam_token_expiry.py | 275 ++++++++++++++++ 3 files changed, 602 insertions(+), 79 deletions(-) create mode 100644 tests/test_litellm/proxy/db/test_rds_iam_token_expiry.py diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index 406ddceabf5..c9c0cfe8f68 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -17,50 +17,141 @@ from litellm.secret_managers.main import str_to_bool class PrismaWrapper: + """ + Wrapper around Prisma client that handles RDS IAM token authentication. + + When iam_token_db_auth is enabled, this wrapper: + 1. Proactively refreshes IAM tokens before they expire (background task) + 2. Falls back to synchronous refresh if a token is found expired + 3. Uses proper locking to prevent race conditions during reconnection + + RDS IAM tokens are valid for 15 minutes. This wrapper refreshes them + 3 minutes before expiration to ensure uninterrupted database connectivity. + """ + + # Buffer time in seconds before token expiration to trigger refresh + # Refresh 3 minutes (180 seconds) before the token expires + TOKEN_REFRESH_BUFFER_SECONDS = 180 + + # Fallback refresh interval if token parsing fails (10 minutes) + FALLBACK_REFRESH_INTERVAL_SECONDS = 600 + def __init__(self, original_prisma: Any, iam_token_db_auth: bool): self._original_prisma = original_prisma self.iam_token_db_auth = iam_token_db_auth + # Background token refresh task management + self._token_refresh_task: Optional[asyncio.Task] = None + self._reconnection_lock = asyncio.Lock() + self._last_refresh_time: Optional[datetime] = None + + def _extract_token_from_db_url(self, db_url: Optional[str]) -> Optional[str]: + """ + Extract the token (password) from the DATABASE_URL. + + The token contains the AWS signature with X-Amz-Date and X-Amz-Expires parameters. + + Important: We must parse the URL while it's still encoded to preserve structure, + then decode the password portion. Otherwise the '?' in the token breaks URL parsing. + """ + if db_url is None: + return None + try: + # Parse URL while still encoded to preserve structure + parsed = urllib.parse.urlparse(db_url) + if parsed.password: + # Now decode just the password/token + return urllib.parse.unquote(parsed.password) + return None + except Exception: + return None + + def _parse_token_expiration(self, token: Optional[str]) -> Optional[datetime]: + """ + Parse the token to extract its expiration time. + + Returns the datetime when the token expires, or None if parsing fails. + """ + if token is None: + return None + + try: + # Token format: ...?X-Amz-Date=YYYYMMDDTHHMMSSZ&X-Amz-Expires=900&... + if "?" not in token: + return None + + query_string = token.split("?", 1)[1] + params = urllib.parse.parse_qs(query_string) + + expires_str = params.get("X-Amz-Expires", [None])[0] + date_str = params.get("X-Amz-Date", [None])[0] + + if not expires_str or not date_str: + return None + + token_created = datetime.strptime(date_str, "%Y%m%dT%H%M%SZ") + expires_in = int(expires_str) + + return token_created + timedelta(seconds=expires_in) + except Exception as e: + verbose_proxy_logger.debug(f"Failed to parse token expiration: {e}") + return None + + def _calculate_seconds_until_refresh(self) -> float: + """ + Calculate exactly how many seconds until we need to refresh the token. + + Uses precise timing: sleeps until (token_expiration - buffer_seconds). + For a 15-minute (900s) token with 180s buffer, this returns ~720s (12 min). + + Returns: + Number of seconds to sleep before the next refresh. + Returns 0 if token should be refreshed immediately. + Returns FALLBACK_REFRESH_INTERVAL_SECONDS if parsing fails. + """ + db_url = os.getenv("DATABASE_URL") + token = self._extract_token_from_db_url(db_url) + expiration_time = self._parse_token_expiration(token) + + if expiration_time is None: + # If we can't parse the token, use fallback interval + verbose_proxy_logger.debug( + f"Could not parse token expiration, using fallback interval of " + f"{self.FALLBACK_REFRESH_INTERVAL_SECONDS}s" + ) + return self.FALLBACK_REFRESH_INTERVAL_SECONDS + + # Calculate when we should refresh (expiration - buffer) + refresh_at = expiration_time - timedelta( + seconds=self.TOKEN_REFRESH_BUFFER_SECONDS + ) + + # How long until refresh time? + now = datetime.utcnow() + seconds_until_refresh = (refresh_at - now).total_seconds() + + # If already past refresh time, return 0 (refresh immediately) + return max(0, seconds_until_refresh) + def is_token_expired(self, token_url: Optional[str]) -> bool: + """Check if the token in the given URL is expired.""" if token_url is None: return True - # Decode the token URL to handle URL-encoded characters - decoded_url = urllib.parse.unquote(token_url) - # Parse the token URL - parsed_url = urllib.parse.urlparse(decoded_url) + token = self._extract_token_from_db_url(token_url) + expiration_time = self._parse_token_expiration(token) - # Parse the query parameters from the path component (if they exist there) - query_params = urllib.parse.parse_qs(parsed_url.query) + if expiration_time is None: + # If we can't parse the token, assume it's expired to trigger refresh + verbose_proxy_logger.debug( + "Could not parse token expiration, treating as expired" + ) + return True - # Get expiration time from the query parameters - expires = query_params.get("X-Amz-Expires", [None])[0] - if expires is None: - raise ValueError("X-Amz-Expires parameter is missing or invalid.") - - expires_int = int(expires) - - # Get the token's creation time from the X-Amz-Date parameter - token_time_str = query_params.get("X-Amz-Date", [""])[0] - if not token_time_str: - raise ValueError("X-Amz-Date parameter is missing or invalid.") - - # Ensure the token time string is parsed correctly - try: - token_time = datetime.strptime(token_time_str, "%Y%m%dT%H%M%SZ") - except ValueError as e: - raise ValueError(f"Invalid X-Amz-Date format: {e}") - - # Calculate the expiration time - expiration_time = token_time + timedelta(seconds=expires_int) - - # Current time in UTC - current_time = datetime.utcnow() - - # Check if the token is expired - return current_time > expiration_time + return datetime.utcnow() > expiration_time def get_rds_iam_token(self) -> Optional[str]: + """Generate a new RDS IAM token and update DATABASE_URL.""" if self.iam_token_db_auth: from litellm.proxy.auth.rds_iam_token import generate_iam_auth_token @@ -74,7 +165,6 @@ class PrismaWrapper: db_host=db_host, db_port=db_port, db_user=db_user ) - # print(f"token: {token}") _db_url = f"postgresql://{db_user}:{token}@{db_host}:{db_port}/{db_name}" if db_schema: _db_url += f"?schema={db_schema}" @@ -86,6 +176,7 @@ class PrismaWrapper: async def recreate_prisma_client( self, new_db_url: str, http_client: Optional[Any] = None ): + """Disconnect and reconnect the Prisma client with a new database URL.""" from prisma import Prisma # type: ignore try: @@ -100,21 +191,159 @@ class PrismaWrapper: await self._original_prisma.connect() + async def start_token_refresh_task(self) -> None: + """ + Start the background token refresh task. + + This task proactively refreshes RDS IAM tokens before they expire, + preventing connection failures. Should be called after the initial + Prisma client connection is established. + """ + if not self.iam_token_db_auth: + verbose_proxy_logger.debug( + "IAM token auth not enabled, skipping token refresh task" + ) + return + + if self._token_refresh_task is not None: + verbose_proxy_logger.debug("Token refresh task already running") + return + + self._token_refresh_task = asyncio.create_task(self._token_refresh_loop()) + verbose_proxy_logger.info( + "Started RDS IAM token proactive refresh background task" + ) + + async def stop_token_refresh_task(self) -> None: + """ + Stop the background token refresh task gracefully. + + Should be called during application shutdown to clean up resources. + """ + if self._token_refresh_task is None: + return + + self._token_refresh_task.cancel() + try: + await self._token_refresh_task + except asyncio.CancelledError: + pass + self._token_refresh_task = None + verbose_proxy_logger.info("Stopped RDS IAM token refresh background task") + + async def _token_refresh_loop(self) -> None: + """ + Background loop that proactively refreshes RDS IAM tokens before expiration. + + Uses precise timing: calculates the exact sleep duration until the token + needs to be refreshed (expiration - 3 minute buffer), then refreshes. + This is more efficient than polling, requiring only 1 wake-up per token cycle. + """ + verbose_proxy_logger.info( + f"RDS IAM token refresh loop started. " + f"Tokens will be refreshed {self.TOKEN_REFRESH_BUFFER_SECONDS}s before expiration." + ) + + while True: + try: + # Calculate exactly how long to sleep until next refresh + sleep_seconds = self._calculate_seconds_until_refresh() + + if sleep_seconds > 0: + verbose_proxy_logger.info( + f"RDS IAM token refresh scheduled in {sleep_seconds:.0f} seconds " + f"({sleep_seconds / 60:.1f} minutes)" + ) + await asyncio.sleep(sleep_seconds) + + # Refresh the token + verbose_proxy_logger.info("Proactively refreshing RDS IAM token...") + await self._safe_refresh_token() + + except asyncio.CancelledError: + verbose_proxy_logger.info("RDS IAM token refresh loop cancelled") + break + except Exception as e: + verbose_proxy_logger.error( + f"Error in RDS IAM token refresh loop: {e}. " + f"Retrying in {self.FALLBACK_REFRESH_INTERVAL_SECONDS}s..." + ) + # On error, wait before retrying to avoid tight error loops + try: + await asyncio.sleep(self.FALLBACK_REFRESH_INTERVAL_SECONDS) + except asyncio.CancelledError: + break + + async def _safe_refresh_token(self) -> None: + """ + Refresh the RDS IAM token with proper locking to prevent race conditions. + + Uses an asyncio lock to ensure only one refresh operation happens at a time, + preventing multiple concurrent reconnection attempts. + """ + async with self._reconnection_lock: + new_db_url = self.get_rds_iam_token() + if new_db_url: + await self.recreate_prisma_client(new_db_url) + self._last_refresh_time = datetime.utcnow() + verbose_proxy_logger.info( + "RDS IAM token refreshed successfully. New token valid for ~15 minutes." + ) + else: + verbose_proxy_logger.error( + "Failed to generate new RDS IAM token during proactive refresh" + ) + def __getattr__(self, name: str): + """ + Proxy attribute access to the underlying Prisma client. + + If IAM token auth is enabled and the token is expired, this method + provides a synchronous fallback to refresh the token. However, this + should rarely be needed since the background task proactively refreshes + tokens before they expire. + + FIXED: Now properly waits for reconnection to complete before returning, + instead of the previous fire-and-forget pattern that caused the bug. + """ original_attr = getattr(self._original_prisma, name) + if self.iam_token_db_auth: db_url = os.getenv("DATABASE_URL") - if self.is_token_expired(db_url): - db_url = self.get_rds_iam_token() - loop = asyncio.get_event_loop() - if db_url: + # Check if token is expired (should be rare if background task is running) + if self.is_token_expired(db_url): + verbose_proxy_logger.warning( + "RDS IAM token expired in __getattr__ - proactive refresh may have failed. " + "Triggering synchronous fallback refresh..." + ) + + new_db_url = self.get_rds_iam_token() + if new_db_url: + loop = asyncio.get_event_loop() + if loop.is_running(): - asyncio.run_coroutine_threadsafe( - self.recreate_prisma_client(db_url), loop + # FIXED: Actually wait for the reconnection to complete! + # The previous code used fire-and-forget which caused the bug. + future = asyncio.run_coroutine_threadsafe( + self.recreate_prisma_client(new_db_url), loop ) + try: + # Wait up to 30 seconds for reconnection + future.result(timeout=30) + verbose_proxy_logger.info( + "Synchronous token refresh completed successfully" + ) + except Exception as e: + verbose_proxy_logger.error( + f"Failed to refresh token synchronously: {e}" + ) + raise else: - asyncio.run(self.recreate_prisma_client(db_url)) + asyncio.run(self.recreate_prisma_client(new_db_url)) + + # Get the NEW attribute after reconnection + original_attr = getattr(self._original_prisma, name) else: raise ValueError("Failed to get RDS IAM token") diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 06525e39133..a56a6379cb7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -533,9 +533,9 @@ except ImportError: server_root_path = os.getenv("SERVER_ROOT_PATH", "") _license_check = LicenseCheck() premium_user: bool = _license_check.is_premium() -premium_user_data: Optional["EnterpriseLicenseData"] = ( - _license_check.airgapped_license_data -) +premium_user_data: Optional[ + "EnterpriseLicenseData" +] = _license_check.airgapped_license_data global_max_parallel_request_retries_env: Optional[str] = os.getenv( "LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES" ) @@ -658,7 +658,7 @@ async def _initialize_shared_aiohttp_session(): @asynccontextmanager -async def proxy_startup_event(app: FastAPI): +async def proxy_startup_event(app: FastAPI): # noqa: PLR0915 global prisma_client, master_key, use_background_health_checks, llm_router, llm_model_list, general_settings, proxy_budget_rescheduler_min_time, proxy_budget_rescheduler_max_time, litellm_proxy_admin_name, db_writer_client, store_model_in_db, premium_user, _license_check, proxy_batch_polling_interval, shared_aiohttp_session import json @@ -788,6 +788,17 @@ async def proxy_startup_event(app: FastAPI): except Exception as e: verbose_proxy_logger.error(f"Error closing shared aiohttp session: {e}") + # Shutdown event - stop RDS IAM token refresh background task + if ( + prisma_client is not None + and hasattr(prisma_client, "db") + and hasattr(prisma_client.db, "stop_token_refresh_task") + ): + try: + await prisma_client.db.stop_token_refresh_task() + except Exception as e: + verbose_proxy_logger.error(f"Error stopping token refresh task: {e}") + await proxy_shutdown_event() # type: ignore[reportGeneralTypeIssues] @@ -1083,9 +1094,7 @@ try: # In non-root Docker, we restructure in /var/lib/litellm/ui. try: _restructure_ui_html_files(ui_path) - verbose_proxy_logger.info( - f"Restructured UI directory: {ui_path}" - ) + verbose_proxy_logger.info(f"Restructured UI directory: {ui_path}") except PermissionError as e: verbose_proxy_logger.exception( f"Permission error while restructuring UI directory {ui_path}: {e}" @@ -1171,9 +1180,9 @@ master_key: Optional[str] = None config_agents: Optional[List[AgentConfig]] = None otel_logging = False prisma_client: Optional[PrismaClient] = None -shared_aiohttp_session: Optional["ClientSession"] = ( - None # Global shared session for connection reuse -) +shared_aiohttp_session: Optional[ + "ClientSession" +] = None # Global shared session for connection reuse user_api_key_cache = DualCache( default_in_memory_ttl=UserAPIKeyCacheTTLEnum.in_memory_cache_ttl.value ) @@ -1181,9 +1190,9 @@ model_max_budget_limiter = _PROXY_VirtualKeyModelMaxBudgetLimiter( dual_cache=user_api_key_cache ) litellm.logging_callback_manager.add_litellm_callback(model_max_budget_limiter) -redis_usage_cache: Optional[RedisCache] = ( - None # redis cache used for tracking spend, tpm/rpm limits -) +redis_usage_cache: Optional[ + RedisCache +] = None # redis cache used for tracking spend, tpm/rpm limits polling_via_cache_enabled: Union[Literal["all"], List[str], bool] = False polling_cache_ttl: int = 3600 # Default 1 hour TTL for polling cache user_custom_auth = None @@ -1522,9 +1531,9 @@ async def update_cache( # noqa: PLR0915 _id = "team_id:{}".format(team_id) try: # Fetch the existing cost for the given user - existing_spend_obj: Optional[LiteLLM_TeamTable] = ( - await user_api_key_cache.async_get_cache(key=_id) - ) + existing_spend_obj: Optional[ + LiteLLM_TeamTable + ] = await user_api_key_cache.async_get_cache(key=_id) if existing_spend_obj is None: # do nothing if team not in api key cache return @@ -1876,7 +1885,6 @@ class ProxyConfig: "environment_variables" in config_to_save and config_to_save["environment_variables"] ): - # decrypt the environment_variables - in case a caller function has already encrypted the environment_variables decrypted_env_vars = self._decrypt_and_set_db_env_variables( environment_variables=config_to_save["environment_variables"], @@ -2794,21 +2802,21 @@ class ProxyConfig: verbose_proxy_logger.debug(f"_alerting_callbacks: {general_settings}") if _alerting_callbacks is None: return - + # Ensure proxy_logging_obj.alerting is set for all alerting types _alerting_value = general_settings.get("alerting", None) - verbose_proxy_logger.debug(f"_load_alerting_settings: Calling update_values with alerting={_alerting_value}") + verbose_proxy_logger.debug( + f"_load_alerting_settings: Calling update_values with alerting={_alerting_value}" + ) proxy_logging_obj.update_values( alerting=_alerting_value, alerting_threshold=general_settings.get("alerting_threshold", 600), alert_types=general_settings.get("alert_types", None), - alert_to_webhook_url=general_settings.get( - "alert_to_webhook_url", None - ), + alert_to_webhook_url=general_settings.get("alert_to_webhook_url", None), alerting_args=general_settings.get("alerting_args", None), redis_cache=redis_usage_cache, ) - + for _alert in _alerting_callbacks: if _alert == "slack": # [OLD] v0 implementation - already handled by update_values above @@ -3279,7 +3287,7 @@ class ProxyConfig: proxy_logging_obj: ProxyLogging """ _general_settings = config_data.get("general_settings", {}) - + if _general_settings is not None and "alerting" in _general_settings: if ( general_settings is not None @@ -3294,7 +3302,8 @@ class ProxyConfig: _merged_alerting = list(_yaml_alerting.union(_db_alerting)) # Preserve order: YAML values first, then DB values _merged_alerting = list(general_settings["alerting"]) + [ - item for item in _general_settings["alerting"] + item + for item in _general_settings["alerting"] if item not in general_settings["alerting"] ] verbose_proxy_logger.debug( @@ -3605,7 +3614,6 @@ class ProxyConfig: await self._init_vector_stores_in_db(prisma_client=prisma_client) if self._should_load_db_object(object_type="vector_store_indexes"): - await self._init_vector_store_indexes_in_db(prisma_client=prisma_client) if self._should_load_db_object(object_type="mcp"): @@ -3804,10 +3812,10 @@ class ProxyConfig: ) try: - guardrails_in_db: List[Guardrail] = ( - await GuardrailRegistry.get_all_guardrails_from_db( - prisma_client=prisma_client - ) + guardrails_in_db: List[ + Guardrail + ] = await GuardrailRegistry.get_all_guardrails_from_db( + prisma_client=prisma_client ) verbose_proxy_logger.debug( "guardrails from the DB %s", str(guardrails_in_db) @@ -4134,9 +4142,9 @@ async def initialize( # noqa: PLR0915 user_api_base = api_base dynamic_config[user_model]["api_base"] = api_base if api_version: - os.environ["AZURE_API_VERSION"] = ( - api_version # set this for azure - litellm can read this from the env - ) + os.environ[ + "AZURE_API_VERSION" + ] = api_version # set this for azure - litellm can read this from the env if max_tokens: # model-specific param dynamic_config[user_model]["max_tokens"] = max_tokens if temperature: # model-specific param @@ -4654,10 +4662,14 @@ class ProxyStartupEvent: replace_existing=True, misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, ) - verbose_proxy_logger.info("Responses cost check job scheduled successfully") + verbose_proxy_logger.info( + "Responses cost check job scheduled successfully" + ) except Exception as e: - verbose_proxy_logger.debug(f"Failed to setup responses cost checking: {e}") + verbose_proxy_logger.debug( + f"Failed to setup responses cost checking: {e}" + ) verbose_proxy_logger.debug( "Checking responses cost for LiteLLM Managed Files is an Enterprise Feature. Skipping..." ) @@ -4826,6 +4838,14 @@ class ProxyStartupEvent: await prisma_client.connect() + ## Start RDS IAM token refresh background task if enabled ## + # This proactively refreshes IAM tokens before they expire, + # preventing the 15-minute connection failure bug (#16220) + if hasattr(prisma_client, "db") and hasattr( + prisma_client.db, "start_token_refresh_task" + ): + await prisma_client.db.start_token_refresh_task() + ## Add necessary views to proxy ## asyncio.create_task( prisma_client.check_view_exists() @@ -5937,7 +5957,6 @@ async def realtime_websocket_endpoint( ), user_api_key_dict=Depends(user_api_key_auth_websocket), ): - await websocket.accept() # Only use explicit parameters, not all query params @@ -9525,9 +9544,9 @@ async def get_config_list( hasattr(sub_field_info, "description") and sub_field_info.description is not None ): - nested_fields[idx].field_description = ( - sub_field_info.description - ) + nested_fields[ + idx + ].field_description = sub_field_info.description idx += 1 _stored_in_db = None diff --git a/tests/test_litellm/proxy/db/test_rds_iam_token_expiry.py b/tests/test_litellm/proxy/db/test_rds_iam_token_expiry.py new file mode 100644 index 00000000000..1492acb0794 --- /dev/null +++ b/tests/test_litellm/proxy/db/test_rds_iam_token_expiry.py @@ -0,0 +1,275 @@ +""" +Tests for the RDS IAM token proactive refresh implementation. + +Tests for GitHub Issue #16220: RDS IAM authentication connection failures after 15 minutes. + +The fix implements: +1. Proactive background token refresh (refreshes 3 min before expiration) +2. Precise sleep timing (1 wake-up per token cycle instead of polling) +3. Proper locking during reconnection +4. Fixed __getattr__ fallback that now waits for reconnection + +Run these tests: + poetry run pytest tests/test_litellm/proxy/db/test_rds_iam_token_expiry.py -v -s +""" + +import asyncio +import os +import urllib.parse +from datetime import datetime, timedelta +from unittest.mock import MagicMock, patch + +import pytest + + +class TestPrismaWrapperTokenRefresh: + """Tests for the PrismaWrapper RDS IAM token refresh implementation.""" + + @pytest.fixture + def setup_env(self): + """Setup environment variables for testing.""" + os.environ["DATABASE_HOST"] = "test-host.rds.amazonaws.com" + os.environ["DATABASE_PORT"] = "5432" + os.environ["DATABASE_USER"] = "test_user" + os.environ["DATABASE_NAME"] = "test_db" + os.environ["IAM_TOKEN_DB_AUTH"] = "True" + yield + # Cleanup + for key in [ + "DATABASE_HOST", + "DATABASE_PORT", + "DATABASE_USER", + "DATABASE_NAME", + "DATABASE_URL", + "IAM_TOKEN_DB_AUTH", + "DATABASE_SCHEMA", + ]: + os.environ.pop(key, None) + + def _generate_mock_token(self, expires_in_seconds: int = 900) -> str: + """Generate a mock IAM token with expiration info.""" + now = datetime.utcnow() + date_str = now.strftime("%Y%m%dT%H%M%SZ") + # Build the token like AWS does + token = f"mock-token?X-Amz-Date={date_str}&X-Amz-Expires={expires_in_seconds}&X-Amz-Signature=abc123" + return urllib.parse.quote(token, safe="") + + def _set_database_url_with_token(self, expires_in_seconds: int = 900): + """Set DATABASE_URL with a mock token.""" + token = self._generate_mock_token(expires_in_seconds) + os.environ[ + "DATABASE_URL" + ] = f"postgresql://test_user:{token}@test-host:5432/test_db" + + @pytest.mark.asyncio + async def test_is_token_expired_fresh(self, setup_env): + """Test that fresh token is not detected as expired.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + mock_prisma = MagicMock() + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=True) + + self._set_database_url_with_token(expires_in_seconds=900) + db_url = os.getenv("DATABASE_URL") + + assert wrapper.is_token_expired(db_url) is False + + @pytest.mark.asyncio + async def test_is_token_expired_old(self, setup_env): + """Test that old token is detected as expired.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + mock_prisma = MagicMock() + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=True) + + # Create an expired token + old_date = datetime.utcnow() - timedelta(seconds=901) + date_str = old_date.strftime("%Y%m%dT%H%M%SZ") + token = ( + f"mock-token?X-Amz-Date={date_str}&X-Amz-Expires=900&X-Amz-Signature=abc" + ) + encoded_token = urllib.parse.quote(token, safe="") + db_url = f"postgresql://test_user:{encoded_token}@test-host:5432/test_db" + + assert wrapper.is_token_expired(db_url) is True + + @pytest.mark.asyncio + async def test_start_stop_token_refresh_task(self, setup_env): + """Test that token refresh task starts and stops correctly.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + mock_prisma = MagicMock() + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=True) + + # Set a valid token + self._set_database_url_with_token(expires_in_seconds=900) + + # Start the task + await wrapper.start_token_refresh_task() + assert wrapper._token_refresh_task is not None + assert not wrapper._token_refresh_task.done() + + # Stop the task + await wrapper.stop_token_refresh_task() + assert wrapper._token_refresh_task is None + + @pytest.mark.asyncio + async def test_start_task_not_enabled(self, setup_env): + """Test that task doesn't start when IAM auth is not enabled.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + mock_prisma = MagicMock() + # IAM auth disabled + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=False) + + await wrapper.start_token_refresh_task() + assert wrapper._token_refresh_task is None + + @pytest.mark.asyncio + async def test_is_token_expired_null(self, setup_env): + """Test that None token is treated as expired.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + mock_prisma = MagicMock() + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=True) + + assert wrapper.is_token_expired(None) is True + + +class TestTokenExpirationParsing: + """Tests for token expiration parsing utilities.""" + + def test_parse_token_expiration_valid(self): + """Test parsing expiration from a valid token.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + mock_prisma = MagicMock() + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=True) + + # Create a token with known expiration + token = "mock-token?X-Amz-Date=20240101T120000Z&X-Amz-Expires=900&X-Amz-Signature=abc" + + expiration = wrapper._parse_token_expiration(token) + + assert expiration is not None + expected = datetime(2024, 1, 1, 12, 0, 0) + timedelta(seconds=900) + assert expiration == expected + + def test_parse_token_expiration_invalid(self): + """Test that invalid token returns None.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + mock_prisma = MagicMock() + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=True) + + # Invalid tokens + assert wrapper._parse_token_expiration(None) is None + assert wrapper._parse_token_expiration("no-query-params") is None + assert wrapper._parse_token_expiration("?missing=params") is None + + +class TestBackgroundRefreshLoop: + """Tests for the background refresh loop timing.""" + + @pytest.fixture + def setup_env(self): + """Setup environment variables for testing.""" + os.environ["DATABASE_HOST"] = "test-host.rds.amazonaws.com" + os.environ["DATABASE_PORT"] = "5432" + os.environ["DATABASE_USER"] = "test_user" + os.environ["DATABASE_NAME"] = "test_db" + yield + # Cleanup + for key in [ + "DATABASE_HOST", + "DATABASE_PORT", + "DATABASE_USER", + "DATABASE_NAME", + "DATABASE_URL", + ]: + os.environ.pop(key, None) + + @pytest.mark.asyncio + async def test_calculate_seconds_fallback_when_no_url(self, setup_env): + """Test that fallback is used when DATABASE_URL is not set.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + mock_prisma = MagicMock() + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=True) + + # Don't set DATABASE_URL + seconds = wrapper._calculate_seconds_until_refresh() + + # Should return fallback interval + assert seconds == wrapper.FALLBACK_REFRESH_INTERVAL_SECONDS + + +# ============================================================================ +# DEMONSTRATION SCRIPT +# ============================================================================ + + +async def demonstrate_fix(): + """ + Demonstrates the fix for the RDS IAM token expiration bug. + + Shows how the proactive refresh prevents the 15-minute connection failure. + """ + # Import the actual implementation + try: + from litellm.proxy.db.prisma_client import PrismaWrapper + except ImportError: + return + + # Setup mock environment + os.environ["DATABASE_HOST"] = "mock-rds.region.rds.amazonaws.com" + os.environ["DATABASE_PORT"] = "5432" + os.environ["DATABASE_USER"] = "iam_user" + os.environ["DATABASE_NAME"] = "litellm" + + # Create initial token (expires in 10 seconds for demo) + now = datetime.utcnow() + date_str = now.strftime("%Y%m%dT%H%M%SZ") + token = f"mock-token?X-Amz-Date={date_str}&X-Amz-Expires=10&X-Amz-Signature=abc123" + encoded_token = urllib.parse.quote(token, safe="") + os.environ[ + "DATABASE_URL" + ] = f"postgresql://iam_user:{encoded_token}@mock-rds:5432/litellm" + + # Create mock prisma client + mock_prisma = MagicMock() + + wrapper = PrismaWrapper(original_prisma=mock_prisma, iam_token_db_auth=True) + + # Override buffer for faster demo + wrapper.TOKEN_REFRESH_BUFFER_SECONDS = 3 + wrapper.FALLBACK_REFRESH_INTERVAL_SECONDS = 5 + _ = wrapper._calculate_seconds_until_refresh() # Verify calculation works + db_url = os.getenv("DATABASE_URL") + is_expired = wrapper.is_token_expired(db_url) + assert is_expired is False, "Fresh token should not be expired!" + + # Mock the _token_refresh_loop to prevent it from actually running + async def mock_loop(): + try: + await asyncio.sleep(1000) + except asyncio.CancelledError: + pass + + with patch.object(wrapper, "_token_refresh_loop", side_effect=mock_loop): + await wrapper.start_token_refresh_task() + await wrapper.stop_token_refresh_task() + + # Cleanup + for key in [ + "DATABASE_HOST", + "DATABASE_PORT", + "DATABASE_USER", + "DATABASE_NAME", + "DATABASE_URL", + ]: + os.environ.pop(key, None) + + +if __name__ == "__main__": + asyncio.run(demonstrate_fix()) From cc3b068f2240f4996984a3c28b4af2cd9d9643f9 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Fri, 9 Jan 2026 05:35:15 +0900 Subject: [PATCH 068/121] fix: mypy error --- litellm/integrations/focus/focus_logger.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/litellm/integrations/focus/focus_logger.py b/litellm/integrations/focus/focus_logger.py index 1589f030fa5..505337ff1c8 100644 --- a/litellm/integrations/focus/focus_logger.py +++ b/litellm/integrations/focus/focus_logger.py @@ -130,10 +130,7 @@ class FocusLogger(CustomLogger): async def initialize_focus_export_job(self) -> None: """Entry point for scheduler jobs to run export cycle with locking.""" - try: - from litellm.proxy.proxy_server import proxy_logging_obj - except ImportError: - proxy_logging_obj = None + from litellm.proxy.proxy_server import proxy_logging_obj pod_lock_manager = None if proxy_logging_obj is not None: From 9c93882365ad85186ef8af66abfc4a3a8f50b1aa Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 8 Jan 2026 13:03:51 -0800 Subject: [PATCH 069/121] fixing build --- .../components/playground/chat_ui/ChatUI.tsx | 119 +++++++++--------- .../llm_calls/chat_completion.test.tsx | 81 +++++++----- .../llm_calls/responses_api.test.tsx | 22 +++- 3 files changed, 128 insertions(+), 94 deletions(-) diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx index 9918040ef64..f747b5b2599 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx @@ -84,10 +84,7 @@ interface ChatUIProps { }; } -const MCP_SUPPORTED_ENDPOINTS = new Set([ - EndpointType.CHAT, - EndpointType.RESPONSES, -]); +const MCP_SUPPORTED_ENDPOINTS = new Set([EndpointType.CHAT, EndpointType.RESPONSES]); const ChatUI: React.FC = ({ accessToken, @@ -131,7 +128,7 @@ const ChatUI: React.FC = ({ }); const [apiKey, setApiKey] = useState(() => sessionStorage.getItem("apiKey") || ""); const [customProxyBaseUrl, setCustomProxyBaseUrl] = useState( - () => sessionStorage.getItem("customProxyBaseUrl") || "" + () => sessionStorage.getItem("customProxyBaseUrl") || "", ); const [inputMessage, setInputMessage] = useState(""); const [chatHistory, setChatHistory] = useState(() => { @@ -215,7 +212,7 @@ const ChatUI: React.FC = ({ const [temperature, setTemperature] = useState(1.0); const [maxTokens, setMaxTokens] = useState(2048); const [useAdvancedParams, setUseAdvancedParams] = useState(false); - + // Code Interpreter state (using custom hook) const codeInterpreter = useCodeInterpreter(); @@ -244,16 +241,15 @@ const ChatUI: React.FC = ({ try { const response = await listMCPTools(userApiKey, serverId); - setServerToolsMap(prev => ({ + setServerToolsMap((prev) => ({ ...prev, - [serverId]: response.tools || [] + [serverId]: response.tools || [], })); } catch (error) { console.error(`Error fetching tools for server ${serverId}:`, error); } }; - useEffect(() => { if (isGetCodeModalVisible) { const code = generateCodeSnippet({ @@ -1007,7 +1003,7 @@ const ChatUI: React.FC = ({ traceId, selectedVectorStores.length > 0 ? selectedVectorStores : undefined, selectedGuardrails.length > 0 ? selectedGuardrails : undefined, - selectedMCPTools, // Pass the selected tools array + selectedMCPServers, // Pass the selected tools array customProxyBaseUrl || undefined, ); } else if (endpointType === EndpointType.EMBEDDINGS) { @@ -1205,9 +1201,7 @@ const ChatUI: React.FC = ({ icon={ApiOutlined} /> {customProxyBaseUrl && ( - - API calls will be sent to: {customProxyBaseUrl} - + API calls will be sent to: {customProxyBaseUrl} )}
@@ -1382,7 +1376,11 @@ const ChatUI: React.FC = ({ optionLabelProp="label" > {agentInfo.map((agent) => ( - +
{agent.agent_name || agent.agent_id} {agent.agent_card_params?.description && ( @@ -1416,10 +1414,7 @@ const ChatUI: React.FC = ({
MCP Servers - + @@ -1435,15 +1430,15 @@ const ChatUI: React.FC = ({ } else { setSelectedMCPServers(value); // Clean up tool restrictions for removed servers - setMCPServerToolRestrictions(prev => { + setMCPServerToolRestrictions((prev) => { const updated = { ...prev }; - Object.keys(updated).forEach(serverId => { + Object.keys(updated).forEach((serverId) => { if (!value.includes(serverId)) delete updated[serverId]; }); return updated; }); // Load tools for newly selected servers - value.forEach(serverId => { + value.forEach((serverId) => { if (!serverToolsMap[serverId]) { loadServerTools(serverId); } @@ -1474,12 +1469,8 @@ const ChatUI: React.FC = ({ disabled={selectedMCPServers.includes("__all__")} >
- - {server.alias || server.server_name || server.server_id} - - {server.description && ( - {server.description} - )} + {server.alias || server.server_name || server.server_id} + {server.description && {server.description}}
))} @@ -1489,40 +1480,40 @@ const ChatUI: React.FC = ({ {selectedMCPServers.length > 0 && !selectedMCPServers.includes("__all__") && MCP_SUPPORTED_ENDPOINTS.has(endpointType as EndpointType) && ( -
- {selectedMCPServers.map(serverId => { - const server = mcpServers.find(s => s.server_id === serverId); - const tools = serverToolsMap[serverId] || []; - if (tools.length === 0) return null; +
+ {selectedMCPServers.map((serverId) => { + const server = mcpServers.find((s) => s.server_id === serverId); + const tools = serverToolsMap[serverId] || []; + if (tools.length === 0) return null; - return ( -
- - Limit tools for {server?.alias || server?.server_name || serverId}: - - { + setMCPServerToolRestrictions((prev) => ({ + ...prev, + [serverId]: selectedTools, + })); + }} + options={tools.map((tool) => ({ + value: tool.name, + label: tool.name, + }))} + maxTagCount={2} + /> +
+ ); + })} +
+ )}
@@ -2073,7 +2064,13 @@ const ChatUI: React.FC = ({ )} {/* Quick Code Interpreter toggle for Responses */} {endpointType === EndpointType.RESPONSES && ( - +