From 5de9bfde53f454bd9f02f9dbe207adfc4cc52974 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Sat, 13 Dec 2025 08:18:43 -0800 Subject: [PATCH] [Fix] CI/CD - mypy & check_code_and_doc_quality & mcp_testing (#17920) * Fix duplicate imports in SAP embedding transformation * fix: add missing prompt_spec parameter to HumanloopLogger.get_chat_completion_prompt - Add prompt_spec: Optional[PromptSpec] = None parameter to match base class signature - Import PromptSpec from litellm.types.prompts.init_prompts - Pass prompt_spec to super().get_chat_completion_prompt() call - Fixes mypy type error: Signature incompatible with supertype CustomLogger * fix: add missing parameters to AnthropicCacheControlHook.async_get_chat_completion_prompt - Add ignore_prompt_manager_model and ignore_prompt_manager_optional_params parameters - Change litellm_logging_obj type from Any to LiteLLMLoggingObj using TYPE_CHECKING pattern - Pass all parameters including prompt_spec to get_chat_completion_prompt call - Fixes mypy type errors: Signature incompatible with supertype CustomLogger and PromptManagementBase * fix: add missing parameters to DotpromptManager.async_get_chat_completion_prompt - Add ignore_prompt_manager_model and ignore_prompt_manager_optional_params parameters - Change litellm_logging_obj type from Any to LiteLLMLoggingObj using TYPE_CHECKING pattern - Pass all parameters including ignore flags to PromptManagementBase.async_get_chat_completion_prompt - Fixes mypy type errors: Signature incompatible with supertype CustomLogger and PromptManagementBase * fix: document envs * fix: add missing parameters to LangfusePromptManagement.async_get_chat_completion_prompt - Add ignore_prompt_manager_model and ignore_prompt_manager_optional_params parameters - Pass all parameters including prompt_spec and ignore flags to get_chat_completion_prompt - Fixes mypy type errors: Signature incompatible with supertype CustomLogger and PromptManagementBase * fix: add missing parameters to prompt management async methods (Category 1) - vector_store_pre_call_hook: add ignore_prompt_manager_model, ignore_prompt_manager_optional_params, prompt_spec - gitlab_prompt_manager: add ignore parameters, fix litellm_logging_obj type - bitbucket_prompt_manager: add ignore parameters, fix litellm_logging_obj type - proxy/custom_prompt_management: add prompt_spec parameter - Fixes mypy type errors: Signature incompatible with supertype * fix: fix arize_phoenix_prompt_manager and custom_prompt_management (Category 2) - arize_phoenix_prompt_manager: add prompt_spec to all methods, fix prompt_id types, implement async_compile_prompt_helper - custom_prompt_management: implement async_compile_prompt_helper abstract method - Fixes mypy type errors: Signature incompatible with supertype and abstract method errors * fix: fix obvious type errors (Category 3 - Quick Wins) - langfuse: change 'callable' to 'Callable' type annotation - presidio: add type narrowing check for Choices vs StreamingChoices - StreamingChoices doesn't have .message attribute, only Choices does - Add hasattr check before accessing choice.message - Fixes mypy type errors: callable? not callable and union-attr errors * fix: handle expires_after None in Azure files handler (Todo 14) - Extract logic to _prepare_create_file_data helper method - Remove expires_after from dict if None to match SDK's Omit pattern - Add type ignore for FileExpiresAfter -> file_create_params.ExpiresAfter mismatch - Fixes mypy error: Argument expires_after has incompatible type * fix: change purpose parameter type to OpenAIFilesPurpose (Todo 18) - Import OpenAIFilesPurpose in storage_backend_service.py - Change upload_file_to_storage_backend purpose parameter from str to OpenAIFilesPurpose - Change _create_file_object_with_storage_metadata purpose parameter from str to OpenAIFilesPurpose - Fixes mypy error: Argument purpose has incompatible type str; expected Literal type - Purpose is already validated in files_endpoints.py before reaching these functions * fix: handle UploadFile | str type for expires_after form fields (Todo 19) - Validate expires_after[anchor] and expires_after[seconds] are strings, not UploadFiles - Validate anchor equals 'created_at' before using literal in TypedDict - Use literal 'created_at' (not variable) in FileExpiresAfter to satisfy Literal type - Add proper error handling for invalid anchor values and int conversion - Fixes mypy errors: Incompatible types for anchor and seconds in FileExpiresAfter * fix: add type narrowing for expires_after_seconds_str to fix mypy error - Add assert statement after UploadFile validation to help mypy narrow type - Use validated variable with explicit str type annotation - Fixes: Argument of type 'UploadFile | str' cannot be assigned to int() * fix: trigger async_success_handler for MCP tool calls to enable cost tracking and logging - Set call_type to CallTypes.call_mcp_tool.value before calling async_success_handler - Update mcp_tool_call_metadata with cost info when server is found - Call async_success_handler to build standard_logging_object and trigger callbacks - Fixes test_mcp_cost_tracking by ensuring standard_logging_payload is populated * refactor: use positive isinstance check for safer type narrowing - Replace assert with positive isinstance(..., str) check - Matches codebase pattern (see pass_through_endpoints.py) - Safer than assert: assertions can be disabled with -O flag - Mypy properly narrows type after positive isinstance check - More explicit and readable than assert statement * fix: add missing REDIS_DAILY_AGENT_SPEND_UPDATE_QUEUE to ServiceTypes enum (Todo 17) - Add REDIS_DAILY_AGENT_SPEND_UPDATE_QUEUE enum value following the pattern of other daily spend queues - Add corresponding entry to DEFAULT_SERVICE_CONFIGS with GAUGE metrics - Fixes mypy error: 'type[ServiceTypes]' has no attribute 'REDIS_DAILY_AGENT_SPEND_UPDATE_QUEUE' - This enum value is already used in redis_update_buffer.py for agent spend tracking --- docs/my-website/docs/proxy/config_settings.md | 1 + .../anthropic_cache_control_hook.py | 14 +++++- .../arize/arize_phoenix_prompt_manager.py | 43 ++++++++++++++--- .../bitbucket/bitbucket_prompt_manager.py | 13 ++++- .../integrations/custom_prompt_management.py | 13 +++++ .../dotprompt/dotprompt_manager.py | 13 ++++- .../gitlab/gitlab_prompt_manager.py | 13 ++++- litellm/integrations/humanloop.py | 3 ++ litellm/integrations/langfuse/langfuse.py | 4 +- .../langfuse/langfuse_prompt_management.py | 5 ++ .../vector_store_pre_call_hook.py | 6 ++- litellm/llms/azure/files/handler.py | 17 ++++++- litellm/llms/sap/embed/transformation.py | 2 - .../proxy/_experimental/mcp_server/server.py | 19 ++++++++ litellm/proxy/custom_prompt_management.py | 2 + .../guardrails/guardrail_hooks/presidio.py | 3 ++ .../openai_files_endpoints/files_endpoints.py | 47 ++++++++++++++++++- .../storage_backend_service.py | 6 +-- litellm/types/services.py | 4 ++ 19 files changed, 202 insertions(+), 26 deletions(-) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index d87c8b3f468..3bffc141fde 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -487,6 +487,7 @@ router_settings: | DEFAULT_CRON_JOB_LOCK_TTL_SECONDS | Time-to-live for cron job locks in seconds. Default is 60 (1 minute) | DEFAULT_DATAFORSEO_LOCATION_CODE | Default location code for DataForSEO search API. Default is 2250 (France) | DEFAULT_FAILURE_THRESHOLD_PERCENT | Threshold percentage of failures to cool down a deployment. Default is 0.5 (50%) +| DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS | Minimum number of requests before applying error rate cooldown. Prevents cooldown from triggering on first failure. Default is 5 | DEFAULT_FLUSH_INTERVAL_SECONDS | Default interval in seconds for flushing operations. Default is 5 | DEFAULT_HEALTH_CHECK_INTERVAL | Default interval in seconds for health checks. Default is 300 (5 minutes) | DEFAULT_HEALTH_CHECK_PROMPT | Default prompt used during health checks for non-image models. Default is "test from litellm" diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 45b932a73af..5df79580d3e 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -7,7 +7,7 @@ Users can define """ import copy -from typing import Any, Dict, List, Optional, Tuple, Union, cast +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger @@ -21,6 +21,11 @@ from litellm.types.llms.openai import AllMessageValues, ChatCompletionCachedCont from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import StandardCallbackDynamicParams +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + class AnthropicCacheControlHook(CustomPromptManagement): def get_chat_completion_prompt( @@ -198,11 +203,13 @@ class AnthropicCacheControlHook(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, - litellm_logging_obj: Any, + litellm_logging_obj: LiteLLMLoggingObj, prompt_spec: Optional[PromptSpec] = None, tools: Optional[List[Dict]] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, + ignore_prompt_manager_model: Optional[bool] = False, + ignore_prompt_manager_optional_params: Optional[bool] = False, ) -> Tuple[str, List[AllMessageValues], dict]: """Async version - delegates to sync since no async operations needed.""" return self.get_chat_completion_prompt( @@ -212,8 +219,11 @@ class AnthropicCacheControlHook(CustomPromptManagement): prompt_id=prompt_id, prompt_variables=prompt_variables, dynamic_callback_params=dynamic_callback_params, + prompt_spec=prompt_spec, prompt_label=prompt_label, prompt_version=prompt_version, + ignore_prompt_manager_model=ignore_prompt_manager_model, + ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params, ) @staticmethod diff --git a/litellm/integrations/arize/arize_phoenix_prompt_manager.py b/litellm/integrations/arize/arize_phoenix_prompt_manager.py index aa028e389ca..19af0bb9552 100644 --- a/litellm/integrations/arize/arize_phoenix_prompt_manager.py +++ b/litellm/integrations/arize/arize_phoenix_prompt_manager.py @@ -13,6 +13,7 @@ from litellm.integrations.prompt_management_base import ( PromptManagementClient, ) from litellm.types.llms.openai import AllMessageValues +from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import StandardCallbackDynamicParams from .arize_phoenix_client import ArizePhoenixClient @@ -362,7 +363,8 @@ class ArizePhoenixPromptManager(CustomPromptManagement): def should_run_prompt_management( self, - prompt_id: str, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], dynamic_callback_params: StandardCallbackDynamicParams, ) -> bool: """ @@ -375,7 +377,8 @@ class ArizePhoenixPromptManager(CustomPromptManagement): def _compile_prompt_helper( self, - prompt_id: str, + prompt_id: Optional[str], + prompt_spec: Optional[PromptSpec], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, prompt_label: Optional[str] = None, @@ -390,6 +393,8 @@ class ArizePhoenixPromptManager(CustomPromptManagement): 3. Returns formatted chat messages 4. Extracts model and optional parameters from metadata """ + if prompt_id is None: + raise ValueError("prompt_id is required for Arize Phoenix prompt manager") try: # Load the prompt from Arize Phoenix if not already loaded if prompt_id not in self.prompt_manager.prompts: @@ -426,6 +431,30 @@ class ArizePhoenixPromptManager(CustomPromptManagement): except Exception as e: raise ValueError(f"Error compiling prompt '{prompt_id}': {e}") + async def async_compile_prompt_helper( + self, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ) -> PromptManagementClient: + """ + Async version of compile prompt helper. Since Arize Phoenix operations are synchronous, + this simply delegates to the sync version. + """ + if prompt_id is None: + raise ValueError("prompt_id is required for Arize Phoenix prompt manager") + return self._compile_prompt_helper( + prompt_id=prompt_id, + prompt_spec=prompt_spec, + prompt_variables=prompt_variables, + dynamic_callback_params=dynamic_callback_params, + prompt_label=prompt_label, + prompt_version=prompt_version, + ) + def get_chat_completion_prompt( self, model: str, @@ -434,6 +463,7 @@ class ArizePhoenixPromptManager(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, ignore_prompt_manager_model: Optional[bool] = False, @@ -450,8 +480,9 @@ class ArizePhoenixPromptManager(CustomPromptManagement): prompt_id, prompt_variables, dynamic_callback_params, - prompt_label, - prompt_version, - self.ignore_prompt_manager_model, - self.ignore_prompt_manager_optional_params, + prompt_spec=prompt_spec, + prompt_label=prompt_label, + prompt_version=prompt_version, + ignore_prompt_manager_model=ignore_prompt_manager_model, + ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params, ) diff --git a/litellm/integrations/bitbucket/bitbucket_prompt_manager.py b/litellm/integrations/bitbucket/bitbucket_prompt_manager.py index aa6bff5509e..701f2273640 100644 --- a/litellm/integrations/bitbucket/bitbucket_prompt_manager.py +++ b/litellm/integrations/bitbucket/bitbucket_prompt_manager.py @@ -3,11 +3,16 @@ BitBucket prompt manager that integrates with LiteLLM's prompt management system Fetches .prompt files from BitBucket repositories and provides team-based access control. """ -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union from jinja2 import DictLoader, Environment, select_autoescape from litellm.integrations.custom_prompt_management import CustomPromptManagement + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any from litellm.integrations.prompt_management_base import ( PromptManagementBase, PromptManagementClient, @@ -550,11 +555,13 @@ class BitBucketPromptManager(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, - litellm_logging_obj: Any, + litellm_logging_obj: LiteLLMLoggingObj, prompt_spec: Optional[PromptSpec] = None, tools: Optional[List[Dict]] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, + ignore_prompt_manager_model: Optional[bool] = False, + ignore_prompt_manager_optional_params: Optional[bool] = False, ) -> Tuple[str, List[AllMessageValues], dict]: """ Async version - delegates to PromptManagementBase async implementation. @@ -572,4 +579,6 @@ class BitBucketPromptManager(CustomPromptManagement): tools=tools, prompt_label=prompt_label, prompt_version=prompt_version, + ignore_prompt_manager_model=ignore_prompt_manager_model, + ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params, ) diff --git a/litellm/integrations/custom_prompt_management.py b/litellm/integrations/custom_prompt_management.py index 875ad8f1ef5..61e619aba65 100644 --- a/litellm/integrations/custom_prompt_management.py +++ b/litellm/integrations/custom_prompt_management.py @@ -68,3 +68,16 @@ class CustomPromptManagement(CustomLogger, PromptManagementBase): raise NotImplementedError( "Custom prompt management does not support compile prompt helper" ) + + async def async_compile_prompt_helper( + self, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ) -> PromptManagementClient: + raise NotImplementedError( + "Custom prompt management does not support async compile prompt helper" + ) diff --git a/litellm/integrations/dotprompt/dotprompt_manager.py b/litellm/integrations/dotprompt/dotprompt_manager.py index e3901bd9359..9412ac3c842 100644 --- a/litellm/integrations/dotprompt/dotprompt_manager.py +++ b/litellm/integrations/dotprompt/dotprompt_manager.py @@ -4,7 +4,7 @@ Builds on top of PromptManagementBase to provide .prompt file support. """ import json -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.integrations.prompt_management_base import PromptManagementClient @@ -12,6 +12,11 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import StandardCallbackDynamicParams +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + from .prompt_manager import PromptManager, PromptTemplate @@ -224,11 +229,13 @@ class DotpromptManager(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, - litellm_logging_obj: Any, + litellm_logging_obj: LiteLLMLoggingObj, prompt_spec: Optional[PromptSpec] = None, tools: Optional[List[Dict]] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, + ignore_prompt_manager_model: Optional[bool] = False, + ignore_prompt_manager_optional_params: Optional[bool] = False, ) -> Tuple[str, List[AllMessageValues], dict]: """ Async version - delegates to PromptManagementBase async implementation. @@ -248,6 +255,8 @@ class DotpromptManager(CustomPromptManagement): tools=tools, prompt_label=prompt_label, prompt_version=prompt_version, + ignore_prompt_manager_model=ignore_prompt_manager_model, + ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params, ) def _convert_to_messages(self, rendered_content: str) -> List[AllMessageValues]: diff --git a/litellm/integrations/gitlab/gitlab_prompt_manager.py b/litellm/integrations/gitlab/gitlab_prompt_manager.py index 85335a811a3..b073948d768 100644 --- a/litellm/integrations/gitlab/gitlab_prompt_manager.py +++ b/litellm/integrations/gitlab/gitlab_prompt_manager.py @@ -2,11 +2,16 @@ GitLab prompt manager with configurable prompts folder. """ -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union from jinja2 import DictLoader, Environment, select_autoescape from litellm.integrations.custom_prompt_management import CustomPromptManagement + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any from litellm.integrations.gitlab.gitlab_client import GitLabClient from litellm.integrations.prompt_management_base import ( PromptManagementBase, @@ -571,11 +576,13 @@ class GitLabPromptManager(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, - litellm_logging_obj: Any, + litellm_logging_obj: LiteLLMLoggingObj, prompt_spec: Optional[PromptSpec] = None, tools: Optional[List[Dict]] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, + ignore_prompt_manager_model: Optional[bool] = False, + ignore_prompt_manager_optional_params: Optional[bool] = False, ) -> Tuple[str, List[AllMessageValues], dict]: """ Async version - delegates to PromptManagementBase async implementation. @@ -593,6 +600,8 @@ class GitLabPromptManager(CustomPromptManagement): tools=tools, prompt_label=prompt_label, prompt_version=prompt_version, + ignore_prompt_manager_model=ignore_prompt_manager_model, + ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params, ) diff --git a/litellm/integrations/humanloop.py b/litellm/integrations/humanloop.py index df967272687..369df5ee0bd 100644 --- a/litellm/integrations/humanloop.py +++ b/litellm/integrations/humanloop.py @@ -14,6 +14,7 @@ from litellm.caching import DualCache from litellm.llms.custom_httpx.http_handler import _get_httpx_client from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues +from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import StandardCallbackDynamicParams from .custom_logger import CustomLogger @@ -156,6 +157,7 @@ class HumanloopLogger(CustomLogger): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, ignore_prompt_manager_model: Optional[bool] = False, @@ -180,6 +182,7 @@ class HumanloopLogger(CustomLogger): prompt_id=prompt_id, prompt_variables=prompt_variables, dynamic_callback_params=dynamic_callback_params, + prompt_spec=prompt_spec, ) prompt_template = prompt_manager._get_prompt_from_id( diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 7d7f5ded614..c8aaa13de51 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -3,7 +3,7 @@ import os import traceback from datetime import datetime -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union, cast from packaging.version import Version @@ -894,7 +894,7 @@ class LangFuseLogger: return Version(self.langfuse_sdk_version) >= Version("2.7.3") @staticmethod - def _apply_masking_function(data: Any, masking_function: callable) -> Any: + def _apply_masking_function(data: Any, masking_function: Callable[[Any], Any]) -> Any: """ Apply a masking function to data, handling different data types. diff --git a/litellm/integrations/langfuse/langfuse_prompt_management.py b/litellm/integrations/langfuse/langfuse_prompt_management.py index 8e562238cc7..adc8ae61d01 100644 --- a/litellm/integrations/langfuse/langfuse_prompt_management.py +++ b/litellm/integrations/langfuse/langfuse_prompt_management.py @@ -188,6 +188,8 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge tools: Optional[List[Dict]] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, + ignore_prompt_manager_model: Optional[bool] = False, + ignore_prompt_manager_optional_params: Optional[bool] = False, ) -> Tuple[str, List[AllMessageValues], dict,]: return self.get_chat_completion_prompt( model, @@ -196,8 +198,11 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge prompt_id, prompt_variables, dynamic_callback_params, + prompt_spec=prompt_spec, prompt_label=prompt_label, prompt_version=prompt_version, + ignore_prompt_manager_model=ignore_prompt_manager_model, + ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params, ) def should_run_prompt_management( diff --git a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py index 218581a41ad..c94b925ea21 100644 --- a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py +++ b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py @@ -12,6 +12,7 @@ import litellm.vector_stores from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage +from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import StandardCallbackDynamicParams from litellm.types.vector_stores import ( LiteLLM_ManagedVectorStore, @@ -23,7 +24,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj else: - LiteLLMLoggingObj = None + LiteLLMLoggingObj = Any class VectorStorePreCallHook(CustomLogger): @@ -49,9 +50,12 @@ class VectorStorePreCallHook(CustomLogger): prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, litellm_logging_obj: LiteLLMLoggingObj, + prompt_spec: Optional[PromptSpec] = None, tools: Optional[List[Dict]] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, + ignore_prompt_manager_model: Optional[bool] = False, + ignore_prompt_manager_optional_params: Optional[bool] = False, ) -> Tuple[str, List[AllMessageValues], dict]: """ Perform vector store search and append results as context to messages. diff --git a/litellm/llms/azure/files/handler.py b/litellm/llms/azure/files/handler.py index 50c122ccf2c..69b2d71753b 100644 --- a/litellm/llms/azure/files/handler.py +++ b/litellm/llms/azure/files/handler.py @@ -24,13 +24,26 @@ class AzureOpenAIFilesAPI(BaseAzureLLM): def __init__(self) -> None: super().__init__() + @staticmethod + def _prepare_create_file_data(create_file_data: CreateFileRequest) -> dict[str, Any]: + """ + Prepare create_file_data for OpenAI SDK. + + Removes expires_after if None to match SDK's Omit pattern. + SDK expects file_create_params.ExpiresAfter | Omit, but FileExpiresAfter works at runtime. + """ + data = dict(create_file_data) + if data.get("expires_after") is None: + data.pop("expires_after", None) + return data + async def acreate_file( self, create_file_data: CreateFileRequest, openai_client: AsyncAzureOpenAI, ) -> OpenAIFileObject: verbose_logger.debug("create_file_data=%s", create_file_data) - response = await openai_client.files.create(**create_file_data) + response = await openai_client.files.create(**self._prepare_create_file_data(create_file_data)) # type: ignore[arg-type] verbose_logger.debug("create_file_response=%s", response) return OpenAIFileObject(**response.model_dump()) @@ -69,7 +82,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM): return self.acreate_file( create_file_data=create_file_data, openai_client=openai_client ) - response = cast(AzureOpenAI, openai_client).files.create(**create_file_data) + response = cast(AzureOpenAI, openai_client).files.create(**self._prepare_create_file_data(create_file_data)) # type: ignore[arg-type] return OpenAIFileObject(**response.model_dump()) async def afile_content( diff --git a/litellm/llms/sap/embed/transformation.py b/litellm/llms/sap/embed/transformation.py index 6a641626a0b..231cc3ceccf 100644 --- a/litellm/llms/sap/embed/transformation.py +++ b/litellm/llms/sap/embed/transformation.py @@ -5,10 +5,8 @@ Translates from OpenAI's `/v1/embeddings` to IBM's `/text/embeddings` route. from typing import Optional, List, Dict, Literal, Union from pydantic import BaseModel, Field from functools import cached_property -from typing import Dict, List, Literal, Optional, Union import httpx -from pydantic import BaseModel, Field from litellm.llms.base_llm.embedding.transformation import ( BaseEmbeddingConfig, diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index edf53e99573..bdff60c932b 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -1280,6 +1280,11 @@ if MCP_AVAILABLE: standard_logging_mcp_tool_call["mcp_server_cost_info"] = ( mcp_server.mcp_info or {} ).get("mcp_server_cost_info") + # Update model_call_details with the cost info + if litellm_logging_obj: + litellm_logging_obj.model_call_details[ + "mcp_tool_call_metadata" + ] = standard_logging_mcp_tool_call response = await _handle_managed_mcp_tool( server_name=server_name, name=original_tool_name, # Pass the full name (potentially prefixed) @@ -1317,6 +1322,20 @@ if MCP_AVAILABLE: start_time=start_time, end_time=end_time, ) + # Set call_type to call_mcp_tool so cost calculator recognizes it + from litellm.types.utils import CallTypes + + litellm_logging_obj.call_type = CallTypes.call_mcp_tool.value + # Trigger success logging to build standard_logging_object and call callbacks + # async_success_handler will: + # 1. Call _success_handler_helper_fn which recognizes call_mcp_tool + # 2. Call _process_hidden_params_and_response_cost which: + # - Calculates cost via _response_cost_calculator -> MCPCostCalculator + # - Builds standard_logging_object + # 3. Call async_log_success_event on all callbacks + await litellm_logging_obj.async_success_handler( + result=response, start_time=start_time, end_time=end_time + ) return response async def mcp_get_prompt( diff --git a/litellm/proxy/custom_prompt_management.py b/litellm/proxy/custom_prompt_management.py index 7cb7e935348..62a51911409 100644 --- a/litellm/proxy/custom_prompt_management.py +++ b/litellm/proxy/custom_prompt_management.py @@ -3,6 +3,7 @@ from typing import List, Optional, Tuple from litellm._logging import verbose_logger from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.types.llms.openai import AllMessageValues +from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import StandardCallbackDynamicParams @@ -15,6 +16,7 @@ class X42PromptManagement(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, ignore_prompt_manager_model: Optional[bool] = False, diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 106e4769915..b850e5cec08 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -724,6 +724,9 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): ) for choice in response.choices: + # Type narrowing: StreamingChoices doesn't have .message attribute + if not hasattr(choice, "message"): + continue content = getattr(choice.message, "content", None) if content is None: continue diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 805b156a7d8..810f5c62720 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -369,9 +369,52 @@ async def create_file( # noqa: PLR0915 "error": "Both expires_after[anchor] and expires_after[seconds] must be provided if expires_after is specified", }, ) + + # Validate expires_after[anchor] is a string (not UploadFile) + if isinstance(expires_after_anchor, UploadFile): + raise HTTPException( + status_code=400, + detail={ + "error": "expires_after[anchor] must be a string, not a file upload", + }, + ) + + # Validate expires_after[seconds] is a string (not UploadFile) + # Use positive isinstance check for proper type narrowing (matches codebase pattern) + if not isinstance(expires_after_seconds_str, str): + raise HTTPException( + status_code=400, + detail={ + "error": "expires_after[seconds] must be a string, not a file upload", + }, + ) + # After this check, mypy knows expires_after_seconds_str is str + expires_after_seconds_str_validated: str = expires_after_seconds_str + + # Validate anchor is "created_at" + if expires_after_anchor != "created_at": + raise HTTPException( + status_code=400, + detail={ + "error": f"expires_after[anchor] must be 'created_at', got '{expires_after_anchor}'", + }, + ) + + # Convert seconds to int + try: + expires_after_seconds = int(expires_after_seconds_str_validated) + except (ValueError, TypeError) as e: + raise HTTPException( + status_code=400, + detail={ + "error": f"expires_after[seconds] must be a valid integer, got '{expires_after_seconds_str}': {e}", + }, + ) + + # Use literal "created_at" (not variable) for TypedDict to satisfy Literal type expires_after = FileExpiresAfter( - anchor=expires_after_anchor, - seconds=int(expires_after_seconds_str), + anchor="created_at", # Literal, not expires_after_anchor variable + seconds=expires_after_seconds, ) # Include original request and headers in the data diff --git a/litellm/proxy/openai_files_endpoints/storage_backend_service.py b/litellm/proxy/openai_files_endpoints/storage_backend_service.py index 727a7876a5c..991fff1d3fc 100644 --- a/litellm/proxy/openai_files_endpoints/storage_backend_service.py +++ b/litellm/proxy/openai_files_endpoints/storage_backend_service.py @@ -15,7 +15,7 @@ from litellm.llms.base_llm.files.storage_backend_factory import get_storage_back from litellm.llms.base_llm.files.transformation import BaseFileEndpoints from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging -from litellm.types.llms.openai import OpenAIFileObject +from litellm.types.llms.openai import OpenAIFileObject, OpenAIFilesPurpose from litellm.types.utils import SpecialEnums @@ -35,7 +35,7 @@ class StorageBackendFileService: file_data: Mapping[str, Any], target_storage: str, target_model_names: List[str], - purpose: str, + purpose: OpenAIFilesPurpose, proxy_logging_obj: ProxyLogging, user_api_key_dict: UserAPIKeyAuth, ) -> OpenAIFileObject: @@ -112,7 +112,7 @@ class StorageBackendFileService: def _create_file_object_with_storage_metadata( file_content: bytes, filename: str, - purpose: str, + purpose: OpenAIFilesPurpose, target_storage: str, storage_url: str, ) -> OpenAIFileObject: diff --git a/litellm/types/services.py b/litellm/types/services.py index bb34f817f29..580d5653dec 100644 --- a/litellm/types/services.py +++ b/litellm/types/services.py @@ -37,6 +37,7 @@ class ServiceTypes(str, enum.Enum): REDIS_DAILY_END_USER_SPEND_UPDATE_QUEUE = "redis_daily_end_user_spend_update_queue" REDIS_DAILY_ORG_SPEND_UPDATE_QUEUE = "redis_daily_org_spend_update_queue" REDIS_DAILY_TEAM_SPEND_UPDATE_QUEUE = "redis_daily_team_spend_update_queue" + REDIS_DAILY_AGENT_SPEND_UPDATE_QUEUE = "redis_daily_agent_spend_update_queue" REDIS_DAILY_TAG_SPEND_UPDATE_QUEUE = "redis_daily_tag_spend_update_queue" # spend update queue - current spend of key, user, team IN_MEMORY_SPEND_UPDATE_QUEUE = "in_memory_spend_update_queue" @@ -93,6 +94,9 @@ DEFAULT_SERVICE_CONFIGS = { ServiceTypes.REDIS_DAILY_END_USER_SPEND_UPDATE_QUEUE.value: { "metrics": [ServiceMetrics.GAUGE] }, + ServiceTypes.REDIS_DAILY_AGENT_SPEND_UPDATE_QUEUE.value: { + "metrics": [ServiceMetrics.GAUGE] + }, ServiceTypes.IN_MEMORY_SPEND_UPDATE_QUEUE.value: { "metrics": [ServiceMetrics.GAUGE] },