diff --git a/litellm/a2a_protocol/main.py b/litellm/a2a_protocol/main.py index a8f8fff2ba9..401b602fef5 100644 --- a/litellm/a2a_protocol/main.py +++ b/litellm/a2a_protocol/main.py @@ -271,7 +271,9 @@ async def asend_message( card_url = getattr(agent_card, "url", None) if agent_card else None context_id = trace_id or str(uuid.uuid4()) - request.params.message.context_id = context_id + if request.params.message.context_id is None: + request.params.message.context_id = context_id + # Retry loop: if connection fails due to localhost URL in agent card, retry with fixed URL a2a_response = None for _ in range(2): # max 2 attempts: original + 1 retry diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 0be89ac67fb..6f587abcdf1 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -352,9 +352,9 @@ class Logging(LiteLLMLoggingBaseClass): ) self.function_id = function_id self.streaming_chunks: List[Any] = [] # for generating complete stream response - self.sync_streaming_chunks: List[ - Any - ] = [] # for generating complete stream response + self.sync_streaming_chunks: List[Any] = ( + [] + ) # for generating complete stream response self.log_raw_request_response = log_raw_request_response # Initialize dynamic callbacks @@ -746,9 +746,9 @@ class Logging(LiteLLMLoggingBaseClass): prompt_spec=prompt_spec, dynamic_callback_params=dynamic_callback_params, ): - self.model_call_details[ - "prompt_integration" - ] = logger.__class__.__name__ + self.model_call_details["prompt_integration"] = ( + logger.__class__.__name__ + ) return logger except Exception: # If check fails, continue to next logger @@ -816,9 +816,9 @@ class Logging(LiteLLMLoggingBaseClass): if anthropic_cache_control_logger := AnthropicCacheControlHook.get_custom_logger_for_anthropic_cache_control_hook( non_default_params ): - self.model_call_details[ - "prompt_integration" - ] = anthropic_cache_control_logger.__class__.__name__ + self.model_call_details["prompt_integration"] = ( + anthropic_cache_control_logger.__class__.__name__ + ) return anthropic_cache_control_logger ######################################################### @@ -830,9 +830,9 @@ class Logging(LiteLLMLoggingBaseClass): internal_usage_cache=None, llm_router=None, ) - self.model_call_details[ - "prompt_integration" - ] = vector_store_custom_logger.__class__.__name__ + self.model_call_details["prompt_integration"] = ( + vector_store_custom_logger.__class__.__name__ + ) # Add to global callbacks so post-call hooks are invoked if ( vector_store_custom_logger @@ -892,9 +892,9 @@ class Logging(LiteLLMLoggingBaseClass): model ): # if model name was changes pre-call, overwrite the initial model call name with the new one self.model_call_details["model"] = model - self.model_call_details["litellm_params"][ - "api_base" - ] = self._get_masked_api_base(additional_args.get("api_base", "")) + self.model_call_details["litellm_params"]["api_base"] = ( + self._get_masked_api_base(additional_args.get("api_base", "")) + ) def pre_call(self, input, api_key, model=None, additional_args={}): # noqa: PLR0915 # Log the exact input to the LLM API @@ -923,10 +923,10 @@ class Logging(LiteLLMLoggingBaseClass): try: # [Non-blocking Extra Debug Information in metadata] if turn_off_message_logging is True: - _metadata[ - "raw_request" - ] = "redacted by litellm. \ + _metadata["raw_request"] = ( + "redacted by litellm. \ 'litellm.turn_off_message_logging=True'" + ) else: curl_command = self._get_request_curl_command( api_base=additional_args.get("api_base", ""), @@ -937,34 +937,34 @@ class Logging(LiteLLMLoggingBaseClass): _metadata["raw_request"] = str(curl_command) # split up, so it's easier to parse in the UI - self.model_call_details[ - "raw_request_typed_dict" - ] = RawRequestTypedDict( - raw_request_api_base=str( - additional_args.get("api_base") or "" - ), - raw_request_body=self._get_raw_request_body( - additional_args.get("complete_input_dict", {}) - ), - # NOTE: setting ignore_sensitive_headers to True will cause - # the Authorization header to be leaked when calls to the health - # endpoint are made and fail. - raw_request_headers=self._get_masked_headers( - additional_args.get("headers", {}) or {}, - ), - error=None, + self.model_call_details["raw_request_typed_dict"] = ( + RawRequestTypedDict( + raw_request_api_base=str( + additional_args.get("api_base") or "" + ), + raw_request_body=self._get_raw_request_body( + additional_args.get("complete_input_dict", {}) + ), + # NOTE: setting ignore_sensitive_headers to True will cause + # the Authorization header to be leaked when calls to the health + # endpoint are made and fail. + raw_request_headers=self._get_masked_headers( + additional_args.get("headers", {}) or {}, + ), + error=None, + ) ) except Exception as e: - self.model_call_details[ - "raw_request_typed_dict" - ] = RawRequestTypedDict( - error=str(e), + self.model_call_details["raw_request_typed_dict"] = ( + RawRequestTypedDict( + error=str(e), + ) ) - _metadata[ - "raw_request" - ] = "Unable to Log \ + _metadata["raw_request"] = ( + "Unable to Log \ raw request: {}".format( - str(e) + str(e) + ) ) if getattr(self, "logger_fn", None) and callable(self.logger_fn): try: @@ -1265,13 +1265,13 @@ class Logging(LiteLLMLoggingBaseClass): for callback in callbacks: try: if isinstance(callback, CustomLogger): - response: Optional[ - MCPPostCallResponseObject - ] = await callback.async_post_mcp_tool_call_hook( - kwargs=kwargs, - response_obj=post_mcp_tool_call_response_obj, - start_time=start_time, - end_time=end_time, + response: Optional[MCPPostCallResponseObject] = ( + await callback.async_post_mcp_tool_call_hook( + kwargs=kwargs, + response_obj=post_mcp_tool_call_response_obj, + start_time=start_time, + end_time=end_time, + ) ) ###################################################################### # if any of the callbacks modify the response, use the modified response @@ -1466,9 +1466,9 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.debug( f"response_cost_failure_debug_information: {debug_info}" ) - self.model_call_details[ - "response_cost_failure_debug_information" - ] = debug_info + self.model_call_details["response_cost_failure_debug_information"] = ( + debug_info + ) return None try: @@ -1494,9 +1494,9 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.debug( f"response_cost_failure_debug_information: {debug_info}" ) - self.model_call_details[ - "response_cost_failure_debug_information" - ] = debug_info + self.model_call_details["response_cost_failure_debug_information"] = ( + debug_info + ) return None @@ -1652,10 +1652,8 @@ class Logging(LiteLLMLoggingBaseClass): result=logging_result ) - self.model_call_details[ - "standard_logging_object" - ] = self._build_standard_logging_payload( - logging_result, start_time, end_time + self.model_call_details["standard_logging_object"] = ( + self._build_standard_logging_payload(logging_result, start_time, end_time) ) if ( @@ -1734,9 +1732,9 @@ class Logging(LiteLLMLoggingBaseClass): end_time = datetime.datetime.now() if self.completion_start_time is None: self.completion_start_time = end_time - self.model_call_details[ - "completion_start_time" - ] = self.completion_start_time + self.model_call_details["completion_start_time"] = ( + self.completion_start_time + ) self.model_call_details["log_event_type"] = "successful_api_call" self.model_call_details["end_time"] = end_time @@ -1773,10 +1771,10 @@ class Logging(LiteLLMLoggingBaseClass): end_time=end_time, ) elif isinstance(result, dict) or isinstance(result, list): - self.model_call_details[ - "standard_logging_object" - ] = self._build_standard_logging_payload( - result, start_time, end_time + self.model_call_details["standard_logging_object"] = ( + self._build_standard_logging_payload( + result, start_time, end_time + ) ) if ( standard_logging_payload := self.model_call_details.get( @@ -1785,9 +1783,9 @@ class Logging(LiteLLMLoggingBaseClass): ) is not None: emit_standard_logging_payload(standard_logging_payload) elif standard_logging_object is not None: - self.model_call_details[ - "standard_logging_object" - ] = standard_logging_object + self.model_call_details["standard_logging_object"] = ( + standard_logging_object + ) else: self.model_call_details["response_cost"] = None @@ -1945,17 +1943,17 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.debug( "Logging Details LiteLLM-Success Call streaming complete" ) - self.model_call_details[ - "complete_streaming_response" - ] = complete_streaming_response - self.model_call_details[ - "response_cost" - ] = self._response_cost_calculator(result=complete_streaming_response) + self.model_call_details["complete_streaming_response"] = ( + complete_streaming_response + ) + self.model_call_details["response_cost"] = ( + self._response_cost_calculator(result=complete_streaming_response) + ) ## STANDARDIZED LOGGING PAYLOAD - self.model_call_details[ - "standard_logging_object" - ] = self._build_standard_logging_payload( - complete_streaming_response, start_time, end_time + self.model_call_details["standard_logging_object"] = ( + self._build_standard_logging_payload( + complete_streaming_response, start_time, end_time + ) ) if ( standard_logging_payload := self.model_call_details.get( @@ -2289,10 +2287,10 @@ class Logging(LiteLLMLoggingBaseClass): ) else: if self.stream and complete_streaming_response: - self.model_call_details[ - "complete_response" - ] = self.model_call_details.get( - "complete_streaming_response", {} + self.model_call_details["complete_response"] = ( + self.model_call_details.get( + "complete_streaming_response", {} + ) ) result = self.model_call_details["complete_response"] openMeterLogger.log_success_event( @@ -2316,10 +2314,10 @@ class Logging(LiteLLMLoggingBaseClass): ) else: if self.stream and complete_streaming_response: - self.model_call_details[ - "complete_response" - ] = self.model_call_details.get( - "complete_streaming_response", {} + self.model_call_details["complete_response"] = ( + self.model_call_details.get( + "complete_streaming_response", {} + ) ) result = self.model_call_details["complete_response"] @@ -2458,9 +2456,9 @@ class Logging(LiteLLMLoggingBaseClass): if complete_streaming_response is not None: print_verbose("Async success callbacks: Got a complete streaming response") - self.model_call_details[ - "async_complete_streaming_response" - ] = complete_streaming_response + self.model_call_details["async_complete_streaming_response"] = ( + complete_streaming_response + ) try: if self.model_call_details.get("cache_hit", False) is True: @@ -2471,10 +2469,10 @@ class Logging(LiteLLMLoggingBaseClass): model_call_details=self.model_call_details ) # base_model defaults to None if not set on model_info - self.model_call_details[ - "response_cost" - ] = self._response_cost_calculator( - result=complete_streaming_response + self.model_call_details["response_cost"] = ( + self._response_cost_calculator( + result=complete_streaming_response + ) ) verbose_logger.debug( @@ -2487,10 +2485,10 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["response_cost"] = None ## STANDARDIZED LOGGING PAYLOAD - self.model_call_details[ - "standard_logging_object" - ] = self._build_standard_logging_payload( - complete_streaming_response, start_time, end_time + self.model_call_details["standard_logging_object"] = ( + self._build_standard_logging_payload( + complete_streaming_response, start_time, end_time + ) ) # print standard logging payload @@ -2517,10 +2515,8 @@ class Logging(LiteLLMLoggingBaseClass): # _success_handler_helper_fn if self.model_call_details.get("standard_logging_object") is None: ## STANDARDIZED LOGGING PAYLOAD - self.model_call_details[ - "standard_logging_object" - ] = self._build_standard_logging_payload( - result, start_time, end_time + self.model_call_details["standard_logging_object"] = ( + self._build_standard_logging_payload(result, start_time, end_time) ) # print standard logging payload @@ -2764,18 +2760,18 @@ class Logging(LiteLLMLoggingBaseClass): ## STANDARDIZED LOGGING PAYLOAD - self.model_call_details[ - "standard_logging_object" - ] = get_standard_logging_object_payload( - kwargs=self.model_call_details, - init_response_obj={}, - start_time=start_time, - end_time=end_time, - logging_obj=self, - status="failure", - error_str=str(exception), - original_exception=exception, - standard_built_in_tools_params=self.standard_built_in_tools_params, + self.model_call_details["standard_logging_object"] = ( + get_standard_logging_object_payload( + kwargs=self.model_call_details, + init_response_obj={}, + start_time=start_time, + end_time=end_time, + logging_obj=self, + status="failure", + error_str=str(exception), + original_exception=exception, + standard_built_in_tools_params=self.standard_built_in_tools_params, + ) ) return start_time, end_time @@ -3739,9 +3735,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 service_name=arize_config.project_name, ) - os.environ[ - "OTEL_EXPORTER_OTLP_TRACES_HEADERS" - ] = f"space_id={arize_config.space_key or arize_config.space_id},api_key={arize_config.api_key}" + os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = ( + f"space_id={arize_config.space_key or arize_config.space_id},api_key={arize_config.api_key}" + ) for callback in _in_memory_loggers: if ( isinstance(callback, ArizeLogger) @@ -3767,13 +3763,13 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "") # Add openinference.project.name attribute if existing_attrs: - os.environ[ - "OTEL_RESOURCE_ATTRIBUTES" - ] = f"{existing_attrs},openinference.project.name={arize_phoenix_config.project_name}" + os.environ["OTEL_RESOURCE_ATTRIBUTES"] = ( + f"{existing_attrs},openinference.project.name={arize_phoenix_config.project_name}" + ) else: - os.environ[ - "OTEL_RESOURCE_ATTRIBUTES" - ] = f"openinference.project.name={arize_phoenix_config.project_name}" + os.environ["OTEL_RESOURCE_ATTRIBUTES"] = ( + f"openinference.project.name={arize_phoenix_config.project_name}" + ) # Set Phoenix project name from environment variable phoenix_project_name = os.environ.get("PHOENIX_PROJECT_NAME", None) @@ -3781,19 +3777,19 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "") # Add openinference.project.name attribute if existing_attrs: - os.environ[ - "OTEL_RESOURCE_ATTRIBUTES" - ] = f"{existing_attrs},openinference.project.name={phoenix_project_name}" + os.environ["OTEL_RESOURCE_ATTRIBUTES"] = ( + f"{existing_attrs},openinference.project.name={phoenix_project_name}" + ) else: - os.environ[ - "OTEL_RESOURCE_ATTRIBUTES" - ] = f"openinference.project.name={phoenix_project_name}" + os.environ["OTEL_RESOURCE_ATTRIBUTES"] = ( + f"openinference.project.name={phoenix_project_name}" + ) # auth can be disabled on local deployments of arize phoenix if arize_phoenix_config.otlp_auth_headers is not None: - os.environ[ - "OTEL_EXPORTER_OTLP_TRACES_HEADERS" - ] = arize_phoenix_config.otlp_auth_headers + os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = ( + arize_phoenix_config.otlp_auth_headers + ) for callback in _in_memory_loggers: if ( @@ -3969,9 +3965,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 exporter="otlp_http", endpoint="https://langtrace.ai/api/trace", ) - os.environ[ - "OTEL_EXPORTER_OTLP_TRACES_HEADERS" - ] = f"api_key={os.getenv('LANGTRACE_API_KEY')}" + os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = ( + f"api_key={os.getenv('LANGTRACE_API_KEY')}" + ) for callback in _in_memory_loggers: if ( isinstance(callback, OpenTelemetry) @@ -4204,8 +4200,7 @@ def _maybe_auto_initialize_arize_phoenix(_in_memory_loggers: list) -> None: litellm.logging_callback_manager.add_litellm_callback(phoenix_logger) verbose_logger.info( - "Auto-initialized Arize Phoenix logger alongside otel " - "(endpoint=%s)", + "Auto-initialized Arize Phoenix logger alongside otel " "(endpoint=%s)", arize_phoenix_config.endpoint, ) except Exception as e: @@ -4768,9 +4763,11 @@ class StandardLoggingPayloadSetup: ).model_dump() if isinstance(_raw, dict): if ResponseAPILoggingUtils._is_response_api_usage(_raw): - return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( - _raw - ).model_dump() + return ( + ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + _raw + ).model_dump() + ) return _raw if isinstance(_raw, Usage): return _raw.model_dump() @@ -4884,10 +4881,10 @@ class StandardLoggingPayloadSetup: for key in StandardLoggingHiddenParams.__annotations__.keys(): if key in hidden_params: if key == "additional_headers": - clean_hidden_params[ - "additional_headers" - ] = StandardLoggingPayloadSetup.get_additional_headers( - hidden_params[key] + clean_hidden_params["additional_headers"] = ( + StandardLoggingPayloadSetup.get_additional_headers( + hidden_params[key] + ) ) else: clean_hidden_params[key] = hidden_params[key] # type: ignore @@ -5039,6 +5036,7 @@ class StandardLoggingPayloadSetup: dynamic_litellm_session_id = litellm_params.get("litellm_session_id") dynamic_litellm_trace_id = litellm_params.get("litellm_trace_id") + # Note: we recommend using `litellm_session_id` for session tracking # `litellm_trace_id` is an internal litellm param if dynamic_litellm_session_id: @@ -5509,9 +5507,9 @@ def scrub_sensitive_keys_in_metadata(litellm_params: Optional[dict]): ): for k, v in metadata["user_api_key_metadata"].items(): if k == "logging": # prevent logging user logging keys - cleaned_user_api_key_metadata[ - k - ] = "scrubbed_by_litellm_for_sensitive_keys" + cleaned_user_api_key_metadata[k] = ( + "scrubbed_by_litellm_for_sensitive_keys" + ) else: cleaned_user_api_key_metadata[k] = v @@ -5623,4 +5621,3 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload: model_parameters={"stream": True}, hidden_params=hidden_params, ) - diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index 7f30277ebca..6bcee14f29e 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -69,6 +69,7 @@ async def _handle_stream_message( from litellm.a2a_protocol.main import A2A_SDK_AVAILABLE if not A2A_SDK_AVAILABLE: + async def _error_stream(): yield json.dumps( { @@ -106,7 +107,12 @@ async def _handle_stream_message( proxy_server_request=proxy_server_request, ) - if use_proxy_hooks and user_api_key_dict is not None and request_data is not None and proxy_logging_obj is not None: + if ( + use_proxy_hooks + and user_api_key_dict is not None + and request_data is not None + and proxy_logging_obj is not None + ): from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, ) @@ -119,20 +125,27 @@ async def _handle_stream_message( return json.dumps(obj) + "\n" def _ndjson_error(proxy_exc: Any) -> str: - return json.dumps( - { - "jsonrpc": "2.0", - "id": request_id, - "error": { - "code": -32603, - "message": getattr( - proxy_exc, "message", f"Streaming error: {proxy_exc!s}" - ), - }, - } - ) + "\n" + return ( + json.dumps( + { + "jsonrpc": "2.0", + "id": request_id, + "error": { + "code": -32603, + "message": getattr( + proxy_exc, + "message", + f"Streaming error: {proxy_exc!s}", + ), + }, + } + ) + + "\n" + ) - async for line in ProxyBaseLLMRequestProcessing.async_streaming_data_generator( + async for ( + line + ) in ProxyBaseLLMRequestProcessing.async_streaming_data_generator( response=a2a_stream, user_api_key_dict=user_api_key_dict, request_data=request_data, @@ -151,7 +164,12 @@ async def _handle_stream_message( yield json.dumps(chunk) + "\n" except Exception as e: verbose_proxy_logger.exception(f"Error streaming A2A response: {e}") - if use_proxy_hooks and proxy_logging_obj is not None and user_api_key_dict is not None and request_data is not None: + if ( + use_proxy_hooks + and proxy_logging_obj is not None + and user_api_key_dict is not None + and request_data is not None + ): transformed_exception = await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, @@ -382,6 +400,7 @@ async def invoke_agent_a2a( agent_id=agent.agent_id, metadata=data.get("metadata", {}), proxy_server_request=data.get("proxy_server_request"), + litellm_logging_obj=logging_obj, ) response = await proxy_logging_obj.post_call_success_hook( diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index ca21d0430a5..9861026ba3e 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -10,16 +10,12 @@ import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging from litellm.litellm_core_utils.safe_json_loads import safe_json_loads -from litellm.proxy._types import ( - AddTeamCallback, - CommonProxyErrors, - LitellmDataForBackendLLMCall, - LitellmUserRoles, - SpecialHeaders, - TeamCallbackMetadata, - UserAPIKeyAuth, -) -from litellm.proxy.common_utils.http_parsing_utils import _safe_get_request_headers +from litellm.proxy._types import (AddTeamCallback, CommonProxyErrors, + LitellmDataForBackendLLMCall, + LitellmUserRoles, SpecialHeaders, + TeamCallbackMetadata, UserAPIKeyAuth) +from litellm.proxy.common_utils.http_parsing_utils import \ + _safe_get_request_headers # Cache special headers as a frozenset for O(1) lookup performance _SPECIAL_HEADERS_CACHE = frozenset( @@ -28,12 +24,9 @@ _SPECIAL_HEADERS_CACHE = frozenset( from litellm.router import Router from litellm.types.llms.anthropic import ANTHROPIC_API_HEADERS from litellm.types.services import ServiceTypes -from litellm.types.utils import ( - LlmProviders, - ProviderSpecificHeader, - StandardLoggingUserAPIKeyMetadata, - SupportedCacheControls, -) +from litellm.types.utils import (LlmProviders, ProviderSpecificHeader, + StandardLoggingUserAPIKeyMetadata, + SupportedCacheControls) service_logger_obj = ServiceLogging() # used for tracking latency on OTEL @@ -577,13 +570,13 @@ class LiteLLMProxyRequestSetup: # Finally update the requests metadata with the `metadata_from_headers` ######################################################################################### - agent_id_from_header = headers.get("x-litellm-agent-id") # x-litellm-trace-id and x-litellm-session-id are interchangeable for call chaining chain_id = headers.get("x-litellm-trace-id") or headers.get( "x-litellm-session-id" ) + if agent_id_from_header: metadata_from_headers["agent_id"] = agent_id_from_header verbose_proxy_logger.debug( @@ -670,8 +663,7 @@ class LiteLLMProxyRequestSetup: return data from litellm.proxy._types import ( LiteLLM_ManagementEndpoint_MetadataFields, - LiteLLM_ManagementEndpoint_MetadataFields_Premium, - ) + LiteLLM_ManagementEndpoint_MetadataFields_Premium) # ignore any special fields added_metadata = {} @@ -840,7 +832,8 @@ async def add_litellm_data_to_request( # noqa: PLR0915 """ from litellm.proxy.proxy_server import llm_router, premium_user - from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields + from litellm.types.proxy.litellm_pre_call_utils import (RedactedDict, + SecretFields) _raw_headers: Dict[str, str] = RedactedDict(_safe_get_request_headers(request)) @@ -1515,7 +1508,8 @@ async def move_guardrails_to_metadata( # Only check policy engine if no local config (avoid import + registry lookup) if not (has_key_config or has_team_config or has_request_config): - from litellm.proxy.policy_engine.policy_registry import get_policy_registry + from litellm.proxy.policy_engine.policy_registry import \ + get_policy_registry if not get_policy_registry().is_initialized(): # Nothing configured anywhere - clean up request body fields and return @@ -1579,14 +1573,16 @@ async def move_guardrails_to_metadata( def _is_policy_version_id(s: str) -> bool: """Return True if string is a policy version ID (starts with policy_ prefix).""" - from litellm.proxy.policy_engine.policy_registry import POLICY_VERSION_ID_PREFIX + from litellm.proxy.policy_engine.policy_registry import \ + POLICY_VERSION_ID_PREFIX return isinstance(s, str) and s.startswith(POLICY_VERSION_ID_PREFIX) def _extract_policy_id(s: str) -> Optional[str]: """Extract raw UUID from policy_ string, or None if not a valid version ID.""" - from litellm.proxy.policy_engine.policy_registry import POLICY_VERSION_ID_PREFIX + from litellm.proxy.policy_engine.policy_registry import \ + POLICY_VERSION_ID_PREFIX if not _is_policy_version_id(s): return None @@ -1607,10 +1603,9 @@ def _match_and_track_policies( """ from litellm._logging import verbose_proxy_logger from litellm.proxy.common_utils.callback_utils import ( - add_policy_sources_to_metadata, - add_policy_to_applied_policies_header, - ) - from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry + add_policy_sources_to_metadata, add_policy_to_applied_policies_header) + from litellm.proxy.policy_engine.attachment_registry import \ + get_attachment_registry from litellm.proxy.policy_engine.policy_matcher import PolicyMatcher # Get matching policies via attachments (with match reasons for attribution) @@ -1755,7 +1750,8 @@ async def add_guardrails_from_policy_engine( user_api_key_dict: The user's API key authentication info """ from litellm._logging import verbose_proxy_logger - from litellm.proxy.common_utils.http_parsing_utils import get_tags_from_request_body + from litellm.proxy.common_utils.http_parsing_utils import \ + get_tags_from_request_body from litellm.proxy.policy_engine.policy_registry import get_policy_registry from litellm.types.proxy.policy_engine import PolicyMatchContext diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 31615a768d7..131841f7b59 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -11,26 +11,21 @@ from pydantic import BaseModel import litellm from litellm._logging import verbose_proxy_logger -from litellm.constants import ( - MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB, -) +from litellm.constants import \ + MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB from litellm.constants import REDACTED_BY_LITELM_STRING from litellm.litellm_core_utils.core_helpers import ( - get_litellm_metadata_from_kwargs, - reconstruct_model_name, -) + get_litellm_metadata_from_kwargs, reconstruct_model_name) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload from litellm.proxy.utils import PrismaClient, hash_token -from litellm.types.utils import ( - CostBreakdown, - StandardLoggingGuardrailInformation, - StandardLoggingMCPToolCall, - StandardLoggingModelInformation, - StandardLoggingPayload, - StandardLoggingVectorStoreRequest, - VectorStoreSearchResponse, -) +from litellm.types.utils import (CostBreakdown, + StandardLoggingGuardrailInformation, + StandardLoggingMCPToolCall, + StandardLoggingModelInformation, + StandardLoggingPayload, + StandardLoggingVectorStoreRequest, + VectorStoreSearchResponse) from litellm.utils import get_end_user_id_for_cost_tracking @@ -116,16 +111,15 @@ def _get_spend_logs_metadata( # Filter the metadata dictionary to include only the specified keys clean_metadata = SpendLogsMetadata( **{ # type: ignore - key: metadata.get(key) - for key in SpendLogsMetadata.__annotations__.keys() + key: metadata.get(key) for key in SpendLogsMetadata.__annotations__.keys() } ) clean_metadata["applied_guardrails"] = applied_guardrails clean_metadata["batch_models"] = batch_models clean_metadata["mcp_tool_call_metadata"] = mcp_tool_call_metadata - clean_metadata[ - "vector_store_request_metadata" - ] = _get_vector_store_request_for_spend_logs_payload(vector_store_request_metadata) + clean_metadata["vector_store_request_metadata"] = ( + _get_vector_store_request_for_spend_logs_payload(vector_store_request_metadata) + ) clean_metadata["guardrail_information"] = guardrail_information clean_metadata["usage_object"] = usage_object clean_metadata["model_map_information"] = model_map_information @@ -372,9 +366,11 @@ def get_logging_payload( # noqa: PLR0915 guardrail_information=( standard_logging_payload.get("guardrail_information", None) if standard_logging_payload is not None - else metadata.get("standard_logging_guardrail_information", None) - if metadata is not None - else None + else ( + metadata.get("standard_logging_guardrail_information", None) + if metadata is not None + else None + ) ), cold_storage_object_key=( standard_logging_payload["metadata"].get("cold_storage_object_key", None) @@ -501,6 +497,7 @@ def _get_session_id_for_spend_log( """ from litellm._uuid import uuid + if ( standard_logging_payload is not None and standard_logging_payload.get("trace_id") is not None @@ -515,9 +512,7 @@ def _get_session_id_for_spend_log( return str(uuid.uuid4()) -def _get_request_duration_ms( - start_time: datetime, end_time: datetime -) -> Optional[int]: +def _get_request_duration_ms(start_time: datetime, end_time: datetime) -> Optional[int]: """Compute request duration in milliseconds from start and end times.""" try: return int((end_time - start_time).total_seconds() * 1000) @@ -709,20 +704,20 @@ def _convert_to_json_serializable_dict( if max_depth <= 0: # Return a placeholder if max depth is exceeded return "" - + if visited is None: visited = set() - + # Get the object's memory address to track visited objects obj_id = id(obj) if obj_id in visited: # Circular reference detected, return placeholder return "" - + # Only track mutable objects (dict, list, objects with __dict__) if isinstance(obj, (dict, list)) or hasattr(obj, "__dict__"): visited.add(obj_id) - + try: if isinstance(obj, BaseModel): # Use Pydantic's model_dump() instead of pickle @@ -741,7 +736,9 @@ def _convert_to_json_serializable_dict( ] elif hasattr(obj, "__dict__"): # Handle objects with __dict__ attribute - return _convert_to_json_serializable_dict(obj.__dict__, visited, max_depth - 1) + return _convert_to_json_serializable_dict( + obj.__dict__, visited, max_depth - 1 + ) else: # Primitives (str, int, float, bool, None) pass through return obj @@ -777,9 +774,7 @@ def _get_proxy_server_request_for_spend_logs_payload( # Apply message redaction if turn_off_message_logging is enabled if kwargs is not None: from litellm.litellm_core_utils.redact_messages import ( - perform_redaction, - should_redact_message_logging, - ) + perform_redaction, should_redact_message_logging) # Build model_call_details dict to check redaction settings model_call_details = { @@ -788,12 +783,12 @@ def _get_proxy_server_request_for_spend_logs_payload( "standard_callback_dynamic_params" ), } - + # If redaction is enabled, convert to serializable dict before redacting if should_redact_message_logging(model_call_details=model_call_details): _request_body = _convert_to_json_serializable_dict(_request_body) perform_redaction(model_call_details=_request_body, result=None) - + _request_body = _sanitize_request_body_for_spend_logs_payload(_request_body) _request_body_json_str = json.dumps(_request_body, default=str) return _request_body_json_str @@ -845,10 +840,8 @@ def _get_response_for_spend_logs_payload( # Apply message redaction if turn_off_message_logging is enabled if kwargs is not None: from litellm.litellm_core_utils.redact_messages import ( - perform_redaction, - should_redact_message_logging, - ) - + perform_redaction, should_redact_message_logging) + litellm_params = kwargs.get("litellm_params", {}) model_call_details = { "litellm_params": litellm_params, @@ -856,11 +849,13 @@ def _get_response_for_spend_logs_payload( "standard_callback_dynamic_params" ), } - + # If redaction is enabled, convert to serializable dict before redacting if should_redact_message_logging(model_call_details=model_call_details): response_obj = _convert_to_json_serializable_dict(response_obj) - response_obj = perform_redaction(model_call_details={}, result=response_obj) + response_obj = perform_redaction( + model_call_details={}, result=response_obj + ) sanitized_wrapper = _sanitize_request_body_for_spend_logs_payload( {"response": response_obj} @@ -882,7 +877,7 @@ def _should_store_prompts_and_responses_in_spend_logs() -> bool: # Check general_settings (from DB or proxy_config.yaml) store_prompts_value = general_settings.get("store_prompts_in_spend_logs") - + # Normalize case: handle True/true/TRUE, False/false/FALSE, None/null if store_prompts_value is True: return True @@ -890,7 +885,7 @@ def _should_store_prompts_and_responses_in_spend_logs() -> bool: # Case-insensitive string comparison if store_prompts_value.lower() == "true": return True - + # Also check environment variable return get_secret_bool("STORE_PROMPTS_IN_SPEND_LOGS") is True diff --git a/litellm/utils.py b/litellm/utils.py index cbe6aa8e793..f19b1261d84 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1454,10 +1454,12 @@ def client(original_function): # noqa: PLR0915 logging_obj, kwargs = function_setup( original_function.__name__, rules_obj, start_time, *args, **kwargs ) - + # Type assertion: logging_obj is guaranteed to be non-None after function_setup - assert logging_obj is not None, "logging_obj should not be None after function_setup" - + assert ( + logging_obj is not None + ), "logging_obj should not be None after function_setup" + ## LOAD CREDENTIALS load_credentials_from_list(kwargs) kwargs["litellm_logging_obj"] = logging_obj @@ -1753,7 +1755,9 @@ def client(original_function): # noqa: PLR0915 print_args_passed_to_litellm(original_function, args, kwargs) start_time = datetime.datetime.now() result = None - _update_response_metadata = getattr(sys.modules[__name__], "update_response_metadata") + _update_response_metadata = getattr( + sys.modules[__name__], "update_response_metadata" + ) logging_obj: Optional[LiteLLMLoggingObject] = kwargs.get( "litellm_logging_obj", None ) @@ -1776,9 +1780,11 @@ def client(original_function): # noqa: PLR0915 logging_obj, kwargs = function_setup( original_function.__name__, rules_obj, start_time, *args, **kwargs ) - + # Type assertion: logging_obj is guaranteed to be non-None after function_setup - assert logging_obj is not None, "logging_obj should not be None after function_setup" + assert ( + logging_obj is not None + ), "logging_obj should not be None after function_setup" modified_kwargs = await async_pre_call_deployment_hook(kwargs, call_type) if modified_kwargs is not None: @@ -1861,6 +1867,7 @@ def client(original_function): # noqa: PLR0915 # MODEL CALL result = await original_function(*args, **kwargs) end_time = datetime.datetime.now() + if _is_streaming_request( kwargs=kwargs, call_type=call_type, @@ -2082,12 +2089,14 @@ def _is_async_request( return False -_STREAMING_CALL_TYPES = frozenset({ - CallTypes.generate_content_stream, - CallTypes.agenerate_content_stream, - CallTypes.generate_content_stream.value, - CallTypes.agenerate_content_stream.value, -}) +_STREAMING_CALL_TYPES = frozenset( + { + CallTypes.generate_content_stream, + CallTypes.agenerate_content_stream, + CallTypes.generate_content_stream.value, + CallTypes.agenerate_content_stream.value, + } +) def _is_streaming_request( @@ -2181,7 +2190,7 @@ def encode(model="", text="", custom_tokenizer: Optional[dict] = None): # Normalize: HuggingFace Tokenizer.encode() returns an Encoding object; # extract .ids so the return type is always List[int]. if hasattr(enc, "ids"): - return enc.ids + return enc.ids # type: ignore return enc @@ -5836,7 +5845,7 @@ def get_model_info( _model_info[key] = value # type: ignore # if verbose_logger.isEnabledFor(logging.DEBUG): - # verbose_logger.debug(f"model_info: {_model_info}") + # verbose_logger.debug(f"model_info: {_model_info}") returned_model_info = ModelInfo( **_model_info, supported_openai_params=supported_openai_params @@ -6179,8 +6188,10 @@ def validate_environment( # noqa: PLR0915 "AWS_ROLE_ARN" in os.environ or "AWS_PROFILE" in os.environ or "AWS_WEB_IDENTITY_TOKEN_FILE" in os.environ - or "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" in os.environ # ECS task role - or "AWS_CONTAINER_CREDENTIALS_FULL_URI" in os.environ # ECS/Fargate full URI credential delivery + or "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" + in os.environ # ECS task role + or "AWS_CONTAINER_CREDENTIALS_FULL_URI" + in os.environ # ECS/Fargate full URI credential delivery ): keys_in_environment = True else: @@ -7386,7 +7397,9 @@ class ModelResponseIterator: if convert_to_delta is True: _stream_response = ModelResponseStream() _stream_response.choices[0].delta.content = model_response.choices[0].message.content # type: ignore - self.model_response: Union[ModelResponse, ModelResponseStream] = _stream_response + self.model_response: Union[ModelResponse, ModelResponseStream] = ( + _stream_response + ) else: self.model_response = model_response self.is_done = False @@ -7457,13 +7470,13 @@ def is_cached_message(message: AllMessageValues) -> bool: Used for anthropic/gemini context caching. Follows the anthropic format {"cache_control": {"type": "ephemeral"}} - + Can be disabled globally by setting litellm.disable_anthropic_gemini_context_caching_transform = True """ # Check if context caching is disabled globally if litellm.disable_anthropic_gemini_context_caching_transform is True: return False - + if "content" not in message: return False @@ -7980,6 +7993,7 @@ class ProviderConfigManager: def _get_azure_ai_config(model: str) -> BaseConfig: """Get Azure AI config based on model type.""" from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo + return AzureFoundryModelInfo.get_azure_ai_config_for_model(model) @staticmethod