From cb5464421f751afd58df39a8dfe1719e20feecc2 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 30 Aug 2025 16:11:39 -0700 Subject: [PATCH 01/40] fix(braintrust_logging.py): filter metadata before logging avoid unserializable json --- litellm/integrations/braintrust_logging.py | 13 +-- litellm/litellm_core_utils/safe_json_dumps.py | 92 +++++++++++++++++++ litellm/proxy/_new_secret_config.yaml | 10 +- 3 files changed, 100 insertions(+), 15 deletions(-) diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index 531da933fcc..5238bfe1dbf 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -12,6 +12,7 @@ from pydantic import BaseModel import litellm from litellm import verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.safe_json_dumps import filter_json_serializable from litellm.llms.custom_httpx.http_handler import ( HTTPHandler, get_async_httpx_client, @@ -45,9 +46,9 @@ class BraintrustLogger(CustomLogger): "Authorization": "Bearer " + self.api_key, "Content-Type": "application/json", } - self._project_id_cache: Dict[ - str, str - ] = {} # Cache mapping project names to IDs + self._project_id_cache: Dict[str, str] = ( + {} + ) # Cache mapping project names to IDs self.global_braintrust_http_handler = get_async_httpx_client( llm_provider=httpxSpecialProvider.LoggingCallback ) @@ -276,7 +277,7 @@ class BraintrustLogger(CustomLogger): # Allow metadata override for span name span_name = metadata.get("span_name", "Chat Completion") - + request_data = { "id": litellm_call_id, "input": prompt["messages"], @@ -431,12 +432,12 @@ class BraintrustLogger(CustomLogger): # Allow metadata override for span name span_name = metadata.get("span_name", "Chat Completion") - + request_data = { "id": litellm_call_id, "input": prompt["messages"], "output": output, - "metadata": clean_metadata, + "metadata": filter_json_serializable(clean_metadata), "tags": tags, "span_attributes": {"name": span_name, "type": "llm"}, } diff --git a/litellm/litellm_core_utils/safe_json_dumps.py b/litellm/litellm_core_utils/safe_json_dumps.py index 7ad0038ecb2..b3b1d7fb3df 100644 --- a/litellm/litellm_core_utils/safe_json_dumps.py +++ b/litellm/litellm_core_utils/safe_json_dumps.py @@ -1,5 +1,6 @@ import json from typing import Any, Union + from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH @@ -49,3 +50,94 @@ def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str: safe_data = _serialize(data, set(), 0) return json.dumps(safe_data, default=str) + + +def filter_json_serializable( + data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH +) -> Any: + """ + Recursively filter data to only include JSON serializable items. + Non-serializable items are completely skipped (not included in the result). + """ + + def _is_json_serializable(obj: Any) -> bool: + """Test if an object is JSON serializable.""" + try: + json.dumps(obj) + return True + except (TypeError, ValueError): + return False + + def _filter(obj: Any, seen: set, depth: int) -> Any: + # Check for maximum depth. + if depth > max_depth: + return None + + # Base-case: if it is a primitive, test if it's serializable + if isinstance(obj, (str, int, float, bool, type(None))): + return obj if _is_json_serializable(obj) else None + + # Check for circular reference. + if id(obj) in seen: + return None + + seen.add(id(obj)) + + try: + if isinstance(obj, dict): + result = {} + for k, v in obj.items(): + # Only include keys that are strings and values that are serializable + if isinstance(k, str): + filtered_value = _filter(v, seen, depth + 1) + # Only add the key-value pair if the value is serializable + if filtered_value is not None or v is None: + if _is_json_serializable(filtered_value): + result[k] = filtered_value + seen.remove(id(obj)) + return result + + elif isinstance(obj, list): + result = [] + for item in obj: + filtered_item = _filter(item, seen, depth + 1) + # Only include items that are serializable + if filtered_item is not None or item is None: + if _is_json_serializable(filtered_item): + result.append(filtered_item) + seen.remove(id(obj)) + return result + + elif isinstance(obj, tuple): + filtered_items = [] + for item in obj: + filtered_item = _filter(item, seen, depth + 1) + # Only include items that are serializable + if filtered_item is not None or item is None: + if _is_json_serializable(filtered_item): + filtered_items.append(filtered_item) + seen.remove(id(obj)) + return tuple(filtered_items) + + elif isinstance(obj, set): + filtered_items = [] + for item in obj: + filtered_item = _filter(item, seen, depth + 1) + # Only include items that are serializable + if filtered_item is not None or item is None: + if _is_json_serializable(filtered_item): + filtered_items.append(filtered_item) + seen.remove(id(obj)) + return sorted(filtered_items) + + else: + # Test if the object is directly serializable + seen.remove(id(obj)) + return obj if _is_json_serializable(obj) else None + + except Exception: + if id(obj) in seen: + seen.remove(id(obj)) + return None + + return _filter(data, set(), 0) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index b3653c31435..f4dc1fca711 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -19,12 +19,4 @@ router_settings: litellm_settings: callbacks: ["otel"] - cache: true - cache_params: - type: redis - ttl: 600 - supported_call_types: ["acompletion", "completion"] - - model_group_settings: - forward_client_headers_to_llm_api: - - fake-openai-endpoint \ No newline at end of file + success_callback: ["braintrust"] \ No newline at end of file From 9cffabb433cae113facae5b54e3f62cf45b8cb4d Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 30 Aug 2025 16:17:57 -0700 Subject: [PATCH 02/40] refactor(braintrust_logging.py): migrate braintrust logging to standard logging payload avoids issue with span in request metadata standard logging payload is always json serializable --- litellm/integrations/braintrust_logging.py | 55 ++++------------------ 1 file changed, 9 insertions(+), 46 deletions(-) diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index 5238bfe1dbf..8e4e31280ee 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -170,6 +170,7 @@ class BraintrustLogger(CustomLogger): verbose_logger.debug("REACHES BRAINTRUST SUCCESS") try: litellm_call_id = kwargs.get("litellm_call_id") + standard_logging_object = kwargs.get("standard_logging_object", {}) prompt = {"messages": kwargs.get("messages")} output = None choices = [] @@ -193,33 +194,13 @@ class BraintrustLogger(CustomLogger): ): output = response_obj["data"] - litellm_params = kwargs.get("litellm_params", {}) - metadata = ( - litellm_params.get("metadata", {}) or {} - ) # if litellm_params['metadata'] == None - metadata = self.add_metadata_from_header(litellm_params, metadata) - clean_metadata = {} - try: - metadata = copy.deepcopy( - metadata - ) # Avoid modifying the original metadata - except Exception: - new_metadata = {} - for key, value in metadata.items(): - if ( - isinstance(value, list) - or isinstance(value, dict) - or isinstance(value, str) - or isinstance(value, int) - or isinstance(value, float) - ): - new_metadata[key] = copy.deepcopy(value) - metadata = new_metadata + litellm_params = kwargs.get("litellm_params", {}) or {} + dynamic_metadata = litellm_params.get("dynamic_metadata", {}) or {} # Get project_id from metadata or create default if needed - project_id = metadata.get("project_id") + project_id = dynamic_metadata.get("project_id") if project_id is None: - project_name = metadata.get("project_name") + project_name = dynamic_metadata.get("project_name") project_id = ( self.get_project_id_sync(project_name) if project_name else None ) @@ -230,8 +211,8 @@ class BraintrustLogger(CustomLogger): project_id = self.default_project_id tags = [] - if isinstance(metadata, dict): - for key, value in metadata.items(): + if isinstance(dynamic_metadata, dict): + for key, value in dynamic_metadata.items(): # generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy if ( litellm.langfuse_default_tags is not None @@ -240,25 +221,7 @@ class BraintrustLogger(CustomLogger): ): tags.append(f"{key}:{value}") - # clean litellm metadata before logging - if key in [ - "headers", - "endpoint", - "caching_groups", - "previous_models", - ]: - continue - else: - clean_metadata[key] = value - cost = kwargs.get("response_cost", None) - if cost is not None: - clean_metadata["litellm_response_cost"] = cost - - # metadata.model is required for braintrust to calculate the "Estimated cost" metric - litellm_model = kwargs.get("model", None) - if litellm_model is not None: - clean_metadata["model"] = litellm_model metrics: Optional[dict] = None usage_obj = getattr(response_obj, "usage", None) @@ -276,12 +239,12 @@ class BraintrustLogger(CustomLogger): } # Allow metadata override for span name - span_name = metadata.get("span_name", "Chat Completion") + span_name = dynamic_metadata.get("span_name", "Chat Completion") request_data = { "id": litellm_call_id, "input": prompt["messages"], - "metadata": clean_metadata, + "metadata": standard_logging_object, "tags": tags, "span_attributes": {"name": span_name, "type": "llm"}, } From 599071321dc0eeae9207b556f8bbc89ea05665ca Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 30 Aug 2025 16:24:03 -0700 Subject: [PATCH 03/40] fix(braintrust_logging.py): refactor to consistently use standard logging payload --- litellm/integrations/braintrust_logging.py | 91 +++------------------- 1 file changed, 10 insertions(+), 81 deletions(-) diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index 8e4e31280ee..1ddf789ed32 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -109,43 +109,6 @@ class BraintrustLogger(CustomLogger): except httpx.HTTPStatusError as e: raise Exception(f"Failed to register project: {e.response.text}") - @staticmethod - def add_metadata_from_header(litellm_params: dict, metadata: dict) -> dict: - """ - Adds metadata from proxy request headers to Braintrust logging if keys start with "braintrust_" - and overwrites litellm_params.metadata if already included. - - For example if you want to append your trace to an existing `trace_id` via header, send - `headers: { ..., langfuse_existing_trace_id: your-existing-trace-id }` via proxy request. - """ - if litellm_params is None: - return metadata - - if litellm_params.get("proxy_server_request") is None: - return metadata - - if metadata is None: - metadata = {} - - proxy_headers = ( - litellm_params.get("proxy_server_request", {}).get("headers", {}) or {} - ) - - for metadata_param_key in proxy_headers: - if metadata_param_key.startswith("braintrust"): - trace_param_key = metadata_param_key.replace("braintrust", "", 1) - if trace_param_key in metadata: - verbose_logger.warning( - f"Overwriting Braintrust `{trace_param_key}` from request header" - ) - else: - verbose_logger.debug( - f"Found Braintrust `{trace_param_key}` in request header" - ) - metadata[trace_param_key] = proxy_headers.get(metadata_param_key) - - return metadata - async def create_default_project_and_experiment(self): project = await self.global_braintrust_http_handler.post( f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"} @@ -172,6 +135,7 @@ class BraintrustLogger(CustomLogger): litellm_call_id = kwargs.get("litellm_call_id") standard_logging_object = kwargs.get("standard_logging_object", {}) prompt = {"messages": kwargs.get("messages")} + output = None choices = [] if response_obj is not None and ( @@ -276,6 +240,7 @@ class BraintrustLogger(CustomLogger): verbose_logger.debug("REACHES BRAINTRUST SUCCESS") try: litellm_call_id = kwargs.get("litellm_call_id") + standard_logging_object = kwargs.get("standard_logging_object", {}) prompt = {"messages": kwargs.get("messages")} output = None choices = [] @@ -300,32 +265,14 @@ class BraintrustLogger(CustomLogger): output = response_obj["data"] litellm_params = kwargs.get("litellm_params", {}) - metadata = ( - litellm_params.get("metadata", {}) or {} - ) # if litellm_params['metadata'] == None - metadata = self.add_metadata_from_header(litellm_params, metadata) + dynamic_metadata = litellm_params.get("dynamic_metadata", {}) or {} + clean_metadata = {} - new_metadata = {} - for key, value in metadata.items(): - if ( - isinstance(value, list) - or isinstance(value, str) - or isinstance(value, int) - or isinstance(value, float) - ): - new_metadata[key] = value - elif isinstance(value, BaseModel): - new_metadata[key] = value.model_dump_json() - elif isinstance(value, dict): - for k, v in value.items(): - if isinstance(v, datetime): - value[k] = v.isoformat() - new_metadata[key] = value # Get project_id from metadata or create default if needed - project_id = metadata.get("project_id") + project_id = dynamic_metadata.get("project_id") if project_id is None: - project_name = metadata.get("project_name") + project_name = dynamic_metadata.get("project_name") project_id = ( await self.get_project_id_async(project_name) if project_name @@ -338,8 +285,8 @@ class BraintrustLogger(CustomLogger): project_id = self.default_project_id tags = [] - if isinstance(metadata, dict): - for key, value in metadata.items(): + if isinstance(dynamic_metadata, dict): + for key, value in dynamic_metadata.items(): # generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy if ( litellm.langfuse_default_tags is not None @@ -348,25 +295,7 @@ class BraintrustLogger(CustomLogger): ): tags.append(f"{key}:{value}") - # clean litellm metadata before logging - if key in [ - "headers", - "endpoint", - "caching_groups", - "previous_models", - ]: - continue - else: - clean_metadata[key] = value - cost = kwargs.get("response_cost", None) - if cost is not None: - clean_metadata["litellm_response_cost"] = cost - - # metadata.model is required for braintrust to calculate the "Estimated cost" metric - litellm_model = kwargs.get("model", None) - if litellm_model is not None: - clean_metadata["model"] = litellm_model metrics: Optional[dict] = None usage_obj = getattr(response_obj, "usage", None) @@ -394,13 +323,13 @@ class BraintrustLogger(CustomLogger): ) # Allow metadata override for span name - span_name = metadata.get("span_name", "Chat Completion") + span_name = dynamic_metadata.get("span_name", "Chat Completion") request_data = { "id": litellm_call_id, "input": prompt["messages"], "output": output, - "metadata": filter_json_serializable(clean_metadata), + "metadata": standard_logging_object, "tags": tags, "span_attributes": {"name": span_name, "type": "llm"}, } From 51c73dc60ba2bc050fe8e7cc17c07f6e75df4c24 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 30 Aug 2025 17:26:18 -0700 Subject: [PATCH 04/40] fix(vertex_and_google_ai_studio_gemini.py): bubble up thoughtsignature back to client --- .../vertex_and_google_ai_studio_gemini.py | 77 ++++++++++----- litellm/types/llms/openai.py | 8 +- litellm/types/llms/vertex_ai.py | 4 +- tests/llm_translation/test_gemini.py | 96 ++++++++++++++----- 4 files changed, 137 insertions(+), 48 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 99a04c20fba..37470a6ee09 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -43,6 +43,7 @@ from litellm.types.llms.gemini import BidiGenerateContentServerMessage from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionResponseMessage, + ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ChatCompletionToolParamFunctionChunk, @@ -792,7 +793,25 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): content_str += _content_str return content_str, reasoning_content_str - + + def _extract_thinking_blocks_from_parts( + self, parts: List[HttpxPartType] + ) -> List[ChatCompletionThinkingBlock]: + """Extract thinking blocks from parts if present""" + thinking_blocks: List[ChatCompletionThinkingBlock] = [] + for part in parts: + if "thoughtSignature" in part: + part_copy = part.copy() + part_copy.pop("thoughtSignature") + thinking_blocks.append( + ChatCompletionThinkingBlock( + type="thinking", + thinking=json.dumps(part_copy), + signature=part["thoughtSignature"], + ) + ) + return thinking_blocks + def _extract_image_response_from_parts( self, parts: List[HttpxPartType] ) -> Optional[ImageURLObject]: @@ -804,10 +823,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if mime_type.startswith("image/"): # Convert base64 data to data URI format data_uri = f"data:{mime_type};base64,{data}" - return ImageURLObject( - url=data_uri, - detail="auto" - ) + return ImageURLObject(url=data_uri, detail="auto") return None def _extract_audio_response_from_parts( @@ -1127,7 +1143,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): elif web_search_queries: web_search_requests = len(grounding_metadata) return web_search_requests - + @staticmethod def _create_streaming_choice( chat_completion_message: ChatCompletionResponseMessage, @@ -1151,9 +1167,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): index=candidate.get("index", idx), delta=Delta( content=chat_completion_message.get("content"), - reasoning_content=chat_completion_message.get( - "reasoning_content" - ), + reasoning_content=chat_completion_message.get("reasoning_content"), tool_calls=tools, image=image_response, function_call=functions, @@ -1164,13 +1178,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return choice @staticmethod - def _extract_candidate_metadata(candidate: Candidates) -> Tuple[List[dict], List[dict], List, List]: + def _extract_candidate_metadata( + candidate: Candidates, + ) -> Tuple[List[dict], List[dict], List, List]: """ Extract metadata from a single candidate response. - + Returns: grounding_metadata: List[dict] - url_context_metadata: List[dict] + url_context_metadata: List[dict] safety_ratings: List citation_metadata: List """ @@ -1178,7 +1194,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): url_context_metadata: List[dict] = [] safety_ratings: List = [] citation_metadata: List = [] - + if "groundingMetadata" in candidate: if isinstance(candidate["groundingMetadata"], list): grounding_metadata.extend(candidate["groundingMetadata"]) # type: ignore @@ -1194,8 +1210,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if "urlContextMetadata" in candidate: # Add URL context metadata to grounding metadata url_context_metadata.append(cast(dict, candidate["urlContextMetadata"])) - - return grounding_metadata, url_context_metadata, safety_ratings, citation_metadata + + return ( + grounding_metadata, + url_context_metadata, + safety_ratings, + citation_metadata, + ) @staticmethod def _process_candidates( @@ -1227,6 +1248,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): tools: Optional[List[ChatCompletionToolCallChunk]] = [] functions: Optional[ChatCompletionToolCallFunctionChunk] = None cumulative_tool_call_index: int = 0 + thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None for idx, candidate in enumerate(_candidates): if "content" not in candidate: @@ -1239,7 +1261,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): candidate_safety_ratings, candidate_citation_metadata, ) = VertexGeminiConfig._extract_candidate_metadata(candidate) - + grounding_metadata.extend(candidate_grounding_metadata) url_context_metadata.extend(candidate_url_context_metadata) safety_ratings.extend(candidate_safety_ratings) @@ -1264,6 +1286,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) ) + thinking_blocks = ( + VertexGeminiConfig()._extract_thinking_blocks_from_parts( + parts=candidate["content"]["parts"] + ) + ) + if audio_response is not None: cast(Dict[str, Any], chat_completion_message)[ "audio" @@ -1271,7 +1299,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): chat_completion_message["content"] = None # OpenAI spec if image_response is not None: # Handle image response - combine with text content into structured format - cast(Dict[str, Any], chat_completion_message)["image"] = image_response + cast(Dict[str, Any], chat_completion_message)[ + "image" + ] = image_response if content is not None: chat_completion_message["content"] = content @@ -1298,15 +1328,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if functions is not None: chat_completion_message["function_call"] = functions + if thinking_blocks is not None: + chat_completion_message["thinking_blocks"] = thinking_blocks # type: ignore + if isinstance(model_response, ModelResponseStream): choice = VertexGeminiConfig._create_streaming_choice( chat_completion_message=chat_completion_message, - candidate=candidate, - idx=idx, - tools=tools, - functions=functions, + candidate=candidate, + idx=idx, + tools=tools, + functions=functions, chat_completion_logprobs=chat_completion_logprobs, - image_response=image_response + image_response=image_response, ) model_response.choices.append(choice) elif isinstance(model_response, ModelResponse): diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index a0c8e5b6295..9b6cad38008 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -43,10 +43,14 @@ from openai.types.responses.response import ( # Handle OpenAI SDK version compatibility for Text type try: - from openai.types.responses.response_create_params import Text as ResponseText + from openai.types.responses.response_create_params import ( + Text as ResponseText, # type: ignore + ) except (ImportError, AttributeError): # Fall back to the concrete config type available in all SDK versions - from openai.types.responses.response_text_config_param import ResponseTextConfigParam as ResponseText + from openai.types.responses.response_text_config_param import ( + ResponseTextConfigParam as ResponseText, + ) from openai.types.responses.response_create_params import ( Reasoning, diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 2931770cd6e..052b872bcde 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -72,6 +72,7 @@ class HttpxPartType(TypedDict, total=False): executableCode: HttpxExecutableCode codeExecutionResult: HttpxCodeExecutionResult thought: bool + thoughtSignature: str class HttpxContentType(TypedDict, total=False): @@ -245,10 +246,11 @@ class UsageMetadata(TypedDict, total=False): class TokenCountDetailsResponse(TypedDict): """ Response structure for token count details with modality breakdown. - + Example: {'totalTokens': 12, 'promptTokensDetails': [{'modality': 'TEXT', 'tokenCount': 12}]} """ + totalTokens: int promptTokensDetails: List[PromptTokensDetails] diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index 22a54b8a56b..c54168e9a6f 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -436,7 +436,10 @@ def test_gemini_with_empty_function_call_arguments(): async def test_claude_tool_use_with_gemini(): response = await litellm.anthropic.messages.acreate( messages=[ - {"role": "user", "content": "Hello, can you tell me the weather in Boston. Please respond with a tool call?"} + { + "role": "user", + "content": "Hello, can you tell me the weather in Boston. Please respond with a tool call?", + } ], model="gemini/gemini-2.5-flash", stream=True, @@ -578,11 +581,17 @@ def test_gemini_tool_use(): assert stop_reason is not None assert stop_reason == "tool_calls" + @pytest.mark.asyncio async def test_gemini_image_generation_async(): litellm._turn_on_debug() response = await litellm.acompletion( - messages=[{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}], + messages=[ + { + "role": "user", + "content": "Generate an image of a banana wearing a costume that says LiteLLM", + } + ], model="gemini/gemini-2.5-flash-image-preview", ) @@ -597,12 +606,16 @@ async def test_gemini_image_generation_async(): assert IMAGE_URL["url"].startswith("data:image/png;base64,") - @pytest.mark.asyncio async def test_gemini_image_generation_async_stream(): - #litellm._turn_on_debug() + # litellm._turn_on_debug() response = await litellm.acompletion( - messages=[{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}], + messages=[ + { + "role": "user", + "content": "Generate an image of a banana wearing a costume that says LiteLLM", + } + ], model="gemini/gemini-2.5-flash-image-preview", stream=True, ) @@ -611,35 +624,72 @@ async def test_gemini_image_generation_async_stream(): model_response_image = None async for chunk in response: print("CHUNK: ", chunk) - if hasattr(chunk.choices[0].delta, "image") and chunk.choices[0].delta.image is not None: + if ( + hasattr(chunk.choices[0].delta, "image") + and chunk.choices[0].delta.image is not None + ): model_response_image = chunk.choices[0].delta.image print("MODEL_RESPONSE_IMAGE: ", model_response_image) assert model_response_image is not None assert model_response_image["url"].startswith("data:image/png;base64,") break - + ######################################################### # Important: Validate we did get an image in the response ######################################################### assert model_response_image is not None assert model_response_image["url"].startswith("data:image/png;base64,") - + def test_system_message_with_no_user_message(): - """ - Test that the system message is translated correctly for non-OpenAI providers. - """ - messages = [ - { - "role": "system", - "content": "Be a good bot!", + """ + Test that the system message is translated correctly for non-OpenAI providers. + """ + messages = [ + { + "role": "system", + "content": "Be a good bot!", + }, + ] + + response = litellm.completion( + model="gemini/gemini-2.5-flash", + messages=messages, + ) + assert response is not None + + assert response.choices[0].message.content is not None + + +def test_gemini_with_thinking(): + from litellm import completion + + litellm._turn_on_debug() + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, }, - ] + } + ] + messages = [{"role": "user", "content": "What's the weather like in Boston today?"}] - response = litellm.completion( - model="gemini/gemini-2.5-flash", - messages=messages, - ) - assert response is not None - - assert response.choices[0].message.content is not None \ No newline at end of file + result = completion( + model="gemini/gemini-2.5-flash", + messages=messages, + tools=tools, + ) + print(f"result: {result}") From b6f6dc5c1c00ca131c5f24149ba398a59db7c390 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 30 Aug 2025 18:49:34 -0700 Subject: [PATCH 05/40] feat(vertex_ai.py): support parsing thinking content into gemini format allows function calls with thought signatures to be sent back to gemini Closes https://github.com/BerriAI/litellm/pull/13842 --- .../llms/vertex_ai/gemini/transformation.py | 95 ++++++++++++++++++- litellm/types/llms/vertex_ai.py | 1 + tests/llm_translation/test_gemini.py | 86 +++++++++++++++-- .../test_vertex_ai_gemini_transformation.py | 75 +++++++++++++++ 4 files changed, 248 insertions(+), 9 deletions(-) create mode 100644 tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 8ab212e2558..267ca61ef5d 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -105,6 +105,64 @@ def _process_gemini_image(image_url: str, format: Optional[str] = None) -> PartT raise e +def _snake_to_camel(snake_str: str) -> str: + """Convert snake_case to camelCase""" + components = snake_str.split("_") + return components[0] + "".join(x.capitalize() for x in components[1:]) + + +def _camel_to_snake(camel_str: str) -> str: + """Convert camelCase to snake_case""" + import re + + return re.sub(r"(? Optional[str]: + """ + Get the equivalent key from available keys, checking both camelCase and snake_case variants + """ + if key in available_keys: + return key + + # Try camelCase version + camel_key = _snake_to_camel(key) + if camel_key in available_keys: + return camel_key + + # Try snake_case version + snake_key = _camel_to_snake(key) + if snake_key in available_keys: + return snake_key + + return None + + +def check_if_part_exists_in_parts( + parts: List[PartType], part: PartType, excluded_keys: List[str] = [] +) -> bool: + """ + Check if a part exists in a list of parts + Handles both camelCase and snake_case key variations (e.g., function_call vs functionCall) + """ + keys_to_compare = set(part.keys()) - set(excluded_keys) + for p in parts: + p_keys = set(p.keys()) + # Check if all keys in part have equivalent values in p + match_found = True + for key in keys_to_compare: + equivalent_key = _get_equivalent_key(key, p_keys) + if equivalent_key is None or p.get(equivalent_key, None) != part.get( + key, None + ): + match_found = False + break + + if match_found: + return True + return False + + def _gemini_convert_messages_with_history( # noqa: PLR0915 messages: List[AllMessageValues], ) -> List[ContentType]: @@ -236,10 +294,33 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 assistant_msg = ChatCompletionAssistantMessage(**msg_dict) # type: ignore _message_content = assistant_msg.get("content", None) reasoning_content = assistant_msg.get("reasoning_content", None) + thinking_blocks = assistant_msg.get("thinking_blocks") if reasoning_content is not None: assistant_content.append( PartType(thought=True, text=reasoning_content) ) + if thinking_blocks is not None: + for block in thinking_blocks: + block_thinking_str = block.get("thinking") + block_signature = block.get("signature") + if ( + block_thinking_str is not None + and block_signature is not None + ): + try: + assistant_content.append( + PartType( + thoughtSignature=block_signature, + **json.loads(block_thinking_str), + ) + ) + except Exception: + assistant_content.append( + PartType( + thoughtSignature=block_signature, + text=block_thinking_str, + ) + ) if _message_content is not None and isinstance(_message_content, list): _parts = [] for element in _message_content: @@ -262,9 +343,17 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 assistant_msg.get("tool_calls", []) is not None or assistant_msg.get("function_call") is not None ): # support assistant tool invoke conversion - assistant_content.extend( - convert_to_gemini_tool_call_invoke(assistant_msg) + gemini_tool_call_parts = convert_to_gemini_tool_call_invoke( + assistant_msg ) + ## check if gemini_tool_call already exists in assistant_content + for gemini_tool_call_part in gemini_tool_call_parts: + if not check_if_part_exists_in_parts( + assistant_content, + gemini_tool_call_part, + excluded_keys=["thoughtSignature"], + ): + assistant_content.append(gemini_tool_call_part) last_message_with_tool_calls = assistant_msg msg_i += 1 @@ -476,6 +565,7 @@ async def async_transform_request_body( optional_params=optional_params, ) + def _default_user_message_when_system_message_passed() -> ChatCompletionUserMessage: """ Returns a default user message when a "system" message is passed in gemini fails. @@ -484,6 +574,7 @@ def _default_user_message_when_system_message_passed() -> ChatCompletionUserMess """ return ChatCompletionUserMessage(content=".", role="user") + def _transform_system_message( supports_system_message: bool, messages: List[AllMessageValues] ) -> Tuple[Optional[SystemInstructions], List[AllMessageValues]]: diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 052b872bcde..1b74ee25803 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -41,6 +41,7 @@ class PartType(TypedDict, total=False): function_call: FunctionCall function_response: FunctionResponse thought: bool + thoughtSignature: str class HttpxFunctionCall(TypedDict): diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index c54168e9a6f..b3f16ecd838 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -661,10 +661,33 @@ def test_system_message_with_no_user_message(): assert response.choices[0].message.content is not None +def get_current_weather(location, unit="fahrenheit"): + """Get the current weather in a given location""" + if "tokyo" in location.lower(): + return json.dumps({"location": "Tokyo", "temperature": "10", "unit": "celsius"}) + elif "san francisco" in location.lower(): + return json.dumps( + {"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"} + ) + elif "paris" in location.lower(): + return json.dumps({"location": "Paris", "temperature": "22", "unit": "celsius"}) + else: + return json.dumps({"location": location, "temperature": "unknown"}) + + def test_gemini_with_thinking(): from litellm import completion litellm._turn_on_debug() + litellm.modify_params = True + model = "gemini/gemini-2.5-flash" + messages = [ + { + "role": "user", + "content": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses", + } + ] + tools = [ { "type": "function", @@ -676,20 +699,69 @@ def test_gemini_with_thinking(): "properties": { "location": { "type": "string", - "description": "The city and state, e.g. San Francisco, CA", + "description": "The city and state", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, }, "required": ["location"], }, }, } ] - messages = [{"role": "user", "content": "What's the weather like in Boston today?"}] - - result = completion( - model="gemini/gemini-2.5-flash", + response = litellm.completion( + model=model, messages=messages, tools=tools, + tool_choice="auto", # auto is default, but we'll be explicit + reasoning_effort="low", ) - print(f"result: {result}") + print("Response\n", response) + response_message = response.choices[0].message + tool_calls = response_message.tool_calls + + print("Expecting there to be 3 tool calls") + assert len(tool_calls) > 0 # this has to call the function for SF, Tokyo and paris + + # Step 2: check if the model wanted to call a function + print(f"tool_calls: {tool_calls}") + if tool_calls: + # Step 3: call the function + # Note: the JSON response may not always be valid; be sure to handle errors + available_functions = { + "get_current_weather": get_current_weather, + } # only one function in this example, but you can have multiple + messages.append(response_message) # extend conversation with assistant's reply + print("Response message\n", response_message) + # Step 4: send the info for each function call and function response to the model + for tool_call in tool_calls: + function_name = tool_call.function.name + if function_name not in available_functions: + # the model called a function that does not exist in available_functions - don't try calling anything + return + function_to_call = available_functions[function_name] + function_args = json.loads(tool_call.function.arguments) + function_response = function_to_call( + location=function_args.get("location"), + unit=function_args.get("unit"), + ) + messages.append( + { + "tool_call_id": tool_call.id, + "role": "tool", + "name": function_name, + "content": function_response, + } + ) # extend conversation with function response + print(f"messages: {messages}") + second_response = litellm.completion( + model=model, + messages=messages, + seed=22, + reasoning_effort="low", + tools=tools, + drop_params=True, + ) # get a new response from the model where it can see the function response + print("second response\n", second_response) diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py new file mode 100644 index 00000000000..d6d33258576 --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -0,0 +1,75 @@ +from litellm.llms.vertex_ai.gemini.transformation import check_if_part_exists_in_parts + + +def test_check_if_part_exists_in_parts(): + parts = [ + {"text": "Hello", "thought": True}, + {"text": "World", "thought": False}, + ] + part = {"text": "Hello", "thought": True} + new_part = {"text": "Hello World", "thought": True} + assert check_if_part_exists_in_parts(parts, part) + assert not check_if_part_exists_in_parts(parts, new_part, ["thought"]) + assert check_if_part_exists_in_parts(parts, new_part, ["text"]) + + +def test_check_if_part_exists_in_parts_camel_case_snake_case(): + """Test that function handles both camelCase and snake_case key variations""" + # Test snake_case to camelCase matching + parts_with_snake_case = [ + { + "function_call": { + "name": "get_current_weather", + "args": {"location": "San Francisco, CA"}, + } + }, + {"text": "Some other content"}, + ] + + part_with_camel_case = { + "functionCall": { + "name": "get_current_weather", + "args": {"location": "San Francisco, CA"}, + } + } + + # Should find match between function_call and functionCall + assert check_if_part_exists_in_parts(parts_with_snake_case, part_with_camel_case) + + # Test camelCase to snake_case matching + parts_with_camel_case = [ + {"functionCall": {"name": "calculate_sum", "args": {"a": 1, "b": 2}}} + ] + + part_with_snake_case = { + "function_call": {"name": "calculate_sum", "args": {"a": 1, "b": 2}} + } + + # Should find match between functionCall and function_call + assert check_if_part_exists_in_parts(parts_with_camel_case, part_with_snake_case) + + # Test no match when values differ + part_with_different_values = { + "function_call": {"name": "different_function", "args": {"x": 5}} + } + + assert not check_if_part_exists_in_parts( + parts_with_snake_case, part_with_different_values + ) + + # Test multiple keys with mixed casing + parts_mixed = [ + { + "function_call": {"name": "test"}, + "thoughtSignature": "reasoning", + "text": "content", + } + ] + + part_mixed_casing = { + "functionCall": {"name": "test"}, + "thought_signature": "reasoning", + "text": "content", + } + + assert check_if_part_exists_in_parts(parts_mixed, part_mixed_casing) From eed235833511ddd584946df62ea1f688dfa0c01e Mon Sep 17 00:00:00 2001 From: tanjiro <56165694+NANDINI-star@users.noreply.github.com> Date: Sun, 31 Aug 2025 14:03:32 +0900 Subject: [PATCH 06/40] move filter inside user table --- .../src/components/view_users.tsx | 233 +++--------------- .../src/components/view_users/table.tsx | 221 ++++++++++++++++- 2 files changed, 247 insertions(+), 207 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_users.tsx b/ui/litellm-dashboard/src/components/view_users.tsx index 1d80e513e35..7c828f756b5 100644 --- a/ui/litellm-dashboard/src/components/view_users.tsx +++ b/ui/litellm-dashboard/src/components/view_users.tsx @@ -79,7 +79,6 @@ const ViewUserDashboard: React.FC = ({ accessToken, toke const [activeTab, setActiveTab] = useState("users") const [filters, setFilters] = useState(initialFilters) const [debouncedFilters, setDebouncedFilters, debouncer] = useDebouncedState(filters, { wait: 300 }) - const [showFilters, setShowFilters] = useState(false) const [isInvitationLinkModalVisible, setIsInvitationLinkModalVisible] = useState(false) const [invitationLinkData, setInvitationLinkData] = useState(null) const [baseUrl, setBaseUrl] = useState(null) @@ -330,209 +329,35 @@ const ViewUserDashboard: React.FC = ({ accessToken, toke -
-
-
- {/* Search and Filter Controls */} -
- {/* Email Search */} -
- updateFilters({ email: e.target.value })} - /> - - - -
- - {/* Filter Button */} - - - {/* Reset Filters Button */} - -
- - {/* Additional Filters */} - {showFilters && ( -
- {/* User ID Search */} -
- updateFilters({ user_id: e.target.value })} - /> - - - -
- - {/* Role Dropdown */} -
- -
- - {/* Team Dropdown */} -
- -
- - {/* SSO ID Search */} -
- updateFilters({ sso_user_id: e.target.value })} - /> -
-
- )} - - {/* Results Count and Pagination */} -
- - Showing{" "} - {userListResponse && userListResponse.users && userListResponse.users.length > 0 - ? (userListResponse.page - 1) * userListResponse.page_size + 1 - : 0}{" "} - -{" "} - {userListResponse && userListResponse.users - ? Math.min(userListResponse.page * userListResponse.page_size, userListResponse.total) - : 0}{" "} - of {userListResponse ? userListResponse.total : 0} results - - - {/* Pagination Buttons */} -
- - -
-
-
-
-
- { - setSelectedUser(user) - setEditModalVisible(true) - }} - handleDelete={handleDelete} - handleResetPassword={handleResetPassword} - enableSelection={selectionMode} - selectedUsers={selectedUsers} - onSelectionChange={handleSelectionChange} - /> -
- -
+ { + setSelectedUser(user) + setEditModalVisible(true) + }} + handleDelete={handleDelete} + handleResetPassword={handleResetPassword} + enableSelection={selectionMode} + selectedUsers={selectedUsers} + onSelectionChange={handleSelectionChange} + filters={filters} + updateFilters={updateFilters} + initialFilters={initialFilters} + teams={teams} + userListResponse={userListResponse} + currentPage={currentPage} + handlePageChange={handlePageChange} + /> diff --git a/ui/litellm-dashboard/src/components/view_users/table.tsx b/ui/litellm-dashboard/src/components/view_users/table.tsx index 0f1193c6e87..0b422ead7b3 100644 --- a/ui/litellm-dashboard/src/components/view_users/table.tsx +++ b/ui/litellm-dashboard/src/components/view_users/table.tsx @@ -15,12 +15,27 @@ import { TableBody, TableRow, TableCell, + Select, + SelectItem, } from "@tremor/react"; import { SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon } from "@heroicons/react/outline"; import { UserInfo } from "./types"; import UserInfoView from "./user_info_view"; import { columns as createColumns } from "./columns"; +interface FilterState { + email: string; + user_id: string; + user_role: string; + sso_user_id: string; + team: string; + model: string; + min_spend: number | null; + max_spend: number | null; + sort_by: string; + sort_order: "asc" | "desc"; +} + interface UserDataTableProps { data: UserInfo[]; columns: ColumnDef[]; @@ -39,6 +54,15 @@ interface UserDataTableProps { selectedUsers?: UserInfo[]; onSelectionChange?: (selectedUsers: UserInfo[]) => void; enableSelection?: boolean; + // Filter-related props + filters: FilterState; + updateFilters: (update: Partial) => void; + initialFilters: FilterState; + teams: any[] | null; + // Pagination props + userListResponse: any; + currentPage: number; + handlePageChange: (newPage: number) => void; } export function UserDataTable({ @@ -56,6 +80,13 @@ export function UserDataTable({ selectedUsers = [], onSelectionChange, enableSelection = false, + filters, + updateFilters, + initialFilters, + teams, + userListResponse, + currentPage, + handlePageChange, }: UserDataTableProps) { const [sorting, setSorting] = React.useState([ { @@ -65,6 +96,7 @@ export function UserDataTable({ ]); const [selectedUserId, setSelectedUserId] = React.useState(null); const [openInEditMode, setOpenInEditMode] = React.useState(false); + const [showFilters, setShowFilters] = React.useState(false); const handleUserClick = (userId: string, openInEditMode: boolean = false) => { setSelectedUserId(userId); @@ -171,9 +203,190 @@ export function UserDataTable({ } return ( -
-
- +
+ {/* Filter Section */} +
+
+ {/* Search and Filter Controls */} +
+ {/* Email Search */} +
+ updateFilters({ email: e.target.value })} + /> + + + +
+ + {/* Filter Button */} + + + {/* Reset Filters Button */} + +
+ + {/* Additional Filters */} + {showFilters && ( +
+ {/* User ID Search */} +
+ updateFilters({ user_id: e.target.value })} + /> + + + +
+ + {/* Role Dropdown */} +
+ +
+ + {/* Team Dropdown */} +
+ +
+ + {/* SSO ID Search */} +
+ updateFilters({ sso_user_id: e.target.value })} + /> +
+
+ )} + + {/* Results Count and Pagination */} +
+ + Showing{" "} + {userListResponse && userListResponse.users && userListResponse.users.length > 0 + ? (userListResponse.page - 1) * userListResponse.page_size + 1 + : 0}{" "} + -{" "} + {userListResponse && userListResponse.users + ? Math.min(userListResponse.page * userListResponse.page_size, userListResponse.total) + : 0}{" "} + of {userListResponse ? userListResponse.total : 0} results + + + {/* Pagination Buttons */} +
+ + +
+
+
+
+ + {/* Table Section */} +
+
+
+
{table.getHeaderGroups().map((headerGroup) => ( @@ -260,6 +473,8 @@ export function UserDataTable({ )}
+
+
); From 130c1dd4fc94f1fa2acdfc418888107fd9fcec52 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sun, 31 Aug 2025 20:25:12 -0700 Subject: [PATCH 07/40] fix(types/openai.py): add default none values to responsesapiresponse object Fixes https://github.com/BerriAI/litellm/issues/14061 --- litellm/types/llms/openai.py | 38 ++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index a0c8e5b6295..6e7c4150774 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -43,10 +43,14 @@ from openai.types.responses.response import ( # Handle OpenAI SDK version compatibility for Text type try: - from openai.types.responses.response_create_params import Text as ResponseText + from openai.types.responses.response_create_params import ( + Text as ResponseText, # type: ignore + ) except (ImportError, AttributeError): # Fall back to the concrete config type available in all SDK versions - from openai.types.responses.response_text_config_param import ResponseTextConfigParam as ResponseText + from openai.types.responses.response_text_config_param import ( + ResponseTextConfigParam as ResponseText, + ) from openai.types.responses.response_create_params import ( Reasoning, @@ -1025,29 +1029,29 @@ class ResponseAPIUsage(BaseLiteLLMOpenAIResponseObject): class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): id: str created_at: int - error: Optional[dict] - incomplete_details: Optional[IncompleteDetails] - instructions: Optional[str] - metadata: Optional[Dict] - model: Optional[str] - object: Optional[str] + error: Optional[dict] = None + incomplete_details: Optional[IncompleteDetails] = None + instructions: Optional[str] = None + metadata: Optional[Dict] = None + model: Optional[str] = None + object: Optional[str] = None output: Union[ List[Union[ResponseOutputItem, Dict]], List[Union[GenericResponseOutputItem, OutputFunctionToolCall]], ] parallel_tool_calls: bool - temperature: Optional[float] + temperature: Optional[float] = None tool_choice: ToolChoice tools: Union[List[Tool], List[ResponseFunctionToolCall], List[Dict[str, Any]]] top_p: Optional[float] - max_output_tokens: Optional[int] - previous_response_id: Optional[str] - reasoning: Optional[Reasoning] - status: Optional[str] - text: Optional[Union["ResponseText", Dict[str, Any]]] - truncation: Optional[Literal["auto", "disabled"]] - usage: Optional[ResponseAPIUsage] - user: Optional[str] + max_output_tokens: Optional[int] = None + previous_response_id: Optional[str] = None + reasoning: Optional[Reasoning] = None + status: Optional[str] = None + text: Optional[Union["ResponseText", Dict[str, Any]]] = None + truncation: Optional[Literal["auto", "disabled"]] = None + usage: Optional[ResponseAPIUsage] = None + user: Optional[str] = None store: Optional[bool] = None # Define private attributes using PrivateAttr _hidden_params: dict = PrivateAttr(default_factory=dict) From 3a68ca5140d5725c04bd83d47de8d3fde7175896 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sun, 31 Aug 2025 20:29:21 -0700 Subject: [PATCH 08/40] fix(ollama/chat): add 'think' param support --- litellm/llms/ollama/chat/transformation.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index d4ce4052a7e..6f8427af665 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -137,6 +137,7 @@ class OllamaChatConfig(BaseConfig): "tool_choice", "functions", "response_format", + "reasoning_effort", ] def map_openai_params( @@ -175,6 +176,8 @@ class OllamaChatConfig(BaseConfig): if value.get("json_schema") and value["json_schema"].get("schema"): optional_params["format"] = value["json_schema"]["schema"] ### FUNCTION CALLING LOGIC ### + if param == "reasoning_effort" and value is not None: + optional_params["think"] = True if param == "tools": ## CHECK IF MODEL SUPPORTS TOOL CALLING ## try: @@ -212,9 +215,9 @@ class OllamaChatConfig(BaseConfig): litellm.add_function_to_prompt = ( True # so that main.py adds the function call to the prompt ) - optional_params[ - "functions_unsupported_model" - ] = non_default_params.get("functions") + optional_params["functions_unsupported_model"] = ( + non_default_params.get("functions") + ) non_default_params.pop("tool_choice", None) # causes ollama requests to hang non_default_params.pop("functions", None) # causes ollama requests to hang return optional_params From 90bd89c7fd01d120e6d04ba87402ccba4e27d72e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sun, 31 Aug 2025 20:38:52 -0700 Subject: [PATCH 09/40] feat(ollama_chat/): add 'think' param support + output parse '' content into 'reasoning_content' Ensures consistent use of thinking --- litellm/llms/ollama/chat/transformation.py | 26 +++++++++++++-- .../index.html} | 0 .../proxy/_experimental/out/onboarding.html | 1 - litellm/proxy/_new_secret_config.yaml | 33 +++++++++++-------- 4 files changed, 43 insertions(+), 17 deletions(-) rename litellm/proxy/_experimental/out/{model_hub_table.html => model_hub_table/index.html} (100%) delete mode 100644 litellm/proxy/_experimental/out/onboarding.html diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 6f8427af665..64d0f30f2a4 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -349,11 +349,31 @@ class OllamaChatConfig(BaseConfig): ## RESPONSE OBJECT model_response.choices[0].finish_reason = "stop" + response_json_message = response_json.get("message") + if response_json_message is not None: + if "thinking" in response_json_message: + # remap 'thinking' to 'reasoning_content' + response_json_message["reasoning_content"] = response_json_message[ + "thinking" + ] + del response_json_message["thinking"] + elif response_json_message.get("content") is not None: + # parse reasoning content from content + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + _parse_content_for_reasoning, + ) + + reasoning_content, content = _parse_content_for_reasoning( + response_json_message["content"] + ) + response_json_message["reasoning_content"] = reasoning_content + response_json_message["content"] = content + if ( request_data.get("format", "") == "json" and litellm_params.get("function_name") is not None ): - function_call = json.loads(response_json["message"]["content"]) + function_call = json.loads(response_json_message["content"]) message = litellm.Message( content=None, tool_calls=[ @@ -370,11 +390,13 @@ class OllamaChatConfig(BaseConfig): "type": "function", } ], + reasoning_content=response_json_message.get("reasoning_content"), ) model_response.choices[0].message = message # type: ignore model_response.choices[0].finish_reason = "tool_calls" else: - _message = litellm.Message(**response_json["message"]) + + _message = litellm.Message(**response_json_message) model_response.choices[0].message = _message # type: ignore model_response.created = int(time.time()) model_response.model = "ollama_chat/" + model diff --git a/litellm/proxy/_experimental/out/model_hub_table.html b/litellm/proxy/_experimental/out/model_hub_table/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model_hub_table.html rename to litellm/proxy/_experimental/out/model_hub_table/index.html diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding.html deleted file mode 100644 index 5c5f1cfe908..00000000000 --- a/litellm/proxy/_experimental/out/onboarding.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index b3653c31435..c49bdbcc15b 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,18 +1,23 @@ model_list: - - model_name: fake-openai-endpoint - litellm_params: - model: openai/fake - api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - - model_name: gpt-5-mini - litellm_params: - model: azure/gpt-5-mini - api_base: os.environ/AZURE_GPT_5_MINI_API_BASE # runs os.getenv("AZURE_API_BASE") - api_key: os.environ/AZURE_GPT_5_MINI_API_KEY # runs os.getenv("AZURE_API_KEY") - stream_timeout: 60 - merge_reasoning_content_in_choices: true - model_info: - mode: chat + - model_name: fake-openai-endpoint + litellm_params: + model: openai/fake + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + - model_name: gpt-5-mini + litellm_params: + model: azure/gpt-5-mini + api_base: os.environ/AZURE_GPT_5_MINI_API_BASE # runs os.getenv("AZURE_API_BASE") + api_key: os.environ/AZURE_GPT_5_MINI_API_KEY # runs os.getenv("AZURE_API_KEY") + stream_timeout: 60 + merge_reasoning_content_in_choices: true + model_info: + mode: chat + - model_name: ollama-deepseek-r1 + litellm_params: + model: ollama_chat/deepseek-r1:1.5b + model_info: + mode: chat router_settings: model_group_alias: {"my-fake-gpt-4": "fake-openai-endpoint"} From e6429f6565c36b68ef1c5eebfca4039e799cc130 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sun, 31 Aug 2025 20:55:01 -0700 Subject: [PATCH 10/40] feat(ollama_chat/transformation.py): handle thinking content on streaming for ollama chat models Output parse correctly to 'reasoning_content' --- litellm/llms/ollama/chat/transformation.py | 46 +++++++++++++++++++++- litellm/proxy/_new_secret_config.yaml | 12 ------ 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 64d0f30f2a4..2ee7d06ae5c 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -437,6 +437,9 @@ class OllamaChatConfig(BaseConfig): class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): + started_reasoning_content: bool = False + finished_reasoning_content: bool = False + def _is_function_call_complete(self, function_args: Union[str, dict]) -> bool: if isinstance(function_args, dict): return True @@ -490,8 +493,49 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): if is_function_call_complete: tool_call["id"] = str(uuid.uuid4()) + # PROCESS REASONING CONTENT + reasoning_content: Optional[str] = None + content: Optional[str] = None + if chunk["message"].get("thinking") is not None: + if self.started_reasoning_content is False: + reasoning_content = chunk["message"].get("thinking") + self.started_reasoning_content = True + elif self.finished_reasoning_content is False: + reasoning_content = chunk["message"].get("thinking") + self.finished_reasoning_content = True + elif chunk["message"].get("content") is not None: + if "" in chunk["message"].get("content"): + reasoning_content = ( + chunk["message"].get("content").replace("", "") + ) + + self.started_reasoning_content = True + + if ( + "" in chunk["message"].get("content") + and self.started_reasoning_content + ): + reasoning_content = chunk["message"].get("content") + remaining_content = ( + chunk["message"].get("content").split("") + ) + if len(remaining_content) > 1: + content = remaining_content[1] + self.finished_reasoning_content = True + + if ( + self.started_reasoning_content is True + and self.finished_reasoning_content is False + ): + reasoning_content = ( + chunk["message"].get("content").replace("", "") + ) + else: + content = chunk["message"].get("content") + delta = Delta( - content=chunk["message"].get("content", ""), + content=content, + reasoning_content=reasoning_content, tool_calls=tool_calls, ) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index c49bdbcc15b..adfee33eba2 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -21,15 +21,3 @@ model_list: router_settings: model_group_alias: {"my-fake-gpt-4": "fake-openai-endpoint"} - -litellm_settings: - callbacks: ["otel"] - cache: true - cache_params: - type: redis - ttl: 600 - supported_call_types: ["acompletion", "completion"] - - model_group_settings: - forward_client_headers_to_llm_api: - - fake-openai-endpoint \ No newline at end of file From 2ad77d9bf608fc4b0a1fe74ec9240055a398486e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sun, 31 Aug 2025 21:17:10 -0700 Subject: [PATCH 11/40] feat(ollama/completion): output parse thinking content on streaming + non-streaming for ollama completion calls Completes 'thinking' param support for ollama --- litellm/llms/ollama/chat/transformation.py | 29 +- .../llms/ollama/completion/transformation.py | 102 +++++-- litellm/proxy/_new_secret_config.yaml | 2 +- .../test_ollama_completion_transformation.py | 264 +++++++++++++++++- 4 files changed, 353 insertions(+), 44 deletions(-) diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 2ee7d06ae5c..c70fb97af74 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -504,34 +504,23 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): reasoning_content = chunk["message"].get("thinking") self.finished_reasoning_content = True elif chunk["message"].get("content") is not None: - if "" in chunk["message"].get("content"): - reasoning_content = ( - chunk["message"].get("content").replace("", "") - ) + message_content = chunk["message"].get("content") + if "" in message_content: + message_content = message_content.replace("", "") self.started_reasoning_content = True - if ( - "" in chunk["message"].get("content") - and self.started_reasoning_content - ): - reasoning_content = chunk["message"].get("content") - remaining_content = ( - chunk["message"].get("content").split("") - ) - if len(remaining_content) > 1: - content = remaining_content[1] + if "" in message_content and self.started_reasoning_content: + message_content = message_content.replace("", "") self.finished_reasoning_content = True if ( - self.started_reasoning_content is True - and self.finished_reasoning_content is False + self.started_reasoning_content + and not self.finished_reasoning_content ): - reasoning_content = ( - chunk["message"].get("content").replace("", "") - ) + reasoning_content = message_content else: - content = chunk["message"].get("content") + content = message_content delta = Delta( content=content, diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 4f7be507cc2..2654d9461ed 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -19,13 +19,13 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMExcepti from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues, ChatCompletionUsageBlock from litellm.types.utils import ( + Delta, GenericStreamingChunk, ModelInfoBase, ModelResponse, ModelResponseStream, ProviderField, StreamingChoices, - Delta, ) from ..common_utils import OllamaError, _convert_image @@ -92,9 +92,9 @@ class OllamaConfig(BaseConfig): repeat_penalty: Optional[float] = None temperature: Optional[float] = None seed: Optional[int] = None - stop: Optional[ - list - ] = None # stop is a list based on this - https://github.com/ollama/ollama/pull/442 + stop: Optional[list] = ( + None # stop is a list based on this - https://github.com/ollama/ollama/pull/442 + ) tfs_z: Optional[float] = None num_predict: Optional[int] = None top_k: Optional[int] = None @@ -154,6 +154,7 @@ class OllamaConfig(BaseConfig): "stop", "response_format", "max_completion_tokens", + "reasoning_effort", ] def map_openai_params( @@ -166,19 +167,21 @@ class OllamaConfig(BaseConfig): for param, value in non_default_params.items(): if param == "max_tokens" or param == "max_completion_tokens": optional_params["num_predict"] = value - if param == "stream": + elif param == "stream": optional_params["stream"] = value - if param == "temperature": + elif param == "temperature": optional_params["temperature"] = value - if param == "seed": + elif param == "seed": optional_params["seed"] = value - if param == "top_p": + elif param == "top_p": optional_params["top_p"] = value - if param == "frequency_penalty": + elif param == "frequency_penalty": optional_params["frequency_penalty"] = value - if param == "stop": + elif param == "stop": optional_params["stop"] = value - if param == "response_format" and isinstance(value, dict): + elif param == "reasoning_effort" and value is not None: + optional_params["think"] = True + elif param == "response_format" and isinstance(value, dict): if value["type"] == "json_object": optional_params["format"] = "json" elif value["type"] == "json_schema": @@ -258,12 +261,17 @@ class OllamaConfig(BaseConfig): api_key: Optional[str] = None, json_mode: Optional[bool] = None, ) -> ModelResponse: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + _parse_content_for_reasoning, + ) + response_json = raw_response.json() ## RESPONSE OBJECT model_response.choices[0].finish_reason = "stop" if request_data.get("format", "") == "json": # Check if response field exists and is not empty before parsing JSON response_text = response_json.get("response", "") + if not response_text or not response_text.strip(): # Handle empty response gracefully - set empty content message = litellm.Message(content="") @@ -288,7 +296,9 @@ class OllamaConfig(BaseConfig): "id": f"call_{str(uuid.uuid4())}", "function": { "name": function_call["name"], - "arguments": json.dumps(function_call["arguments"]), + "arguments": json.dumps( + function_call["arguments"] + ), }, "type": "function", } @@ -305,11 +315,26 @@ class OllamaConfig(BaseConfig): model_response.choices[0].finish_reason = "stop" except json.JSONDecodeError: # If JSON parsing fails, treat as regular text response - message = litellm.Message(content=response_text) + ## output parse reasoning content from response_text + reasoning_content: Optional[str] = None + content: Optional[str] = None + if response_text is not None: + reasoning_content, content = _parse_content_for_reasoning( + response_text + ) + message = litellm.Message( + content=content, reasoning_content=reasoning_content + ) model_response.choices[0].message = message # type: ignore model_response.choices[0].finish_reason = "stop" else: - model_response.choices[0].message.content = response_json["response"] # type: ignore + response_text = response_json.get("response", "") + content: Optional[str] = None + reasoning_content: Optional[str] = None + if response_text is not None: + reasoning_content, content = _parse_content_for_reasoning(response_text) + model_response.choices[0].message.content = content # type: ignore + model_response.choices[0].message.reasoning_content = reasoning_content # type: ignore model_response.created = int(time.time()) model_response.model = "ollama/" + model _prompt = request_data.get("prompt", "") @@ -434,12 +459,21 @@ class OllamaConfig(BaseConfig): class OllamaTextCompletionResponseIterator(BaseModelResponseIterator): + def __init__( + self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False + ): + super().__init__(streaming_response, sync_stream, json_mode) + self.started_reasoning_content: bool = False + self.finished_reasoning_content: bool = False + def _handle_string_chunk( self, str_line: str ) -> Union[GenericStreamingChunk, ModelResponseStream]: return self.chunk_parser(json.loads(str_line)) - def chunk_parser(self, chunk: dict) -> Union[GenericStreamingChunk, ModelResponseStream]: + def chunk_parser( + self, chunk: dict + ) -> Union[GenericStreamingChunk, ModelResponseStream]: try: if "error" in chunk: raise Exception(f"Ollama Error - {chunk}") @@ -469,12 +503,42 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator): ) elif chunk["response"]: text = chunk["response"] - return GenericStreamingChunk( - text=text, - is_finished=is_finished, - finish_reason="stop", + reasoning_content: Optional[str] = None + content: Optional[str] = None + if text is not None: + if "" in text: + text = text.replace("", "") + self.started_reasoning_content = True + elif "" in text: + text = text.replace("", "") + self.finished_reasoning_content = True + + if ( + self.started_reasoning_content + and not self.finished_reasoning_content + ): + reasoning_content = text + else: + content = text + + return ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta( + reasoning_content=reasoning_content, content=content + ), + ) + ], + finish_reason=finish_reason, usage=None, ) + # return GenericStreamingChunk( + # text=text, + # is_finished=is_finished, + # finish_reason="stop", + # usage=None, + # ) elif "thinking" in chunk and not chunk["response"]: # Return reasoning content as ModelResponseStream so UIs can render it thinking_content = chunk.get("thinking") or "" diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index adfee33eba2..324b4866305 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -15,7 +15,7 @@ model_list: mode: chat - model_name: ollama-deepseek-r1 litellm_params: - model: ollama_chat/deepseek-r1:1.5b + model: ollama/deepseek-r1:1.5b model_info: mode: chat diff --git a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py index 985d51f99da..452f5a94024 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py @@ -159,6 +159,261 @@ class TestOllamaConfig: assert result.choices[0]["finish_reason"] == "stop" # No usage assertions here as we don't need to test them in every case + def test_transform_response_with_thinking_tags(self): + """Test that responses with ... tags parse reasoning content correctly.""" + # Initialize config + config = OllamaConfig() + + # Create mock response with thinking tags + raw_response = MagicMock() + raw_response.json.return_value = { + "response": "I need to think about this problem step by stepHere is my answer", + "prompt_eval_count": 15, + "eval_count": 8, + } + + # Create properly structured model response object + model_response = ModelResponse( + id="test_id", + choices=[{"message": Message(content="")}], + ) + + # Create mock encoding + mock_encoding = MagicMock() + mock_encoding.encode.return_value = [1, 2, 3] + + # Transform response + result = config.transform_response( + model="llama2", + raw_response=raw_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=mock_encoding, + ) + + # Verify reasoning content is extracted + assert ( + result.choices[0]["message"].reasoning_content + == "I need to think about this problem step by step" + ) + assert result.choices[0]["message"].content == "Here is my answer" + assert result.choices[0]["finish_reason"] == "stop" + + def test_transform_response_with_thinking_tags_alternative(self): + """Test that responses with ... tags parse reasoning content correctly.""" + # Initialize config + config = OllamaConfig() + + # Create mock response with thinking tags (alternative format) + raw_response = MagicMock() + raw_response.json.return_value = { + "response": "Let me analyze this carefullyThe solution is X", + } + + # Create properly structured model response object + model_response = ModelResponse( + id="test_id", + choices=[{"message": Message(content="")}], + ) + + # Create mock encoding + mock_encoding = MagicMock() + mock_encoding.encode.return_value = [1, 2, 3] + + # Transform response + result = config.transform_response( + model="llama2", + raw_response=raw_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=mock_encoding, + ) + + # Verify reasoning content is extracted + assert ( + result.choices[0]["message"].reasoning_content + == "Let me analyze this carefully" + ) + assert result.choices[0]["message"].content == "The solution is X" + assert result.choices[0]["finish_reason"] == "stop" + + def test_transform_response_with_multiline_thinking_tags(self): + """Test that responses with multiline thinking content work correctly.""" + # Initialize config + config = OllamaConfig() + + # Create mock response with multiline thinking content + raw_response = MagicMock() + raw_response.json.return_value = { + "response": "\nThis is a complex problem.\nI need to break it down:\n1. First step\n2. Second step\nBased on my analysis, the answer is Y", + } + + # Create properly structured model response object + model_response = ModelResponse( + id="test_id", + choices=[{"message": Message(content="")}], + ) + + # Create mock encoding + mock_encoding = MagicMock() + mock_encoding.encode.return_value = [1, 2, 3] + + # Transform response + result = config.transform_response( + model="llama2", + raw_response=raw_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=mock_encoding, + ) + + # Verify multiline reasoning content is extracted + expected_reasoning = "\nThis is a complex problem.\nI need to break it down:\n1. First step\n2. Second step\n" + assert result.choices[0]["message"].reasoning_content == expected_reasoning + assert ( + result.choices[0]["message"].content + == "Based on my analysis, the answer is Y" + ) + assert result.choices[0]["finish_reason"] == "stop" + + def test_transform_response_thinking_only(self): + """Test response with only thinking content and no additional content.""" + # Initialize config + config = OllamaConfig() + + # Create mock response with only thinking content + raw_response = MagicMock() + raw_response.json.return_value = { + "response": "Just internal thoughts, no response", + } + + # Create properly structured model response object + model_response = ModelResponse( + id="test_id", + choices=[{"message": Message(content="")}], + ) + + # Create mock encoding + mock_encoding = MagicMock() + mock_encoding.encode.return_value = [1, 2, 3] + + # Transform response + result = config.transform_response( + model="llama2", + raw_response=raw_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=mock_encoding, + ) + + # Verify reasoning content is extracted and content is empty + assert ( + result.choices[0]["message"].reasoning_content + == "Just internal thoughts, no response" + ) + assert result.choices[0]["message"].content == "" + assert result.choices[0]["finish_reason"] == "stop" + + def test_transform_response_json_mode_with_thinking_tags(self): + """Test JSON mode with thinking tags - should handle as text when JSON parsing fails.""" + # Initialize config + config = OllamaConfig() + + # Create mock response with thinking tags in JSON mode + raw_response = MagicMock() + raw_response.json.return_value = { + "response": "Planning my JSON responseThis is not valid JSON", + } + + # Create properly structured model response object + model_response = ModelResponse( + id="test_id", + choices=[{"message": Message(content="")}], + ) + + # Create mock encoding + mock_encoding = MagicMock() + mock_encoding.encode.return_value = [1, 2, 3] + + # Transform response + result = config.transform_response( + model="llama2", + raw_response=raw_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={"format": "json"}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=mock_encoding, + ) + + # Verify reasoning content is extracted even in JSON mode when JSON parsing fails + assert ( + result.choices[0]["message"].reasoning_content + == "Planning my JSON response" + ) + assert result.choices[0]["message"].content == "This is not valid JSON" + assert result.choices[0]["finish_reason"] == "stop" + + def test_transform_response_no_thinking_tags(self): + """Test that responses without thinking tags work normally.""" + # Initialize config + config = OllamaConfig() + + # Create mock response without thinking tags + raw_response = MagicMock() + raw_response.json.return_value = { + "response": "Regular response without any thinking tags", + } + + # Create properly structured model response object + model_response = ModelResponse( + id="test_id", + choices=[{"message": Message(content="")}], + ) + + # Create mock encoding + mock_encoding = MagicMock() + mock_encoding.encode.return_value = [1, 2, 3] + + # Transform response + result = config.transform_response( + model="llama2", + raw_response=raw_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=mock_encoding, + ) + + # Verify no reasoning content is extracted + assert result.choices[0]["message"].reasoning_content is None + assert ( + result.choices[0]["message"].content + == "Regular response without any thinking tags" + ) + assert result.choices[0]["finish_reason"] == "stop" + class TestOllamaTextCompletionResponseIterator: def test_chunk_parser_with_thinking_field(self): @@ -199,10 +454,11 @@ class TestOllamaTextCompletionResponseIterator: result = iterator.chunk_parser(normal_chunk) - assert result["text"] == "Hello world" - assert result["is_finished"] is False - assert result["finish_reason"] == "stop" - assert result["usage"] is None + # Updated to handle ModelResponseStream return type + assert isinstance(result, ModelResponseStream) + assert result.choices and result.choices[0].delta is not None + assert result.choices[0].delta.content == "Hello world" + assert getattr(result.choices[0].delta, "reasoning_content", None) is None def test_chunk_parser_done_chunk(self): """Test that done chunks work correctly.""" From 002c2f16dc679c34944dfb9f68dc7595dc1ab9a9 Mon Sep 17 00:00:00 2001 From: retanoj Date: Mon, 1 Sep 2025 20:02:53 +0800 Subject: [PATCH 12/40] fix token count error when proxy gemini cli to openai like model --- litellm/google_genai/adapters/handler.py | 4 ++ litellm/proxy/google_endpoints/endpoints.py | 23 +++++++-- .../proxy/google_endpoints/__init__.py | 0 .../proxy/google_endpoints/test_endpoints.py | 49 +++++++++++++++++++ 4 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 tests/test_litellm/proxy/google_endpoints/__init__.py create mode 100644 tests/test_litellm/proxy/google_endpoints/test_endpoints.py diff --git a/litellm/google_genai/adapters/handler.py b/litellm/google_genai/adapters/handler.py index 1f575f27591..c5f378554b2 100644 --- a/litellm/google_genai/adapters/handler.py +++ b/litellm/google_genai/adapters/handler.py @@ -37,6 +37,10 @@ class GenerateContentToCompletionHandler: completion_kwargs: Dict[str, Any] = dict(completion_request) + # feed metadata for custom callback + # if 'metadata' in extra_kwargs: + # completion_kwargs['metadata'] = extra_kwargs['metadata'] + if stream: completion_kwargs["stream"] = stream diff --git a/litellm/proxy/google_endpoints/endpoints.py b/litellm/proxy/google_endpoints/endpoints.py index 373232e22d2..4f57e1e7ce8 100644 --- a/litellm/proxy/google_endpoints/endpoints.py +++ b/litellm/proxy/google_endpoints/endpoints.py @@ -173,15 +173,22 @@ async def google_count_tokens(request: Request, model_name: str): """ from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.proxy_server import token_counter as internal_token_counter + from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter data = await _read_request_body(request=request) contents = data.get("contents", []) #Create TokenCountRequest for the internal endpoint from litellm.proxy._types import TokenCountRequest + # Translate contents to openai format messages using the adapter + messages = (GoogleGenAIAdapter() + .translate_generate_content_to_completion(model_name, contents) + .get("messages", [])) + token_request = TokenCountRequest( model=model_name, - contents=contents + contents=contents, + messages=messages, # compatibility when use openai-like endpoint ) # Call the internal token counter function with direct request flag set to False @@ -192,10 +199,16 @@ async def google_count_tokens(request: Request, model_name: str): if token_response is not None: # cast the response to the well known format original_response: dict = token_response.original_response or {} - return TokenCountDetailsResponse( - totalTokens=original_response.get("totalTokens", 0), - promptTokensDetails=original_response.get("promptTokensDetails", []), - ) + if original_response: + return TokenCountDetailsResponse( + totalTokens=original_response.get("totalTokens", 0), + promptTokensDetails=original_response.get("promptTokensDetails", []), + ) + else: + return TokenCountDetailsResponse( + totalTokens=token_response.total_tokens or 0, + promptTokensDetails=[], + ) ######################################################### # Return the response in the well known format diff --git a/tests/test_litellm/proxy/google_endpoints/__init__.py b/tests/test_litellm/proxy/google_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/proxy/google_endpoints/test_endpoints.py b/tests/test_litellm/proxy/google_endpoints/test_endpoints.py new file mode 100644 index 00000000000..2f2538bf9aa --- /dev/null +++ b/tests/test_litellm/proxy/google_endpoints/test_endpoints.py @@ -0,0 +1,49 @@ +""" +Test for google_endpoints/endpoints.py +""" +import pytest +import sys, os +from dotenv import load_dotenv + + +from litellm.proxy.google_endpoints.endpoints import google_count_tokens +from litellm.types.llms.vertex_ai import TokenCountDetailsResponse +from starlette.requests import Request + +load_dotenv() + +sys.path.insert( + 0, os.path.abspath("../../../..") +) + +@pytest.mark.asyncio +async def test_proxy_gemini_to_openai_like_model_token_counting(): + """ + Test the token counting endpoint for proxing gemini to openai-like models. + """ + response: TokenCountDetailsResponse = await google_count_tokens( + request=Request( + scope={ + "type": "http", + "parsed_body": ( + [ + "contents" + ], + { + "contents": [ + { + "parts": [ + { + "text": "Hello, how are you?" + } + ] + } + ] + } + ) + } + ), + model_name="volcengine/foo", + ) + + assert response.get("totalTokens") > 0 \ No newline at end of file From e05ffcfb4c2b3bbcb9de75a8f1609ffa141f6326 Mon Sep 17 00:00:00 2001 From: retanoj Date: Mon, 1 Sep 2025 20:09:19 +0800 Subject: [PATCH 13/40] fix back --- litellm/google_genai/adapters/handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/google_genai/adapters/handler.py b/litellm/google_genai/adapters/handler.py index c5f378554b2..c15d0cb9deb 100644 --- a/litellm/google_genai/adapters/handler.py +++ b/litellm/google_genai/adapters/handler.py @@ -38,8 +38,8 @@ class GenerateContentToCompletionHandler: completion_kwargs: Dict[str, Any] = dict(completion_request) # feed metadata for custom callback - # if 'metadata' in extra_kwargs: - # completion_kwargs['metadata'] = extra_kwargs['metadata'] + if 'metadata' in extra_kwargs: + completion_kwargs['metadata'] = extra_kwargs['metadata'] if stream: completion_kwargs["stream"] = stream From 4a422ca897b0bc7a7cf878266ac9dab892500f32 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 1 Sep 2025 17:38:15 -0700 Subject: [PATCH 14/40] fix: support logging dynamic metadata values to braintrust --- litellm/integrations/braintrust_logging.py | 18 +- .../integrations/test_braintrust_span_name.py | 162 ++++++++++-------- 2 files changed, 106 insertions(+), 74 deletions(-) diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index 1ddf789ed32..c362b513281 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -159,7 +159,7 @@ class BraintrustLogger(CustomLogger): output = response_obj["data"] litellm_params = kwargs.get("litellm_params", {}) or {} - dynamic_metadata = litellm_params.get("dynamic_metadata", {}) or {} + dynamic_metadata = litellm_params.get("metadata", {}) or {} # Get project_id from metadata or create default if needed project_id = dynamic_metadata.get("project_id") @@ -175,6 +175,7 @@ class BraintrustLogger(CustomLogger): project_id = self.default_project_id tags = [] + if isinstance(dynamic_metadata, dict): for key, value in dynamic_metadata.items(): # generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy @@ -185,6 +186,11 @@ class BraintrustLogger(CustomLogger): ): tags.append(f"{key}:{value}") + if ( + isinstance(value, str) and key not in standard_logging_object + ): # support logging dynamic metadata to braintrust + standard_logging_object[key] = value + cost = kwargs.get("response_cost", None) metrics: Optional[dict] = None @@ -265,9 +271,7 @@ class BraintrustLogger(CustomLogger): output = response_obj["data"] litellm_params = kwargs.get("litellm_params", {}) - dynamic_metadata = litellm_params.get("dynamic_metadata", {}) or {} - - clean_metadata = {} + dynamic_metadata = litellm_params.get("metadata", {}) or {} # Get project_id from metadata or create default if needed project_id = dynamic_metadata.get("project_id") @@ -285,6 +289,7 @@ class BraintrustLogger(CustomLogger): project_id = self.default_project_id tags = [] + if isinstance(dynamic_metadata, dict): for key, value in dynamic_metadata.items(): # generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy @@ -295,6 +300,11 @@ class BraintrustLogger(CustomLogger): ): tags.append(f"{key}:{value}") + if ( + isinstance(value, str) and key not in standard_logging_object + ): # support logging dynamic metadata to braintrust + standard_logging_object[key] = value + cost = kwargs.get("response_cost", None) metrics: Optional[dict] = None diff --git a/tests/test_litellm/integrations/test_braintrust_span_name.py b/tests/test_litellm/integrations/test_braintrust_span_name.py index 10e512fc0ca..30381e99783 100644 --- a/tests/test_litellm/integrations/test_braintrust_span_name.py +++ b/tests/test_litellm/integrations/test_braintrust_span_name.py @@ -11,7 +11,7 @@ from litellm.integrations.braintrust_logging import BraintrustLogger class TestBraintrustSpanName(unittest.TestCase): """Test custom span_name functionality in Braintrust logging.""" - @patch('litellm.integrations.braintrust_logging.HTTPHandler') + @patch("litellm.integrations.braintrust_logging.HTTPHandler") def test_default_span_name(self, MockHTTPHandler): """Test that default span name is 'Chat Completion' when not provided.""" # Mock HTTP response @@ -22,39 +22,43 @@ class TestBraintrustSpanName(unittest.TestCase): # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - + # Create a properly structured mock response response_obj = litellm.ModelResponse( id="test-id", object="chat.completion", created=1234567890, model="gpt-3.5-turbo", - choices=[{ - "index": 0, - "message": {"role": "assistant", "content": "test response"}, - "finish_reason": "stop" - }], - usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30} + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "test response"}, + "finish_reason": "stop", + } + ], + usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, ) - + kwargs = { "litellm_call_id": "test-call-id", "messages": [{"role": "user", "content": "test"}], "litellm_params": {"metadata": {}}, "model": "gpt-3.5-turbo", - "response_cost": 0.001 + "response_cost": 0.001, } - + # Execute logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now()) - + # Verify call_args = mock_http_handler.post.call_args self.assertIsNotNone(call_args) - json_data = call_args.kwargs['json'] - self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Chat Completion') + json_data = call_args.kwargs["json"] + self.assertEqual( + json_data["events"][0]["span_attributes"]["name"], "Chat Completion" + ) - @patch('litellm.integrations.braintrust_logging.HTTPHandler') + @patch("litellm.integrations.braintrust_logging.HTTPHandler") def test_custom_span_name(self, MockHTTPHandler): """Test that custom span name is used when provided in metadata.""" # Mock HTTP response @@ -65,39 +69,43 @@ class TestBraintrustSpanName(unittest.TestCase): # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - + # Create a properly structured mock response response_obj = litellm.ModelResponse( id="test-id", object="chat.completion", created=1234567890, model="gpt-3.5-turbo", - choices=[{ - "index": 0, - "message": {"role": "assistant", "content": "test response"}, - "finish_reason": "stop" - }], - usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30} + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "test response"}, + "finish_reason": "stop", + } + ], + usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, ) - + kwargs = { "litellm_call_id": "test-call-id", "messages": [{"role": "user", "content": "test"}], "litellm_params": {"metadata": {"span_name": "Custom Operation"}}, "model": "gpt-3.5-turbo", - "response_cost": 0.001 + "response_cost": 0.001, } - + # Execute logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now()) - + # Verify call_args = mock_http_handler.post.call_args self.assertIsNotNone(call_args) - json_data = call_args.kwargs['json'] - self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Custom Operation') + json_data = call_args.kwargs["json"] + self.assertEqual( + json_data["events"][0]["span_attributes"]["name"], "Custom Operation" + ) - @patch('litellm.integrations.braintrust_logging.HTTPHandler') + @patch("litellm.integrations.braintrust_logging.HTTPHandler") def test_span_name_with_other_metadata(self, MockHTTPHandler): """Test that span_name works alongside other metadata fields.""" # Mock HTTP response @@ -108,21 +116,23 @@ class TestBraintrustSpanName(unittest.TestCase): # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - + # Create a properly structured mock response response_obj = litellm.ModelResponse( id="test-id", object="chat.completion", created=1234567890, model="gpt-3.5-turbo", - choices=[{ - "index": 0, - "message": {"role": "assistant", "content": "test response"}, - "finish_reason": "stop" - }], - usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30} + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "test response"}, + "finish_reason": "stop", + } + ], + usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, ) - + kwargs = { "litellm_call_id": "test-call-id", "messages": [{"role": "user", "content": "test"}], @@ -132,34 +142,40 @@ class TestBraintrustSpanName(unittest.TestCase): "project_id": "custom-project", "user_id": "user123", "session_id": "session456", - "environment": "production" + "environment": "production", } }, "model": "gpt-3.5-turbo", - "response_cost": 0.001 + "response_cost": 0.001, + "standard_logging_object": { + "user_id": "user123", + }, } - + # Execute logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now()) - + # Verify call_args = mock_http_handler.post.call_args self.assertIsNotNone(call_args) - json_data = call_args.kwargs['json'] - - # Check span name - self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Multi Metadata Test') - - # Check that other metadata is preserved (except for filtered keys) - event_metadata = json_data['events'][0]['metadata'] - self.assertEqual(event_metadata['user_id'], 'user123') - self.assertEqual(event_metadata['session_id'], 'session456') - self.assertEqual(event_metadata['environment'], 'production') - - # Span name should be in span_attributes, not in metadata - self.assertIn('span_name', event_metadata) # span_name is also kept in metadata + json_data = call_args.kwargs["json"] - @patch('litellm.integrations.braintrust_logging.get_async_httpx_client') + # Check span name + self.assertEqual( + json_data["events"][0]["span_attributes"]["name"], "Multi Metadata Test" + ) + + # Check that other metadata is preserved (except for filtered keys) + event_metadata = json_data["events"][0]["metadata"] + print(event_metadata) + self.assertEqual(event_metadata["user_id"], "user123") + self.assertEqual(event_metadata["session_id"], "session456") + self.assertEqual(event_metadata["environment"], "production") + + # Span name should be in span_attributes, not in metadata + self.assertIn("span_name", event_metadata) # span_name is also kept in metadata + + @patch("litellm.integrations.braintrust_logging.get_async_httpx_client") async def test_async_custom_span_name(self, mock_get_http_handler): """Test async logging with custom span name.""" # Mock async HTTP response @@ -170,38 +186,44 @@ class TestBraintrustSpanName(unittest.TestCase): # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - + # Create a properly structured mock response response_obj = litellm.ModelResponse( id="test-id", object="chat.completion", created=1234567890, model="gpt-3.5-turbo", - choices=[{ - "index": 0, - "message": {"role": "assistant", "content": "test response"}, - "finish_reason": "stop" - }], - usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30} + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "test response"}, + "finish_reason": "stop", + } + ], + usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, ) - + kwargs = { "litellm_call_id": "test-call-id", "messages": [{"role": "user", "content": "test"}], "litellm_params": {"metadata": {"span_name": "Async Custom Operation"}}, "model": "gpt-3.5-turbo", - "response_cost": 0.001 + "response_cost": 0.001, } - + # Execute - await logger.async_log_success_event(kwargs, response_obj, datetime.now(), datetime.now()) - + await logger.async_log_success_event( + kwargs, response_obj, datetime.now(), datetime.now() + ) + # Verify call_args = mock_http_handler.post.call_args self.assertIsNotNone(call_args) - json_data = call_args.kwargs['json'] - self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Async Custom Operation') + json_data = call_args.kwargs["json"] + self.assertEqual( + json_data["events"][0]["span_attributes"]["name"], "Async Custom Operation" + ) if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() From c54c41f726686b921e6ba771ff70e4bd6e988032 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 1 Sep 2025 17:38:39 -0700 Subject: [PATCH 15/40] fix: fix ruff errors --- litellm/integrations/braintrust_logging.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index c362b513281..5bc6afb6dbc 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -1,18 +1,15 @@ # What is this? ## Log success + failure events to Braintrust -import copy import os from datetime import datetime from typing import Dict, Optional import httpx -from pydantic import BaseModel import litellm from litellm import verbose_logger from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.safe_json_dumps import filter_json_serializable from litellm.llms.custom_httpx.http_handler import ( HTTPHandler, get_async_httpx_client, @@ -25,7 +22,6 @@ API_BASE = "https://api.braintrustdata.com/v1" def get_utc_datetime(): import datetime as dt - from datetime import datetime if hasattr(dt, "UTC"): return datetime.now(dt.UTC) # type: ignore From 5d6532419febfdc0be03fbc9e8c0b16c043b034e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 1 Sep 2025 17:46:38 -0700 Subject: [PATCH 16/40] refactor: remove unused function --- litellm/litellm_core_utils/safe_json_dumps.py | 91 ------------------- 1 file changed, 91 deletions(-) diff --git a/litellm/litellm_core_utils/safe_json_dumps.py b/litellm/litellm_core_utils/safe_json_dumps.py index b3b1d7fb3df..c714e36b5f9 100644 --- a/litellm/litellm_core_utils/safe_json_dumps.py +++ b/litellm/litellm_core_utils/safe_json_dumps.py @@ -50,94 +50,3 @@ def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str: safe_data = _serialize(data, set(), 0) return json.dumps(safe_data, default=str) - - -def filter_json_serializable( - data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH -) -> Any: - """ - Recursively filter data to only include JSON serializable items. - Non-serializable items are completely skipped (not included in the result). - """ - - def _is_json_serializable(obj: Any) -> bool: - """Test if an object is JSON serializable.""" - try: - json.dumps(obj) - return True - except (TypeError, ValueError): - return False - - def _filter(obj: Any, seen: set, depth: int) -> Any: - # Check for maximum depth. - if depth > max_depth: - return None - - # Base-case: if it is a primitive, test if it's serializable - if isinstance(obj, (str, int, float, bool, type(None))): - return obj if _is_json_serializable(obj) else None - - # Check for circular reference. - if id(obj) in seen: - return None - - seen.add(id(obj)) - - try: - if isinstance(obj, dict): - result = {} - for k, v in obj.items(): - # Only include keys that are strings and values that are serializable - if isinstance(k, str): - filtered_value = _filter(v, seen, depth + 1) - # Only add the key-value pair if the value is serializable - if filtered_value is not None or v is None: - if _is_json_serializable(filtered_value): - result[k] = filtered_value - seen.remove(id(obj)) - return result - - elif isinstance(obj, list): - result = [] - for item in obj: - filtered_item = _filter(item, seen, depth + 1) - # Only include items that are serializable - if filtered_item is not None or item is None: - if _is_json_serializable(filtered_item): - result.append(filtered_item) - seen.remove(id(obj)) - return result - - elif isinstance(obj, tuple): - filtered_items = [] - for item in obj: - filtered_item = _filter(item, seen, depth + 1) - # Only include items that are serializable - if filtered_item is not None or item is None: - if _is_json_serializable(filtered_item): - filtered_items.append(filtered_item) - seen.remove(id(obj)) - return tuple(filtered_items) - - elif isinstance(obj, set): - filtered_items = [] - for item in obj: - filtered_item = _filter(item, seen, depth + 1) - # Only include items that are serializable - if filtered_item is not None or item is None: - if _is_json_serializable(filtered_item): - filtered_items.append(filtered_item) - seen.remove(id(obj)) - return sorted(filtered_items) - - else: - # Test if the object is directly serializable - seen.remove(id(obj)) - return obj if _is_json_serializable(obj) else None - - except Exception: - if id(obj) in seen: - seen.remove(id(obj)) - return None - - return _filter(data, set(), 0) From f1f9f2a594e3a169b2b18a8cd781cb10f896a7b6 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 1 Sep 2025 17:52:22 -0700 Subject: [PATCH 17/40] fix: fix linting error --- litellm/llms/ollama/completion/transformation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 2654d9461ed..5689864017f 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -329,8 +329,8 @@ class OllamaConfig(BaseConfig): model_response.choices[0].finish_reason = "stop" else: response_text = response_json.get("response", "") - content: Optional[str] = None - reasoning_content: Optional[str] = None + content = None + reasoning_content = None if response_text is not None: reasoning_content, content = _parse_content_for_reasoning(response_text) model_response.choices[0].message.content = content # type: ignore From 3b524ba5f1084d5efc68dcc2b73f37160d37eace Mon Sep 17 00:00:00 2001 From: retanoj Date: Tue, 2 Sep 2025 09:52:50 +0800 Subject: [PATCH 18/40] format --- litellm/google_genai/adapters/handler.py | 4 ++-- litellm/proxy/google_endpoints/endpoints.py | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/litellm/google_genai/adapters/handler.py b/litellm/google_genai/adapters/handler.py index c15d0cb9deb..ee9083cbbf0 100644 --- a/litellm/google_genai/adapters/handler.py +++ b/litellm/google_genai/adapters/handler.py @@ -38,8 +38,8 @@ class GenerateContentToCompletionHandler: completion_kwargs: Dict[str, Any] = dict(completion_request) # feed metadata for custom callback - if 'metadata' in extra_kwargs: - completion_kwargs['metadata'] = extra_kwargs['metadata'] + if "metadata" in extra_kwargs: + completion_kwargs["metadata"] = extra_kwargs["metadata"] if stream: completion_kwargs["stream"] = stream diff --git a/litellm/proxy/google_endpoints/endpoints.py b/litellm/proxy/google_endpoints/endpoints.py index 4f57e1e7ce8..eb481b0a4f0 100644 --- a/litellm/proxy/google_endpoints/endpoints.py +++ b/litellm/proxy/google_endpoints/endpoints.py @@ -181,9 +181,11 @@ async def google_count_tokens(request: Request, model_name: str): from litellm.proxy._types import TokenCountRequest # Translate contents to openai format messages using the adapter - messages = (GoogleGenAIAdapter() - .translate_generate_content_to_completion(model_name, contents) - .get("messages", [])) + messages = ( + GoogleGenAIAdapter() + .translate_generate_content_to_completion(model_name, contents) + .get("messages", []) + ) token_request = TokenCountRequest( model=model_name, @@ -209,7 +211,7 @@ async def google_count_tokens(request: Request, model_name: str): totalTokens=token_response.total_tokens or 0, promptTokensDetails=[], ) - + ######################################################### # Return the response in the well known format ######################################################### From b1686ecbe5fd6d6bc8590cbe1294389548ff572b Mon Sep 17 00:00:00 2001 From: retanoj Date: Tue, 2 Sep 2025 09:59:31 +0800 Subject: [PATCH 19/40] format --- litellm/google_genai/adapters/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/google_genai/adapters/handler.py b/litellm/google_genai/adapters/handler.py index ee9083cbbf0..dcf707ebd51 100644 --- a/litellm/google_genai/adapters/handler.py +++ b/litellm/google_genai/adapters/handler.py @@ -38,7 +38,7 @@ class GenerateContentToCompletionHandler: completion_kwargs: Dict[str, Any] = dict(completion_request) # feed metadata for custom callback - if "metadata" in extra_kwargs: + if extra_kwargs is not None and "metadata" in extra_kwargs: completion_kwargs["metadata"] = extra_kwargs["metadata"] if stream: From ae3931f5f3a233b75c1ebb8990355f28d970e542 Mon Sep 17 00:00:00 2001 From: Yuji Arakawa Date: Tue, 2 Sep 2025 18:32:45 +0900 Subject: [PATCH 20/40] FIx https://github.com/BerriAI/litellm/issues/14158 --- litellm/llms/oci/chat/transformation.py | 16 +- .../llms/oci/chat/test_transformation.py | 229 ++++++++++++++++++ 2 files changed, 237 insertions(+), 8 deletions(-) create mode 100644 tests/test_litellm/llms/oci/chat/test_transformation.py diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 915d2029afe..3be373ca5e5 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -772,7 +772,14 @@ def adapt_messages_to_generic_oci_standard( tool_calls = message.get("tool_calls") tool_call_id = message.get("tool_call_id") - if role in ["system", "user", "assistant"] and content is not None: + if role == "assistant" and tool_calls is not None: + if not isinstance(tool_calls, list): + raise Exception("Prop `tool_calls` must be a list of tool calls") + new_messages.append( + adapt_messages_to_generic_oci_standard_tool_call(role, tool_calls) + ) + + elif role in ["system", "user", "assistant"] and content is not None: if not isinstance(content, (str, list)): raise Exception( "Prop `content` must be a string or a list of content items" @@ -781,13 +788,6 @@ def adapt_messages_to_generic_oci_standard( adapt_messages_to_generic_oci_standard_content_message(role, content) ) - elif role == "assistant" and tool_calls is not None: - if not isinstance(tool_calls, list): - raise Exception("Prop `tool_calls` must be a list of tool calls") - new_messages.append( - adapt_messages_to_generic_oci_standard_tool_call(role, tool_calls) - ) - elif role == "tool": if not isinstance(tool_call_id, str): raise Exception("Prop `tool_call_id` is required and must be a string") diff --git a/tests/test_litellm/llms/oci/chat/test_transformation.py b/tests/test_litellm/llms/oci/chat/test_transformation.py new file mode 100644 index 00000000000..950c1fcb4c9 --- /dev/null +++ b/tests/test_litellm/llms/oci/chat/test_transformation.py @@ -0,0 +1,229 @@ +import pytest +from litellm.llms.oci.chat.transformation import adapt_messages_to_generic_oci_standard + +def test_adapt_messages_with_empty_content_and_tool_calls(): + """Test that assistant messages with empty content and tool_calls are processed correctly.""" + # Arrange + messages_with_empty_content = [ + {"role": "user", "content": "Tell me the weather in Tokyo."}, + { + "role": "assistant", + "content": "", # Empty string + "tool_calls": [ + { + "id": "call_test_empty", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Tokyo"}' + } + } + ] + }, + { + "role": "tool", + "content": '{"weather": "Sunny", "temperature": "25°C"}', + "tool_call_id": "call_test_empty" + } + ] + + # Act + result = adapt_messages_to_generic_oci_standard(messages_with_empty_content) + + # Assert + assert len(result) == 3 + + # Check user message + assert result[0].role == "USER" + assert result[0].content[0].type == "TEXT" + assert result[0].content[0].text == "Tell me the weather in Tokyo." + + # Check assistant message with tool_calls (should prioritize tool_calls over empty content) + assert result[1].role == "ASSISTANT" + assert result[1].toolCalls is not None + assert len(result[1].toolCalls) == 1 + assert result[1].toolCalls[0].id == "call_test_empty" + assert result[1].toolCalls[0].name == "get_weather" + + # Check tool response message + assert result[2].role == "TOOL" # Tool responses have TOOL role, not USER + assert result[2].content[0].type == "TEXT" + assert "weather" in result[2].content[0].text + assert result[2].toolCallId == "call_test_empty" # Tool call ID is in separate field + +def test_adapt_messages_with_none_content_and_tool_calls(): + """Test that assistant messages with None content and tool_calls are processed correctly.""" + # Arrange + messages_with_none_content = [ + {"role": "user", "content": "Tell me the weather in Tokyo."}, + { + "role": "assistant", + "content": None, # None value + "tool_calls": [ + { + "id": "call_test_none", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Tokyo"}' + } + } + ] + }, + { + "role": "tool", + "content": '{"weather": "Sunny", "temperature": "25°C"}', + "tool_call_id": "call_test_none" + } + ] + + # Act + result = adapt_messages_to_generic_oci_standard(messages_with_none_content) + + # Assert + assert len(result) == 3 + + # Check assistant message prioritizes tool_calls over None content + assert result[1].role == "ASSISTANT" + assert result[1].toolCalls is not None + assert len(result[1].toolCalls) == 1 + assert result[1].toolCalls[0].id == "call_test_none" + +def test_adapt_messages_with_tool_calls_only(): + """Test that assistant messages with only tool_calls (no content field) are processed correctly.""" + # Arrange + messages_no_content = [ + {"role": "user", "content": "Tell me the weather in Tokyo."}, + { + "role": "assistant", + # No content field at all + "tool_calls": [ + { + "id": "call_test_no_content", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Tokyo"}' + } + } + ] + }, + { + "role": "tool", + "content": '{"weather": "Sunny", "temperature": "25°C"}', + "tool_call_id": "call_test_no_content" + } + ] + + # Act + result = adapt_messages_to_generic_oci_standard(messages_no_content) + + # Assert + assert len(result) == 3 + + # Check assistant message processes tool_calls correctly + assert result[1].role == "ASSISTANT" + assert result[1].toolCalls is not None + assert len(result[1].toolCalls) == 1 + assert result[1].toolCalls[0].id == "call_test_no_content" + +def test_adapt_messages_with_content_only(): + """Test that assistant messages with only content (no tool_calls) are processed correctly.""" + # Arrange + messages_content_only = [ + {"role": "user", "content": "Hello"}, + { + "role": "assistant", + "content": "Hello! How can I help you today?" + } + ] + + # Act + result = adapt_messages_to_generic_oci_standard(messages_content_only) + + # Assert + assert len(result) == 2 + + # Check assistant message with content only + assert result[1].role == "ASSISTANT" + assert result[1].content[0].type == "TEXT" + assert result[1].content[0].text == "Hello! How can I help you today?" + assert result[1].toolCalls is None + +def test_adapt_messages_tool_id_tracking(): + """Test that tool call IDs are properly tracked for validation.""" + # Arrange + messages = [ + {"role": "user", "content": "Test"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "function": { + "name": "test_func", + "arguments": '{"param": "value"}' + } + } + ] + }, + { + "role": "tool", + "content": "Result", + "tool_call_id": "call_123" + } + ] + + # Act + result = adapt_messages_to_generic_oci_standard(messages) + + # Assert + # Tool call should be processed and ID should be available for validation + assert result[1].toolCalls[0].id == "call_123" + + # Tool response should reference the same ID + tool_response_text = result[2].content[0].text + # Tool response text is just the content, tool_call_id is separate + assert tool_response_text == "Result" # The actual content + assert result[2].toolCallId == "call_123" # Tool call ID is in separate field + +def test_adapt_messages_multiple_tool_calls(): + """Test that multiple tool calls in a single message are processed correctly.""" + # Arrange + messages = [ + {"role": "user", "content": "Test multiple tools"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "func1", + "arguments": '{"param": "value1"}' + } + }, + { + "id": "call_2", + "type": "function", + "function": { + "name": "func2", + "arguments": '{"param": "value2"}' + } + } + ] + } + ] + + # Act + result = adapt_messages_to_generic_oci_standard(messages) + + # Assert + assert len(result) == 2 + assert result[1].role == "ASSISTANT" + assert len(result[1].toolCalls) == 2 + assert result[1].toolCalls[0].id == "call_1" + assert result[1].toolCalls[1].id == "call_2" + From c7109609fd03095ad4ecb3e2bddc2b12be5537a3 Mon Sep 17 00:00:00 2001 From: Yuji Arakawa Date: Tue, 2 Sep 2025 19:11:37 +0900 Subject: [PATCH 21/40] renamed unit test code --- ...ransformation.py => test_oci_chat_transformation_for_14158.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/test_litellm/llms/oci/chat/{test_transformation.py => test_oci_chat_transformation_for_14158.py} (100%) diff --git a/tests/test_litellm/llms/oci/chat/test_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation_for_14158.py similarity index 100% rename from tests/test_litellm/llms/oci/chat/test_transformation.py rename to tests/test_litellm/llms/oci/chat/test_oci_chat_transformation_for_14158.py From 61b2209827fa5d65f6cc588db2f4369984a8dd5f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 2 Sep 2025 07:22:37 -0700 Subject: [PATCH 22/40] test_proxy_function_calling_support_consistency --- tests/test_litellm/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index ed9a2729282..9e487939284 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -981,8 +981,8 @@ class TestProxyFunctionCalling: ( "groq/llama-3.3-70b-versatile", "litellm_proxy/groq/llama-3.3-70b-versatile", - False, - ), # This model doesn't support function calling + True, + ), # Cohere models (generally don't support function calling) ("command-nightly", "litellm_proxy/command-nightly", False), ], From 793c9668733982104eb8342fb045640676a4d26b Mon Sep 17 00:00:00 2001 From: Marcelo Mendoza Date: Tue, 2 Sep 2025 14:36:57 +0000 Subject: [PATCH 23/40] feat: added alert type to alert messate to slack for easier handling on slack side. --- .../SlackAlerting/slack_alerting.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 41db4a551bd..0953fb768cb 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -805,9 +805,9 @@ class SlackAlerting(CustomBatchLogger): ### UNIQUE CACHE KEY ### cache_key = provider + region_name - outage_value: Optional[ProviderRegionOutageModel] = ( - await self.internal_usage_cache.async_get_cache(key=cache_key) - ) + outage_value: Optional[ + ProviderRegionOutageModel + ] = await self.internal_usage_cache.async_get_cache(key=cache_key) if ( getattr(exception, "status_code", None) is None @@ -1367,12 +1367,11 @@ Model Info: # Get the current timestamp current_time = datetime.now().strftime("%H:%M:%S") _proxy_base_url = os.getenv("PROXY_BASE_URL", None) + alert_type_formatted = f"Alert type: `{alert_type}`\n" if alert_type == "daily_reports" or alert_type == "new_model_added": - formatted_message = message + formatted_message = alert_type_formatted + message else: - formatted_message = ( - f"Level: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}" - ) + formatted_message = f"{alert_type_formatted}\n Level: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}" if kwargs: for key, value in kwargs.items(): @@ -1388,9 +1387,9 @@ Model Info: self.alert_to_webhook_url is not None and alert_type in self.alert_to_webhook_url ): - slack_webhook_url: Optional[Union[str, List[str]]] = ( - self.alert_to_webhook_url[alert_type] - ) + slack_webhook_url: Optional[ + Union[str, List[str]] + ] = self.alert_to_webhook_url[alert_type] elif self.default_webhook_url is not None: slack_webhook_url = self.default_webhook_url else: From 92f631d57c8821c899004822e6a6aaa9d79f8f55 Mon Sep 17 00:00:00 2001 From: Marcelo Mendoza Date: Tue, 2 Sep 2025 15:12:11 +0000 Subject: [PATCH 24/40] fix: correct formatting of alert messages in Slack notifications --- .../SlackAlerting/slack_alerting.py | 2 +- .../SlackAlerting/test_slack_alerting.py | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 0953fb768cb..6d705642667 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1371,7 +1371,7 @@ Model Info: if alert_type == "daily_reports" or alert_type == "new_model_added": formatted_message = alert_type_formatted + message else: - formatted_message = f"{alert_type_formatted}\n Level: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}" + formatted_message = f"{alert_type_formatted}\nLevel: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}" if kwargs: for key, value in kwargs.items(): diff --git a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py index d389be79618..7e1bd897908 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py @@ -172,3 +172,27 @@ class TestSlackAlerting(unittest.TestCase): self.slack_alerting.update_values(alerting_args={"slack_alerting": "True"}) assert self.slack_alerting.periodic_started == True + + @patch("litellm.integrations.SlackAlerting.slack_alerting.datetime") + def test_alert_type_in_formatted_message(self, mock_datetime): + # Setup mocks + mock_datetime.now.return_value.strftime.return_value = "12:34:56" + + # Import required types + from litellm.types.integrations.slack_alerting import AlertType + + # Create a simple test message to check formatting + alert_type = AlertType.llm_exceptions + level = "Medium" + message = "Test alert message" + current_time = "12:34:56" + + # Test the specific formatting logic we're interested in + alert_type_formatted = f"Alert type: `{alert_type}`\n" + formatted_message = f"{alert_type_formatted}\n Level: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}" + + # Verify alert_type is in the formatted message as expected + self.assertIn("Alert type: `AlertType.llm_exceptions`", formatted_message) + self.assertIn("Level: `Medium`", formatted_message) + self.assertIn("Timestamp: `12:34:56`", formatted_message) + self.assertIn("Message: Test alert message", formatted_message) From 5dcdbb35dadb0fb2b3592e380fa2830c3486c4be Mon Sep 17 00:00:00 2001 From: Keith Decker Date: Tue, 2 Sep 2025 09:20:25 -0600 Subject: [PATCH 25/40] add metrics and logs (events) with semconv attributes --- docs/my-website/docs/proxy/config_settings.md | 4 + litellm/integrations/opentelemetry.py | 325 ++++++++++++++-- .../open_telemetry/data/captured_kwargs.json | 1 + .../data/captured_response.json | 1 + .../integrations/test_opentelemetry.py | 354 +++++++++++++++++- 5 files changed, 643 insertions(+), 42 deletions(-) create mode 100644 tests/test_litellm/integrations/open_telemetry/data/captured_kwargs.json create mode 100644 tests/test_litellm/integrations/open_telemetry/data/captured_response.json diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 541dc6fb3c8..7eb355d39f2 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -573,6 +573,10 @@ router_settings: | LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM | LITELLM_LOG | Enable detailed logging for LiteLLM | LITELLM_LOG_FILE | File path to write LiteLLM logs to. When set, logs will be written to both console and the specified file +| LITELLM_LOGGER_NAME | Name for OTEL logger +| LITELLM_METER_NAME | Name for OTEL Meter +| LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS | Optionally enable semantic logs for OTEL +| LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL | LITELLM_MASTER_KEY | Master key for proxy authentication | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development) | LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60 diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 22ab3092901..e6f265ded58 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -15,6 +15,8 @@ from litellm.types.utils import ( StandardLoggingPayload, ) +# OpenTelemetry imports moved to individual functions to avoid import errors when not installed + if TYPE_CHECKING: from opentelemetry.sdk.trace.export import SpanExporter as _SpanExporter from opentelemetry.trace import Context as _Context @@ -41,6 +43,8 @@ else: Context = Any LITELLM_TRACER_NAME = os.getenv("OTEL_TRACER_NAME", "litellm") +LITELLM_METER_NAME = os.getenv("LITELLM_METER_NAME", "litellm") +LITELLM_LOGGER_NAME = os.getenv("LITELLM_LOGGER_NAME", "litellm") # Remove the hardcoded LITELLM_RESOURCE dictionary - we'll create it properly later RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request" LITELLM_REQUEST_SPAN_NAME = "litellm_request" @@ -83,6 +87,8 @@ class OpenTelemetryConfig: exporter: Union[str, SpanExporter] = "console" endpoint: Optional[str] = None headers: Optional[str] = None + enable_metrics: bool = False + enable_events: bool = False @classmethod def from_env(cls): @@ -104,6 +110,14 @@ class OpenTelemetryConfig: headers = os.getenv( "OTEL_EXPORTER_OTLP_HEADERS", os.getenv("OTEL_HEADERS") ) # example: OTEL_HEADERS=x-honeycomb-team=B85YgLm96***" + enable_metrics: bool = ( + os.getenv("LITELLM_OTEL_INTEGRATION_ENABLE_METRICS", "false").lower() + == "true" + ) + enable_events: bool = ( + os.getenv("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", "false").lower() + == "true" + ) if exporter == "in_memory": return cls(exporter=InMemorySpanExporter()) @@ -111,6 +125,8 @@ class OpenTelemetryConfig: exporter=exporter, endpoint=endpoint, headers=headers, # example: OTEL_HEADERS=x-honeycomb-team=B85YgLm96***" + enable_metrics=enable_metrics, + enable_events=enable_events, ) @@ -119,27 +135,22 @@ class OpenTelemetry(CustomLogger): self, config: Optional[OpenTelemetryConfig] = None, callback_name: Optional[str] = None, + # injection points for testing + tracer_provider: Optional[Any] = None, + logger_provider: Optional[Any] = None, + meter_provider: Optional[Any] = None, **kwargs, ): - from opentelemetry import trace - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.trace import SpanKind if config is None: config = OpenTelemetryConfig.from_env() self.config = config + self.callback_name = callback_name self.OTEL_EXPORTER = self.config.exporter self.OTEL_ENDPOINT = self.config.endpoint self.OTEL_HEADERS = self.config.headers - provider = TracerProvider(resource=_get_litellm_resource()) - provider.add_span_processor(self._get_span_processor()) - self.callback_name = callback_name - - trace.set_tracer_provider(provider) - self.tracer = trace.get_tracer(LITELLM_TRACER_NAME) - - self.span_kind = SpanKind + self._init_tracing(tracer_provider) _debug_otel = str(os.getenv("DEBUG_OTEL", "False")).lower() @@ -156,6 +167,8 @@ class OpenTelemetry(CustomLogger): # init CustomLogger params super().__init__(**kwargs) + self._init_metrics(meter_provider) + self._init_logs(logger_provider) self._init_otel_logger_on_litellm_proxy() def _init_otel_logger_on_litellm_proxy(self): @@ -178,14 +191,109 @@ class OpenTelemetry(CustomLogger): litellm.service_callback.append("otel") setattr(proxy_server, "open_telemetry_logger", self) + def _init_tracing(self, tracer_provider): + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.trace import SpanKind + + # use provided tracer or create a new one + if tracer_provider is None: + tracer_provider = TracerProvider(resource=_get_litellm_resource()) + # Only add OTLP span processor if we created the tracer provider ourselves + tracer_provider.add_span_processor(self._get_span_processor()) + + # register global provider and grab our tracer + trace.set_tracer_provider(tracer_provider) + self.tracer = trace.get_tracer(LITELLM_TRACER_NAME) + self.span_kind = SpanKind + + def _init_metrics(self, meter_provider): + if not self.config.enable_metrics: + self._operation_duration_histogram = None + self._token_usage_histogram = None + self._cost_histogram = None + return + + from opentelemetry import metrics + from opentelemetry.sdk.metrics import Histogram, MeterProvider + + # Only create OTLP infrastructure if no custom meter provider is provided + if meter_provider is None: + from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, + ) + from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + PeriodicExportingMetricReader, + ) + + _metric_exporter = OTLPMetricExporter( + endpoint=self.config.endpoint, + headers=OpenTelemetry._get_headers_dictionary(self.config.headers), + preferred_temporality={Histogram: AggregationTemporality.DELTA}, + ) + _metric_reader = PeriodicExportingMetricReader( + _metric_exporter, export_interval_millis=10000 + ) + + meter_provider = MeterProvider( + metric_readers=[_metric_reader], resource=_get_litellm_resource() + ) + meter = meter_provider.get_meter(__name__) + else: + # Use the provided meter provider as-is, without creating additional OTLP infrastructure + meter = meter_provider.get_meter(__name__) + + metrics.set_meter_provider(meter_provider) + + self._operation_duration_histogram = meter.create_histogram( + name="gen_ai.client.operation.duration", # Replace with semconv constant in otel 1.38 + description="GenAI operation duration", + unit="s", + ) + self._token_usage_histogram = meter.create_histogram( + name="gen_ai.client.token.usage", # Replace with semconv constant in otel 1.38 + description="GenAI token usage", + unit="{token}", + ) + self._cost_histogram = meter.create_histogram( + name="gen_ai.client.token.cost", + description="GenAI request cost", + unit="USD", + ) + + def _init_logs(self, logger_provider): + # nothing to do if events disabled + if not self.config.enable_events: + return + + from opentelemetry._logs import set_logger_provider + from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter + from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + + # set up log pipeline + if logger_provider is None: + logger_provider = OTLoggerProvider() + # Only add OTLP exporter if we created the logger provider ourselves + logger_provider.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter( + endpoint=self.config.endpoint, + headers=self._get_headers_dictionary(self.config.headers), + ) + ) + ) + set_logger_provider(logger_provider) + def log_success_event(self, kwargs, response_obj, start_time, end_time): - self._handle_sucess(kwargs, response_obj, start_time, end_time) + self._handle_success(kwargs, response_obj, start_time, end_time) def log_failure_event(self, kwargs, response_obj, start_time, end_time): self._handle_failure(kwargs, response_obj, start_time, end_time) async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - self._handle_sucess(kwargs, response_obj, start_time, end_time) + self._handle_success(kwargs, response_obj, start_time, end_time) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): self._handle_failure(kwargs, response_obj, start_time, end_time) @@ -372,9 +480,9 @@ class OpenTelemetry(CustomLogger): def _get_dynamic_otel_headers_from_kwargs(self, kwargs) -> Optional[dict]: """Extract dynamic headers from kwargs if available.""" - standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = ( - kwargs.get("standard_callback_dynamic_params") - ) + standard_callback_dynamic_params: Optional[ + StandardCallbackDynamicParams + ] = kwargs.get("standard_callback_dynamic_params") if not standard_callback_dynamic_params: return None @@ -414,50 +522,185 @@ class OpenTelemetry(CustomLogger): # End of Team/Key Based Logging Control Flow ######################################################### - def _handle_sucess(self, kwargs, response_obj, start_time, end_time): - from opentelemetry import trace - from opentelemetry.trace import Status, StatusCode + def _handle_success(self, kwargs, response_obj, start_time, end_time): verbose_logger.debug( "OpenTelemetry Logger: Logging kwargs: %s, OTEL config settings=%s", kwargs, self.config, ) + ctx, parent_span = self._get_span_context(kwargs) + + # 1. Primary span + span = self._start_primary_span(kwargs, response_obj, start_time, end_time, ctx) + + # 2. Raw‐request sub-span (if enabled) + self._maybe_log_raw_request(kwargs, response_obj, start_time, end_time, span) + + # 3. Guardrail span + self._create_guardrail_span(kwargs=kwargs, context=ctx) + + # 4. Metrics & cost recording + self._record_metrics(kwargs, response_obj, start_time, end_time) + + # 5. Semantic logs. + if self.config.enable_events: + self._emit_semantic_logs(kwargs, response_obj, span) + + # 6. End parent span + if parent_span is not None: + parent_span.end(end_time=self._to_ns(datetime.now())) + + def _start_primary_span(self, kwargs, response_obj, start_time, end_time, context): + from opentelemetry.trace import Status, StatusCode - _parent_context, parent_otel_span = self._get_span_context(kwargs) - # Span 1: Request sent to litellm SDK otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs) span = otel_tracer.start_span( name=self._get_span_name(kwargs), start_time=self._to_ns(start_time), - context=_parent_context, + context=context, ) span.set_status(Status(StatusCode.OK)) self.set_attributes(span, kwargs, response_obj) + span.end(end_time=self._to_ns(end_time)) + return span - if litellm.turn_off_message_logging is True: - pass - elif self.message_logging is not True: - pass - else: - # Span 2: Raw Request / Response to LLM - raw_request_span = otel_tracer.start_span( - name=RAW_REQUEST_SPAN_NAME, - start_time=self._to_ns(start_time), - context=trace.set_span_in_context(span), + def _maybe_log_raw_request( + self, kwargs, response_obj, start_time, end_time, parent_span + ): + from opentelemetry import trace + from opentelemetry.trace import Status, StatusCode + + # only log raw LLM request/response if message_logging is on and not globally turned off + if litellm.turn_off_message_logging or not self.message_logging: + return + + otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs) + raw_span = otel_tracer.start_span( + name=RAW_REQUEST_SPAN_NAME, + start_time=self._to_ns(start_time), + context=trace.set_span_in_context(parent_span), + ) + raw_span.set_status(Status(StatusCode.OK)) + self.set_raw_request_attributes(raw_span, kwargs, response_obj) + raw_span.end(end_time=self._to_ns(end_time)) + + def _record_metrics(self, kwargs, response_obj, start_time, end_time): + duration_s = (end_time - start_time).total_seconds() + params = kwargs.get("litellm_params") or {} + provider = params.get("custom_llm_provider", "Unknown") + + common_attrs = { + "gen_ai.operation.name": "chat", + "gen_ai.system": provider, + "gen_ai.request.model": kwargs.get("model"), + "gen_ai.framework": "litellm", + } + + std_log = kwargs.get("standard_logging_object") + md = getattr(std_log, "metadata", None) or (std_log or {}).get("metadata", {}) + for key in [ + "user_api_key_hash", + "user_api_key_alias", + "user_api_key_team_id", + "user_api_key_org_id", + "user_api_key_user_id", + "user_api_key_team_alias", + "user_api_key_user_email", + "spend_logs_metadata", + "requester_ip_address", + "requester_metadata", + "user_api_key_end_user_id", + "prompt_management_metadata", + "applied_guardrails", + "mcp_tool_call_metadata", + "vector_store_request_metadata", + ]: + if md.get(key) is not None: + common_attrs[f"metadata.{key}"] = str(md[key]) + + if self._operation_duration_histogram: + self._operation_duration_histogram.record( + duration_s, attributes=common_attrs + ) + if ( + response_obj + and (usage := response_obj.get("usage")) + and self._token_usage_histogram + ): + in_attrs = {**common_attrs, "gen_ai.token.type": "input"} + out_attrs = {**common_attrs, "gen_ai.token.type": "completion"} + self._token_usage_histogram.record( + usage.get("prompt_tokens", 0), attributes=in_attrs + ) + self._token_usage_histogram.record( + usage.get("completion_tokens", 0), attributes=out_attrs + ) + + cost = kwargs.get("response_cost") + if self._cost_histogram and cost: + self._cost_histogram.record(cost, attributes=common_attrs) + + def _emit_semantic_logs(self, kwargs, response_obj, span: Span): + if not self.config.enable_events: + return + + from opentelemetry._logs import get_logger, LogRecord + otel_logger = get_logger(LITELLM_LOGGER_NAME) + + parent_ctx = span.get_span_context() + provider = (kwargs.get("litellm_params") or {}).get( + "custom_llm_provider", "Unknown" + ) + + # per-message events + for msg in kwargs.get("messages", []): + role = msg.get("role", "user") + attrs = {"event_name": "gen_ai.content.prompt", "gen_ai.system": provider} + if role == "tool" and msg.get("id"): + attrs["id"] = msg["id"] + if self.message_logging and msg.get("content"): + attrs["gen_ai.prompt"] = msg["content"] + + otel_logger.emit( + LogRecord( + attributes=attrs, + body=msg.copy(), + trace_id=parent_ctx.trace_id, + span_id=parent_ctx.span_id, + trace_flags=parent_ctx.trace_flags, + ) ) - raw_request_span.set_status(Status(StatusCode.OK)) - self.set_raw_request_attributes(raw_request_span, kwargs, response_obj) - raw_request_span.end(end_time=self._to_ns(end_time)) + # per-choice events + for idx, choice in enumerate(response_obj.get("choices", [])): + attrs = { + "event_name": "gen_ai.content.completion", + "gen_ai.system": provider, + "index": idx, + "finish_reason": choice.get("finish_reason"), + } + body_msg = choice.get("message", {}) + if self.message_logging and body_msg.get("content"): + attrs["message.content"] = body_msg["content"] + body = { + "index": idx, + "finish_reason": choice.get("finish_reason"), + "message": {"role": body_msg.get("role", "assistant")}, + } + if self.message_logging and body_msg.get("content"): + body["message"]["content"] = body_msg["content"] - span.end(end_time=self._to_ns(end_time)) + otel_logger.emit( + LogRecord( + attributes=attrs, + body=body, + trace_id=parent_ctx.trace_id, + span_id=parent_ctx.span_id, + trace_flags=parent_ctx.trace_flags, + ) + ) - # Create span for guardrail information - self._create_guardrail_span(kwargs=kwargs, context=_parent_context) - - if parent_otel_span is not None: - parent_otel_span.end(end_time=self._to_ns(datetime.now())) def _create_guardrail_span( self, kwargs: Optional[dict], context: Optional[Context] diff --git a/tests/test_litellm/integrations/open_telemetry/data/captured_kwargs.json b/tests/test_litellm/integrations/open_telemetry/data/captured_kwargs.json new file mode 100644 index 00000000000..913e3bfedae --- /dev/null +++ b/tests/test_litellm/integrations/open_telemetry/data/captured_kwargs.json @@ -0,0 +1 @@ +{"litellm_trace_id": null, "litellm_call_id": "dbecd23a-e71a-49cf-90d4-712a8a8e29c5", "input": [{"role": "user", "content": "What is the capital of France?"}], "litellm_params": {"acompletion": true, "api_key": null, "force_timeout": 600, "logger_fn": null, "verbose": false, "custom_llm_provider": "bedrock", "api_base": "https://bedrock-runtime.us-west-2.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-west-2%3A1234567890123%3Ainference-profile%2Fus.anthropic.claude-3-7-sonnet-20250219-v1%3A0/converse", "litellm_call_id": "dbecd23a-e71a-49cf-90d4-712a8a8e29c5", "model_alias_map": {}, "completion_call_id": null, "aembedding": null, "metadata": {"requester_metadata": {}, "user_api_key_hash": "unused-for-aws-bedrock", "user_api_key_alias": null, "user_api_key_team_id": null, "user_api_key_user_id": null, "user_api_key_org_id": null, "user_api_key_team_alias": null, "user_api_key_end_user_id": null, "user_api_key_user_email": null, "user_api_key": "unused-for-aws-bedrock", "user_api_end_user_max_budget": null, "litellm_api_version": "1.72.3", "global_max_parallel_requests": null, "user_api_key_team_max_budget": null, "user_api_key_team_spend": null, "user_api_key_spend": 0.0, "user_api_key_max_budget": null, "user_api_key_model_max_budget": {}, "user_api_key_metadata": {}, "headers": {"host": "0.0.0.0:44444", "accept-encoding": "gzip, deflate, zstd", "connection": "keep-alive", "accept": "application/json", "content-type": "application/json", "user-agent": "AsyncOpenAI/Python 1.84.0", "x-stainless-lang": "python", "x-stainless-package-version": "1.84.0", "x-stainless-os": "MacOS", "x-stainless-arch": "arm64", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.10", "x-stainless-async": "async:asyncio", "x-stainless-retry-count": "0", "x-stainless-read-timeout": "600", "content-length": "116"}, "endpoint": "http://0.0.0.0:44444/chat/completions", "litellm_parent_otel_span": null, "requester_ip_address": "", "model_group": "claude-3-7-sonnet", "model_group_size": 1, "deployment": "bedrock/arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "model_info": {"id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "db_model": false}, "api_base": null, "caching_groups": null, "hidden_params": {"custom_llm_provider": "bedrock", "region_name": null, "optional_params": {"stream": false, "max_retries": 0, "provider": "aws", "region": "us-west-2"}, "litellm_call_id": "dbecd23a-e71a-49cf-90d4-712a8a8e29c5", "api_base": null, "model_id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "response_cost": 0.001047, "additional_headers": {"x-litellm-model-group": "claude-3-7-sonnet", "x-litellm-attempted-retries": 0, "x-litellm-attempted-fallbacks": 0}, "litellm_model_name": "bedrock/arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "litellm_overhead_time_ms": 231.156, "_response_ms": 236.798}}, "model_info": {"id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "db_model": false}, "proxy_server_request": {"url": "http://0.0.0.0:44444/chat/completions", "method": "POST", "headers": {"host": "0.0.0.0:44444", "accept-encoding": "gzip, deflate, zstd", "connection": "keep-alive", "accept": "application/json", "content-type": "application/json", "user-agent": "AsyncOpenAI/Python 1.84.0", "x-stainless-lang": "python", "x-stainless-package-version": "1.84.0", "x-stainless-os": "MacOS", "x-stainless-arch": "arm64", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.10", "x-stainless-async": "async:asyncio", "x-stainless-retry-count": "0", "x-stainless-read-timeout": "600", "content-length": "116"}, "body": {"messages": [{"role": "user", "content": "What is the capital of France?"}], "model": "claude-3-7-sonnet", "stream": false}}, "preset_cache_key": null, "no-log": null, "stream_response": {}, "input_cost_per_token": null, "input_cost_per_second": null, "output_cost_per_token": null, "output_cost_per_second": null, "cooldown_time": null, "text_completion": null, "azure_ad_token_provider": null, "user_continue_message": null, "base_model": null, "litellm_trace_id": "4c97150b-b1a3-4dec-bd7a-734786b1b3bc", "litellm_session_id": null, "hf_model_name": null, "custom_prompt_dict": {}, "litellm_metadata": null, "disable_add_transform_inline_image_block": null, "drop_params": null, "prompt_id": null, "prompt_variables": null, "async_call": null, "ssl_verify": null, "merge_reasoning_content_in_choices": false, "api_version": null, "azure_ad_token": null, "tenant_id": null, "client_id": null, "client_secret": null, "azure_username": null, "azure_password": null, "max_retries": 0, "timeout": 6000.0, "bucket_name": null, "vertex_credentials": null, "vertex_project": null, "use_litellm_proxy": false}, "applied_guardrails": [], "model": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "messages": [{"role": "user", "content": "What is the capital of France?"}], "optional_params": {"stream": false, "max_retries": 0, "provider": "aws", "region": "us-west-2"}, "start_time": "2025-06-22 10:59:08.159939", "stream": false, "user": null, "call_type": "acompletion", "completion_start_time": "2025-06-22 10:59:08.399523", "standard_callback_dynamic_params": {}, "stream_options": null, "max_retries": 0, "provider": "aws", "region": "us-west-2", "custom_llm_provider": "bedrock", "api_key": "", "additional_args": {"complete_input_dict": "{\"messages\": [{\"role\": \"user\", \"content\": [{\"text\": \"What is the capital of France?\"}]}], \"additionalModelRequestFields\": {\"provider\": \"aws\", \"region\": \"us-west-2\"}, \"system\": [], \"inferenceConfig\": {}}"}, "log_event_type": "post_api_call", "api_call_start_time": "2025-06-22 10:59:08.387641", "llm_api_duration_ms": 5.642, "original_response": "{\"metrics\":{\"latencyMs\":1513},\"output\":{\"message\":{\"content\":[{\"text\":\"The capital of France is Paris. Paris has been the capital city of France since 987 CE when Hugh Capet, the first king of the Capetian dynasty, made the city his seat of government. Today, Paris is not only the political capital but also the cultural and economic center of France.\"}],\"role\":\"assistant\"}},\"stopReason\":\"end_turn\",\"usage\":{\"cacheReadInputTokenCount\":0,\"cacheReadInputTokens\":0,\"cacheWriteInputTokenCount\":0,\"cacheWriteInputTokens\":0,\"inputTokens\":14,\"outputTokens\":67,\"totalTokens\":81}}", "end_time": "2025-06-22 10:59:08.399523", "cache_hit": null, "response_cost": 0.001047, "standard_logging_object": {"id": "chatcmpl-fa9be5b7-9487-46ab-86de-6462d578fea1", "trace_id": "4c97150b-b1a3-4dec-bd7a-734786b1b3bc", "call_type": "acompletion", "cache_hit": null, "stream": true, "status": "success", "custom_llm_provider": "bedrock", "saved_cache_cost": 0.0, "startTime": 1750615148.162725, "endTime": 1750615148.399523, "completionStartTime": 1750615148.399523, "response_time": 0.23679804801940918, "model": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "metadata": {"user_api_key_hash": "unused-for-aws-bedrock", "user_api_key_alias": null, "user_api_key_team_id": null, "user_api_key_org_id": null, "user_api_key_user_id": null, "user_api_key_team_alias": null, "user_api_key_user_email": null, "spend_logs_metadata": null, "requester_ip_address": "", "requester_metadata": {}, "user_api_key_end_user_id": null, "prompt_management_metadata": null, "applied_guardrails": [], "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "usage_object": {"completion_tokens": 67, "prompt_tokens": 14, "total_tokens": 81, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "requester_custom_headers": {"x-stainless-lang": "python", "x-stainless-package-version": "1.84.0", "x-stainless-os": "MacOS", "x-stainless-arch": "arm64", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.10", "x-stainless-async": "async:asyncio", "x-stainless-retry-count": "0", "x-stainless-read-timeout": "600"}}, "cache_key": null, "response_cost": 0.001047, "total_tokens": 81, "prompt_tokens": 14, "completion_tokens": 67, "request_tags": [], "end_user": "", "api_base": "https://bedrock-runtime.us-west-2.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-west-2%3A1234567890123%3Ainference-profile%2Fus.anthropic.claude-3-7-sonnet-20250219-v1%3A0/converse", "model_group": "claude-3-7-sonnet", "model_id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "requester_ip_address": "", "messages": [{"role": "user", "content": "What is the capital of France?"}], "response": {"id": "chatcmpl-fa9be5b7-9487-46ab-86de-6462d578fea1", "created": 1750615148, "model": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "object": "chat.completion", "system_fingerprint": null, "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "The capital of France is Paris. Paris has been the capital city of France since 987 CE when Hugh Capet, the first king of the Capetian dynasty, made the city his seat of government. Today, Paris is not only the political capital but also the cultural and economic center of France.", "role": "assistant", "tool_calls": null, "function_call": null}}], "usage": {"completion_tokens": 67, "prompt_tokens": 14, "total_tokens": 81, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}, "model_parameters": {"stream": false}, "hidden_params": {"model_id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "cache_key": null, "api_base": null, "response_cost": 0.001047, "additional_headers": {}, "litellm_overhead_time_ms": 231.156, "batch_models": null, "litellm_model_name": "bedrock/arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "usage_object": null}, "model_map_information": {"model_map_key": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "model_map_value": {"key": "anthropic.claude-3-7-sonnet-20250219-v1:0", "max_tokens": 8192, "max_input_tokens": 200000, "max_output_tokens": 8192, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_reasoning_token": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "bedrock_converse", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": null, "supports_audio_output": null, "supports_pdf_input": true, "supports_embedding_image_input": null, "supports_native_streaming": null, "supports_web_search": null, "supports_url_context": null, "supports_reasoning": true, "supports_computer_use": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["max_tokens", "max_completion_tokens", "stream", "stream_options", "stop", "temperature", "top_p", "extra_headers", "response_format", "tools", "tool_choice", "thinking", "reasoning_effort"]}}, "error_str": null, "error_information": {"error_code": "", "error_class": "", "llm_provider": "", "traceback": "", "error_message": ""}, "response_cost_failure_debug_info": null, "guardrail_information": null, "standard_built_in_tools_params": {"web_search_options": null, "file_search": null}}, "async_complete_streaming_response": "ModelResponse(id='chatcmpl-fa9be5b7-9487-46ab-86de-6462d578fea1', created=1750615148, model='arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='The capital of France is Paris. Paris has been the capital city of France since 987 CE when Hugh Capet, the first king of the Capetian dynasty, made the city his seat of government. Today, Paris is not only the political capital but also the cultural and economic center of France.', role='assistant', tool_calls=None, function_call=None, provider_specific_fields=None))], usage=Usage(completion_tokens=67, prompt_tokens=14, total_tokens=81, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=0, text_tokens=None, image_tokens=None), cache_creation_input_tokens=0, cache_read_input_tokens=0))"} \ No newline at end of file diff --git a/tests/test_litellm/integrations/open_telemetry/data/captured_response.json b/tests/test_litellm/integrations/open_telemetry/data/captured_response.json new file mode 100644 index 00000000000..3cf77781cc2 --- /dev/null +++ b/tests/test_litellm/integrations/open_telemetry/data/captured_response.json @@ -0,0 +1 @@ +{"id": "chatcmpl-fa9be5b7-9487-46ab-86de-6462d578fea1", "created": 1750615148, "model": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "object": "chat.completion", "system_fingerprint": null, "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "The capital of France is Paris. Paris has been the capital city of France since 987 CE when Hugh Capet, the first king of the Capetian dynasty, made the city his seat of government. Today, Paris is not only the political capital but also the cultural and economic center of France.", "role": "assistant", "tool_calls": null, "function_call": null}}], "usage": {"completion_tokens": 67, "prompt_tokens": 14, "total_tokens": 81, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}} \ No newline at end of file diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index e11895e30ea..7fb91f274d0 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -1,15 +1,142 @@ +import json import os import sys import unittest from unittest.mock import MagicMock, patch +from datetime import datetime, timedelta +import time # Adds the grandparent directory to sys.path to allow importing project modules sys.path.insert(0, os.path.abspath("../..")) from litellm.integrations.opentelemetry import OpenTelemetry from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider +from opentelemetry.sdk._logs.export import SimpleLogRecordProcessor, InMemoryLogExporter +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + +class TestOpenTelemetryGuardrails(unittest.TestCase): + @patch("litellm.integrations.opentelemetry.datetime") + def test_create_guardrail_span_with_valid_info(self, mock_datetime): + # Setup + otel = OpenTelemetry() + otel.tracer = MagicMock() + mock_span = MagicMock() + otel.tracer.start_span.return_value = mock_span + + # Create guardrail information + guardrail_info = { + "guardrail_name": "test_guardrail", + "guardrail_mode": "input", + "masked_entity_count": {"CREDIT_CARD": 2}, + "guardrail_response": "filtered_content", + "start_time": 1609459200.0, + "end_time": 1609459201.0, + } + + # Create a kwargs dict with standard_logging_object containing guardrail information + kwargs = {"standard_logging_object": {"guardrail_information": guardrail_info}} + + # Call the method + otel._create_guardrail_span(kwargs=kwargs, context=None) + + # Assertions + otel.tracer.start_span.assert_called_once() + + # print all calls to mock_span.set_attribute + print("Calls to mock_span.set_attribute:") + for call in mock_span.set_attribute.call_args_list: + print(call) + + # Check that the span has the correct attributes set + mock_span.set_attribute.assert_any_call("guardrail_name", "test_guardrail") + mock_span.set_attribute.assert_any_call("guardrail_mode", "input") + mock_span.set_attribute.assert_any_call( + "guardrail_response", "filtered_content" + ) + mock_span.set_attribute.assert_any_call( + "masked_entity_count", safe_dumps({"CREDIT_CARD": 2}) + ) + + # Verify that the span was ended + mock_span.end.assert_called_once() + + def test_create_guardrail_span_with_no_info(self): + # Setup + otel = OpenTelemetry() + otel.tracer = MagicMock() + + # Test with no guardrail information + kwargs = {"standard_logging_object": {}} + otel._create_guardrail_span(kwargs=kwargs, context=None) + + # Verify that start_span was never called + otel.tracer.start_span.assert_not_called() + class TestOpenTelemetry(unittest.TestCase): + POLL_INTERVAL = 0.05 + POLL_TIMEOUT = 2.0 + MODEL = "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0" + HERE = os.path.dirname(__file__) + + def wait_for_spans(self, exporter: InMemorySpanExporter, prefix: str): + """Poll until we see at least one span with an attribute key starting with `prefix`.""" + deadline = time.time() + self.POLL_TIMEOUT + while time.time() < deadline: + spans = exporter.get_finished_spans() + matches = [ + s + for s in spans + if s.attributes and any(str(k).startswith(prefix) for k in s.attributes) + ] + if matches: + return matches + time.sleep(self.POLL_INTERVAL) + return [] + + def wait_for_metric(self, reader: InMemoryMetricReader, name: str): + """Poll until we see a metric with the given name.""" + deadline = time.time() + self.POLL_TIMEOUT + while time.time() < deadline: + data = reader.get_metrics_data() + # guard against None or missing attribute + if not data or not hasattr(data, "resource_metrics"): + time.sleep(self.POLL_INTERVAL) + continue + + for rm in data.resource_metrics: + for sm in rm.scope_metrics: + for m in sm.metrics: + if m.name == name: + return m + + time.sleep(self.POLL_INTERVAL) + return None + + def wait_for_log(self, reader: InMemoryLogExporter, name: str): + """Poll until we see a log with the given name.""" + deadline = time.time() + self.POLL_TIMEOUT + while time.time() < deadline: + logs = reader.get_finished_logs() + if not logs: + time.sleep(self.POLL_INTERVAL) + continue + matches = [ + log + for log in logs + # if log.attributes and any(str(k).startswith(prefix) for k in log.attributes) + ] + if matches: + return matches + time.sleep(self.POLL_INTERVAL) + return [] + @patch("litellm.integrations.opentelemetry.datetime") def test_create_guardrail_span_with_valid_info(self, mock_datetime): # Setup @@ -79,7 +206,6 @@ class TestOpenTelemetry(unittest.TestCase): ) as mock_get_headers, patch.object( otel, "_get_tracer_with_dynamic_headers" ) as mock_get_tracer: - # Test case 1: With dynamic headers mock_get_headers.return_value = { "arize-space-id": "test-space", @@ -399,3 +525,229 @@ class TestOpenTelemetry(unittest.TestCase): self.assertEqual(attributes.get("service.name"), "litellm-service") # But other attributes from OTEL_RESOURCE_ATTRIBUTES should still be present self.assertEqual(attributes.get("extra.attr"), "extra-value") + + def test_handle_success_generates_spans_metrics_and_events(self): + # force both metrics & events on + os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS"] = "true" + os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_METRICS"] = "true" + + # ─── build in‐memory OTEL providers/exporters ───────────────────────────── + span_exporter = InMemorySpanExporter() + tracer_provider = TracerProvider() + tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + + log_exporter = InMemoryLogExporter() + logger_provider = OTLoggerProvider() + logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) + + metric_reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[metric_reader]) + + # ─── instantiate our OpenTelemetry logger with test providers ─────────── + otel = OpenTelemetry( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + + # OpenTelemetry attempts to set a global tracer provider, which can be set only once. + # so we hack here to set a local tracer deriver from the provider we created. + otel.tracer = tracer_provider.get_tracer(__name__) + + # ─── minimal input / output for a chat call ────────────────────────────── + start = datetime.utcnow() + end = start + timedelta(seconds=1) + + with open( + os.path.join(self.HERE, "open_telemetry", "data", "captured_kwargs.json") + ) as f: + kwargs = json.load(f) + with open( + os.path.join(self.HERE, "open_telemetry", "data", "captured_response.json") + ) as f: + response_obj = json.load(f) + + # ─── exercise the hook ─────────────────────────────────────────────────── + otel._handle_success(kwargs, response_obj, start, end) + + # ─── assert spans ──────────────────────────────────────────────────────── + spans = self.wait_for_spans(span_exporter, "gen_ai.") + self.assertTrue(spans, "Expected at least one gen_ai span") + + # verify our top‐level litellm_request span is present + names = [s.name for s in spans] + self.assertIn("litellm_request", names) + + # ─── assert metrics ────────────────────────────────────────────────────── + duration_metric = self.wait_for_metric( + metric_reader, "gen_ai.client.operation.duration" + ) + self.assertIsNotNone(duration_metric, "duration histogram was not recorded") + + # check that our model attribute made it onto at least one data point + found_dp = False + if ( + duration_metric + and hasattr(duration_metric, "data") + and hasattr(duration_metric.data, "data_points") + ): + found_dp = any( + dp.attributes.get("gen_ai.request.model") == self.MODEL + for dp in duration_metric.data.data_points + ) + self.assertTrue( + found_dp, "expected gen_ai.request.model attribute on a data point" + ) + + # ─── assert logs ─────────────────────────────────────────────────────── + logs = [] + logs = self.wait_for_log(log_exporter, "gen_ai.") + self.assertTrue(logs, "Expected at least one gen_ai log") + + user_logs = [log for log in logs if log.log_record.attributes.get("event_name") == "gen_ai.content.prompt"] + self.assertTrue(user_logs, "did not see a gen_ai.content.prompt log") + # check log bodies + user_prompt = user_logs[0].log_record.attributes.get("gen_ai.prompt") + self.assertEqual("What is the capital of France?", user_prompt, "did not see a prompt message") + + choice_logs = [log for log in logs if log.log_record.attributes.get("event_name") == "gen_ai.content.completion"] + self.assertTrue(choice_logs, "did not see a gen_ai.content.completion event") + + choice_response = choice_logs[0].log_record.body + self.assertIsNotNone(choice_response, "did not see a response message") + self.assertEqual("stop", choice_response.get("finish_reason"), "did not see expected finish reason") + + + def test_handle_success_spans_only(self): + # make sure neither events nor metrics is on + os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", None) + os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_METRICS", None) + + # ─── build in‐memory OTEL providers/exporters ───────────────────────────── + span_exporter = InMemorySpanExporter() + tracer_provider = TracerProvider() + tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + + # no logs / no metrics + log_exporter = InMemoryLogExporter() + logger_provider = OTLoggerProvider() + logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) + metric_reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[metric_reader]) + + # ─── instantiate our OpenTelemetry logger with test providers ─────────── + otel = OpenTelemetry( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, # pass even if events disabled (safe) + ) + # bind our tracer to the test tracer provider (global registration is a no-op after the first time) + otel.tracer = tracer_provider.get_tracer(__name__) + + # ─── minimal input / output for a chat call ────────────────────────────── + start = datetime.utcnow() + end = start + timedelta(seconds=1) + with open( + os.path.join(self.HERE, "open_telemetry", "data", "captured_kwargs.json") + ) as f: + kwargs = json.load(f) + with open( + os.path.join(self.HERE, "open_telemetry", "data", "captured_response.json") + ) as f: + response_obj = json.load(f) + + # ─── exercise the hook ─────────────────────────────────────────────────── + otel._handle_success(kwargs, response_obj, start, end) + + # ─── assert spans only ─────────────────────────────────────────────────── + spans = span_exporter.get_finished_spans() + self.assertTrue(spans, "Expected at least one span") + # must have the top‐level litellm_request span + # self.assertIn( + # LITELLM_REQUEST_SPAN_NAME, + # [s.name for s in spans], + # "litellm_request span missing", + # ) + # model attribute should be on that span + found = any( + s.attributes + and s.attributes.get("gen_ai.request.model") == self.MODEL + for s in spans + ) + self.assertTrue(found, "expected gen_ai.request.model on span attributes") + + # no metrics recorded + self.assertIsNone( + self.wait_for_metric(metric_reader, "gen_ai.client.operation.duration"), + "Did not expect any metrics", + ) + # no logs emitted + logs = log_exporter.get_finished_logs() + self.assertFalse(logs, "Did not expect any logs") + + def test_handle_success_spans_and_metrics(self): + # only metrics on + os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", None) + os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_METRICS"] = "true" + + # ─── build in‐memory OTEL providers/exporters ───────────────────────────── + span_exporter = InMemorySpanExporter() + tracer_provider = TracerProvider() + tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + + log_exporter = InMemoryLogExporter() + logger_provider = OTLoggerProvider() + logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) + metric_reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[metric_reader]) + + # ─── instantiate our OpenTelemetry logger with test providers ─────────── + otel = OpenTelemetry( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, # needed if events were enabled + ) + otel.tracer = tracer_provider.get_tracer(__name__) + + # ─── minimal input / output for a chat call ────────────────────────────── + start = datetime.utcnow() + end = start + timedelta(seconds=1) + with open( + os.path.join(self.HERE, "open_telemetry", "data", "captured_kwargs.json") + ) as f: + kwargs = json.load(f) + with open( + os.path.join(self.HERE, "open_telemetry", "data", "captured_response.json") + ) as f: + response_obj = json.load(f) + + # ─── exercise the hook ─────────────────────────────────────────────────── + otel._handle_success(kwargs, response_obj, start, end) + + # ─── assert spans ──────────────────────────────────────────────────────── + spans = span_exporter.get_finished_spans() + self.assertTrue(spans, "Expected at least one span") + + # ─── assert metrics ────────────────────────────────────────────────────── + duration_metric = self.wait_for_metric( + metric_reader, "gen_ai.client.operation.duration" + ) + self.assertIsNotNone(duration_metric, "duration histogram was not recorded") + # model attribute should be present on a data point + found_dp = False + if ( + duration_metric + and hasattr(duration_metric, "data") + and hasattr(duration_metric.data, "data_points") + ): + found_dp = any( + dp.attributes.get("gen_ai.request.model") == self.MODEL + for dp in duration_metric.data.data_points + ) + self.assertTrue( + found_dp, "expected gen_ai.request.model attribute on a data point" + ) + + # ─── no events when only metrics enabled ───────────────────────────────── + logs = log_exporter.get_finished_logs() + self.assertFalse(logs, "Did not expect any logs") From db1b418b156850df20686b31f11c61f7beab20f2 Mon Sep 17 00:00:00 2001 From: Marcelo Mendoza Date: Tue, 2 Sep 2025 17:54:54 +0200 Subject: [PATCH 26/40] fix: update alert type formatting to handle enum names in Slack messages --- litellm/integrations/SlackAlerting/slack_alerting.py | 4 +++- .../integrations/SlackAlerting/test_slack_alerting.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 6d705642667..7da38e193b6 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1367,7 +1367,9 @@ Model Info: # Get the current timestamp current_time = datetime.now().strftime("%H:%M:%S") _proxy_base_url = os.getenv("PROXY_BASE_URL", None) - alert_type_formatted = f"Alert type: `{alert_type}`\n" + # Use .name if it's an enum, otherwise use as is + alert_type_name = getattr(alert_type, 'name', alert_type) + alert_type_formatted = f"Alert type: `{alert_type_name}`" if alert_type == "daily_reports" or alert_type == "new_model_added": formatted_message = alert_type_formatted + message else: diff --git a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py index 7e1bd897908..9cccdb51799 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py @@ -188,11 +188,11 @@ class TestSlackAlerting(unittest.TestCase): current_time = "12:34:56" # Test the specific formatting logic we're interested in - alert_type_formatted = f"Alert type: `{alert_type}`\n" + alert_type_formatted = f"Alert type: `{alert_type.name}`\n" formatted_message = f"{alert_type_formatted}\n Level: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}" # Verify alert_type is in the formatted message as expected - self.assertIn("Alert type: `AlertType.llm_exceptions`", formatted_message) + self.assertIn("Alert type: `llm_exceptions`", formatted_message) self.assertIn("Level: `Medium`", formatted_message) self.assertIn("Timestamp: `12:34:56`", formatted_message) self.assertIn("Message: Test alert message", formatted_message) From 4adfd18bc6483272b3a123a1d43913d9fdfea8c1 Mon Sep 17 00:00:00 2001 From: Sameer Kankute <135028480+kankute-sameer@users.noreply.github.com> Date: Tue, 2 Sep 2025 22:07:08 +0530 Subject: [PATCH 27/40] [Feat]Add support for safety_identifier parameter in chat.completions.create (#14174) * Add support for safety_identifier parameter in chat.completions.create * make sure param is getting actually passed to the raw api --- docs/my-website/docs/completion/input.md | 3 + litellm/constants.py | 5 +- .../llms/openai/chat/gpt_transformation.py | 1 + litellm/main.py | 4 ++ litellm/types/llms/openai.py | 1 + litellm/utils.py | 9 ++- tests/llm_translation/test_openai.py | 59 +++++++++++++++++++ 7 files changed, 76 insertions(+), 6 deletions(-) diff --git a/docs/my-website/docs/completion/input.md b/docs/my-website/docs/completion/input.md index 26629a0b8f8..9699d97b352 100644 --- a/docs/my-website/docs/completion/input.md +++ b/docs/my-website/docs/completion/input.md @@ -106,6 +106,7 @@ def completion( parallel_tool_calls: Optional[bool] = None, logprobs: Optional[bool] = None, top_logprobs: Optional[int] = None, + safety_identifier: Optional[str] = None, deployment_id=None, # soon to be deprecated params by OpenAI functions: Optional[List] = None, @@ -196,6 +197,8 @@ def completion( - `top_logprobs`: *int (optional)* - An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. `logprobs` must be set to true if this parameter is used. +- `safety_identifier`: *string (optional)* - A unique identifier for tracking and managing safety-related requests. This parameter helps with safety monitoring and compliance tracking. + - `headers`: *dict (optional)* - A dictionary of headers to be sent with the request. - `extra_headers`: *dict (optional)* - Alternative to `headers`, used to send extra headers in LLM API request. diff --git a/litellm/constants.py b/litellm/constants.py index 0655473301f..0803b61a2e8 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -14,7 +14,9 @@ DEFAULT_S3_BATCH_SIZE = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512)) DEFAULT_SQS_FLUSH_INTERVAL_SECONDS = int( os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10) ) -DEFAULT_NUM_WORKERS_LITELLM_PROXY = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 4)) +DEFAULT_NUM_WORKERS_LITELLM_PROXY = int( + os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 4) +) DEFAULT_SQS_BATCH_SIZE = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512)) SQS_SEND_MESSAGE_ACTION = "SendMessage" SQS_API_VERSION = "2012-11-05" @@ -395,6 +397,7 @@ DEFAULT_CHAT_COMPLETION_PARAM_VALUES = { "reasoning_effort": None, "thinking": None, "web_search_options": None, + "safety_identifier": None, } openai_compatible_endpoints: List = [ diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index be0ca3a7086..204916e3a48 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -158,6 +158,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): "parallel_tool_calls", "audio", "web_search_options", + "safety_identifier", ] # works across all models model_specific_params = [] diff --git a/litellm/main.py b/litellm/main.py index 786a0196e5e..9c2aa678369 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -357,6 +357,7 @@ async def acompletion( top_logprobs: Optional[int] = None, deployment_id=None, reasoning_effort: Optional[Literal["minimal", "low", "medium", "high"]] = None, + safety_identifier: Optional[str] = None, # set api_base, api_version, api_key base_url: Optional[str] = None, api_version: Optional[str] = None, @@ -493,6 +494,7 @@ async def acompletion( "api_key": api_key, "model_list": model_list, "reasoning_effort": reasoning_effort, + "safety_identifier": safety_identifier, "extra_headers": extra_headers, "acompletion": True, # assuming this is a required parameter "thinking": thinking, @@ -906,6 +908,7 @@ def completion( # type: ignore # noqa: PLR0915 web_search_options: Optional[OpenAIWebSearchOptions] = None, deployment_id=None, extra_headers: Optional[dict] = None, + safety_identifier: Optional[str] = None, # soon to be deprecated params by OpenAI functions: Optional[List] = None, function_call: Optional[str] = None, @@ -1243,6 +1246,7 @@ def completion( # type: ignore # noqa: PLR0915 "reasoning_effort": reasoning_effort, "thinking": thinking, "web_search_options": web_search_options, + "safety_identifier": safety_identifier, "allowed_openai_params": kwargs.get("allowed_openai_params"), } optional_params = get_optional_params( diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 6e7c4150774..5b58c232964 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -788,6 +788,7 @@ class ChatCompletionRequest(TypedDict, total=False): response_format: dict seed: int service_tier: str + safety_identifier: str stop: Union[str, List[str]] stream_options: dict temperature: float diff --git a/litellm/utils.py b/litellm/utils.py index 69f4603fea0..1601d01d3f0 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -837,15 +837,13 @@ async def _client_async_logging_helper( # Async Logging Worker ################################################ from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( - async_coroutine = logging_obj.async_success_handler( - result=result, - start_time=start_time, - end_time=end_time + async_coroutine=logging_obj.async_success_handler( + result=result, start_time=start_time, end_time=end_time ) ) - ################################################ # Sync Logging Worker ################################################ @@ -3304,6 +3302,7 @@ def get_optional_params( # noqa: PLR0915 messages: Optional[List[AllMessageValues]] = None, thinking: Optional[AnthropicThinkingParam] = None, web_search_options: Optional[OpenAIWebSearchOptions] = None, + safety_identifier: Optional[str] = None, **kwargs, ): passed_params = locals().copy() diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 0121eccaac3..619ae338e50 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -664,3 +664,62 @@ async def test_openai_gpt5_reasoning(): ) print("response: ", response) assert response.choices[0].message.content is not None + + +@pytest.mark.asyncio +async def test_openai_safety_identifier_parameter(): + """Test that safety_identifier parameter is correctly passed to the OpenAI API.""" + from openai import AsyncOpenAI + + litellm.set_verbose = True + client = AsyncOpenAI(api_key="fake-api-key") + + with patch.object( + client.chat.completions.with_raw_response, "create" + ) as mock_client: + try: + await litellm.acompletion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Hello, how are you?"}], + safety_identifier="user_code_123456", + client=client, + ) + except Exception as e: + print(f"Error: {e}") + + mock_client.assert_called_once() + request_body = mock_client.call_args.kwargs + + # Verify the request contains the safety_identifier parameter + assert "safety_identifier" in request_body + # Verify safety_identifier is correctly sent to the API + assert request_body["safety_identifier"] == "user_code_123456" + + +def test_openai_safety_identifier_parameter_sync(): + """Test that safety_identifier parameter is correctly passed to the OpenAI API.""" + from openai import OpenAI + + litellm.set_verbose = True + client = OpenAI(api_key="fake-api-key") + + with patch.object( + client.chat.completions.with_raw_response, "create" + ) as mock_client: + try: + litellm.completion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Hello, how are you?"}], + safety_identifier="user_code_123456", + client=client, + ) + except Exception as e: + print(f"Error: {e}") + + mock_client.assert_called_once() + request_body = mock_client.call_args.kwargs + + # Verify the request contains the safety_identifier parameter + assert "safety_identifier" in request_body + # Verify safety_identifier is correctly sent to the API + assert request_body["safety_identifier"] == "user_code_123456" From f0f84d6c5c8f9c040d49d628615ceab8969fa764 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 2 Sep 2025 12:12:21 -0700 Subject: [PATCH 28/40] refactor: BEDROCK_CONVERSE_MODELS --- litellm/__init__.py | 35 +---------------------------------- litellm/constants.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 34 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 79865c83513..453396f9a0f 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -67,6 +67,7 @@ from litellm.constants import ( bedrock_embedding_models, known_tokenizer_config, BEDROCK_INVOKE_PROVIDERS_LITERAL, + BEDROCK_CONVERSE_MODELS, DEFAULT_MAX_TOKENS, DEFAULT_SOFT_BUDGET, DEFAULT_ALLOWED_FAILS, @@ -432,40 +433,6 @@ organization = None project = None config_path = None vertex_ai_safety_settings: Optional[dict] = None -BEDROCK_CONVERSE_MODELS = [ - "openai.gpt-oss-20b-1:0", - "openai.gpt-oss-120b-1:0", - "anthropic.claude-opus-4-1-20250805-v1:0", - "anthropic.claude-opus-4-20250514-v1:0", - "anthropic.claude-sonnet-4-20250514-v1:0", - "anthropic.claude-3-7-sonnet-20250219-v1:0", - "anthropic.claude-3-5-haiku-20241022-v1:0", - "anthropic.claude-3-5-sonnet-20241022-v2:0", - "anthropic.claude-3-5-sonnet-20240620-v1:0", - "anthropic.claude-3-opus-20240229-v1:0", - "anthropic.claude-3-sonnet-20240229-v1:0", - "anthropic.claude-3-haiku-20240307-v1:0", - "anthropic.claude-v2", - "anthropic.claude-v2:1", - "anthropic.claude-v1", - "anthropic.claude-instant-v1", - "ai21.jamba-instruct-v1:0", - "ai21.jamba-1-5-mini-v1:0", - "ai21.jamba-1-5-large-v1:0", - "meta.llama3-70b-instruct-v1:0", - "meta.llama3-8b-instruct-v1:0", - "meta.llama3-1-8b-instruct-v1:0", - "meta.llama3-1-70b-instruct-v1:0", - "meta.llama3-1-405b-instruct-v1:0", - "meta.llama3-70b-instruct-v1:0", - "mistral.mistral-large-2407-v1:0", - "mistral.mistral-large-2402-v1:0", - "mistral.mistral-small-2402-v1:0", - "meta.llama3-2-1b-instruct-v1:0", - "meta.llama3-2-3b-instruct-v1:0", - "meta.llama3-2-11b-instruct-v1:0", - "meta.llama3-2-90b-instruct-v1:0", -] ####### COMPLETION MODELS ################### from typing import Set diff --git a/litellm/constants.py b/litellm/constants.py index 0803b61a2e8..21e30bef32b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -748,6 +748,42 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[ "deepseek_r1", ] +BEDROCK_CONVERSE_MODELS = [ + "openai.gpt-oss-20b-1:0", + "openai.gpt-oss-120b-1:0", + "anthropic.claude-opus-4-1-20250805-v1:0", + "anthropic.claude-opus-4-20250514-v1:0", + "anthropic.claude-sonnet-4-20250514-v1:0", + "anthropic.claude-3-7-sonnet-20250219-v1:0", + "anthropic.claude-3-5-haiku-20241022-v1:0", + "anthropic.claude-3-5-sonnet-20241022-v2:0", + "anthropic.claude-3-5-sonnet-20240620-v1:0", + "anthropic.claude-3-opus-20240229-v1:0", + "anthropic.claude-3-sonnet-20240229-v1:0", + "anthropic.claude-3-haiku-20240307-v1:0", + "anthropic.claude-v2", + "anthropic.claude-v2:1", + "anthropic.claude-v1", + "anthropic.claude-instant-v1", + "ai21.jamba-instruct-v1:0", + "ai21.jamba-1-5-mini-v1:0", + "ai21.jamba-1-5-large-v1:0", + "meta.llama3-70b-instruct-v1:0", + "meta.llama3-8b-instruct-v1:0", + "meta.llama3-1-8b-instruct-v1:0", + "meta.llama3-1-70b-instruct-v1:0", + "meta.llama3-1-405b-instruct-v1:0", + "meta.llama3-70b-instruct-v1:0", + "mistral.mistral-large-2407-v1:0", + "mistral.mistral-large-2402-v1:0", + "mistral.mistral-small-2402-v1:0", + "meta.llama3-2-1b-instruct-v1:0", + "meta.llama3-2-3b-instruct-v1:0", + "meta.llama3-2-11b-instruct-v1:0", + "meta.llama3-2-90b-instruct-v1:0", +] + + open_ai_embedding_models: set = set(["text-embedding-ada-002"]) cohere_embedding_models: set = set( [ From d282e7eaac853f5f4a475f211c66cc12d6dc5af9 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 2 Sep 2025 12:38:05 -0700 Subject: [PATCH 29/40] fix: handle non-str case --- litellm/llms/ollama/completion/transformation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 5689864017f..4a491c88963 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -331,8 +331,10 @@ class OllamaConfig(BaseConfig): response_text = response_json.get("response", "") content = None reasoning_content = None - if response_text is not None: + if response_text is not None and isinstance(response_text, str): reasoning_content, content = _parse_content_for_reasoning(response_text) + else: + content = response_text # type: ignore model_response.choices[0].message.content = content # type: ignore model_response.choices[0].message.reasoning_content = reasoning_content # type: ignore model_response.created = int(time.time()) From 47c04a8a8c415314db54b2d8af3076a0eb0534d3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 2 Sep 2025 14:13:17 -0700 Subject: [PATCH 30/40] proxy_logging_guardrails_model_info_tests --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b1c630cdcb3..c9de5adcc9f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1913,6 +1913,7 @@ jobs: -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ From 4b7c114c2af24e769154bfa57aface6b9245e06c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 2 Sep 2025 14:15:00 -0700 Subject: [PATCH 31/40] google-cloud-aiplatform --- .github/workflows/test-litellm.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml index 7e67aee8d73..0d3a9f2b5d4 100644 --- a/.github/workflows/test-litellm.yml +++ b/.github/workflows/test-litellm.yml @@ -31,6 +31,7 @@ jobs: poetry run pip install "pytest-retry==1.6.3" poetry run pip install pytest-xdist poetry run pip install "google-genai==1.22.0" + poetry run pip install "google-cloud-aiplatform>=1.38" poetry run pip install "fastapi-offline==1.7.3" - name: Setup litellm-enterprise as local package run: | From c821f1ddf1d5f7610d684e3b2b3270d5822d81be Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 2 Sep 2025 14:15:26 -0700 Subject: [PATCH 32/40] [Feature]: Support GPT-OSS models on vertex ai (#14184) * add VertexAIGPTOSSTransformation * fix: optional_params * fix: is_vertex_partner_model * test_partner_models_httpx * docs GPT oss docs * test_vertex_ai_gpt_oss_reasoning_effort * add vertex ai models --- .../docs/providers/vertex_partner.md | 136 +++++++++++ litellm/__init__.py | 4 + .../gpt_oss/transformation.py | 27 +++ .../vertex_ai_partner_models/main.py | 2 + ...odel_prices_and_context_window_backup.json | 22 ++ litellm/proxy/proxy_config.yaml | 1 - litellm/utils.py | 16 ++ model_prices_and_context_window.json | 22 ++ .../test_amazing_vertex_completion.py | 4 +- .../test_vertex_ai_gpt_oss_transformation.py | 223 ++++++++++++++++++ 10 files changed, 455 insertions(+), 2 deletions(-) create mode 100644 litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/transformation.py create mode 100644 tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py diff --git a/docs/my-website/docs/providers/vertex_partner.md b/docs/my-website/docs/providers/vertex_partner.md index cf780e35dbd..856f054b8e6 100644 --- a/docs/my-website/docs/providers/vertex_partner.md +++ b/docs/my-website/docs/providers/vertex_partner.md @@ -15,6 +15,7 @@ import TabItem from '@theme/TabItem'; | Mistral | `vertex_ai/mistral-*` | [Vertex AI - Mistral Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/mistral) | | AI21 (Jamba) | `vertex_ai/jamba-*` | [Vertex AI - AI21 Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/ai21) | | Qwen | `vertex_ai/qwen/*` | [Vertex AI - Qwen Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/qwen) | +| OpenAI (GPT-OSS) | `vertex_ai/openai/gpt-oss-*` | [Vertex AI - GPT-OSS Models](https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/) | | Model Garden | `vertex_ai/openai/{MODEL_ID}` or `vertex_ai/{MODEL_ID}` | [Vertex Model Garden](https://cloud.google.com/model-garden?hl=en) | ## Vertex AI - Anthropic (Claude) @@ -658,6 +659,141 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ +## VertexAI GPT-OSS Models + +| Property | Details | +|----------|---------| +| Provider Route | `vertex_ai/openai/{MODEL}` | +| Vertex Documentation | [Vertex AI - GPT-OSS Models](https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/) | + +**LiteLLM Supports all Vertex AI GPT-OSS Models.** Ensure you use the `vertex_ai/openai/` prefix for all Vertex AI GPT-OSS models. + +| Model Name | Usage | +|------------------|------------------------------| +| vertex_ai/openai/gpt-oss-20b-maas | `completion('vertex_ai/openai/gpt-oss-20b-maas', messages)` | + +#### Usage + + + + +```python +from litellm import completion +import os + +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "" + +model = "openai/gpt-oss-20b-maas" + +vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"] +vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"] + +response = completion( + model="vertex_ai/" + model, + messages=[{"role": "user", "content": "hi"}], + vertex_ai_project=vertex_ai_project, + vertex_ai_location=vertex_ai_location, +) +print("\nModel Response", response) +``` + + + +**1. Add to config** + +```yaml +model_list: + - model_name: gpt-oss + litellm_params: + model: vertex_ai/openai/gpt-oss-20b-maas + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-central1" +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-oss", # 👈 the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + +#### Usage - `reasoning_effort` + +GPT-OSS models support the `reasoning_effort` parameter for enhanced reasoning capabilities. + + + + +```python +from litellm import completion + +response = completion( + model="vertex_ai/openai/gpt-oss-20b-maas", + messages=[{"role": "user", "content": "Solve this complex problem step by step"}], + reasoning_effort="low", # Options: "minimal", "low", "medium", "high" + vertex_ai_project="your-vertex-project", + vertex_ai_location="us-central1", +) +``` + + + + + +1. Setup config.yaml + +```yaml +model_list: +- model_name: gpt-oss + litellm_params: + model: vertex_ai/openai/gpt-oss-20b-maas + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-central1" +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer " \ + -d '{ + "model": "gpt-oss", + "messages": [{"role": "user", "content": "Solve this complex problem step by step"}], + "reasoning_effort": "low" + }' +``` + + + + ## Model Garden :::tip diff --git a/litellm/__init__.py b/litellm/__init__.py index 453396f9a0f..6a184d70b5a 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -458,6 +458,7 @@ vertex_llama3_models: Set = set() vertex_deepseek_models: Set = set() vertex_ai_ai21_models: Set = set() vertex_mistral_models: Set = set() +vertex_openai_models: Set = set() ai21_models: Set = set() ai21_chat_models: Set = set() nlp_cloud_models: Set = set() @@ -604,6 +605,9 @@ def add_known_models(): elif value.get("litellm_provider") == "vertex_ai-image-models": key = key.replace("vertex_ai/", "") vertex_ai_image_models.add(key) + elif value.get("litellm_provider") == "vertex_ai-openai_models": + key = key.replace("vertex_ai/", "") + vertex_openai_models.add(key) elif value.get("litellm_provider") == "ai21": if value.get("mode") == "chat": ai21_chat_models.add(key) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/transformation.py new file mode 100644 index 00000000000..86e36e802ed --- /dev/null +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/transformation.py @@ -0,0 +1,27 @@ +import litellm +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + +class VertexAIGPTOSSTransformation(OpenAIGPTConfig): + """ + Transformation for GPT-OSS model from VertexAI + + https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/gpt-oss-120b-maas?hl=id + """ + def __init__(self): + super().__init__() + + def get_supported_openai_params(self, model: str) -> list: + base_gpt_series_params = super().get_supported_openai_params(model=model) + gpt_oss_only_params = ["reasoning_effort"] + base_gpt_series_params.extend(gpt_oss_only_params) + + ######################################################### + # VertexAI - GPT-OSS does not support tool calls + ######################################################### + if litellm.supports_function_calling(model=model) is False: + TOOL_CALLING_PARAMS_TO_REMOVE = ["tool", "tool_choice", "function_call", "functions"] + base_gpt_series_params = [param for param in base_gpt_series_params if param not in TOOL_CALLING_PARAMS_TO_REMOVE] + + return base_gpt_series_params + diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py index f281cab3b58..ee30c1749a8 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py @@ -49,6 +49,7 @@ class VertexAIPartnerModels(VertexBase): or model.startswith("jamba") or model.startswith("claude") or model.startswith("qwen") + or model.startswith("openai") ): return True return False @@ -59,6 +60,7 @@ class VertexAIPartnerModels(VertexBase): "llama", "deepseek-ai", "qwen", + "openai", ] if any(provider in model for provider in OPENAI_LIKE_VERTEX_PROVIDERS): return True diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 57309554621..7cedffd93f3 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9884,6 +9884,28 @@ "supports_tool_choice": true, "supports_prompt_caching": true }, + "vertex_ai/openai/gpt-oss-20b-maas": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 0.075e-06, + "output_cost_per_token": 0.30e-06, + "litellm_provider": "vertex_ai-openai_models", + "mode": "chat", + "supports_reasoning": true, + "source": "https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/gpt-oss-120b-maas" + }, + "vertex_ai/openai/gpt-oss-120b-maas": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 0.15e-06, + "output_cost_per_token": 0.60e-06, + "litellm_provider": "vertex_ai-openai_models", + "mode": "chat", + "supports_reasoning": true, + "source": "https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/gpt-oss-120b-maas" + }, "vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas": { "max_tokens": 32768, "max_input_tokens": 262144, diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index acff522196e..72c69a28e95 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -3,4 +3,3 @@ model_list: litellm_params: model: openai/* api_base: https://exampleopenaiendpoint-production-0ee2.up.railway.app/ - mock_response: "hi" diff --git a/litellm/utils.py b/litellm/utils.py index 1601d01d3f0..405d4cb98e4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3601,6 +3601,17 @@ def get_optional_params( # noqa: PLR0915 else False ), ) + elif provider_config is not None: + optional_params = provider_config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=( + drop_params + if drop_params is not None and isinstance(drop_params, bool) + else False + ), + ) else: # use generic openai-like param mapping optional_params = litellm.VertexAILlama3Config().map_openai_params( non_default_params=non_default_params, @@ -6864,6 +6875,11 @@ class ProviderConfigManager: return litellm.VertexGeminiConfig() elif "claude" in model: return litellm.VertexAIAnthropicConfig() + elif "gpt-oss" in model: + from litellm.llms.vertex_ai.vertex_ai_partner_models.gpt_oss.transformation import ( + VertexAIGPTOSSTransformation, + ) + return VertexAIGPTOSSTransformation() elif model in litellm.vertex_mistral_models: if "codestral" in model: return litellm.CodestralTextCompletionConfig() diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 57309554621..7cedffd93f3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -9884,6 +9884,28 @@ "supports_tool_choice": true, "supports_prompt_caching": true }, + "vertex_ai/openai/gpt-oss-20b-maas": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 0.075e-06, + "output_cost_per_token": 0.30e-06, + "litellm_provider": "vertex_ai-openai_models", + "mode": "chat", + "supports_reasoning": true, + "source": "https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/gpt-oss-120b-maas" + }, + "vertex_ai/openai/gpt-oss-120b-maas": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 0.15e-06, + "output_cost_per_token": 0.60e-06, + "litellm_provider": "vertex_ai-openai_models", + "mode": "chat", + "supports_reasoning": true, + "source": "https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/gpt-oss-120b-maas" + }, "vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas": { "max_tokens": 32768, "max_input_tokens": 262144, diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index b908eabd0cf..9b6fa868677 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -840,7 +840,8 @@ from test_completion import response_format_tests [ ("vertex_ai/mistral-large-2411", "us-central1"), ("vertex_ai/mistral-nemo@2407", "us-central1"), - ("vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas", "us-south1") + ("vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas", "us-south1"), + ("vertex_ai/openai/gpt-oss-20b-maas", "us-central1"), ], ) @pytest.mark.parametrize( @@ -911,6 +912,7 @@ async def test_partner_models_httpx(model, region, sync_mode): ("vertex_ai/meta/llama-4-scout-17b-16e-instruct-maas", "us-east5"), ("vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas", "us-south1"), ("vertex_ai/mistral-large-2411", "us-central1"), # critical - we had this issue: https://github.com/BerriAI/litellm/issues/13888 + ("vertex_ai/openai/gpt-oss-20b-maas", "us-central1"), ], ) @pytest.mark.parametrize( diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py new file mode 100644 index 00000000000..6743258bae6 --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py @@ -0,0 +1,223 @@ +import json +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../../..") +) # Adds the parent directory to the system path + +import litellm +from litellm.llms.vertex_ai.vertex_ai_partner_models.gpt_oss.transformation import ( + VertexAIGPTOSSTransformation, +) + + +class TestVertexAIGPTOSSTransformation: + """Test class for VertexAI GPT-OSS transformation functionality.""" + + def test_supports_reasoning_effort(self): + """Test that reasoning_effort parameter is supported for GPT-OSS models.""" + config = VertexAIGPTOSSTransformation() + supported_params = config.get_supported_openai_params(model="openai/gpt-oss-20b-maas") + + assert "reasoning_effort" in supported_params + + def test_removes_tool_calling_params_when_not_supported(self): + """Test that tool calling parameters are removed when function calling is not supported.""" + config = VertexAIGPTOSSTransformation() + + # Mock litellm.supports_function_calling to return False + with patch('litellm.supports_function_calling', return_value=False): + supported_params = config.get_supported_openai_params(model="openai/gpt-oss-20b-maas") + + # Tool calling params should be removed + assert "tool" not in supported_params + assert "tool_choice" not in supported_params + assert "function_call" not in supported_params + assert "functions" not in supported_params + + # But reasoning_effort should still be there + assert "reasoning_effort" in supported_params + + +@pytest.mark.asyncio +async def test_vertex_ai_gpt_oss_simple_request(): + """ + Test that a simple request to vertex_ai/openai/gpt-oss-20b-maas lands at the correct URL + with the correct request body. + """ + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + # Mock response + mock_response = AsyncMock() + mock_response.status_code = 200 + mock_response.headers = {} + mock_response.json.return_value = { + "id": "chatcmpl-test123", + "object": "chat.completion", + "created": 1234567890, + "model": "openai/gpt-oss-20b-maas", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! I'm Litellm Bot, a helpful assistant. I don't have access to real-time weather information, but I'd be happy to help you with other questions or tasks!" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 42, + "completion_tokens": 28, + "total_tokens": 70 + } + } + + client = AsyncHTTPHandler() + + with patch.object(client, "post", return_value=mock_response) as mock_post: + response = await litellm.acompletion( + model="vertex_ai/openai/gpt-oss-20b-maas", + messages=[ + { + "role": "system", + "content": "Your name is Litellm Bot, you are a helpful assistant" + }, + { + "role": "user", + "content": "Hello, what is your name and can you tell me the weather?" + } + ], + vertex_ai_location="us-central1", + vertex_ai_project="pathrise-convert-1606954137718", + client=client + ) + + # Verify the mock was called + mock_post.assert_called_once() + + # Get the call arguments + call_args = mock_post.call_args + called_url = call_args[0][0] # First positional argument is the URL + request_body = json.loads(call_args.kwargs["data"]) + + # Verify the URL + expected_url = "https://us-central1-aiplatform.googleapis.com/v1/projects/pathrise-convert-1606954137718/locations/us-central1/endpoints/openapi/chat/completions" + assert called_url == expected_url + + # Verify the request body + expected_request_body = { + 'model': 'openai/gpt-oss-20b-maas', + 'messages': [ + { + 'role': 'system', + 'content': 'Your name is Litellm Bot, you are a helpful assistant' + }, + { + 'role': 'user', + 'content': 'Hello, what is your name and can you tell me the weather?' + } + ], + 'stream': False + } + assert request_body == expected_request_body + + # Verify response structure + assert response.model == "vertex_ai/openai/gpt-oss-20b-maas" + assert len(response.choices) == 1 + assert response.choices[0].message.role == "assistant" + + +@pytest.mark.asyncio +async def test_vertex_ai_gpt_oss_reasoning_effort(): + """ + Test that reasoning_effort parameter is correctly passed in the request body + for GPT-OSS models. + """ + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + # Mock response + mock_response = AsyncMock() + mock_response.status_code = 200 + mock_response.headers = {} + mock_response.json.return_value = { + "id": "chatcmpl-test456", + "object": "chat.completion", + "created": 1234567890, + "model": "openai/gpt-oss-20b-maas", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "I need to think about this carefully. The weather varies by location and time, so I would need to know your specific location to provide accurate weather information." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 35, + "completion_tokens": 32, + "total_tokens": 67 + } + } + + client = AsyncHTTPHandler() + + with patch.object(client, "post", return_value=mock_response) as mock_post: + response = await litellm.acompletion( + model="vertex_ai/openai/gpt-oss-20b-maas", + messages=[ + { + "role": "system", + "content": "Your name is Litellm Bot, you are a helpful assistant" + }, + { + "role": "user", + "content": "Hello, what is your name and can you tell me the weather?" + } + ], + reasoning_effort="low", + vertex_ai_location="us-central1", + vertex_ai_project="pathrise-convert-1606954137718", + client=client + ) + + # Verify the mock was called + mock_post.assert_called_once() + + # Get the call arguments + call_args = mock_post.call_args + request_body = json.loads(call_args.kwargs["data"]) + + # Verify reasoning_effort is in the request body + assert "reasoning_effort" in request_body + assert request_body["reasoning_effort"] == "low" + + # Verify other expected fields + expected_request_body = { + 'model': 'openai/gpt-oss-20b-maas', + 'messages': [ + { + 'role': 'system', + 'content': 'Your name is Litellm Bot, you are a helpful assistant' + }, + { + 'role': 'user', + 'content': 'Hello, what is your name and can you tell me the weather?' + } + ], + 'reasoning_effort': 'low', + 'stream': False + } + assert request_body == expected_request_body + + # Verify response structure + assert response.model == "vertex_ai/openai/gpt-oss-20b-maas" + assert len(response.choices) == 1 + assert response.choices[0].message.role == "assistant" From 128d9a348816ed7a2bc0444016f613340bfaeded Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 2 Sep 2025 15:13:15 -0700 Subject: [PATCH 33/40] [Feature]: Add header support for spend_logs_metadata (#14186) * fix: allow settings spend_logs_metadata * fix add_litellm_data_for_backend_llm_call * fix: add add_litellm_metadata_from_request_headers * fix add_litellm_metadata_from_request_headers * test_add_litellm_metadata_from_request_headers * add_litellm_metadata_from_request_headers * docs Tracking Spend with custom metadata * add_litellm_metadata_from_request_headers * add_litellm_metadata_from_request_headers --- docs/my-website/docs/proxy/enterprise.md | 87 ++++++++++++ docs/my-website/docs/proxy/request_headers.md | 2 + litellm/proxy/_types.py | 6 + litellm/proxy/litellm_pre_call_utils.py | 52 +++++++- .../proxy/test_litellm_pre_call_utils.py | 124 ++++++++++++++++++ 5 files changed, 266 insertions(+), 5 deletions(-) diff --git a/docs/my-website/docs/proxy/enterprise.md b/docs/my-website/docs/proxy/enterprise.md index 468bcad2cf8..7d50aedb424 100644 --- a/docs/my-website/docs/proxy/enterprise.md +++ b/docs/my-website/docs/proxy/enterprise.md @@ -439,6 +439,33 @@ response = client.chat.completions.create( print(response) ``` + +**Using Headers:** + +```python +import openai +client = openai.OpenAI( + api_key="sk-1234", + base_url="http://0.0.0.0:4000" +) + +# Pass spend logs metadata via headers +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_headers={ + "x-litellm-spend-logs-metadata": '{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}' + } +) + +print(response) +``` + @@ -478,6 +505,43 @@ async function runOpenAI() { // Call the asynchronous function runOpenAI(); ``` + +**Using Headers:** + +```js +const openai = require('openai'); + +async function runOpenAI() { + const client = new openai.OpenAI({ + apiKey: 'sk-1234', + baseURL: 'http://0.0.0.0:4000' + }); + + try { + const response = await client.chat.completions.create({ + model: 'gpt-3.5-turbo', + messages: [ + { + role: 'user', + content: "this is a test request, write a short poem" + }, + ] + }, { + headers: { + 'x-litellm-spend-logs-metadata': '{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}' + } + }); + console.log(response); + } catch (error) { + console.log("got this exception from server"); + console.error(error); + } +} + +// Call the asynchronous function +runOpenAI(); +``` + @@ -502,6 +566,29 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ } }' ``` + + + + + +Pass `x-litellm-spend-logs-metadata` as a request header with JSON string + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'x-litellm-spend-logs-metadata: {"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}' \ + --data '{ + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + diff --git a/docs/my-website/docs/proxy/request_headers.md b/docs/my-website/docs/proxy/request_headers.md index c250d42f7bb..347dbe6a1bc 100644 --- a/docs/my-website/docs/proxy/request_headers.md +++ b/docs/my-website/docs/proxy/request_headers.md @@ -14,6 +14,8 @@ Special headers that are supported by LiteLLM. `x-litellm-num-retries`: Optional[int]: The number of retries for the request. +`x-litellm-spend-logs-metadata`: Optional[str]: JSON string containing custom metadata to include in spend logs. Example: `{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}`. [Learn More](./logging#tracking-spend-with-custom-metadata) + ## Anthropic Headers `anthropic-version` Optional[str]: The version of the Anthropic API to use. diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 87c5bd7a7c5..7206d3dcb0e 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2908,6 +2908,12 @@ class LitellmDataForBackendLLMCall(TypedDict, total=False): user: Optional[str] num_retries: Optional[int] +class LitellmMetadataFromRequestHeaders(TypedDict, total=False): + """ + Headers a user can pass that will get added to litellm metadata for the request + """ + spend_logs_metadata: Optional[dict] + class JWTKeyItem(TypedDict, total=False): kid: str diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 32a283066ed..557aa3ceb9a 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -291,6 +291,17 @@ class LiteLLMProxyRequestSetup: if num_retries_header is not None: return int(num_retries_header) return None + + @staticmethod + def _get_spend_logs_metadata_from_request_headers(headers: dict) -> Optional[dict]: + """ + Get the `spend_logs_metadata` from the request headers. + """ + from litellm.litellm_core_utils.safe_json_loads import safe_json_loads + spend_logs_metadata_header = headers.get("x-litellm-spend-logs-metadata", None) + if spend_logs_metadata_header is not None: + return safe_json_loads(spend_logs_metadata_header) + return None @staticmethod def _get_forwardable_headers( @@ -459,6 +470,30 @@ class LiteLLMProxyRequestSetup: data["num_retries"] = num_retries return data + + @staticmethod + def add_litellm_metadata_from_request_headers( + headers: dict, + data: dict, + _metadata_variable_name: str, + ) -> dict: + """ + Add litellm metadata from request headers + + Relevant issue: https://github.com/BerriAI/litellm/issues/14008 + """ + from litellm.proxy._types import LitellmMetadataFromRequestHeaders + metadata_from_headers = LitellmMetadataFromRequestHeaders() + spend_logs_metadata = LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers(headers) + if spend_logs_metadata is not None: + metadata_from_headers["spend_logs_metadata"] = spend_logs_metadata + + ######################################################################################### + # Finally update the requests metadata with the `metadata_from_headers` + ######################################################################################### + if isinstance(data[_metadata_variable_name], dict): + data[_metadata_variable_name].update(metadata_from_headers) + return data @staticmethod def get_sanitized_user_information_from_key( @@ -643,6 +678,10 @@ async def add_litellm_data_to_request( # noqa: PLR0915 from litellm.types.proxy.litellm_pre_call_utils import SecretFields safe_add_api_version_from_query_params(data, request) + _metadata_variable_name = _get_metadata_variable_name(request) + if data.get(_metadata_variable_name, None) is None: + data[_metadata_variable_name] = {} + _headers = clean_headers( request.headers, @@ -661,6 +700,14 @@ async def add_litellm_data_to_request( # noqa: PLR0915 ) ) + data.update( + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers=_headers, + data=data, + _metadata_variable_name=_metadata_variable_name, + ) + ) + # check for forwardable headers data = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group( data=data, headers=_headers, user_api_key_dict=user_api_key_dict @@ -711,11 +758,6 @@ async def add_litellm_data_to_request( # noqa: PLR0915 verbose_proxy_logger.debug("receiving data: %s", data) - _metadata_variable_name = _get_metadata_variable_name(request) - - if data.get(_metadata_variable_name, None) is None: - data[_metadata_variable_name] = {} - # Parse metadata if it's a string (e.g., from multipart/form-data) if "metadata" in data and data["metadata"] is not None: if isinstance(data["metadata"], str): diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 5104ffd80de..817f19d8d7d 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -8,6 +8,7 @@ from unittest.mock import MagicMock, patch import pytest from fastapi import Request +import litellm from litellm.proxy._types import TeamCallbackMetadata, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import ( KeyAndTeamLoggingSettings, @@ -935,3 +936,126 @@ def test_add_headers_to_llm_call_by_model_group_existing_headers_in_data(): finally: # Restore original model_group_settings litellm.model_group_settings = original_model_group_settings + +import json +import time +from typing import Optional +from unittest.mock import AsyncMock + +from fastapi.responses import Response + +from litellm.integrations.custom_logger import CustomLogger +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.utils import ProxyLogging +from litellm.types.utils import StandardLoggingPayload + + +class TestCustomLogger(CustomLogger): + def __init__(self): + self.standard_logging_object: Optional[StandardLoggingPayload] = None + super().__init__() + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + print(f"SUCCESS CALLBACK CALLED! kwargs keys: {list(kwargs.keys())}") + self.standard_logging_object = kwargs.get("standard_logging_object") + print(f"Captured standard_logging_object: {self.standard_logging_object}") + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + print(f"FAILURE CALLBACK CALLED! kwargs keys: {list(kwargs.keys())}") + +@pytest.mark.asyncio +async def test_add_litellm_metadata_from_request_headers(): + """ + Test that add_litellm_metadata_from_request_headers properly adds litellm metadata from request headers, + makes an LLM request using base_process_llm_request, sleeps for 3 seconds, and checks standard_logging_payload has spend_logs_metadata from headers + + Relevant issue: https://github.com/BerriAI/litellm/issues/14008 + """ + # Set up test logger + litellm._turn_on_debug() + test_logger = TestCustomLogger() + litellm.callbacks = [test_logger] + + # Prepare test data (ensure no streaming, add mock_response and api_key to route to litellm.acompletion) + headers = {"x-litellm-spend-logs-metadata": '{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion", "timestamp": "2025-09-02T10:30:00Z"}'} + data = {"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}], "stream": False, "mock_response": "Hi", "api_key": "fake-key"} + + # Create mock request with headers + mock_request = MagicMock(spec=Request) + mock_request.headers = headers + mock_request.url.path = "/chat/completions" + + # Create mock response + mock_fastapi_response = MagicMock(spec=Response) + + # Create mock user API key dict + mock_user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + user_id="test-user", + org_id="test-org" + ) + + # Create mock proxy logging object + mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) + + # Create async functions for the hooks + async def mock_during_call_hook(*args, **kwargs): + return None + + async def mock_pre_call_hook(*args, **kwargs): + return data + + async def mock_post_call_success_hook(*args, **kwargs): + # Return the response unchanged + return kwargs.get('response', args[2] if len(args) > 2 else None) + + mock_proxy_logging_obj.during_call_hook = mock_during_call_hook + mock_proxy_logging_obj.pre_call_hook = mock_pre_call_hook + mock_proxy_logging_obj.post_call_success_hook = mock_post_call_success_hook + + # Create mock proxy config + mock_proxy_config = MagicMock() + + # Create mock general settings + general_settings = {} + + # Create mock select_data_generator with correct signature + def mock_select_data_generator(response=None, user_api_key_dict=None, request_data=None): + async def mock_generator(): + yield "data: " + json.dumps({"choices": [{"delta": {"content": "Hello"}}]}) + "\n\n" + yield "data: [DONE]\n\n" + return mock_generator() + + # Create the processor + processor = ProxyBaseLLMRequestProcessing(data=data) + + # Call base_process_llm_request (it will use the mock_response="Hi" parameter) + result = await processor.base_process_llm_request( + request=mock_request, + fastapi_response=mock_fastapi_response, + user_api_key_dict=mock_user_api_key_dict, + route_type="acompletion", + proxy_logging_obj=mock_proxy_logging_obj, + general_settings=general_settings, + proxy_config=mock_proxy_config, + select_data_generator=mock_select_data_generator, + llm_router=None, + model="gpt-4", + is_streaming_request=False + ) + + # Sleep for 3 seconds to allow logging to complete + await asyncio.sleep(3) + + # Check if standard_logging_object was set + assert test_logger.standard_logging_object is not None, "standard_logging_object should be populated after LLM request" + + # Verify the logging object contains expected metadata + standard_logging_obj = test_logger.standard_logging_object + + print(f"Standard logging object captured: {json.dumps(standard_logging_obj, indent=4, default=str)}") + + SPEND_LOGS_METADATA = standard_logging_obj["metadata"]["spend_logs_metadata"] + assert SPEND_LOGS_METADATA == dict(json.loads(headers["x-litellm-spend-logs-metadata"])), "spend_logs_metadata should be the same as the headers" + + From 212ca20edfe4af1cbd117d4f950e862b69407f73 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 2 Sep 2025 15:19:35 -0700 Subject: [PATCH 34/40] docs fix --- docs/my-website/docs/proxy/request_headers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/request_headers.md b/docs/my-website/docs/proxy/request_headers.md index 347dbe6a1bc..eea66e5fa93 100644 --- a/docs/my-website/docs/proxy/request_headers.md +++ b/docs/my-website/docs/proxy/request_headers.md @@ -14,7 +14,7 @@ Special headers that are supported by LiteLLM. `x-litellm-num-retries`: Optional[int]: The number of retries for the request. -`x-litellm-spend-logs-metadata`: Optional[str]: JSON string containing custom metadata to include in spend logs. Example: `{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}`. [Learn More](./logging#tracking-spend-with-custom-metadata) +`x-litellm-spend-logs-metadata`: Optional[str]: JSON string containing custom metadata to include in spend logs. Example: `{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}`. [Learn More](../proxy/enterprise#tracking-spend-with-custom-metadata) ## Anthropic Headers From edf047b6bb9f0592dfb819b76177c4c7b5d6354c Mon Sep 17 00:00:00 2001 From: kayoch1n Date: Wed, 3 Sep 2025 10:00:16 +0800 Subject: [PATCH 35/40] Replace "/" with "-" in model name when being used as a h11 header name --- litellm/proxy/common_utils/callback_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index e718255750a..46c9afef3cc 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -316,18 +316,21 @@ def get_remaining_tokens_and_requests_from_request_data(data: Dict) -> Dict[str, headers = {} _metadata = data.get("metadata", None) or {} model_group = get_model_group_from_request_data(data) + + # The h11 package considers "/" or ":" invalid and raise a LocalProtocolError + h11_model_group_name = model_group.replace('/', '-').replace(':', '-') # Remaining Requests remaining_requests_variable_name = f"litellm-key-remaining-requests-{model_group}" remaining_requests = _metadata.get(remaining_requests_variable_name, None) if remaining_requests: - headers[f"x-litellm-key-remaining-requests-{model_group}"] = remaining_requests + headers[f"x-litellm-key-remaining-requests-{h11_model_group_name}"] = remaining_requests # Remaining Tokens remaining_tokens_variable_name = f"litellm-key-remaining-tokens-{model_group}" remaining_tokens = _metadata.get(remaining_tokens_variable_name, None) if remaining_tokens: - headers[f"x-litellm-key-remaining-tokens-{model_group}"] = remaining_tokens + headers[f"x-litellm-key-remaining-tokens-{h11_model_group_name}"] = remaining_tokens return headers From ffbe5cd899d883bed3f1c354a85d0992729ba535 Mon Sep 17 00:00:00 2001 From: kayoch1n Date: Wed, 3 Sep 2025 10:57:09 +0800 Subject: [PATCH 36/40] Add a testcase --- .../proxy/common_utils/test_callback_utils.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tests/test_litellm/proxy/common_utils/test_callback_utils.py diff --git a/tests/test_litellm/proxy/common_utils/test_callback_utils.py b/tests/test_litellm/proxy/common_utils/test_callback_utils.py new file mode 100644 index 00000000000..ffcab13571b --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_callback_utils.py @@ -0,0 +1,27 @@ +import sys +import os + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path + +from litellm.proxy.common_utils.callback_utils import get_remaining_tokens_and_requests_from_request_data + +def test_get_remaining_tokens_and_requests_from_request_data(): + model_group = "openrouter/google/gemini-2.0-flash-001" + casedata = { + "metadata": { + "model_group": model_group, + f"litellm-key-remaining-requests-{model_group}": 100, + f"litellm-key-remaining-tokens-{model_group}": 200 + } + } + + headers = get_remaining_tokens_and_requests_from_request_data(casedata) + + expected_name = "openrouter-google-gemini-2.0-flash-001" + assert headers == { + f"x-litellm-key-remaining-requests-{expected_name}": 100, + f"x-litellm-key-remaining-tokens-{expected_name}": 200 + } + From 1a97a80c519ab57413293fd45849613838ed0a8e Mon Sep 17 00:00:00 2001 From: kayoch1n Date: Wed, 3 Sep 2025 11:18:57 +0800 Subject: [PATCH 37/40] Format code --- litellm/proxy/common_utils/callback_utils.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 46c9afef3cc..d52592952bc 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -316,21 +316,27 @@ def get_remaining_tokens_and_requests_from_request_data(data: Dict) -> Dict[str, headers = {} _metadata = data.get("metadata", None) or {} model_group = get_model_group_from_request_data(data) - + # The h11 package considers "/" or ":" invalid and raise a LocalProtocolError - h11_model_group_name = model_group.replace('/', '-').replace(':', '-') + h11_model_group_name = ( + model_group.replace("/", "-").replace(":", "-") if model_group else None + ) # Remaining Requests remaining_requests_variable_name = f"litellm-key-remaining-requests-{model_group}" remaining_requests = _metadata.get(remaining_requests_variable_name, None) if remaining_requests: - headers[f"x-litellm-key-remaining-requests-{h11_model_group_name}"] = remaining_requests + headers[f"x-litellm-key-remaining-requests-{h11_model_group_name}"] = ( + remaining_requests + ) # Remaining Tokens remaining_tokens_variable_name = f"litellm-key-remaining-tokens-{model_group}" remaining_tokens = _metadata.get(remaining_tokens_variable_name, None) if remaining_tokens: - headers[f"x-litellm-key-remaining-tokens-{h11_model_group_name}"] = remaining_tokens + headers[f"x-litellm-key-remaining-tokens-{h11_model_group_name}"] = ( + remaining_tokens + ) return headers From 76555cad81633b9bba5938419af22644da8674a0 Mon Sep 17 00:00:00 2001 From: kayoch1n Date: Wed, 3 Sep 2025 11:20:52 +0800 Subject: [PATCH 38/40] Format code --- .../proxy/common_utils/test_callback_utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/proxy/common_utils/test_callback_utils.py b/tests/test_litellm/proxy/common_utils/test_callback_utils.py index ffcab13571b..b9ed4b9b508 100644 --- a/tests/test_litellm/proxy/common_utils/test_callback_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_callback_utils.py @@ -5,7 +5,10 @@ sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system path -from litellm.proxy.common_utils.callback_utils import get_remaining_tokens_and_requests_from_request_data +from litellm.proxy.common_utils.callback_utils import ( + get_remaining_tokens_and_requests_from_request_data, +) + def test_get_remaining_tokens_and_requests_from_request_data(): model_group = "openrouter/google/gemini-2.0-flash-001" @@ -13,7 +16,7 @@ def test_get_remaining_tokens_and_requests_from_request_data(): "metadata": { "model_group": model_group, f"litellm-key-remaining-requests-{model_group}": 100, - f"litellm-key-remaining-tokens-{model_group}": 200 + f"litellm-key-remaining-tokens-{model_group}": 200, } } @@ -22,6 +25,5 @@ def test_get_remaining_tokens_and_requests_from_request_data(): expected_name = "openrouter-google-gemini-2.0-flash-001" assert headers == { f"x-litellm-key-remaining-requests-{expected_name}": 100, - f"x-litellm-key-remaining-tokens-{expected_name}": 200 + f"x-litellm-key-remaining-tokens-{expected_name}": 200, } - From 54cca0cc7e12819727a98f9c8ed61177f9ff0dda Mon Sep 17 00:00:00 2001 From: zhxlp <1573635222@qq.com> Date: Wed, 3 Sep 2025 11:34:58 +0800 Subject: [PATCH 39/40] fix: Log page parameter passing error --- ui/litellm-dashboard/src/components/view_logs/index.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ui/litellm-dashboard/src/components/view_logs/index.tsx b/ui/litellm-dashboard/src/components/view_logs/index.tsx index c45116c0ddd..6c9ef66e95b 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/index.tsx @@ -80,6 +80,7 @@ export default function SpendLogsTable({ const [selectedKeyInfo, setSelectedKeyInfo] = useState(null) const [selectedKeyIdInfoView, setSelectedKeyIdInfoView] = useState(null) const [selectedStatus, setSelectedStatus] = useState("") + const [selectedEndUser, setSelectedEndUser] = useState("") const [filterByCurrentUser, setFilterByCurrentUser] = useState(userRole && internalUserRoles.includes(userRole)) const [activeTab, setActiveTab] = useState("request logs") @@ -193,6 +194,7 @@ export default function SpendLogsTable({ currentPage, pageSize, filterByCurrentUser ? userID : undefined, + selectedEndUser, selectedStatus, selectedModel, ) @@ -280,6 +282,7 @@ export default function SpendLogsTable({ } setSelectedStatus(filters["Status"] || "") setSelectedModel(filters["Model"] || "") + setSelectedEndUser(filters["End User"] || "") if (filters["Key Hash"]) { setSelectedKeyHash(filters["Key Hash"]) From 63c4a30564b06f2aa4a8c6011069c6f6b22ea1f2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 2 Sep 2025 23:07:32 -0700 Subject: [PATCH 40/40] TestVertexAIGPTOSSTransformation --- .../test_vertex_ai_gpt_oss_transformation.py | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py index 6743258bae6..34046a00ee8 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py @@ -1,7 +1,7 @@ import json import os import sys -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import MagicMock, patch import httpx import pytest @@ -51,9 +51,12 @@ async def test_vertex_ai_gpt_oss_simple_request(): with the correct request body. """ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexLLM, + ) # Mock response - mock_response = AsyncMock() + mock_response = MagicMock() mock_response.status_code = 200 mock_response.headers = {} mock_response.json.return_value = { @@ -80,7 +83,11 @@ async def test_vertex_ai_gpt_oss_simple_request(): client = AsyncHTTPHandler() - with patch.object(client, "post", return_value=mock_response) as mock_post: + async def mock_post_func(*args, **kwargs): + return mock_response + + with patch.object(client, "post", side_effect=mock_post_func) as mock_post, \ + patch.object(VertexLLM, "_ensure_access_token", return_value=("fake-token", "pathrise-convert-1606954137718")): response = await litellm.acompletion( model="vertex_ai/openai/gpt-oss-20b-maas", messages=[ @@ -103,7 +110,8 @@ async def test_vertex_ai_gpt_oss_simple_request(): # Get the call arguments call_args = mock_post.call_args - called_url = call_args[0][0] # First positional argument is the URL + # For side_effect, the URL is passed as kwargs['url'] + called_url = call_args.kwargs["url"] request_body = json.loads(call_args.kwargs["data"]) # Verify the URL @@ -128,7 +136,7 @@ async def test_vertex_ai_gpt_oss_simple_request(): assert request_body == expected_request_body # Verify response structure - assert response.model == "vertex_ai/openai/gpt-oss-20b-maas" + assert response.model == "openai/gpt-oss-20b-maas" assert len(response.choices) == 1 assert response.choices[0].message.role == "assistant" @@ -140,9 +148,12 @@ async def test_vertex_ai_gpt_oss_reasoning_effort(): for GPT-OSS models. """ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexLLM, + ) # Mock response - mock_response = AsyncMock() + mock_response = MagicMock() mock_response.status_code = 200 mock_response.headers = {} mock_response.json.return_value = { @@ -169,7 +180,11 @@ async def test_vertex_ai_gpt_oss_reasoning_effort(): client = AsyncHTTPHandler() - with patch.object(client, "post", return_value=mock_response) as mock_post: + async def mock_post_func(*args, **kwargs): + return mock_response + + with patch.object(client, "post", side_effect=mock_post_func) as mock_post, \ + patch.object(VertexLLM, "_ensure_access_token", return_value=("fake-token", "pathrise-convert-1606954137718")): response = await litellm.acompletion( model="vertex_ai/openai/gpt-oss-20b-maas", messages=[ @@ -218,6 +233,6 @@ async def test_vertex_ai_gpt_oss_reasoning_effort(): assert request_body == expected_request_body # Verify response structure - assert response.model == "vertex_ai/openai/gpt-oss-20b-maas" + assert response.model == "openai/gpt-oss-20b-maas" assert len(response.choices) == 1 assert response.choices[0].message.role == "assistant"