From 8a5df2eb744c381231a799df3f905cd485b9cafc Mon Sep 17 00:00:00 2001 From: Zain Hasan Date: Sat, 30 Aug 2025 00:44:42 -0400 Subject: [PATCH 01/46] update together models - add tool choice for models - add deepseek v3.1 --- model_prices_and_context_window.json | 54 +++++++++++++++++----------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c9658c6cc47..d249341954b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14972,10 +14972,10 @@ "output_cost_per_token": 6e-06, "max_input_tokens": 262000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, "mode": "chat", - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8" }, "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { @@ -14983,10 +14983,10 @@ "output_cost_per_token": 2e-06, "max_input_tokens": 256000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, "mode": "chat", - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct" }, "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { @@ -14994,10 +14994,10 @@ "output_cost_per_token": 3e-06, "max_input_tokens": 256000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, "mode": "chat", - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507" }, "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": { @@ -15040,10 +15040,10 @@ "output_cost_per_token": 2.19e-06, "max_input_tokens": 128000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, "mode": "chat", - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.together.ai/models/deepseek-r1-0528-throughput" }, "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": { @@ -15068,9 +15068,9 @@ "output_cost_per_token": 6e-07, "max_input_tokens": 128000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_tool_choice": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_parallel_function_calling": true, "mode": "chat", "source": "https://www.together.ai/models/gpt-oss-120b" }, @@ -15079,9 +15079,9 @@ "output_cost_per_token": 2e-07, "max_input_tokens": 128000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_tool_choice": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_parallel_function_calling": true, "mode": "chat", "source": "https://www.together.ai/models/gpt-oss-20b" }, @@ -15090,12 +15090,24 @@ "output_cost_per_token": 1.1e-06, "max_input_tokens": 128000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_tool_choice": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_parallel_function_calling": true, "mode": "chat", "source": "https://www.together.ai/models/glm-4-5-air" }, + "together_ai/deepseek-ai/DeepSeek-V3.1": { + "input_cost_per_token": 0.6e-06, + "output_cost_per_token": 1.7e-06, + "max_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "mode": "chat", + "supports_tool_choice": true, + "source": "https://www.together.ai/models/deepseek-v3-1" + }, "ollama/codegemma": { "max_tokens": 8192, "max_input_tokens": 8192, From cb5464421f751afd58df39a8dfe1719e20feecc2 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 30 Aug 2025 16:11:39 -0700 Subject: [PATCH 02/46] fix(braintrust_logging.py): filter metadata before logging avoid unserializable json --- litellm/integrations/braintrust_logging.py | 13 +-- litellm/litellm_core_utils/safe_json_dumps.py | 92 +++++++++++++++++++ litellm/proxy/_new_secret_config.yaml | 10 +- 3 files changed, 100 insertions(+), 15 deletions(-) diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index 531da933fcc..5238bfe1dbf 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -12,6 +12,7 @@ from pydantic import BaseModel import litellm from litellm import verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.safe_json_dumps import filter_json_serializable from litellm.llms.custom_httpx.http_handler import ( HTTPHandler, get_async_httpx_client, @@ -45,9 +46,9 @@ class BraintrustLogger(CustomLogger): "Authorization": "Bearer " + self.api_key, "Content-Type": "application/json", } - self._project_id_cache: Dict[ - str, str - ] = {} # Cache mapping project names to IDs + self._project_id_cache: Dict[str, str] = ( + {} + ) # Cache mapping project names to IDs self.global_braintrust_http_handler = get_async_httpx_client( llm_provider=httpxSpecialProvider.LoggingCallback ) @@ -276,7 +277,7 @@ class BraintrustLogger(CustomLogger): # Allow metadata override for span name span_name = metadata.get("span_name", "Chat Completion") - + request_data = { "id": litellm_call_id, "input": prompt["messages"], @@ -431,12 +432,12 @@ class BraintrustLogger(CustomLogger): # Allow metadata override for span name span_name = metadata.get("span_name", "Chat Completion") - + request_data = { "id": litellm_call_id, "input": prompt["messages"], "output": output, - "metadata": clean_metadata, + "metadata": filter_json_serializable(clean_metadata), "tags": tags, "span_attributes": {"name": span_name, "type": "llm"}, } diff --git a/litellm/litellm_core_utils/safe_json_dumps.py b/litellm/litellm_core_utils/safe_json_dumps.py index 7ad0038ecb2..b3b1d7fb3df 100644 --- a/litellm/litellm_core_utils/safe_json_dumps.py +++ b/litellm/litellm_core_utils/safe_json_dumps.py @@ -1,5 +1,6 @@ import json from typing import Any, Union + from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH @@ -49,3 +50,94 @@ def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str: safe_data = _serialize(data, set(), 0) return json.dumps(safe_data, default=str) + + +def filter_json_serializable( + data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH +) -> Any: + """ + Recursively filter data to only include JSON serializable items. + Non-serializable items are completely skipped (not included in the result). + """ + + def _is_json_serializable(obj: Any) -> bool: + """Test if an object is JSON serializable.""" + try: + json.dumps(obj) + return True + except (TypeError, ValueError): + return False + + def _filter(obj: Any, seen: set, depth: int) -> Any: + # Check for maximum depth. + if depth > max_depth: + return None + + # Base-case: if it is a primitive, test if it's serializable + if isinstance(obj, (str, int, float, bool, type(None))): + return obj if _is_json_serializable(obj) else None + + # Check for circular reference. + if id(obj) in seen: + return None + + seen.add(id(obj)) + + try: + if isinstance(obj, dict): + result = {} + for k, v in obj.items(): + # Only include keys that are strings and values that are serializable + if isinstance(k, str): + filtered_value = _filter(v, seen, depth + 1) + # Only add the key-value pair if the value is serializable + if filtered_value is not None or v is None: + if _is_json_serializable(filtered_value): + result[k] = filtered_value + seen.remove(id(obj)) + return result + + elif isinstance(obj, list): + result = [] + for item in obj: + filtered_item = _filter(item, seen, depth + 1) + # Only include items that are serializable + if filtered_item is not None or item is None: + if _is_json_serializable(filtered_item): + result.append(filtered_item) + seen.remove(id(obj)) + return result + + elif isinstance(obj, tuple): + filtered_items = [] + for item in obj: + filtered_item = _filter(item, seen, depth + 1) + # Only include items that are serializable + if filtered_item is not None or item is None: + if _is_json_serializable(filtered_item): + filtered_items.append(filtered_item) + seen.remove(id(obj)) + return tuple(filtered_items) + + elif isinstance(obj, set): + filtered_items = [] + for item in obj: + filtered_item = _filter(item, seen, depth + 1) + # Only include items that are serializable + if filtered_item is not None or item is None: + if _is_json_serializable(filtered_item): + filtered_items.append(filtered_item) + seen.remove(id(obj)) + return sorted(filtered_items) + + else: + # Test if the object is directly serializable + seen.remove(id(obj)) + return obj if _is_json_serializable(obj) else None + + except Exception: + if id(obj) in seen: + seen.remove(id(obj)) + return None + + return _filter(data, set(), 0) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index b3653c31435..f4dc1fca711 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -19,12 +19,4 @@ router_settings: litellm_settings: callbacks: ["otel"] - cache: true - cache_params: - type: redis - ttl: 600 - supported_call_types: ["acompletion", "completion"] - - model_group_settings: - forward_client_headers_to_llm_api: - - fake-openai-endpoint \ No newline at end of file + success_callback: ["braintrust"] \ No newline at end of file From 9cffabb433cae113facae5b54e3f62cf45b8cb4d Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 30 Aug 2025 16:17:57 -0700 Subject: [PATCH 03/46] refactor(braintrust_logging.py): migrate braintrust logging to standard logging payload avoids issue with span in request metadata standard logging payload is always json serializable --- litellm/integrations/braintrust_logging.py | 55 ++++------------------ 1 file changed, 9 insertions(+), 46 deletions(-) diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index 5238bfe1dbf..8e4e31280ee 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -170,6 +170,7 @@ class BraintrustLogger(CustomLogger): verbose_logger.debug("REACHES BRAINTRUST SUCCESS") try: litellm_call_id = kwargs.get("litellm_call_id") + standard_logging_object = kwargs.get("standard_logging_object", {}) prompt = {"messages": kwargs.get("messages")} output = None choices = [] @@ -193,33 +194,13 @@ class BraintrustLogger(CustomLogger): ): output = response_obj["data"] - litellm_params = kwargs.get("litellm_params", {}) - metadata = ( - litellm_params.get("metadata", {}) or {} - ) # if litellm_params['metadata'] == None - metadata = self.add_metadata_from_header(litellm_params, metadata) - clean_metadata = {} - try: - metadata = copy.deepcopy( - metadata - ) # Avoid modifying the original metadata - except Exception: - new_metadata = {} - for key, value in metadata.items(): - if ( - isinstance(value, list) - or isinstance(value, dict) - or isinstance(value, str) - or isinstance(value, int) - or isinstance(value, float) - ): - new_metadata[key] = copy.deepcopy(value) - metadata = new_metadata + litellm_params = kwargs.get("litellm_params", {}) or {} + dynamic_metadata = litellm_params.get("dynamic_metadata", {}) or {} # Get project_id from metadata or create default if needed - project_id = metadata.get("project_id") + project_id = dynamic_metadata.get("project_id") if project_id is None: - project_name = metadata.get("project_name") + project_name = dynamic_metadata.get("project_name") project_id = ( self.get_project_id_sync(project_name) if project_name else None ) @@ -230,8 +211,8 @@ class BraintrustLogger(CustomLogger): project_id = self.default_project_id tags = [] - if isinstance(metadata, dict): - for key, value in metadata.items(): + if isinstance(dynamic_metadata, dict): + for key, value in dynamic_metadata.items(): # generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy if ( litellm.langfuse_default_tags is not None @@ -240,25 +221,7 @@ class BraintrustLogger(CustomLogger): ): tags.append(f"{key}:{value}") - # clean litellm metadata before logging - if key in [ - "headers", - "endpoint", - "caching_groups", - "previous_models", - ]: - continue - else: - clean_metadata[key] = value - cost = kwargs.get("response_cost", None) - if cost is not None: - clean_metadata["litellm_response_cost"] = cost - - # metadata.model is required for braintrust to calculate the "Estimated cost" metric - litellm_model = kwargs.get("model", None) - if litellm_model is not None: - clean_metadata["model"] = litellm_model metrics: Optional[dict] = None usage_obj = getattr(response_obj, "usage", None) @@ -276,12 +239,12 @@ class BraintrustLogger(CustomLogger): } # Allow metadata override for span name - span_name = metadata.get("span_name", "Chat Completion") + span_name = dynamic_metadata.get("span_name", "Chat Completion") request_data = { "id": litellm_call_id, "input": prompt["messages"], - "metadata": clean_metadata, + "metadata": standard_logging_object, "tags": tags, "span_attributes": {"name": span_name, "type": "llm"}, } From 599071321dc0eeae9207b556f8bbc89ea05665ca Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 30 Aug 2025 16:24:03 -0700 Subject: [PATCH 04/46] fix(braintrust_logging.py): refactor to consistently use standard logging payload --- litellm/integrations/braintrust_logging.py | 91 +++------------------- 1 file changed, 10 insertions(+), 81 deletions(-) diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index 8e4e31280ee..1ddf789ed32 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -109,43 +109,6 @@ class BraintrustLogger(CustomLogger): except httpx.HTTPStatusError as e: raise Exception(f"Failed to register project: {e.response.text}") - @staticmethod - def add_metadata_from_header(litellm_params: dict, metadata: dict) -> dict: - """ - Adds metadata from proxy request headers to Braintrust logging if keys start with "braintrust_" - and overwrites litellm_params.metadata if already included. - - For example if you want to append your trace to an existing `trace_id` via header, send - `headers: { ..., langfuse_existing_trace_id: your-existing-trace-id }` via proxy request. - """ - if litellm_params is None: - return metadata - - if litellm_params.get("proxy_server_request") is None: - return metadata - - if metadata is None: - metadata = {} - - proxy_headers = ( - litellm_params.get("proxy_server_request", {}).get("headers", {}) or {} - ) - - for metadata_param_key in proxy_headers: - if metadata_param_key.startswith("braintrust"): - trace_param_key = metadata_param_key.replace("braintrust", "", 1) - if trace_param_key in metadata: - verbose_logger.warning( - f"Overwriting Braintrust `{trace_param_key}` from request header" - ) - else: - verbose_logger.debug( - f"Found Braintrust `{trace_param_key}` in request header" - ) - metadata[trace_param_key] = proxy_headers.get(metadata_param_key) - - return metadata - async def create_default_project_and_experiment(self): project = await self.global_braintrust_http_handler.post( f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"} @@ -172,6 +135,7 @@ class BraintrustLogger(CustomLogger): litellm_call_id = kwargs.get("litellm_call_id") standard_logging_object = kwargs.get("standard_logging_object", {}) prompt = {"messages": kwargs.get("messages")} + output = None choices = [] if response_obj is not None and ( @@ -276,6 +240,7 @@ class BraintrustLogger(CustomLogger): verbose_logger.debug("REACHES BRAINTRUST SUCCESS") try: litellm_call_id = kwargs.get("litellm_call_id") + standard_logging_object = kwargs.get("standard_logging_object", {}) prompt = {"messages": kwargs.get("messages")} output = None choices = [] @@ -300,32 +265,14 @@ class BraintrustLogger(CustomLogger): output = response_obj["data"] litellm_params = kwargs.get("litellm_params", {}) - metadata = ( - litellm_params.get("metadata", {}) or {} - ) # if litellm_params['metadata'] == None - metadata = self.add_metadata_from_header(litellm_params, metadata) + dynamic_metadata = litellm_params.get("dynamic_metadata", {}) or {} + clean_metadata = {} - new_metadata = {} - for key, value in metadata.items(): - if ( - isinstance(value, list) - or isinstance(value, str) - or isinstance(value, int) - or isinstance(value, float) - ): - new_metadata[key] = value - elif isinstance(value, BaseModel): - new_metadata[key] = value.model_dump_json() - elif isinstance(value, dict): - for k, v in value.items(): - if isinstance(v, datetime): - value[k] = v.isoformat() - new_metadata[key] = value # Get project_id from metadata or create default if needed - project_id = metadata.get("project_id") + project_id = dynamic_metadata.get("project_id") if project_id is None: - project_name = metadata.get("project_name") + project_name = dynamic_metadata.get("project_name") project_id = ( await self.get_project_id_async(project_name) if project_name @@ -338,8 +285,8 @@ class BraintrustLogger(CustomLogger): project_id = self.default_project_id tags = [] - if isinstance(metadata, dict): - for key, value in metadata.items(): + if isinstance(dynamic_metadata, dict): + for key, value in dynamic_metadata.items(): # generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy if ( litellm.langfuse_default_tags is not None @@ -348,25 +295,7 @@ class BraintrustLogger(CustomLogger): ): tags.append(f"{key}:{value}") - # clean litellm metadata before logging - if key in [ - "headers", - "endpoint", - "caching_groups", - "previous_models", - ]: - continue - else: - clean_metadata[key] = value - cost = kwargs.get("response_cost", None) - if cost is not None: - clean_metadata["litellm_response_cost"] = cost - - # metadata.model is required for braintrust to calculate the "Estimated cost" metric - litellm_model = kwargs.get("model", None) - if litellm_model is not None: - clean_metadata["model"] = litellm_model metrics: Optional[dict] = None usage_obj = getattr(response_obj, "usage", None) @@ -394,13 +323,13 @@ class BraintrustLogger(CustomLogger): ) # Allow metadata override for span name - span_name = metadata.get("span_name", "Chat Completion") + span_name = dynamic_metadata.get("span_name", "Chat Completion") request_data = { "id": litellm_call_id, "input": prompt["messages"], "output": output, - "metadata": filter_json_serializable(clean_metadata), + "metadata": standard_logging_object, "tags": tags, "span_attributes": {"name": span_name, "type": "llm"}, } From 51c73dc60ba2bc050fe8e7cc17c07f6e75df4c24 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 30 Aug 2025 17:26:18 -0700 Subject: [PATCH 05/46] fix(vertex_and_google_ai_studio_gemini.py): bubble up thoughtsignature back to client --- .../vertex_and_google_ai_studio_gemini.py | 77 ++++++++++----- litellm/types/llms/openai.py | 8 +- litellm/types/llms/vertex_ai.py | 4 +- tests/llm_translation/test_gemini.py | 96 ++++++++++++++----- 4 files changed, 137 insertions(+), 48 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 99a04c20fba..37470a6ee09 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -43,6 +43,7 @@ from litellm.types.llms.gemini import BidiGenerateContentServerMessage from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionResponseMessage, + ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ChatCompletionToolParamFunctionChunk, @@ -792,7 +793,25 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): content_str += _content_str return content_str, reasoning_content_str - + + def _extract_thinking_blocks_from_parts( + self, parts: List[HttpxPartType] + ) -> List[ChatCompletionThinkingBlock]: + """Extract thinking blocks from parts if present""" + thinking_blocks: List[ChatCompletionThinkingBlock] = [] + for part in parts: + if "thoughtSignature" in part: + part_copy = part.copy() + part_copy.pop("thoughtSignature") + thinking_blocks.append( + ChatCompletionThinkingBlock( + type="thinking", + thinking=json.dumps(part_copy), + signature=part["thoughtSignature"], + ) + ) + return thinking_blocks + def _extract_image_response_from_parts( self, parts: List[HttpxPartType] ) -> Optional[ImageURLObject]: @@ -804,10 +823,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if mime_type.startswith("image/"): # Convert base64 data to data URI format data_uri = f"data:{mime_type};base64,{data}" - return ImageURLObject( - url=data_uri, - detail="auto" - ) + return ImageURLObject(url=data_uri, detail="auto") return None def _extract_audio_response_from_parts( @@ -1127,7 +1143,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): elif web_search_queries: web_search_requests = len(grounding_metadata) return web_search_requests - + @staticmethod def _create_streaming_choice( chat_completion_message: ChatCompletionResponseMessage, @@ -1151,9 +1167,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): index=candidate.get("index", idx), delta=Delta( content=chat_completion_message.get("content"), - reasoning_content=chat_completion_message.get( - "reasoning_content" - ), + reasoning_content=chat_completion_message.get("reasoning_content"), tool_calls=tools, image=image_response, function_call=functions, @@ -1164,13 +1178,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return choice @staticmethod - def _extract_candidate_metadata(candidate: Candidates) -> Tuple[List[dict], List[dict], List, List]: + def _extract_candidate_metadata( + candidate: Candidates, + ) -> Tuple[List[dict], List[dict], List, List]: """ Extract metadata from a single candidate response. - + Returns: grounding_metadata: List[dict] - url_context_metadata: List[dict] + url_context_metadata: List[dict] safety_ratings: List citation_metadata: List """ @@ -1178,7 +1194,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): url_context_metadata: List[dict] = [] safety_ratings: List = [] citation_metadata: List = [] - + if "groundingMetadata" in candidate: if isinstance(candidate["groundingMetadata"], list): grounding_metadata.extend(candidate["groundingMetadata"]) # type: ignore @@ -1194,8 +1210,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if "urlContextMetadata" in candidate: # Add URL context metadata to grounding metadata url_context_metadata.append(cast(dict, candidate["urlContextMetadata"])) - - return grounding_metadata, url_context_metadata, safety_ratings, citation_metadata + + return ( + grounding_metadata, + url_context_metadata, + safety_ratings, + citation_metadata, + ) @staticmethod def _process_candidates( @@ -1227,6 +1248,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): tools: Optional[List[ChatCompletionToolCallChunk]] = [] functions: Optional[ChatCompletionToolCallFunctionChunk] = None cumulative_tool_call_index: int = 0 + thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None for idx, candidate in enumerate(_candidates): if "content" not in candidate: @@ -1239,7 +1261,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): candidate_safety_ratings, candidate_citation_metadata, ) = VertexGeminiConfig._extract_candidate_metadata(candidate) - + grounding_metadata.extend(candidate_grounding_metadata) url_context_metadata.extend(candidate_url_context_metadata) safety_ratings.extend(candidate_safety_ratings) @@ -1264,6 +1286,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) ) + thinking_blocks = ( + VertexGeminiConfig()._extract_thinking_blocks_from_parts( + parts=candidate["content"]["parts"] + ) + ) + if audio_response is not None: cast(Dict[str, Any], chat_completion_message)[ "audio" @@ -1271,7 +1299,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): chat_completion_message["content"] = None # OpenAI spec if image_response is not None: # Handle image response - combine with text content into structured format - cast(Dict[str, Any], chat_completion_message)["image"] = image_response + cast(Dict[str, Any], chat_completion_message)[ + "image" + ] = image_response if content is not None: chat_completion_message["content"] = content @@ -1298,15 +1328,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if functions is not None: chat_completion_message["function_call"] = functions + if thinking_blocks is not None: + chat_completion_message["thinking_blocks"] = thinking_blocks # type: ignore + if isinstance(model_response, ModelResponseStream): choice = VertexGeminiConfig._create_streaming_choice( chat_completion_message=chat_completion_message, - candidate=candidate, - idx=idx, - tools=tools, - functions=functions, + candidate=candidate, + idx=idx, + tools=tools, + functions=functions, chat_completion_logprobs=chat_completion_logprobs, - image_response=image_response + image_response=image_response, ) model_response.choices.append(choice) elif isinstance(model_response, ModelResponse): diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index a0c8e5b6295..9b6cad38008 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -43,10 +43,14 @@ from openai.types.responses.response import ( # Handle OpenAI SDK version compatibility for Text type try: - from openai.types.responses.response_create_params import Text as ResponseText + from openai.types.responses.response_create_params import ( + Text as ResponseText, # type: ignore + ) except (ImportError, AttributeError): # Fall back to the concrete config type available in all SDK versions - from openai.types.responses.response_text_config_param import ResponseTextConfigParam as ResponseText + from openai.types.responses.response_text_config_param import ( + ResponseTextConfigParam as ResponseText, + ) from openai.types.responses.response_create_params import ( Reasoning, diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 2931770cd6e..052b872bcde 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -72,6 +72,7 @@ class HttpxPartType(TypedDict, total=False): executableCode: HttpxExecutableCode codeExecutionResult: HttpxCodeExecutionResult thought: bool + thoughtSignature: str class HttpxContentType(TypedDict, total=False): @@ -245,10 +246,11 @@ class UsageMetadata(TypedDict, total=False): class TokenCountDetailsResponse(TypedDict): """ Response structure for token count details with modality breakdown. - + Example: {'totalTokens': 12, 'promptTokensDetails': [{'modality': 'TEXT', 'tokenCount': 12}]} """ + totalTokens: int promptTokensDetails: List[PromptTokensDetails] diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index 22a54b8a56b..c54168e9a6f 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -436,7 +436,10 @@ def test_gemini_with_empty_function_call_arguments(): async def test_claude_tool_use_with_gemini(): response = await litellm.anthropic.messages.acreate( messages=[ - {"role": "user", "content": "Hello, can you tell me the weather in Boston. Please respond with a tool call?"} + { + "role": "user", + "content": "Hello, can you tell me the weather in Boston. Please respond with a tool call?", + } ], model="gemini/gemini-2.5-flash", stream=True, @@ -578,11 +581,17 @@ def test_gemini_tool_use(): assert stop_reason is not None assert stop_reason == "tool_calls" + @pytest.mark.asyncio async def test_gemini_image_generation_async(): litellm._turn_on_debug() response = await litellm.acompletion( - messages=[{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}], + messages=[ + { + "role": "user", + "content": "Generate an image of a banana wearing a costume that says LiteLLM", + } + ], model="gemini/gemini-2.5-flash-image-preview", ) @@ -597,12 +606,16 @@ async def test_gemini_image_generation_async(): assert IMAGE_URL["url"].startswith("data:image/png;base64,") - @pytest.mark.asyncio async def test_gemini_image_generation_async_stream(): - #litellm._turn_on_debug() + # litellm._turn_on_debug() response = await litellm.acompletion( - messages=[{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}], + messages=[ + { + "role": "user", + "content": "Generate an image of a banana wearing a costume that says LiteLLM", + } + ], model="gemini/gemini-2.5-flash-image-preview", stream=True, ) @@ -611,35 +624,72 @@ async def test_gemini_image_generation_async_stream(): model_response_image = None async for chunk in response: print("CHUNK: ", chunk) - if hasattr(chunk.choices[0].delta, "image") and chunk.choices[0].delta.image is not None: + if ( + hasattr(chunk.choices[0].delta, "image") + and chunk.choices[0].delta.image is not None + ): model_response_image = chunk.choices[0].delta.image print("MODEL_RESPONSE_IMAGE: ", model_response_image) assert model_response_image is not None assert model_response_image["url"].startswith("data:image/png;base64,") break - + ######################################################### # Important: Validate we did get an image in the response ######################################################### assert model_response_image is not None assert model_response_image["url"].startswith("data:image/png;base64,") - + def test_system_message_with_no_user_message(): - """ - Test that the system message is translated correctly for non-OpenAI providers. - """ - messages = [ - { - "role": "system", - "content": "Be a good bot!", + """ + Test that the system message is translated correctly for non-OpenAI providers. + """ + messages = [ + { + "role": "system", + "content": "Be a good bot!", + }, + ] + + response = litellm.completion( + model="gemini/gemini-2.5-flash", + messages=messages, + ) + assert response is not None + + assert response.choices[0].message.content is not None + + +def test_gemini_with_thinking(): + from litellm import completion + + litellm._turn_on_debug() + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, }, - ] + } + ] + messages = [{"role": "user", "content": "What's the weather like in Boston today?"}] - response = litellm.completion( - model="gemini/gemini-2.5-flash", - messages=messages, - ) - assert response is not None - - assert response.choices[0].message.content is not None \ No newline at end of file + result = completion( + model="gemini/gemini-2.5-flash", + messages=messages, + tools=tools, + ) + print(f"result: {result}") From b6f6dc5c1c00ca131c5f24149ba398a59db7c390 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 30 Aug 2025 18:49:34 -0700 Subject: [PATCH 06/46] feat(vertex_ai.py): support parsing thinking content into gemini format allows function calls with thought signatures to be sent back to gemini Closes https://github.com/BerriAI/litellm/pull/13842 --- .../llms/vertex_ai/gemini/transformation.py | 95 ++++++++++++++++++- litellm/types/llms/vertex_ai.py | 1 + tests/llm_translation/test_gemini.py | 86 +++++++++++++++-- .../test_vertex_ai_gemini_transformation.py | 75 +++++++++++++++ 4 files changed, 248 insertions(+), 9 deletions(-) create mode 100644 tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 8ab212e2558..267ca61ef5d 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -105,6 +105,64 @@ def _process_gemini_image(image_url: str, format: Optional[str] = None) -> PartT raise e +def _snake_to_camel(snake_str: str) -> str: + """Convert snake_case to camelCase""" + components = snake_str.split("_") + return components[0] + "".join(x.capitalize() for x in components[1:]) + + +def _camel_to_snake(camel_str: str) -> str: + """Convert camelCase to snake_case""" + import re + + return re.sub(r"(? Optional[str]: + """ + Get the equivalent key from available keys, checking both camelCase and snake_case variants + """ + if key in available_keys: + return key + + # Try camelCase version + camel_key = _snake_to_camel(key) + if camel_key in available_keys: + return camel_key + + # Try snake_case version + snake_key = _camel_to_snake(key) + if snake_key in available_keys: + return snake_key + + return None + + +def check_if_part_exists_in_parts( + parts: List[PartType], part: PartType, excluded_keys: List[str] = [] +) -> bool: + """ + Check if a part exists in a list of parts + Handles both camelCase and snake_case key variations (e.g., function_call vs functionCall) + """ + keys_to_compare = set(part.keys()) - set(excluded_keys) + for p in parts: + p_keys = set(p.keys()) + # Check if all keys in part have equivalent values in p + match_found = True + for key in keys_to_compare: + equivalent_key = _get_equivalent_key(key, p_keys) + if equivalent_key is None or p.get(equivalent_key, None) != part.get( + key, None + ): + match_found = False + break + + if match_found: + return True + return False + + def _gemini_convert_messages_with_history( # noqa: PLR0915 messages: List[AllMessageValues], ) -> List[ContentType]: @@ -236,10 +294,33 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 assistant_msg = ChatCompletionAssistantMessage(**msg_dict) # type: ignore _message_content = assistant_msg.get("content", None) reasoning_content = assistant_msg.get("reasoning_content", None) + thinking_blocks = assistant_msg.get("thinking_blocks") if reasoning_content is not None: assistant_content.append( PartType(thought=True, text=reasoning_content) ) + if thinking_blocks is not None: + for block in thinking_blocks: + block_thinking_str = block.get("thinking") + block_signature = block.get("signature") + if ( + block_thinking_str is not None + and block_signature is not None + ): + try: + assistant_content.append( + PartType( + thoughtSignature=block_signature, + **json.loads(block_thinking_str), + ) + ) + except Exception: + assistant_content.append( + PartType( + thoughtSignature=block_signature, + text=block_thinking_str, + ) + ) if _message_content is not None and isinstance(_message_content, list): _parts = [] for element in _message_content: @@ -262,9 +343,17 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 assistant_msg.get("tool_calls", []) is not None or assistant_msg.get("function_call") is not None ): # support assistant tool invoke conversion - assistant_content.extend( - convert_to_gemini_tool_call_invoke(assistant_msg) + gemini_tool_call_parts = convert_to_gemini_tool_call_invoke( + assistant_msg ) + ## check if gemini_tool_call already exists in assistant_content + for gemini_tool_call_part in gemini_tool_call_parts: + if not check_if_part_exists_in_parts( + assistant_content, + gemini_tool_call_part, + excluded_keys=["thoughtSignature"], + ): + assistant_content.append(gemini_tool_call_part) last_message_with_tool_calls = assistant_msg msg_i += 1 @@ -476,6 +565,7 @@ async def async_transform_request_body( optional_params=optional_params, ) + def _default_user_message_when_system_message_passed() -> ChatCompletionUserMessage: """ Returns a default user message when a "system" message is passed in gemini fails. @@ -484,6 +574,7 @@ def _default_user_message_when_system_message_passed() -> ChatCompletionUserMess """ return ChatCompletionUserMessage(content=".", role="user") + def _transform_system_message( supports_system_message: bool, messages: List[AllMessageValues] ) -> Tuple[Optional[SystemInstructions], List[AllMessageValues]]: diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 052b872bcde..1b74ee25803 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -41,6 +41,7 @@ class PartType(TypedDict, total=False): function_call: FunctionCall function_response: FunctionResponse thought: bool + thoughtSignature: str class HttpxFunctionCall(TypedDict): diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index c54168e9a6f..b3f16ecd838 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -661,10 +661,33 @@ def test_system_message_with_no_user_message(): assert response.choices[0].message.content is not None +def get_current_weather(location, unit="fahrenheit"): + """Get the current weather in a given location""" + if "tokyo" in location.lower(): + return json.dumps({"location": "Tokyo", "temperature": "10", "unit": "celsius"}) + elif "san francisco" in location.lower(): + return json.dumps( + {"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"} + ) + elif "paris" in location.lower(): + return json.dumps({"location": "Paris", "temperature": "22", "unit": "celsius"}) + else: + return json.dumps({"location": location, "temperature": "unknown"}) + + def test_gemini_with_thinking(): from litellm import completion litellm._turn_on_debug() + litellm.modify_params = True + model = "gemini/gemini-2.5-flash" + messages = [ + { + "role": "user", + "content": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses", + } + ] + tools = [ { "type": "function", @@ -676,20 +699,69 @@ def test_gemini_with_thinking(): "properties": { "location": { "type": "string", - "description": "The city and state, e.g. San Francisco, CA", + "description": "The city and state", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, }, "required": ["location"], }, }, } ] - messages = [{"role": "user", "content": "What's the weather like in Boston today?"}] - - result = completion( - model="gemini/gemini-2.5-flash", + response = litellm.completion( + model=model, messages=messages, tools=tools, + tool_choice="auto", # auto is default, but we'll be explicit + reasoning_effort="low", ) - print(f"result: {result}") + print("Response\n", response) + response_message = response.choices[0].message + tool_calls = response_message.tool_calls + + print("Expecting there to be 3 tool calls") + assert len(tool_calls) > 0 # this has to call the function for SF, Tokyo and paris + + # Step 2: check if the model wanted to call a function + print(f"tool_calls: {tool_calls}") + if tool_calls: + # Step 3: call the function + # Note: the JSON response may not always be valid; be sure to handle errors + available_functions = { + "get_current_weather": get_current_weather, + } # only one function in this example, but you can have multiple + messages.append(response_message) # extend conversation with assistant's reply + print("Response message\n", response_message) + # Step 4: send the info for each function call and function response to the model + for tool_call in tool_calls: + function_name = tool_call.function.name + if function_name not in available_functions: + # the model called a function that does not exist in available_functions - don't try calling anything + return + function_to_call = available_functions[function_name] + function_args = json.loads(tool_call.function.arguments) + function_response = function_to_call( + location=function_args.get("location"), + unit=function_args.get("unit"), + ) + messages.append( + { + "tool_call_id": tool_call.id, + "role": "tool", + "name": function_name, + "content": function_response, + } + ) # extend conversation with function response + print(f"messages: {messages}") + second_response = litellm.completion( + model=model, + messages=messages, + seed=22, + reasoning_effort="low", + tools=tools, + drop_params=True, + ) # get a new response from the model where it can see the function response + print("second response\n", second_response) diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py new file mode 100644 index 00000000000..d6d33258576 --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -0,0 +1,75 @@ +from litellm.llms.vertex_ai.gemini.transformation import check_if_part_exists_in_parts + + +def test_check_if_part_exists_in_parts(): + parts = [ + {"text": "Hello", "thought": True}, + {"text": "World", "thought": False}, + ] + part = {"text": "Hello", "thought": True} + new_part = {"text": "Hello World", "thought": True} + assert check_if_part_exists_in_parts(parts, part) + assert not check_if_part_exists_in_parts(parts, new_part, ["thought"]) + assert check_if_part_exists_in_parts(parts, new_part, ["text"]) + + +def test_check_if_part_exists_in_parts_camel_case_snake_case(): + """Test that function handles both camelCase and snake_case key variations""" + # Test snake_case to camelCase matching + parts_with_snake_case = [ + { + "function_call": { + "name": "get_current_weather", + "args": {"location": "San Francisco, CA"}, + } + }, + {"text": "Some other content"}, + ] + + part_with_camel_case = { + "functionCall": { + "name": "get_current_weather", + "args": {"location": "San Francisco, CA"}, + } + } + + # Should find match between function_call and functionCall + assert check_if_part_exists_in_parts(parts_with_snake_case, part_with_camel_case) + + # Test camelCase to snake_case matching + parts_with_camel_case = [ + {"functionCall": {"name": "calculate_sum", "args": {"a": 1, "b": 2}}} + ] + + part_with_snake_case = { + "function_call": {"name": "calculate_sum", "args": {"a": 1, "b": 2}} + } + + # Should find match between functionCall and function_call + assert check_if_part_exists_in_parts(parts_with_camel_case, part_with_snake_case) + + # Test no match when values differ + part_with_different_values = { + "function_call": {"name": "different_function", "args": {"x": 5}} + } + + assert not check_if_part_exists_in_parts( + parts_with_snake_case, part_with_different_values + ) + + # Test multiple keys with mixed casing + parts_mixed = [ + { + "function_call": {"name": "test"}, + "thoughtSignature": "reasoning", + "text": "content", + } + ] + + part_mixed_casing = { + "functionCall": {"name": "test"}, + "thought_signature": "reasoning", + "text": "content", + } + + assert check_if_part_exists_in_parts(parts_mixed, part_mixed_casing) From 15baaf68db76ebcdb8f173b8e2b090636753994d Mon Sep 17 00:00:00 2001 From: tanjiro <56165694+NANDINI-star@users.noreply.github.com> Date: Sun, 31 Aug 2025 17:09:11 +0900 Subject: [PATCH 07/46] back to logs added conditionally --- .../src/components/templates/key_info_view.tsx | 6 ++++-- ui/litellm-dashboard/src/components/view_logs/index.tsx | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index b4a74bf846b..f5e18b68046 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -43,6 +43,7 @@ interface KeyInfoViewProps { teams: any[] | null premiumUser: boolean setAccessToken?: (token: string) => void + backButtonText?: string } export default function KeyInfoView({ @@ -57,6 +58,7 @@ export default function KeyInfoView({ onDelete, premiumUser, setAccessToken, + backButtonText = "Back to Keys", }: KeyInfoViewProps) { const [isEditing, setIsEditing] = useState(false) const [form] = Form.useForm() @@ -92,7 +94,7 @@ export default function KeyInfoView({ return (
Key not found
@@ -261,7 +263,7 @@ export default function KeyInfoView({
{currentKeyData.key_alias || "API Key"} diff --git a/ui/litellm-dashboard/src/components/view_logs/index.tsx b/ui/litellm-dashboard/src/components/view_logs/index.tsx index b227d493aa0..72ad8732b6b 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/index.tsx @@ -500,6 +500,7 @@ export default function SpendLogsTable({ teams={allTeams} onClose={() => setSelectedKeyIdInfoView(null)} premiumUser={premiumUser} + backButtonText="Back to Logs" /> ) : selectedSessionId ? (
From 0c9051abbad18316b7fe2e2cc98070e880ad8887 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sun, 31 Aug 2025 21:24:30 -0700 Subject: [PATCH 08/46] docs(docker_quick_start.md): add docker compose to quick start --- docs/my-website/docs/proxy/deploy.md | 5 +--- .../docs/proxy/docker_quick_start.md | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index 7f893068645..cdb6f7018fc 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -12,10 +12,7 @@ To start using Litellm, run the following commands in a shell: ```bash # Get the code -git clone https://github.com/BerriAI/litellm - -# Go to folder -cd litellm +curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml # Add the master key - you can change this after setup echo 'LITELLM_MASTER_KEY="sk-1234"' > .env diff --git a/docs/my-website/docs/proxy/docker_quick_start.md b/docs/my-website/docs/proxy/docker_quick_start.md index 09f7dfbaf78..b21105bb829 100644 --- a/docs/my-website/docs/proxy/docker_quick_start.md +++ b/docs/my-website/docs/proxy/docker_quick_start.md @@ -35,6 +35,30 @@ $ pip install 'litellm[proxy]' + + +Use this docker compose to spin up the proxy with a postgres database running locally. + +```bash +# Get the docker compose file +curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml + +# Add the master key - you can change this after setup +echo 'LITELLM_MASTER_KEY="sk-1234"' > .env + +# Add the litellm salt key - you cannot change this after adding a model +# It is used to encrypt / decrypt your LLM API Key credentials +# We recommend - https://1password.com/password-generator/ +# password generator to get a random hash for litellm salt key +echo 'LITELLM_SALT_KEY="sk-1234"' >> .env + +source .env + +# Start +docker-compose up +``` + + ## 1. Add a model @@ -43,6 +67,8 @@ Control LiteLLM Proxy with a config.yaml file. Setup your config.yaml with your azure model. +Note: When using the proxy with a database, you can also **just add models via UI** (UI is available on `/ui` route). + ```yaml model_list: - model_name: gpt-4o From 6cd5afa8b11ea3654a2c26d3d64cd2a2e3eff3f9 Mon Sep 17 00:00:00 2001 From: "mubashir1osmani (aider)" Date: Mon, 1 Sep 2025 00:56:22 -0400 Subject: [PATCH 09/46] feat: add persistence and management for guardrails on virtual keys --- .../proxy/guardrails/endpoints.py | 70 ++++++++++++++++++- .../proxy/guardrails/guardrail_registry.py | 30 ++++++++ 2 files changed, 97 insertions(+), 3 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py b/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py index cdf86dcea67..fcd5e82ab3a 100644 --- a/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py +++ b/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py @@ -6,16 +6,28 @@ To see all free guardrails see litellm/proxy/guardrails/* Exposed Routes: - /mask_pii +- /virtual_key/guardrails """ -from typing import Optional +from typing import Dict, List, Optional -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.guardrails.guardrail_endpoints import GUARDRAIL_REGISTRY -from litellm.types.guardrails import ApplyGuardrailRequest, ApplyGuardrailResponse +from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER +from litellm.types.guardrails import ApplyGuardrailRequest, ApplyGuardrailResponse, Guardrail + +# Models for virtual key guardrail management +class VirtualKeyGuardrailRequest(BaseModel): + virtual_key_id: str + guardrail_id: str + +class VirtualKeyGuardrailsResponse(BaseModel): + virtual_key_id: str + guardrails: List[Guardrail] router = APIRouter(tags=["guardrails"], prefix="/guardrails") @@ -39,3 +51,55 @@ async def apply_guardrail( return await active_guardrail.apply_guardrail( text=request.text, language=request.language, entities=request.entities ) + +@router.post("/virtual_key/associate", response_model=Dict[str, str]) +async def associate_guardrail_with_virtual_key( + request: VirtualKeyGuardrailRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Associate a guardrail with a virtual key + """ + # Check if guardrail exists + guardrail = IN_MEMORY_GUARDRAIL_HANDLER.get_guardrail_by_id(request.guardrail_id) + if not guardrail: + raise HTTPException(status_code=404, detail=f"Guardrail {request.guardrail_id} not found") + + # Associate guardrail with virtual key + IN_MEMORY_GUARDRAIL_HANDLER.associate_guardrail_with_virtual_key( + virtual_key_id=request.virtual_key_id, + guardrail_id=request.guardrail_id + ) + + return {"message": f"Guardrail {request.guardrail_id} associated with virtual key {request.virtual_key_id}"} + +@router.post("/virtual_key/disassociate", response_model=Dict[str, str]) +async def disassociate_guardrail_from_virtual_key( + request: VirtualKeyGuardrailRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Disassociate a guardrail from a virtual key + """ + # Disassociate guardrail from virtual key + IN_MEMORY_GUARDRAIL_HANDLER.disassociate_guardrail_from_virtual_key( + virtual_key_id=request.virtual_key_id, + guardrail_id=request.guardrail_id + ) + + return {"message": f"Guardrail {request.guardrail_id} disassociated from virtual key {request.virtual_key_id}"} + +@router.get("/virtual_key/{virtual_key_id}", response_model=VirtualKeyGuardrailsResponse) +async def get_guardrails_for_virtual_key( + virtual_key_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Get all guardrails associated with a virtual key + """ + guardrails = IN_MEMORY_GUARDRAIL_HANDLER.get_guardrails_for_virtual_key(virtual_key_id) + + return VirtualKeyGuardrailsResponse( + virtual_key_id=virtual_key_id, + guardrails=guardrails + ) diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 21429f462d4..523ff9f4cd8 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -369,6 +369,11 @@ class InMemoryGuardrailHandler: """ Guardrail id to CustomGuardrail object mapping """ + + self.virtual_key_to_guardrails: Dict[str, List[str]] = {} + """ + Virtual key id to list of guardrail ids mapping + """ def initialize_guardrail( self, @@ -538,6 +543,31 @@ class InMemoryGuardrailHandler: Get a guardrail by its ID from memory """ return self.IN_MEMORY_GUARDRAILS.get(guardrail_id) + + def associate_guardrail_with_virtual_key(self, virtual_key_id: str, guardrail_id: str) -> None: + """ + Associate a guardrail with a virtual key + """ + if virtual_key_id not in self.virtual_key_to_guardrails: + self.virtual_key_to_guardrails[virtual_key_id] = [] + + if guardrail_id not in self.virtual_key_to_guardrails[virtual_key_id]: + self.virtual_key_to_guardrails[virtual_key_id].append(guardrail_id) + + def disassociate_guardrail_from_virtual_key(self, virtual_key_id: str, guardrail_id: str) -> None: + """ + Disassociate a guardrail from a virtual key + """ + if virtual_key_id in self.virtual_key_to_guardrails: + if guardrail_id in self.virtual_key_to_guardrails[virtual_key_id]: + self.virtual_key_to_guardrails[virtual_key_id].remove(guardrail_id) + + def get_guardrails_for_virtual_key(self, virtual_key_id: str) -> List[Guardrail]: + """ + Get all guardrails associated with a virtual key + """ + guardrail_ids = self.virtual_key_to_guardrails.get(virtual_key_id, []) + return [self.IN_MEMORY_GUARDRAILS[gid] for gid in guardrail_ids if gid in self.IN_MEMORY_GUARDRAILS] ######################################################## From 83654c56089208db695b5b4821e7ae41118aba34 Mon Sep 17 00:00:00 2001 From: "mubashir1osmani (aider)" Date: Mon, 1 Sep 2025 01:17:08 -0400 Subject: [PATCH 10/46] feat: add UI support for displaying and managing guardrails on virtual keys --- .../proxy/guardrails/endpoints.py | 2 + .../proxy/guardrails/guardrail_registry.py | 1 + .../virtual_keys/virtual_key_detail.tsx | 160 ++++++++++++++++++ .../virtual_keys/virtual_key_guardrails.tsx | 93 ++++++++++ .../src/services/guardrails.ts | 27 +++ .../src/services/virtual_key_guardrails.ts | 57 +++++++ ui/litellm-dashboard/src/types/guardrail.ts | 11 ++ ui/litellm-dashboard/src/types/virtual_key.ts | 13 ++ 8 files changed, 364 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/virtual_keys/virtual_key_detail.tsx create mode 100644 ui/litellm-dashboard/src/components/virtual_keys/virtual_key_guardrails.tsx create mode 100644 ui/litellm-dashboard/src/services/guardrails.ts create mode 100644 ui/litellm-dashboard/src/services/virtual_key_guardrails.ts create mode 100644 ui/litellm-dashboard/src/types/guardrail.ts create mode 100644 ui/litellm-dashboard/src/types/virtual_key.ts diff --git a/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py b/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py index fcd5e82ab3a..47bec519c40 100644 --- a/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py +++ b/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py @@ -97,7 +97,9 @@ async def get_guardrails_for_virtual_key( """ Get all guardrails associated with a virtual key """ + verbose_proxy_logger.debug(f"Getting guardrails for virtual key: {virtual_key_id}") guardrails = IN_MEMORY_GUARDRAIL_HANDLER.get_guardrails_for_virtual_key(virtual_key_id) + verbose_proxy_logger.debug(f"Found {len(guardrails)} guardrails for virtual key {virtual_key_id}") return VirtualKeyGuardrailsResponse( virtual_key_id=virtual_key_id, diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 523ff9f4cd8..7d3432d5cb6 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -567,6 +567,7 @@ class InMemoryGuardrailHandler: Get all guardrails associated with a virtual key """ guardrail_ids = self.virtual_key_to_guardrails.get(virtual_key_id, []) + verbose_proxy_logger.debug(f"Getting guardrails for virtual key {virtual_key_id}: {guardrail_ids}") return [self.IN_MEMORY_GUARDRAILS[gid] for gid in guardrail_ids if gid in self.IN_MEMORY_GUARDRAILS] diff --git a/ui/litellm-dashboard/src/components/virtual_keys/virtual_key_detail.tsx b/ui/litellm-dashboard/src/components/virtual_keys/virtual_key_detail.tsx new file mode 100644 index 00000000000..111bd26841c --- /dev/null +++ b/ui/litellm-dashboard/src/components/virtual_keys/virtual_key_detail.tsx @@ -0,0 +1,160 @@ +import React, { useState, useEffect } from 'react'; +import { Card, Flex, Text, Heading, Button, Dialog, TextField, Select, Box, Tabs } from '@radix-ui/themes'; +import { toast } from 'react-hot-toast'; +import { VirtualKey } from '../../types/virtual_key'; +import { fetchVirtualKey, updateVirtualKey } from '../../services/virtual_keys'; +import { VirtualKeyGuardrails } from './virtual_key_guardrails'; +import { fetchGuardrails } from '../../services/guardrails'; +import { Guardrail } from '../../types/guardrail'; +import { associateGuardrailWithVirtualKey } from '../../services/virtual_key_guardrails'; + +interface VirtualKeyDetailProps { + virtualKeyId: string; + onClose: () => void; +} + +export const VirtualKeyDetail: React.FC = ({ virtualKeyId, onClose }) => { + const [virtualKey, setVirtualKey] = useState(null); + const [loading, setLoading] = useState(true); + const [availableGuardrails, setAvailableGuardrails] = useState([]); + const [selectedGuardrailId, setSelectedGuardrailId] = useState(''); + const [refreshTrigger, setRefreshTrigger] = useState(0); + + useEffect(() => { + const loadVirtualKey = async () => { + try { + const data = await fetchVirtualKey(virtualKeyId); + setVirtualKey(data); + } catch (error) { + console.error('Error loading virtual key:', error); + toast.error('Failed to load virtual key details'); + } finally { + setLoading(false); + } + }; + + const loadGuardrails = async () => { + try { + const data = await fetchGuardrails(); + setAvailableGuardrails(data.guardrails); + } catch (error) { + console.error('Error loading guardrails:', error); + } + }; + + loadVirtualKey(); + loadGuardrails(); + }, [virtualKeyId]); + + const handleAddGuardrail = async () => { + if (!selectedGuardrailId) { + toast.error('Please select a guardrail'); + return; + } + + try { + await associateGuardrailWithVirtualKey(virtualKeyId, selectedGuardrailId); + toast.success('Guardrail added to virtual key'); + setSelectedGuardrailId(''); + setRefreshTrigger(prev => prev + 1); + } catch (error) { + console.error('Error adding guardrail:', error); + toast.error('Failed to add guardrail'); + } + }; + + if (loading) { + return ( + + + Virtual Key Details + Loading... + + + ); + } + + if (!virtualKey) { + return ( + + + Error + Virtual key not found + + + + + + ); + } + + return ( + + + Virtual Key: {virtualKey.key_name || 'Unnamed Key'} + + + + Details + Guardrails + + + + + + + + Key ID: + {virtualKey.key_id} + + + Team ID: + {virtualKey.team_id || 'N/A'} + + + Models: + {virtualKey.models?.join(', ') || 'All models'} + + + Spend: + ${virtualKey.spend?.toFixed(2) || '0.00'} + + + + + + + + + Add Guardrail + + + + + {availableGuardrails.map(guardrail => ( + + {guardrail.guardrail_name || `Guardrail ${guardrail.guardrail_id.substring(0, 8)}`} + + ))} + + + + + + + + + + + + + + + + + + ); +}; diff --git a/ui/litellm-dashboard/src/components/virtual_keys/virtual_key_guardrails.tsx b/ui/litellm-dashboard/src/components/virtual_keys/virtual_key_guardrails.tsx new file mode 100644 index 00000000000..0acbc9d3f4a --- /dev/null +++ b/ui/litellm-dashboard/src/components/virtual_keys/virtual_key_guardrails.tsx @@ -0,0 +1,93 @@ +import React, { useEffect, useState } from 'react'; +import { Badge, Button, Card, Flex, Heading, Text, Box, Spinner } from '@radix-ui/themes'; +import { fetchGuardrailsForVirtualKey, associateGuardrailWithVirtualKey, disassociateGuardrailFromVirtualKey } from '../../services/virtual_key_guardrails'; +import { Guardrail } from '../../types/guardrail'; +import { toast } from 'react-hot-toast'; + +interface VirtualKeyGuardrailsProps { + virtualKeyId: string; + refreshTrigger?: number; +} + +export const VirtualKeyGuardrails: React.FC = ({ + virtualKeyId, + refreshTrigger = 0 +}) => { + const [guardrails, setGuardrails] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + const loadGuardrails = async () => { + if (!virtualKeyId) return; + + setLoading(true); + try { + const data = await fetchGuardrailsForVirtualKey(virtualKeyId); + setGuardrails(data.guardrails); + setError(null); + } catch (err) { + console.error('Error loading guardrails for virtual key:', err); + setError('Failed to load guardrails'); + setGuardrails([]); + } finally { + setLoading(false); + } + }; + + loadGuardrails(); + }, [virtualKeyId, refreshTrigger]); + + const handleRemoveGuardrail = async (guardrailId: string) => { + try { + await disassociateGuardrailFromVirtualKey(virtualKeyId, guardrailId); + setGuardrails(guardrails.filter(g => g.guardrail_id !== guardrailId)); + toast.success('Guardrail removed from virtual key'); + } catch (err) { + console.error('Error removing guardrail:', err); + toast.error('Failed to remove guardrail'); + } + }; + + if (loading) { + return ( + + + + + + ); + } + + if (error) { + return ( + + {error} + + ); + } + + return ( + + Attached Guardrails + {guardrails.length === 0 ? ( + No guardrails attached to this virtual key + ) : ( + + {guardrails.map((guardrail) => ( + + + {guardrail.guardrail_name || 'Unnamed Guardrail'} + Type: {guardrail.litellm_params?.guardrail} + Mode: {guardrail.litellm_params?.mode} + + + + ))} + + )} + + ); +}; diff --git a/ui/litellm-dashboard/src/services/guardrails.ts b/ui/litellm-dashboard/src/services/guardrails.ts new file mode 100644 index 00000000000..2cb7b80a4b1 --- /dev/null +++ b/ui/litellm-dashboard/src/services/guardrails.ts @@ -0,0 +1,27 @@ +import { Guardrail } from '../types/guardrail'; + +interface GuardrailsResponse { + guardrails: Guardrail[]; +} + +export async function fetchGuardrails(): Promise { + const response = await fetch('/api/guardrails'); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Failed to fetch guardrails: ${errorText}`); + } + + return await response.json(); +} + +export async function fetchGuardrail(guardrailId: string): Promise { + const response = await fetch(`/api/guardrails/${guardrailId}`); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Failed to fetch guardrail: ${errorText}`); + } + + return await response.json(); +} diff --git a/ui/litellm-dashboard/src/services/virtual_key_guardrails.ts b/ui/litellm-dashboard/src/services/virtual_key_guardrails.ts new file mode 100644 index 00000000000..de7ce30d890 --- /dev/null +++ b/ui/litellm-dashboard/src/services/virtual_key_guardrails.ts @@ -0,0 +1,57 @@ +import { Guardrail } from '../types/guardrail'; + +interface VirtualKeyGuardrailsResponse { + virtual_key_id: string; + guardrails: Guardrail[]; +} + +export async function fetchGuardrailsForVirtualKey(virtualKeyId: string): Promise { + const response = await fetch(`/api/guardrails/virtual_key/${virtualKeyId}`); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Failed to fetch guardrails: ${errorText}`); + } + + return await response.json(); +} + +export async function associateGuardrailWithVirtualKey(virtualKeyId: string, guardrailId: string): Promise<{ message: string }> { + const response = await fetch('/api/guardrails/virtual_key/associate', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + virtual_key_id: virtualKeyId, + guardrail_id: guardrailId, + }), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Failed to associate guardrail: ${errorText}`); + } + + return await response.json(); +} + +export async function disassociateGuardrailFromVirtualKey(virtualKeyId: string, guardrailId: string): Promise<{ message: string }> { + const response = await fetch('/api/guardrails/virtual_key/disassociate', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + virtual_key_id: virtualKeyId, + guardrail_id: guardrailId, + }), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Failed to disassociate guardrail: ${errorText}`); + } + + return await response.json(); +} diff --git a/ui/litellm-dashboard/src/types/guardrail.ts b/ui/litellm-dashboard/src/types/guardrail.ts new file mode 100644 index 00000000000..6cb26d04973 --- /dev/null +++ b/ui/litellm-dashboard/src/types/guardrail.ts @@ -0,0 +1,11 @@ +export interface Guardrail { + guardrail_id: string; + guardrail_name: string | null; + litellm_params: { + guardrail: string; + mode: string; + default_on: boolean; + [key: string]: any; + }; + guardrail_info?: Record; +} diff --git a/ui/litellm-dashboard/src/types/virtual_key.ts b/ui/litellm-dashboard/src/types/virtual_key.ts new file mode 100644 index 00000000000..e130853b1d1 --- /dev/null +++ b/ui/litellm-dashboard/src/types/virtual_key.ts @@ -0,0 +1,13 @@ +export interface VirtualKey { + key_id: string; + key_name?: string; + team_id?: string; + models?: string[]; + spend?: number; + max_budget?: number; + max_parallel_requests?: number; + metadata?: Record; + expires?: string; + created_at?: string; + updated_at?: string; +} From a8f2ebed9cda1c53c3a21196577e2e3639fe9eeb Mon Sep 17 00:00:00 2001 From: "mubashir1osmani (aider)" Date: Mon, 1 Sep 2025 01:17:46 -0400 Subject: [PATCH 11/46] fix: import verbose_proxy_logger to resolve undefined name errors --- enterprise/litellm_enterprise/proxy/guardrails/endpoints.py | 1 + 1 file changed, 1 insertion(+) diff --git a/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py b/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py index 47bec519c40..85f8bf22417 100644 --- a/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py +++ b/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py @@ -13,6 +13,7 @@ from typing import Dict, List, Optional from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel +from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth From e334f48699f54a32a81ec4e75ca43ed52ecadf48 Mon Sep 17 00:00:00 2001 From: TomeHirata Date: Mon, 1 Sep 2025 16:46:54 +0900 Subject: [PATCH 12/46] Add supported text field to anthropic citation response --- litellm/llms/anthropic/chat/handler.py | 2 ++ litellm/llms/anthropic/chat/transformation.py | 10 +++++++++- .../test_anthropic_completion.py | 18 +++++++++++++----- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 5618c50923e..253f5d9be2c 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -743,6 +743,8 @@ class ModelResponseIterator: ) text, tool_use = self._handle_json_mode_chunk(text=text, tool_use=tool_use) + if type_chunk: + provider_specific_fields["chunk_type"] = type_chunk returned_chunk = ModelResponseStream( choices=[ diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index ce874bfde9a..378ca75da5f 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -797,7 +797,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if content.get("citations") is not None: if citations is None: citations = [] - citations.append(content["citations"]) + citations.append( + [ + { + **citation, + "supported_text": content.get("text", ""), + } + for citation in content["citations"] + ] + ) if thinking_blocks is not None: reasoning_content = "" for block in thinking_blocks: diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 45702a261e2..f4bd7531b0b 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -920,6 +920,14 @@ def test_anthropic_citations_api(): citations = resp.choices[0].message.provider_specific_fields["citations"] assert citations is not None + if citations: + citation = citations[0][0] + assert "supported_text" in citation + assert "cited_text" in citation + assert "document_index" in citation + assert "document_title" in citation + assert "start_char_index" in citation + assert "end_char_index" in citation def test_anthropic_citations_api_streaming(): @@ -955,11 +963,11 @@ def test_anthropic_citations_api_streaming(): has_citations = False for chunk in resp: print(f"returned chunk: {chunk}") - if ( - chunk.choices[0].delta.provider_specific_fields - and "citation" in chunk.choices[0].delta.provider_specific_fields - ): - has_citations = True + if provider_specific_fields := chunk.choices[0].delta.provider_specific_fields: + if "citation" in provider_specific_fields: + has_citations = True + + assert "chunk_type" in provider_specific_fields assert has_citations From 1ac7f875c9e2d59f77080161e8f4badb3cb541c7 Mon Sep 17 00:00:00 2001 From: TomeHirata Date: Mon, 1 Sep 2025 17:21:47 +0900 Subject: [PATCH 13/46] fix test --- litellm/llms/anthropic/chat/handler.py | 2 - .../test_anthropic_chat_transformation.py | 43 +++++++++++++------ 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 253f5d9be2c..5618c50923e 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -743,8 +743,6 @@ class ModelResponseIterator: ) text, tool_use = self._handle_json_mode_chunk(text=text, tool_use=tool_use) - if type_chunk: - provider_specific_fields["chunk_type"] = type_chunk returned_chunk = ModelResponseStream( choices=[ diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index ff454968d9c..dcca87baf32 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -115,16 +115,11 @@ def test_calculate_usage_nulls(usage_object, expected_usage): assert hasattr(usage, k) assert getattr(usage, k) == v -@pytest.mark.parametrize("usage_object", [ - { - "server_tool_use": { - "web_search_requests": None - } - }, - { - "server_tool_use": None - } -]) + +@pytest.mark.parametrize( + "usage_object", + [{"server_tool_use": {"web_search_requests": None}}, {"server_tool_use": None}], +) def test_calculate_usage_server_tool_null(usage_object): """ Correctly deal with null values in usage object @@ -132,10 +127,11 @@ def test_calculate_usage_server_tool_null(usage_object): Fixes https://github.com/BerriAI/litellm/issues/11920 """ config = AnthropicConfig() - + usage = config.calculate_usage(usage_object=usage_object, reasoning_content=None) assert not hasattr(usage, "server_tool_use") + def test_extract_response_content_with_citations(): config = AnthropicConfig() @@ -188,7 +184,30 @@ def test_extract_response_content_with_citations(): } _, citations, _, _, _ = config.extract_response_content(completion_response) - assert citations is not None + assert citations == [ + [ + { + "type": "char_location", + "cited_text": "The grass is green. ", + "document_index": 0, + "document_title": "My Document", + "start_char_index": 0, + "end_char_index": 20, + "supported_text": "the grass is green", + }, + ], + [ + { + "type": "char_location", + "cited_text": "The sky is blue.", + "document_index": 0, + "document_title": "My Document", + "start_char_index": 20, + "end_char_index": 36, + "supported_text": "the sky is blue", + }, + ], + ] def test_map_tool_helper(): From 002c2f16dc679c34944dfb9f68dc7595dc1ab9a9 Mon Sep 17 00:00:00 2001 From: retanoj Date: Mon, 1 Sep 2025 20:02:53 +0800 Subject: [PATCH 14/46] fix token count error when proxy gemini cli to openai like model --- litellm/google_genai/adapters/handler.py | 4 ++ litellm/proxy/google_endpoints/endpoints.py | 23 +++++++-- .../proxy/google_endpoints/__init__.py | 0 .../proxy/google_endpoints/test_endpoints.py | 49 +++++++++++++++++++ 4 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 tests/test_litellm/proxy/google_endpoints/__init__.py create mode 100644 tests/test_litellm/proxy/google_endpoints/test_endpoints.py diff --git a/litellm/google_genai/adapters/handler.py b/litellm/google_genai/adapters/handler.py index 1f575f27591..c5f378554b2 100644 --- a/litellm/google_genai/adapters/handler.py +++ b/litellm/google_genai/adapters/handler.py @@ -37,6 +37,10 @@ class GenerateContentToCompletionHandler: completion_kwargs: Dict[str, Any] = dict(completion_request) + # feed metadata for custom callback + # if 'metadata' in extra_kwargs: + # completion_kwargs['metadata'] = extra_kwargs['metadata'] + if stream: completion_kwargs["stream"] = stream diff --git a/litellm/proxy/google_endpoints/endpoints.py b/litellm/proxy/google_endpoints/endpoints.py index 373232e22d2..4f57e1e7ce8 100644 --- a/litellm/proxy/google_endpoints/endpoints.py +++ b/litellm/proxy/google_endpoints/endpoints.py @@ -173,15 +173,22 @@ async def google_count_tokens(request: Request, model_name: str): """ from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.proxy_server import token_counter as internal_token_counter + from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter data = await _read_request_body(request=request) contents = data.get("contents", []) #Create TokenCountRequest for the internal endpoint from litellm.proxy._types import TokenCountRequest + # Translate contents to openai format messages using the adapter + messages = (GoogleGenAIAdapter() + .translate_generate_content_to_completion(model_name, contents) + .get("messages", [])) + token_request = TokenCountRequest( model=model_name, - contents=contents + contents=contents, + messages=messages, # compatibility when use openai-like endpoint ) # Call the internal token counter function with direct request flag set to False @@ -192,10 +199,16 @@ async def google_count_tokens(request: Request, model_name: str): if token_response is not None: # cast the response to the well known format original_response: dict = token_response.original_response or {} - return TokenCountDetailsResponse( - totalTokens=original_response.get("totalTokens", 0), - promptTokensDetails=original_response.get("promptTokensDetails", []), - ) + if original_response: + return TokenCountDetailsResponse( + totalTokens=original_response.get("totalTokens", 0), + promptTokensDetails=original_response.get("promptTokensDetails", []), + ) + else: + return TokenCountDetailsResponse( + totalTokens=token_response.total_tokens or 0, + promptTokensDetails=[], + ) ######################################################### # Return the response in the well known format diff --git a/tests/test_litellm/proxy/google_endpoints/__init__.py b/tests/test_litellm/proxy/google_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/proxy/google_endpoints/test_endpoints.py b/tests/test_litellm/proxy/google_endpoints/test_endpoints.py new file mode 100644 index 00000000000..2f2538bf9aa --- /dev/null +++ b/tests/test_litellm/proxy/google_endpoints/test_endpoints.py @@ -0,0 +1,49 @@ +""" +Test for google_endpoints/endpoints.py +""" +import pytest +import sys, os +from dotenv import load_dotenv + + +from litellm.proxy.google_endpoints.endpoints import google_count_tokens +from litellm.types.llms.vertex_ai import TokenCountDetailsResponse +from starlette.requests import Request + +load_dotenv() + +sys.path.insert( + 0, os.path.abspath("../../../..") +) + +@pytest.mark.asyncio +async def test_proxy_gemini_to_openai_like_model_token_counting(): + """ + Test the token counting endpoint for proxing gemini to openai-like models. + """ + response: TokenCountDetailsResponse = await google_count_tokens( + request=Request( + scope={ + "type": "http", + "parsed_body": ( + [ + "contents" + ], + { + "contents": [ + { + "parts": [ + { + "text": "Hello, how are you?" + } + ] + } + ] + } + ) + } + ), + model_name="volcengine/foo", + ) + + assert response.get("totalTokens") > 0 \ No newline at end of file From e05ffcfb4c2b3bbcb9de75a8f1609ffa141f6326 Mon Sep 17 00:00:00 2001 From: retanoj Date: Mon, 1 Sep 2025 20:09:19 +0800 Subject: [PATCH 15/46] fix back --- litellm/google_genai/adapters/handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/google_genai/adapters/handler.py b/litellm/google_genai/adapters/handler.py index c5f378554b2..c15d0cb9deb 100644 --- a/litellm/google_genai/adapters/handler.py +++ b/litellm/google_genai/adapters/handler.py @@ -38,8 +38,8 @@ class GenerateContentToCompletionHandler: completion_kwargs: Dict[str, Any] = dict(completion_request) # feed metadata for custom callback - # if 'metadata' in extra_kwargs: - # completion_kwargs['metadata'] = extra_kwargs['metadata'] + if 'metadata' in extra_kwargs: + completion_kwargs['metadata'] = extra_kwargs['metadata'] if stream: completion_kwargs["stream"] = stream From af2f447698bed16210c8bc452714876455d51ebb Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Mon, 1 Sep 2025 10:28:48 -0400 Subject: [PATCH 16/46] added prometheus metrics --- .../integrations/prometheus.py | 20 +++++++------ litellm/types/integrations/prometheus.py | 28 +++++++++++++++++-- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/enterprise/litellm_enterprise/integrations/prometheus.py b/enterprise/litellm_enterprise/integrations/prometheus.py index a2d781fa1c4..1aa2359bc15 100644 --- a/enterprise/litellm_enterprise/integrations/prometheus.py +++ b/enterprise/litellm_enterprise/integrations/prometheus.py @@ -95,13 +95,14 @@ class PrometheusLogger(CustomLogger): self.litellm_llm_api_time_to_first_token_metric = self._histogram_factory( "litellm_llm_api_time_to_first_token_metric", "Time to first token for a models LLM API call", - labelnames=[ - "model", - "hashed_api_key", - "api_key_alias", - "team", - "team_alias", - ], + # labelnames=[ + # "model", + # "hashed_api_key", + # "api_key_alias", + # "team", + # "team_alias", + # ], + labelnames=self.get_labels_for_metric("litellm_llm_api_time_to_first_token_metric"), buckets=LATENCY_BUCKETS, ) @@ -255,13 +256,14 @@ class PrometheusLogger(CustomLogger): self.litellm_deployment_state = self._gauge_factory( "litellm_deployment_state", "LLM Deployment Analytics - The state of the deployment: 0 = healthy, 1 = partial outage, 2 = complete outage", - labelnames=_logged_llm_labels, + labelnames=self.get_labels_for_metric("litellm_deployment_state") ) self.litellm_deployment_cooled_down = self._counter_factory( "litellm_deployment_cooled_down", "LLM Deployment Analytics - Number of times a deployment has been cooled down by LiteLLM load balancing logic. exception_status is the status of the exception that caused the deployment to be cooled down", - labelnames=_logged_llm_labels + [EXCEPTION_STATUS], + # labelnames=_logged_llm_labels + [EXCEPTION_STATUS], + labelnames=self.get_labels_for_metric("litellm_deployment_cooled_down") ) self.litellm_deployment_success_responses = self._counter_factory( diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index e0ee950d260..4ab0ccf5022 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -154,6 +154,7 @@ class UserAPIKeyLabelNames(Enum): DEFINED_PROMETHEUS_METRICS = Literal[ "litellm_llm_api_latency_metric", + "litellm_llm_api_time_to_first_token_metric", "litellm_request_total_latency_metric", "litellm_overhead_latency_metric", "litellm_remaining_requests_metric", @@ -173,9 +174,11 @@ DEFINED_PROMETHEUS_METRICS = Literal[ "litellm_remaining_api_key_budget_metric", "litellm_api_key_max_budget_metric", "litellm_api_key_budget_remaining_hours_metric", + "litellm_deployment_state", "litellm_deployment_failure_responses", "litellm_deployment_total_requests", "litellm_deployment_success_responses", + "litellm_deployment_cooled_down", "litellm_pod_lock_manager_size", "litellm_in_memory_daily_spend_update_queue_size", "litellm_redis_daily_spend_update_queue_size", @@ -191,9 +194,14 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.API_KEY_ALIAS.value, UserAPIKeyLabelNames.TEAM.value, UserAPIKeyLabelNames.TEAM_ALIAS.value, - UserAPIKeyLabelNames.REQUESTED_MODEL.value, - UserAPIKeyLabelNames.END_USER.value, - UserAPIKeyLabelNames.USER.value, + ] + + litellm_llm_api_time_to_first_token_metric = [ + UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value, + UserAPIKeyLabelNames.API_KEY_HASH.value, + UserAPIKeyLabelNames.API_KEY_ALIAS.value, + UserAPIKeyLabelNames.TEAM.value, + UserAPIKeyLabelNames.TEAM_ALIAS.value, ] litellm_request_total_latency_metric = [ @@ -315,6 +323,20 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.REQUESTED_MODEL.value, ] + litellm_deployment_state = [ + UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value, + UserAPIKeyLabelNames.MODEL_ID.value, + UserAPIKeyLabelNames.API_BASE.value, + UserAPIKeyLabelNames.API_PROVIDER.value, + ] + + litellm_deployment_cooled_down = [ + UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value, + UserAPIKeyLabelNames.MODEL_ID.value, + UserAPIKeyLabelNames.API_BASE.value, + UserAPIKeyLabelNames.API_PROVIDER.value, + ] + litellm_deployment_successful_fallbacks = [ UserAPIKeyLabelNames.REQUESTED_MODEL.value, UserAPIKeyLabelNames.FALLBACK_MODEL.value, From 8de65e84480573ef88f28054ba6b83a39a19de3f Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Mon, 1 Sep 2025 10:36:51 -0400 Subject: [PATCH 17/46] fix: added prometheus missing metrics --- enterprise/litellm_enterprise/integrations/prometheus.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/enterprise/litellm_enterprise/integrations/prometheus.py b/enterprise/litellm_enterprise/integrations/prometheus.py index 1aa2359bc15..742f2dbbe9b 100644 --- a/enterprise/litellm_enterprise/integrations/prometheus.py +++ b/enterprise/litellm_enterprise/integrations/prometheus.py @@ -244,14 +244,6 @@ class PrometheusLogger(CustomLogger): labelnames=["api_provider"], ) - # Get all keys - _logged_llm_labels = [ - UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value, - UserAPIKeyLabelNames.MODEL_ID.value, - UserAPIKeyLabelNames.API_BASE.value, - UserAPIKeyLabelNames.API_PROVIDER.value, - ] - # Metric for deployment state self.litellm_deployment_state = self._gauge_factory( "litellm_deployment_state", From 31cfe1fe0a7816979ae37b0f0eea9332ce09a5b3 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Mon, 1 Sep 2025 10:45:38 -0400 Subject: [PATCH 18/46] Revert "fix: import verbose_proxy_logger to resolve undefined name errors" This reverts commit a8f2ebed9cda1c53c3a21196577e2e3639fe9eeb. --- enterprise/litellm_enterprise/proxy/guardrails/endpoints.py | 1 - 1 file changed, 1 deletion(-) diff --git a/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py b/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py index 85f8bf22417..47bec519c40 100644 --- a/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py +++ b/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py @@ -13,7 +13,6 @@ from typing import Dict, List, Optional from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel -from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth From a6790ac158a8ce1e20167e7495f8c77cdbbb6b94 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Mon, 1 Sep 2025 10:45:44 -0400 Subject: [PATCH 19/46] Revert "feat: add UI support for displaying and managing guardrails on virtual keys" This reverts commit 83654c56089208db695b5b4821e7ae41118aba34. --- .../proxy/guardrails/endpoints.py | 2 - .../proxy/guardrails/guardrail_registry.py | 1 - .../virtual_keys/virtual_key_detail.tsx | 160 ------------------ .../virtual_keys/virtual_key_guardrails.tsx | 93 ---------- .../src/services/guardrails.ts | 27 --- .../src/services/virtual_key_guardrails.ts | 57 ------- ui/litellm-dashboard/src/types/guardrail.ts | 11 -- ui/litellm-dashboard/src/types/virtual_key.ts | 13 -- 8 files changed, 364 deletions(-) delete mode 100644 ui/litellm-dashboard/src/components/virtual_keys/virtual_key_detail.tsx delete mode 100644 ui/litellm-dashboard/src/components/virtual_keys/virtual_key_guardrails.tsx delete mode 100644 ui/litellm-dashboard/src/services/guardrails.ts delete mode 100644 ui/litellm-dashboard/src/services/virtual_key_guardrails.ts delete mode 100644 ui/litellm-dashboard/src/types/guardrail.ts delete mode 100644 ui/litellm-dashboard/src/types/virtual_key.ts diff --git a/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py b/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py index 47bec519c40..fcd5e82ab3a 100644 --- a/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py +++ b/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py @@ -97,9 +97,7 @@ async def get_guardrails_for_virtual_key( """ Get all guardrails associated with a virtual key """ - verbose_proxy_logger.debug(f"Getting guardrails for virtual key: {virtual_key_id}") guardrails = IN_MEMORY_GUARDRAIL_HANDLER.get_guardrails_for_virtual_key(virtual_key_id) - verbose_proxy_logger.debug(f"Found {len(guardrails)} guardrails for virtual key {virtual_key_id}") return VirtualKeyGuardrailsResponse( virtual_key_id=virtual_key_id, diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 7d3432d5cb6..523ff9f4cd8 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -567,7 +567,6 @@ class InMemoryGuardrailHandler: Get all guardrails associated with a virtual key """ guardrail_ids = self.virtual_key_to_guardrails.get(virtual_key_id, []) - verbose_proxy_logger.debug(f"Getting guardrails for virtual key {virtual_key_id}: {guardrail_ids}") return [self.IN_MEMORY_GUARDRAILS[gid] for gid in guardrail_ids if gid in self.IN_MEMORY_GUARDRAILS] diff --git a/ui/litellm-dashboard/src/components/virtual_keys/virtual_key_detail.tsx b/ui/litellm-dashboard/src/components/virtual_keys/virtual_key_detail.tsx deleted file mode 100644 index 111bd26841c..00000000000 --- a/ui/litellm-dashboard/src/components/virtual_keys/virtual_key_detail.tsx +++ /dev/null @@ -1,160 +0,0 @@ -import React, { useState, useEffect } from 'react'; -import { Card, Flex, Text, Heading, Button, Dialog, TextField, Select, Box, Tabs } from '@radix-ui/themes'; -import { toast } from 'react-hot-toast'; -import { VirtualKey } from '../../types/virtual_key'; -import { fetchVirtualKey, updateVirtualKey } from '../../services/virtual_keys'; -import { VirtualKeyGuardrails } from './virtual_key_guardrails'; -import { fetchGuardrails } from '../../services/guardrails'; -import { Guardrail } from '../../types/guardrail'; -import { associateGuardrailWithVirtualKey } from '../../services/virtual_key_guardrails'; - -interface VirtualKeyDetailProps { - virtualKeyId: string; - onClose: () => void; -} - -export const VirtualKeyDetail: React.FC = ({ virtualKeyId, onClose }) => { - const [virtualKey, setVirtualKey] = useState(null); - const [loading, setLoading] = useState(true); - const [availableGuardrails, setAvailableGuardrails] = useState([]); - const [selectedGuardrailId, setSelectedGuardrailId] = useState(''); - const [refreshTrigger, setRefreshTrigger] = useState(0); - - useEffect(() => { - const loadVirtualKey = async () => { - try { - const data = await fetchVirtualKey(virtualKeyId); - setVirtualKey(data); - } catch (error) { - console.error('Error loading virtual key:', error); - toast.error('Failed to load virtual key details'); - } finally { - setLoading(false); - } - }; - - const loadGuardrails = async () => { - try { - const data = await fetchGuardrails(); - setAvailableGuardrails(data.guardrails); - } catch (error) { - console.error('Error loading guardrails:', error); - } - }; - - loadVirtualKey(); - loadGuardrails(); - }, [virtualKeyId]); - - const handleAddGuardrail = async () => { - if (!selectedGuardrailId) { - toast.error('Please select a guardrail'); - return; - } - - try { - await associateGuardrailWithVirtualKey(virtualKeyId, selectedGuardrailId); - toast.success('Guardrail added to virtual key'); - setSelectedGuardrailId(''); - setRefreshTrigger(prev => prev + 1); - } catch (error) { - console.error('Error adding guardrail:', error); - toast.error('Failed to add guardrail'); - } - }; - - if (loading) { - return ( - - - Virtual Key Details - Loading... - - - ); - } - - if (!virtualKey) { - return ( - - - Error - Virtual key not found - - - - - - ); - } - - return ( - - - Virtual Key: {virtualKey.key_name || 'Unnamed Key'} - - - - Details - Guardrails - - - - - - - - Key ID: - {virtualKey.key_id} - - - Team ID: - {virtualKey.team_id || 'N/A'} - - - Models: - {virtualKey.models?.join(', ') || 'All models'} - - - Spend: - ${virtualKey.spend?.toFixed(2) || '0.00'} - - - - - - - - - Add Guardrail - - - - - {availableGuardrails.map(guardrail => ( - - {guardrail.guardrail_name || `Guardrail ${guardrail.guardrail_id.substring(0, 8)}`} - - ))} - - - - - - - - - - - - - - - - - - ); -}; diff --git a/ui/litellm-dashboard/src/components/virtual_keys/virtual_key_guardrails.tsx b/ui/litellm-dashboard/src/components/virtual_keys/virtual_key_guardrails.tsx deleted file mode 100644 index 0acbc9d3f4a..00000000000 --- a/ui/litellm-dashboard/src/components/virtual_keys/virtual_key_guardrails.tsx +++ /dev/null @@ -1,93 +0,0 @@ -import React, { useEffect, useState } from 'react'; -import { Badge, Button, Card, Flex, Heading, Text, Box, Spinner } from '@radix-ui/themes'; -import { fetchGuardrailsForVirtualKey, associateGuardrailWithVirtualKey, disassociateGuardrailFromVirtualKey } from '../../services/virtual_key_guardrails'; -import { Guardrail } from '../../types/guardrail'; -import { toast } from 'react-hot-toast'; - -interface VirtualKeyGuardrailsProps { - virtualKeyId: string; - refreshTrigger?: number; -} - -export const VirtualKeyGuardrails: React.FC = ({ - virtualKeyId, - refreshTrigger = 0 -}) => { - const [guardrails, setGuardrails] = useState([]); - const [loading, setLoading] = useState(true); - const [error, setError] = useState(null); - - useEffect(() => { - const loadGuardrails = async () => { - if (!virtualKeyId) return; - - setLoading(true); - try { - const data = await fetchGuardrailsForVirtualKey(virtualKeyId); - setGuardrails(data.guardrails); - setError(null); - } catch (err) { - console.error('Error loading guardrails for virtual key:', err); - setError('Failed to load guardrails'); - setGuardrails([]); - } finally { - setLoading(false); - } - }; - - loadGuardrails(); - }, [virtualKeyId, refreshTrigger]); - - const handleRemoveGuardrail = async (guardrailId: string) => { - try { - await disassociateGuardrailFromVirtualKey(virtualKeyId, guardrailId); - setGuardrails(guardrails.filter(g => g.guardrail_id !== guardrailId)); - toast.success('Guardrail removed from virtual key'); - } catch (err) { - console.error('Error removing guardrail:', err); - toast.error('Failed to remove guardrail'); - } - }; - - if (loading) { - return ( - - - - - - ); - } - - if (error) { - return ( - - {error} - - ); - } - - return ( - - Attached Guardrails - {guardrails.length === 0 ? ( - No guardrails attached to this virtual key - ) : ( - - {guardrails.map((guardrail) => ( - - - {guardrail.guardrail_name || 'Unnamed Guardrail'} - Type: {guardrail.litellm_params?.guardrail} - Mode: {guardrail.litellm_params?.mode} - - - - ))} - - )} - - ); -}; diff --git a/ui/litellm-dashboard/src/services/guardrails.ts b/ui/litellm-dashboard/src/services/guardrails.ts deleted file mode 100644 index 2cb7b80a4b1..00000000000 --- a/ui/litellm-dashboard/src/services/guardrails.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Guardrail } from '../types/guardrail'; - -interface GuardrailsResponse { - guardrails: Guardrail[]; -} - -export async function fetchGuardrails(): Promise { - const response = await fetch('/api/guardrails'); - - if (!response.ok) { - const errorText = await response.text(); - throw new Error(`Failed to fetch guardrails: ${errorText}`); - } - - return await response.json(); -} - -export async function fetchGuardrail(guardrailId: string): Promise { - const response = await fetch(`/api/guardrails/${guardrailId}`); - - if (!response.ok) { - const errorText = await response.text(); - throw new Error(`Failed to fetch guardrail: ${errorText}`); - } - - return await response.json(); -} diff --git a/ui/litellm-dashboard/src/services/virtual_key_guardrails.ts b/ui/litellm-dashboard/src/services/virtual_key_guardrails.ts deleted file mode 100644 index de7ce30d890..00000000000 --- a/ui/litellm-dashboard/src/services/virtual_key_guardrails.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { Guardrail } from '../types/guardrail'; - -interface VirtualKeyGuardrailsResponse { - virtual_key_id: string; - guardrails: Guardrail[]; -} - -export async function fetchGuardrailsForVirtualKey(virtualKeyId: string): Promise { - const response = await fetch(`/api/guardrails/virtual_key/${virtualKeyId}`); - - if (!response.ok) { - const errorText = await response.text(); - throw new Error(`Failed to fetch guardrails: ${errorText}`); - } - - return await response.json(); -} - -export async function associateGuardrailWithVirtualKey(virtualKeyId: string, guardrailId: string): Promise<{ message: string }> { - const response = await fetch('/api/guardrails/virtual_key/associate', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - virtual_key_id: virtualKeyId, - guardrail_id: guardrailId, - }), - }); - - if (!response.ok) { - const errorText = await response.text(); - throw new Error(`Failed to associate guardrail: ${errorText}`); - } - - return await response.json(); -} - -export async function disassociateGuardrailFromVirtualKey(virtualKeyId: string, guardrailId: string): Promise<{ message: string }> { - const response = await fetch('/api/guardrails/virtual_key/disassociate', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - virtual_key_id: virtualKeyId, - guardrail_id: guardrailId, - }), - }); - - if (!response.ok) { - const errorText = await response.text(); - throw new Error(`Failed to disassociate guardrail: ${errorText}`); - } - - return await response.json(); -} diff --git a/ui/litellm-dashboard/src/types/guardrail.ts b/ui/litellm-dashboard/src/types/guardrail.ts deleted file mode 100644 index 6cb26d04973..00000000000 --- a/ui/litellm-dashboard/src/types/guardrail.ts +++ /dev/null @@ -1,11 +0,0 @@ -export interface Guardrail { - guardrail_id: string; - guardrail_name: string | null; - litellm_params: { - guardrail: string; - mode: string; - default_on: boolean; - [key: string]: any; - }; - guardrail_info?: Record; -} diff --git a/ui/litellm-dashboard/src/types/virtual_key.ts b/ui/litellm-dashboard/src/types/virtual_key.ts deleted file mode 100644 index e130853b1d1..00000000000 --- a/ui/litellm-dashboard/src/types/virtual_key.ts +++ /dev/null @@ -1,13 +0,0 @@ -export interface VirtualKey { - key_id: string; - key_name?: string; - team_id?: string; - models?: string[]; - spend?: number; - max_budget?: number; - max_parallel_requests?: number; - metadata?: Record; - expires?: string; - created_at?: string; - updated_at?: string; -} From 5649c0f0b99e3a627bd55c7d158bdff2639ec987 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Mon, 1 Sep 2025 10:45:46 -0400 Subject: [PATCH 20/46] Revert "feat: add persistence and management for guardrails on virtual keys" This reverts commit 6cd5afa8b11ea3654a2c26d3d64cd2a2e3eff3f9. --- .../proxy/guardrails/endpoints.py | 70 +------------------ .../proxy/guardrails/guardrail_registry.py | 30 -------- 2 files changed, 3 insertions(+), 97 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py b/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py index fcd5e82ab3a..cdf86dcea67 100644 --- a/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py +++ b/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py @@ -6,28 +6,16 @@ To see all free guardrails see litellm/proxy/guardrails/* Exposed Routes: - /mask_pii -- /virtual_key/guardrails """ -from typing import Dict, List, Optional +from typing import Optional -from fastapi import APIRouter, Depends, HTTPException -from pydantic import BaseModel +from fastapi import APIRouter, Depends from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.guardrails.guardrail_endpoints import GUARDRAIL_REGISTRY -from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER -from litellm.types.guardrails import ApplyGuardrailRequest, ApplyGuardrailResponse, Guardrail - -# Models for virtual key guardrail management -class VirtualKeyGuardrailRequest(BaseModel): - virtual_key_id: str - guardrail_id: str - -class VirtualKeyGuardrailsResponse(BaseModel): - virtual_key_id: str - guardrails: List[Guardrail] +from litellm.types.guardrails import ApplyGuardrailRequest, ApplyGuardrailResponse router = APIRouter(tags=["guardrails"], prefix="/guardrails") @@ -51,55 +39,3 @@ async def apply_guardrail( return await active_guardrail.apply_guardrail( text=request.text, language=request.language, entities=request.entities ) - -@router.post("/virtual_key/associate", response_model=Dict[str, str]) -async def associate_guardrail_with_virtual_key( - request: VirtualKeyGuardrailRequest, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -): - """ - Associate a guardrail with a virtual key - """ - # Check if guardrail exists - guardrail = IN_MEMORY_GUARDRAIL_HANDLER.get_guardrail_by_id(request.guardrail_id) - if not guardrail: - raise HTTPException(status_code=404, detail=f"Guardrail {request.guardrail_id} not found") - - # Associate guardrail with virtual key - IN_MEMORY_GUARDRAIL_HANDLER.associate_guardrail_with_virtual_key( - virtual_key_id=request.virtual_key_id, - guardrail_id=request.guardrail_id - ) - - return {"message": f"Guardrail {request.guardrail_id} associated with virtual key {request.virtual_key_id}"} - -@router.post("/virtual_key/disassociate", response_model=Dict[str, str]) -async def disassociate_guardrail_from_virtual_key( - request: VirtualKeyGuardrailRequest, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -): - """ - Disassociate a guardrail from a virtual key - """ - # Disassociate guardrail from virtual key - IN_MEMORY_GUARDRAIL_HANDLER.disassociate_guardrail_from_virtual_key( - virtual_key_id=request.virtual_key_id, - guardrail_id=request.guardrail_id - ) - - return {"message": f"Guardrail {request.guardrail_id} disassociated from virtual key {request.virtual_key_id}"} - -@router.get("/virtual_key/{virtual_key_id}", response_model=VirtualKeyGuardrailsResponse) -async def get_guardrails_for_virtual_key( - virtual_key_id: str, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -): - """ - Get all guardrails associated with a virtual key - """ - guardrails = IN_MEMORY_GUARDRAIL_HANDLER.get_guardrails_for_virtual_key(virtual_key_id) - - return VirtualKeyGuardrailsResponse( - virtual_key_id=virtual_key_id, - guardrails=guardrails - ) diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 523ff9f4cd8..21429f462d4 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -369,11 +369,6 @@ class InMemoryGuardrailHandler: """ Guardrail id to CustomGuardrail object mapping """ - - self.virtual_key_to_guardrails: Dict[str, List[str]] = {} - """ - Virtual key id to list of guardrail ids mapping - """ def initialize_guardrail( self, @@ -543,31 +538,6 @@ class InMemoryGuardrailHandler: Get a guardrail by its ID from memory """ return self.IN_MEMORY_GUARDRAILS.get(guardrail_id) - - def associate_guardrail_with_virtual_key(self, virtual_key_id: str, guardrail_id: str) -> None: - """ - Associate a guardrail with a virtual key - """ - if virtual_key_id not in self.virtual_key_to_guardrails: - self.virtual_key_to_guardrails[virtual_key_id] = [] - - if guardrail_id not in self.virtual_key_to_guardrails[virtual_key_id]: - self.virtual_key_to_guardrails[virtual_key_id].append(guardrail_id) - - def disassociate_guardrail_from_virtual_key(self, virtual_key_id: str, guardrail_id: str) -> None: - """ - Disassociate a guardrail from a virtual key - """ - if virtual_key_id in self.virtual_key_to_guardrails: - if guardrail_id in self.virtual_key_to_guardrails[virtual_key_id]: - self.virtual_key_to_guardrails[virtual_key_id].remove(guardrail_id) - - def get_guardrails_for_virtual_key(self, virtual_key_id: str) -> List[Guardrail]: - """ - Get all guardrails associated with a virtual key - """ - guardrail_ids = self.virtual_key_to_guardrails.get(virtual_key_id, []) - return [self.IN_MEMORY_GUARDRAILS[gid] for gid in guardrail_ids if gid in self.IN_MEMORY_GUARDRAILS] ######################################################## From 2626123aca31e6ed7c59208c460376c35f7ba251 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Mon, 1 Sep 2025 11:33:34 -0400 Subject: [PATCH 21/46] added litellm_spend_metric --- .../litellm_enterprise/integrations/prometheus.py | 11 ++--------- litellm/types/integrations/prometheus.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/enterprise/litellm_enterprise/integrations/prometheus.py b/enterprise/litellm_enterprise/integrations/prometheus.py index 742f2dbbe9b..efee1a7783e 100644 --- a/enterprise/litellm_enterprise/integrations/prometheus.py +++ b/enterprise/litellm_enterprise/integrations/prometheus.py @@ -110,15 +110,7 @@ class PrometheusLogger(CustomLogger): self.litellm_spend_metric = self._counter_factory( "litellm_spend_metric", "Total spend on LLM requests", - labelnames=[ - "end_user", - "hashed_api_key", - "api_key_alias", - "model", - "team", - "team_alias", - "user", - ], + labelnames=self.get_labels_for_metric("litellm_spend_metric"), ) # Counter for total_output_tokens @@ -321,6 +313,7 @@ class PrometheusLogger(CustomLogger): documentation="deprecated - use litellm_proxy_total_requests_metric. Total number of LLM calls to litellm - track total per API Key, team, user", labelnames=self.get_labels_for_metric("litellm_requests_metric"), ) + except Exception as e: print_verbose(f"Got exception on init prometheus client {str(e)}") raise e diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index 4ab0ccf5022..4637c05d144 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -163,6 +163,7 @@ DEFINED_PROMETHEUS_METRICS = Literal[ "litellm_proxy_failed_requests_metric", "litellm_deployment_latency_per_output_token", "litellm_requests_metric", + "litellm_spend_metric", "litellm_total_tokens_metric", "litellm_input_tokens_metric", "litellm_output_tokens_metric", @@ -290,6 +291,16 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.USER_EMAIL.value, ] + litellm_spend_metric = [ + UserAPIKeyLabelNames.END_USER.value, + UserAPIKeyLabelNames.API_KEY_HASH.value, + UserAPIKeyLabelNames.API_KEY_ALIAS.value, + UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value, + UserAPIKeyLabelNames.TEAM.value, + UserAPIKeyLabelNames.TEAM_ALIAS.value, + UserAPIKeyLabelNames.USER.value, + ] + litellm_input_tokens_metric = [ UserAPIKeyLabelNames.END_USER.value, UserAPIKeyLabelNames.API_KEY_HASH.value, From 2622a0634a4f8c59db87ac307923063202a6b72b Mon Sep 17 00:00:00 2001 From: Ivan Krakhmaliuk Date: Mon, 1 Sep 2025 19:51:54 +0300 Subject: [PATCH 22/46] GPT-5: Remove params unsupported with reasoning models --- litellm/llms/openai/chat/gpt_5_transformation.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 9a8bb74d447..3902304a3b4 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -28,7 +28,18 @@ class OpenAIGPT5Config(OpenAIGPTConfig): base_gpt_series_params.extend(gpt_5_only_params) if not supports_tool_choice(model=model): base_gpt_series_params.remove("tool_choice") - return base_gpt_series_params + + non_supported_params = [ + "logprobs", + "top_p", + "presence_penalty", + "frequency_penalty", + "top_logprobs", + ] + + return [ + param for param in base_gpt_series_params if param not in non_supported_params + ] def map_openai_params( self, From 76dd9d81100dbda65384cb6030c9c63fc0f676ee Mon Sep 17 00:00:00 2001 From: Ivan Krakhmaliuk Date: Mon, 1 Sep 2025 20:30:43 +0300 Subject: [PATCH 23/46] GPT-5: Test for unsupported params --- .../llms/openai/test_gpt5_transformation.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index 3bdab355977..3e6a6a23468 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -41,3 +41,14 @@ def test_gpt5_temperature_error(config: OpenAIConfig): model="gpt-5", drop_params=False, ) + + +def test_gpt5_unsupported_params_drop(config: OpenAIConfig): + assert "top_p" not in config.get_supported_openai_params(model="gpt-5") + params = config.map_openai_params( + non_default_params={"top_p": 0.5}, + optional_params={}, + model="gpt-5", + drop_params=True, + ) + assert "top_p" not in params From ca70b689afa25da7c7d438f86ab76f4531ad4db1 Mon Sep 17 00:00:00 2001 From: tanjiro <56165694+NANDINI-star@users.noreply.github.com> Date: Tue, 2 Sep 2025 03:08:09 +0900 Subject: [PATCH 24/46] Logs page screen size fixed (#14135) * width fixed * tailwind css instead of inline css * minor width fixes * time button fixed * fixed width for the "showing..." part --- .../view_logs/RequestResponsePanel.tsx | 14 +++--- .../src/components/view_logs/index.tsx | 48 ++++++++++--------- .../src/components/view_logs/table.tsx | 10 ++-- 3 files changed, 38 insertions(+), 34 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestResponsePanel.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestResponsePanel.tsx index 4aaaabba58d..13f2547bd75 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestResponsePanel.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestResponsePanel.tsx @@ -72,9 +72,9 @@ export function RequestResponsePanel({ }; return ( -
+
{/* Request Side */} -
+

Request

-
- 
{JSON.stringify(getRawRequest(), null, 2)}
+
+ 
{JSON.stringify(getRawRequest(), null, 2)}
{/* Response Side */} -
+

Response @@ -116,9 +116,9 @@ export function RequestResponsePanel({

-
+
{hasResponse ? ( - 
{JSON.stringify(formattedResponse(), null, 2)}
+ 
{JSON.stringify(formattedResponse(), null, 2)}
) : (
Response data not available
)} diff --git a/ui/litellm-dashboard/src/components/view_logs/index.tsx b/ui/litellm-dashboard/src/components/view_logs/index.tsx index b227d493aa0..07de686fc48 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/index.tsx @@ -465,7 +465,7 @@ export default function SpendLogsTable({ const displayLabel = isCustomDate ? getTimeRangeDisplay(isCustomDate, startTime, endTime) : selectedOption?.label return ( -
+
setActiveTab(index === 0 ? "request logs" : "audit logs")}> Request Logs @@ -518,11 +518,11 @@ export default function SpendLogsTable({ onApplyFilters={handleFilterChange} onResetFilters={handleFilterReset} /> -
-
-
-
-
+
+
+
+
+
-
-
+
+
{quickSelectOpen && ( -
+
{quickSelectOptions.map((option) => (