From 16a7e0ce8fe83402b4ed00d8b02ae2475f6cd906 Mon Sep 17 00:00:00 2001 From: Chesars Date: Wed, 26 Nov 2025 17:01:23 -0300 Subject: [PATCH 001/418] fix: filter empty SSE lines in BaseModelResponseIterator to prevent extra empty chunks When streaming with stream_options={"include_usage": True}, xAI and other providers using BaseLLMHTTPHandler were returning an extra empty chunk after the usage chunk. This was caused by empty SSE lines (separators between events) being processed as empty GenericStreamingChunks. The fix adds a loop in __next__ and __anext__ to skip empty lines before processing, ensuring only meaningful SSE data events are converted to chunks. Fixes #17136 --- litellm/llms/base_llm/base_model_iterator.py | 89 +++++++++++--------- 1 file changed, 50 insertions(+), 39 deletions(-) diff --git a/litellm/llms/base_llm/base_model_iterator.py b/litellm/llms/base_llm/base_model_iterator.py index 6953b1c5878..62cd503a89e 100644 --- a/litellm/llms/base_llm/base_model_iterator.py +++ b/litellm/llms/base_llm/base_model_iterator.py @@ -125,26 +125,32 @@ class BaseModelResponseIterator: ) def __next__(self): - try: - chunk = self.response_iterator.__next__() - except StopIteration: - raise StopIteration - except ValueError as e: - raise RuntimeError(f"Error receiving chunk from stream: {e}") + while True: + try: + chunk = self.response_iterator.__next__() + except StopIteration: + raise StopIteration + except ValueError as e: + raise RuntimeError(f"Error receiving chunk from stream: {e}") - try: - str_line = chunk - if isinstance(chunk, bytes): # Handle binary data - str_line = chunk.decode("utf-8") # Convert bytes to string - index = str_line.find("data:") - if index != -1: - str_line = str_line[index:] - # chunk is a str at this point - return self._handle_string_chunk(str_line=str_line) - except StopIteration: - raise StopIteration - except ValueError as e: - raise RuntimeError(f"Error parsing chunk: {e},\nReceived chunk: {chunk}") + try: + str_line = chunk + if isinstance(chunk, bytes): # Handle binary data + str_line = chunk.decode("utf-8") # Convert bytes to string + index = str_line.find("data:") + if index != -1: + str_line = str_line[index:] + + # Skip empty lines (common in SSE streams between events) + if not str_line or not str_line.strip(): + continue + + # chunk is a str at this point + return self._handle_string_chunk(str_line=str_line) + except StopIteration: + raise StopIteration + except ValueError as e: + raise RuntimeError(f"Error parsing chunk: {e},\nReceived chunk: {chunk}") # Async iterator def __aiter__(self): @@ -152,30 +158,35 @@ class BaseModelResponseIterator: return self async def __anext__(self): - try: - chunk = await self.async_response_iterator.__anext__() + while True: + try: + chunk = await self.async_response_iterator.__anext__() - except StopAsyncIteration: - raise StopAsyncIteration - except ValueError as e: - raise RuntimeError(f"Error receiving chunk from stream: {e}") + except StopAsyncIteration: + raise StopAsyncIteration + except ValueError as e: + raise RuntimeError(f"Error receiving chunk from stream: {e}") - try: - str_line = chunk - if isinstance(chunk, bytes): # Handle binary data - str_line = chunk.decode("utf-8") # Convert bytes to string - index = str_line.find("data:") - if index != -1: - str_line = str_line[index:] + try: + str_line = chunk + if isinstance(chunk, bytes): # Handle binary data + str_line = chunk.decode("utf-8") # Convert bytes to string + index = str_line.find("data:") + if index != -1: + str_line = str_line[index:] - # chunk is a str at this point - chunk = self._handle_string_chunk(str_line=str_line) + # Skip empty lines (common in SSE streams between events) + if not str_line or not str_line.strip(): + continue - return chunk - except StopAsyncIteration: - raise StopAsyncIteration - except ValueError as e: - raise RuntimeError(f"Error parsing chunk: {e},\nReceived chunk: {chunk}") + # chunk is a str at this point + chunk = self._handle_string_chunk(str_line=str_line) + + return chunk + except StopAsyncIteration: + raise StopAsyncIteration + except ValueError as e: + raise RuntimeError(f"Error parsing chunk: {e},\nReceived chunk: {chunk}") class MockResponseIterator: # for returning ai21 streaming responses From 8c128edb5d3790096376c086f9fa1027f6344e08 Mon Sep 17 00:00:00 2001 From: Chesars Date: Wed, 26 Nov 2025 17:05:40 -0300 Subject: [PATCH 002/418] test: add unit tests for BaseModelResponseIterator empty SSE line filtering Tests verify that empty lines between SSE events are properly filtered and don't produce extra empty chunks in streaming responses. --- .../llms/base_llm/test_base_model_iterator.py | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 tests/test_litellm/llms/base_llm/test_base_model_iterator.py diff --git a/tests/test_litellm/llms/base_llm/test_base_model_iterator.py b/tests/test_litellm/llms/base_llm/test_base_model_iterator.py new file mode 100644 index 00000000000..d5166c4690e --- /dev/null +++ b/tests/test_litellm/llms/base_llm/test_base_model_iterator.py @@ -0,0 +1,117 @@ +""" +Tests for BaseModelResponseIterator - specifically testing that empty SSE lines are filtered +""" + +import pytest +from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator +from litellm.types.utils import GenericStreamingChunk, ModelResponseStream + + +class TestBaseModelResponseIterator: + """Test cases for BaseModelResponseIterator empty line filtering""" + + def test_filter_empty_sse_lines_sync(self): + """ + Test that empty SSE lines (common between SSE events) are filtered out + and don't produce empty chunks. + + This fixes the bug where providers using BaseLLMHTTPHandler (like xAI) + would return extra empty chunks when streaming with include_usage=True. + + Related: GitHub Issue #17136 + """ + # Simulate SSE stream with empty lines between events (normal SSE format) + sse_lines = [ + 'data: {"id":"1","choices":[{"delta":{"content":"Hello"}}]}', + '', # Empty line (SSE separator) + 'data: {"id":"1","choices":[{"delta":{"content":" World"}}]}', + '', # Empty line (SSE separator) + 'data: {"id":"1","choices":[],"usage":{"prompt_tokens":10,"completion_tokens":5}}', + '', # Empty line (SSE separator) + 'data: [DONE]', + '', # Empty line after DONE + ] + + iterator = BaseModelResponseIterator( + streaming_response=iter(sse_lines), + sync_stream=True + ) + + chunks = list(iterator) + + # Should have 4 chunks: 2 content + 1 usage + 1 DONE + # Empty lines should be filtered out + assert len(chunks) == 4, f"Expected 4 chunks, got {len(chunks)}" + + # Verify no empty/None chunks were included + # The base iterator returns ModelResponseStream objects + for i, chunk in enumerate(chunks): + assert chunk is not None, f"Chunk {i} should not be None" + + def test_filter_whitespace_only_lines_sync(self): + """Test that lines with only whitespace are also filtered""" + sse_lines = [ + 'data: {"id":"1","choices":[{"delta":{"content":"Hi"}}]}', + ' ', # Whitespace only + '\t', # Tab only + 'data: [DONE]', + ] + + iterator = BaseModelResponseIterator( + streaming_response=iter(sse_lines), + sync_stream=True + ) + + chunks = list(iterator) + + # Should have 2 chunks: 1 content + 1 DONE + assert len(chunks) == 2, f"Expected 2 chunks, got {len(chunks)}" + + def test_valid_chunks_not_filtered_sync(self): + """Test that valid data chunks are not filtered""" + sse_lines = [ + 'data: {"id":"1","choices":[{"delta":{"content":"A"}}]}', + 'data: {"id":"1","choices":[{"delta":{"content":"B"}}]}', + 'data: {"id":"1","choices":[{"delta":{"content":"C"}}]}', + 'data: [DONE]', + ] + + iterator = BaseModelResponseIterator( + streaming_response=iter(sse_lines), + sync_stream=True + ) + + chunks = list(iterator) + + # All 4 chunks should be present + assert len(chunks) == 4, f"Expected 4 chunks, got {len(chunks)}" + + +@pytest.mark.asyncio +async def test_filter_empty_sse_lines_async(): + """ + Test async version: empty SSE lines should be filtered out + """ + async def async_sse_generator(): + lines = [ + 'data: {"id":"1","choices":[{"delta":{"content":"Hello"}}]}', + '', # Empty line + 'data: {"id":"1","choices":[{"delta":{"content":" World"}}]}', + '', # Empty line + 'data: [DONE]', + '', # Empty line + ] + for line in lines: + yield line + + iterator = BaseModelResponseIterator( + streaming_response=async_sse_generator(), + sync_stream=False + ) + + chunks = [] + async for chunk in iterator: + chunks.append(chunk) + + # Should have 3 chunks: 2 content + 1 DONE + assert len(chunks) == 3, f"Expected 3 chunks, got {len(chunks)}" From 265a08823c833858be10386c271107719c9aadbc Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Tue, 9 Dec 2025 08:00:07 -0800 Subject: [PATCH 003/418] refactor(files): add type aliases for provider parameters Introduces 5 type aliases for provider Literal types in the Files API: - FileCreateProvider, FileRetrieveProvider, FileDeleteProvider - FileListProvider, FileContentProvider Updates 10 function signatures to use the new aliases. Reduces duplication and improves readability. Closes #17608 --- litellm/files/main.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/litellm/files/main.py b/litellm/files/main.py index acf545e4319..b66096b013b 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -13,6 +13,13 @@ from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast import httpx +# Type aliases for provider parameters +FileCreateProvider = Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] +FileRetrieveProvider = Literal["openai", "azure", "hosted_vllm"] +FileDeleteProvider = Literal["openai", "azure"] +FileListProvider = Literal["openai", "azure"] +FileContentProvider = Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic"] + import litellm from litellm import get_secret_str from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider @@ -58,7 +65,7 @@ anthropic_files_instance = AnthropicFilesHandler() async def acreate_file( file: FileTypes, purpose: Literal["assistants", "batch", "fine-tune"], - custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai", + custom_llm_provider: FileCreateProvider = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -102,7 +109,7 @@ async def acreate_file( def create_file( file: FileTypes, purpose: Literal["assistants", "batch", "fine-tune"], - custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"]] = None, + custom_llm_provider: Optional[FileCreateProvider] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -281,7 +288,7 @@ def create_file( @client async def afile_retrieve( file_id: str, - custom_llm_provider: Literal["openai", "azure", "hosted_vllm"] = "openai", + custom_llm_provider: FileRetrieveProvider = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -322,7 +329,7 @@ async def afile_retrieve( @client def file_retrieve( file_id: str, - custom_llm_provider: Literal["openai", "azure", "hosted_vllm"] = "openai", + custom_llm_provider: FileRetrieveProvider = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -438,7 +445,7 @@ def file_retrieve( @client async def afile_delete( file_id: str, - custom_llm_provider: Literal["openai", "azure"] = "openai", + custom_llm_provider: FileDeleteProvider = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -482,7 +489,7 @@ async def afile_delete( def file_delete( file_id: str, model: Optional[str] = None, - custom_llm_provider: Union[Literal["openai", "azure"], str] = "openai", + custom_llm_provider: Union[FileDeleteProvider, str] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -604,7 +611,7 @@ def file_delete( # List files @client async def afile_list( - custom_llm_provider: Literal["openai", "azure"] = "openai", + custom_llm_provider: FileListProvider = "openai", purpose: Optional[str] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, @@ -645,7 +652,7 @@ async def afile_list( @client def file_list( - custom_llm_provider: Literal["openai", "azure"] = "openai", + custom_llm_provider: FileListProvider = "openai", purpose: Optional[str] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, @@ -759,7 +766,7 @@ def file_list( @client async def afile_content( file_id: str, - custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic"] = "openai", + custom_llm_provider: FileContentProvider = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -803,9 +810,7 @@ async def afile_content( def file_content( file_id: str, model: Optional[str] = None, - custom_llm_provider: Optional[ - Union[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic"], str] - ] = None, + custom_llm_provider: Optional[Union[FileContentProvider, str]] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, From 68ba9a6a99eac3ea91012fec313edbddf12cf6e5 Mon Sep 17 00:00:00 2001 From: Chesars Date: Mon, 5 Jan 2026 10:29:55 -0300 Subject: [PATCH 004/418] fix: enforce Black formatting in CI instead of auto-formatting Changed CI workflow to use `black --check` instead of `black .` This makes the CI fail if code is not formatted, rather than auto-formatting and discarding changes. Aligns with README.md promise that "all checks must pass" and follows Black best practices for CI/CD pipelines. --- .github/workflows/test-linting.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 35ebffeada3..26f8a2efb68 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -34,10 +34,10 @@ jobs: poetry install --with dev poetry run pip install openai==1.100.1 - - name: Run Black formatting + - name: Check Black formatting run: | cd litellm - poetry run black . + poetry run black --check . cd .. - name: Debug - Check file state From 43054a239059cbc695a0f0215aedfa15615750cc Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Thu, 26 Feb 2026 19:03:49 +0530 Subject: [PATCH 005/418] fix: langfuse trace leak key on model params --- litellm/integrations/langfuse/langfuse.py | 31 +++++++++++++++++------ 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 7bf97665fd2..e2db8be0450 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -25,6 +25,7 @@ from litellm.litellm_core_utils.core_helpers import ( reconstruct_model_name, filter_exceptions_from_params, ) +from litellm.litellm_core_utils.model_param_helper import ModelParamHelper from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info from litellm.integrations.langfuse.langfuse_mock_client import ( create_mock_langfuse_client, @@ -123,7 +124,7 @@ class LangFuseLogger: self.langfuse_flush_interval = LangFuseLogger._get_langfuse_flush_interval( flush_interval ) - + if should_use_langfuse_mock(): self.langfuse_client = create_mock_langfuse_client() self.is_mock_mode = True @@ -291,8 +292,6 @@ class LangFuseLogger: functions = optional_params.pop("functions", None) tools = optional_params.pop("tools", None) - # Remove secret_fields to prevent leaking sensitive data (e.g., authorization headers) - optional_params.pop("secret_fields", None) if functions is not None: prompt["functions"] = functions if tools is not None: @@ -505,13 +504,18 @@ class LangFuseLogger: kwargs.get("model", ""), custom_llm_provider, metadata ) + # Use whitelisted model parameters to prevent leaking secrets + sanitized_model_params = ModelParamHelper.get_standard_logging_model_parameters( + optional_params + ) + trace.generation( CreateGeneration( name=metadata.get("generation_name", "litellm-completion"), startTime=start_time, endTime=end_time, model=model_name, - modelParameters=optional_params, + modelParameters=sanitized_model_params, prompt=input, completion=output, usage={ @@ -607,9 +611,7 @@ class LangFuseLogger: # Use standard_logging_object.trace_id if available (when trace_id from metadata is None) # This allows standard trace_id to be used when provided in standard_logging_object if trace_id is None and standard_logging_object is not None: - trace_id = cast( - Optional[str], standard_logging_object.get("trace_id") - ) + trace_id = cast(Optional[str], standard_logging_object.get("trace_id")) # Fallback to litellm_call_id if no trace_id found if trace_id is None: trace_id = litellm_call_id @@ -833,13 +835,26 @@ class LangFuseLogger: kwargs.get("model", ""), custom_llm_provider, metadata ) + # Use whitelisted model_parameters from StandardLoggingPayload + # to prevent leaking secrets (api_key, auth headers, etc.) + if standard_logging_object is not None: + sanitized_model_params = standard_logging_object.get( + "model_parameters", optional_params + ) + else: + sanitized_model_params = ( + ModelParamHelper.get_standard_logging_model_parameters( + optional_params + ) + ) + generation_params = { "name": generation_name, "id": clean_metadata.pop("generation_id", generation_id), "start_time": start_time, "end_time": end_time, "model": model_name, - "model_parameters": optional_params, + "model_parameters": sanitized_model_params, "input": input if not mask_input else "redacted-by-litellm", "output": output if not mask_output else "redacted-by-litellm", "usage": usage, From 7e9930cc3b4dbee6970124eb6122fd351f1593db Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Sat, 28 Feb 2026 17:22:01 +0530 Subject: [PATCH 006/418] Fix Langfuse trace_id mapping for failed logs and prioritize session_id This fix addresses Bug 1 where failed LiteLLM logs were using request_id instead of session_id for Langfuse trace mapping, breaking trace correlation. Changes: 1. Fix kwargs inconsistency in failure path (litellm_logging.py:2956) - Changed from passing self.model_call_details to passing local kwargs variable - Matches success path behavior and excludes original_response (potentially a coroutine) 2. Prioritize session_id as trace_id fallback (langfuse.py:607-615) - When no explicit trace_id is provided, now uses session_id from metadata - This ensures traces with the same session_id are grouped together in Langfuse - Maintains backward compatibility: only activates when session_id is set Testing: - All 28 existing Langfuse tests pass (excluding pre-existing test_langfuse_e2e_sync which fails due to missing API key) - Specifically verified trace_id resolution tests still pass Co-Authored-By: Claude Haiku 4.5 --- litellm/integrations/langfuse/langfuse.py | 5 ++++- litellm/litellm_core_utils/litellm_logging.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 7bf97665fd2..70f1161792a 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -604,9 +604,12 @@ class LangFuseLogger: session_id = clean_metadata.pop("session_id", None) trace_name = cast(Optional[str], clean_metadata.pop("trace_name", None)) trace_id = clean_metadata.pop("trace_id", None) + # If session_id is provided, use it as trace_id for consistent trace mapping + if trace_id is None and session_id is not None: + trace_id = session_id # Use standard_logging_object.trace_id if available (when trace_id from metadata is None) # This allows standard trace_id to be used when provided in standard_logging_object - if trace_id is None and standard_logging_object is not None: + elif trace_id is None and standard_logging_object is not None: trace_id = cast( Optional[str], standard_logging_object.get("trace_id") ) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e450b233c7e..d5419c98685 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2953,7 +2953,7 @@ class Logging(LiteLLMLoggingBaseClass): user_id=kwargs.get("user", None), status_message=str(exception), level="ERROR", - kwargs=self.model_call_details, + kwargs=kwargs, ) if _response is not None and isinstance(_response, dict): _trace_id = _response.get("trace_id", None) From 315b00fd193a361e581877253c545a127ad5f31c Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Sat, 28 Feb 2026 17:34:59 +0530 Subject: [PATCH 007/418] Fix Langfuse failure path kwargs and add session_id trace tests Fix: The Langfuse failure logging path was passing self.model_call_details (which includes original_response, potentially a coroutine) instead of the clean local kwargs copy. This aligns the failure path with the success path behavior (litellm_logging.py:2956). Reverted the session_id-as-trace_id approach as it causes trace collisions in Langfuse (multiple calls in the same session would overwrite each other). Instead, session_id is correctly used only for Langfuse session grouping via trace_params["session_id"], while each call retains its own unique trace_id. Added 4 tests: - session_id correctly passed as trace session_id (not trace_id) - session_id preserved for ERROR level (failure) logs - explicit trace_id takes priority over session_id - failure path kwargs excludes original_response Co-Authored-By: Claude Opus 4.6 --- litellm/integrations/langfuse/langfuse.py | 5 +- .../integrations/test_langfuse.py | 182 ++++++++++++++++++ 2 files changed, 183 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 70f1161792a..7bf97665fd2 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -604,12 +604,9 @@ class LangFuseLogger: session_id = clean_metadata.pop("session_id", None) trace_name = cast(Optional[str], clean_metadata.pop("trace_name", None)) trace_id = clean_metadata.pop("trace_id", None) - # If session_id is provided, use it as trace_id for consistent trace mapping - if trace_id is None and session_id is not None: - trace_id = session_id # Use standard_logging_object.trace_id if available (when trace_id from metadata is None) # This allows standard trace_id to be used when provided in standard_logging_object - elif trace_id is None and standard_logging_object is not None: + if trace_id is None and standard_logging_object is not None: trace_id = cast( Optional[str], standard_logging_object.get("trace_id") ) diff --git a/tests/test_litellm/integrations/test_langfuse.py b/tests/test_litellm/integrations/test_langfuse.py index 10d3323a255..1c47087e521 100644 --- a/tests/test_litellm/integrations/test_langfuse.py +++ b/tests/test_litellm/integrations/test_langfuse.py @@ -468,6 +468,188 @@ class TestLangfuseUsageDetails(unittest.TestCase): assert self.last_trace_kwargs.get("id") == "call-id-xyz" + def test_log_langfuse_v2_session_id_passed_as_trace_session_id(self): + """ + Test that metadata.session_id is correctly passed as trace_params["session_id"] + for Langfuse session grouping, and does NOT override trace_id. + Each LLM call should get its own unique trace_id while sharing the session_id. + """ + payload = self._build_standard_logging_payload(trace_id="std-trace-123") + kwargs = self._build_langfuse_kwargs(payload) + self.last_trace_kwargs = {} + + with patch( + "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", + side_effect=lambda generation_params, **kwargs: generation_params, + create=True, + ): + self.logger._log_langfuse_v2( + user_id="user-1", + metadata={"session_id": "my-session-abc"}, + litellm_params={"metadata": {"session_id": "my-session-abc"}}, + output=None, + start_time=datetime.datetime.utcnow(), + end_time=datetime.datetime.utcnow(), + kwargs=kwargs, + optional_params={}, + input=None, + response_obj=None, + level="INFO", + litellm_call_id="call-id-456", + ) + + # session_id should be set for Langfuse session grouping + assert self.last_trace_kwargs.get("session_id") == "my-session-abc" + # trace_id should remain the standard trace_id, NOT the session_id + assert self.last_trace_kwargs.get("id") == "std-trace-123" + + def test_log_langfuse_v2_session_id_preserved_for_error_level(self): + """ + Test that session_id is correctly passed in trace_params even when + the log level is ERROR (failure case). This verifies the fix for + failed requests losing session_id mapping in Langfuse. + """ + payload = self._build_standard_logging_payload(trace_id="std-trace-err") + kwargs = self._build_langfuse_kwargs(payload) + self.last_trace_kwargs = {} + + with patch( + "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", + side_effect=lambda generation_params, **kwargs: generation_params, + create=True, + ): + self.logger._log_langfuse_v2( + user_id="user-1", + metadata={"session_id": "error-session-xyz"}, + litellm_params={"metadata": {"session_id": "error-session-xyz"}}, + output="BadRequestError: model not found", + start_time=datetime.datetime.utcnow(), + end_time=datetime.datetime.utcnow(), + kwargs=kwargs, + optional_params={}, + input={"messages": [{"role": "user", "content": "test"}]}, + response_obj=None, + level="ERROR", + litellm_call_id="call-id-err-789", + ) + + # session_id must be preserved even for ERROR level logs + assert self.last_trace_kwargs.get("session_id") == "error-session-xyz" + # trace_id should be the standard trace_id, not the session_id + assert self.last_trace_kwargs.get("id") == "std-trace-err" + # status_message should be set for error traces + assert self.last_trace_kwargs.get("status_message") is not None + + def test_log_langfuse_v2_explicit_trace_id_takes_priority_over_session_id(self): + """ + Test that when both trace_id and session_id are provided in metadata, + trace_id takes priority as the trace identifier. + """ + payload = self._build_standard_logging_payload() + kwargs = self._build_langfuse_kwargs(payload) + self.last_trace_kwargs = {} + + with patch( + "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", + side_effect=lambda generation_params, **kwargs: generation_params, + create=True, + ): + self.logger._log_langfuse_v2( + user_id="user-1", + metadata={ + "session_id": "session-999", + "trace_id": "explicit-trace-id-777", + }, + litellm_params={ + "metadata": { + "session_id": "session-999", + "trace_id": "explicit-trace-id-777", + } + }, + output=None, + start_time=datetime.datetime.utcnow(), + end_time=datetime.datetime.utcnow(), + kwargs=kwargs, + optional_params={}, + input=None, + response_obj=None, + level="DEFAULT", + litellm_call_id="call-id-aaa", + ) + + # Explicit trace_id must take priority + assert self.last_trace_kwargs.get("id") == "explicit-trace-id-777" + # session_id must still be set for session grouping + assert self.last_trace_kwargs.get("session_id") == "session-999" + + +def test_failure_handler_langfuse_kwargs_excludes_original_response(): + """ + Test that the Langfuse failure logging path passes the local kwargs copy + (which excludes 'original_response') rather than self.model_call_details directly. + This prevents passing coroutines or large response objects to the Langfuse logger. + """ + from litellm.litellm_core_utils.litellm_logging import Logging + + # Create a mock coroutine to simulate original_response + mock_coroutine = MagicMock() + mock_coroutine.__class__.__name__ = "coroutine" + + model_call_details = { + "litellm_call_id": "test-call-id", + "litellm_trace_id": None, + "model": "gpt-4", + "messages": [{"role": "user", "content": "test"}], + "litellm_params": { + "metadata": {"session_id": "test-session"}, + "litellm_session_id": None, + }, + "original_response": mock_coroutine, + "optional_params": {}, + "stream": False, + "call_type": "completion", + "input": [{"role": "user", "content": "test"}], + } + + captured_kwargs = {} + + class MockLangfuseLogger: + def log_event_on_langfuse(self, **log_kwargs): + captured_kwargs.update(log_kwargs) + return {"trace_id": "mock-trace-id", "generation_id": "mock-gen-id"} + + mock_logger = MockLangfuseLogger() + + # Simulate the failure path logic from litellm_logging.py + # This mirrors lines 2937-2957 of the failure_handler + kwargs = {} + for k, v in model_call_details.items(): + if k != "original_response": + kwargs[k] = v + + # Verify the local kwargs does NOT contain original_response + assert "original_response" not in kwargs + # Verify session_id is present in kwargs metadata + assert kwargs["litellm_params"]["metadata"]["session_id"] == "test-session" + + # Call with the local kwargs (as the fix does) + mock_logger.log_event_on_langfuse( + start_time=datetime.datetime.utcnow(), + end_time=datetime.datetime.utcnow(), + response_obj=None, + user_id=kwargs.get("user", None), + status_message="TestError: something failed", + level="ERROR", + kwargs=kwargs, + ) + + # Verify original_response is NOT in the kwargs passed to Langfuse + assert "original_response" not in captured_kwargs.get("kwargs", {}) + # Verify session_id metadata is preserved in the kwargs passed to Langfuse + langfuse_metadata = captured_kwargs["kwargs"]["litellm_params"]["metadata"] + assert langfuse_metadata["session_id"] == "test-session" + + def test_max_langfuse_clients_limit(): """ Test that the max langfuse clients limit is respected when initializing multiple clients From 83cab3f54a3f68f5bddac633e2184953f4dfbc2a Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Sat, 28 Feb 2026 17:47:25 +0530 Subject: [PATCH 008/418] Rewrite test to exercise actual failure_handler code path Replace simulated test with one that invokes the real Logging.failure_handler(), mocks LangFuseHandler to capture kwargs, and asserts original_response is excluded and session_id is preserved. This ensures the test catches regressions if the production code changes. Co-Authored-By: Claude Opus 4.6 --- .../integrations/test_langfuse.py | 133 ++++++++++-------- 1 file changed, 76 insertions(+), 57 deletions(-) diff --git a/tests/test_litellm/integrations/test_langfuse.py b/tests/test_litellm/integrations/test_langfuse.py index 1c47087e521..15874113b41 100644 --- a/tests/test_litellm/integrations/test_langfuse.py +++ b/tests/test_litellm/integrations/test_langfuse.py @@ -585,69 +585,88 @@ class TestLangfuseUsageDetails(unittest.TestCase): def test_failure_handler_langfuse_kwargs_excludes_original_response(): """ - Test that the Langfuse failure logging path passes the local kwargs copy - (which excludes 'original_response') rather than self.model_call_details directly. - This prevents passing coroutines or large response objects to the Langfuse logger. + Test that the actual Logging.failure_handler() passes kwargs without + 'original_response' to the Langfuse logger. Exercises the real code path + rather than simulating the filtering logic. """ + import litellm from litellm.litellm_core_utils.litellm_logging import Logging - # Create a mock coroutine to simulate original_response - mock_coroutine = MagicMock() - mock_coroutine.__class__.__name__ = "coroutine" - - model_call_details = { - "litellm_call_id": "test-call-id", - "litellm_trace_id": None, - "model": "gpt-4", - "messages": [{"role": "user", "content": "test"}], - "litellm_params": { - "metadata": {"session_id": "test-session"}, - "litellm_session_id": None, - }, - "original_response": mock_coroutine, - "optional_params": {}, - "stream": False, - "call_type": "completion", - "input": [{"role": "user", "content": "test"}], - } - - captured_kwargs = {} - - class MockLangfuseLogger: - def log_event_on_langfuse(self, **log_kwargs): - captured_kwargs.update(log_kwargs) - return {"trace_id": "mock-trace-id", "generation_id": "mock-gen-id"} - - mock_logger = MockLangfuseLogger() - - # Simulate the failure path logic from litellm_logging.py - # This mirrors lines 2937-2957 of the failure_handler - kwargs = {} - for k, v in model_call_details.items(): - if k != "original_response": - kwargs[k] = v - - # Verify the local kwargs does NOT contain original_response - assert "original_response" not in kwargs - # Verify session_id is present in kwargs metadata - assert kwargs["litellm_params"]["metadata"]["session_id"] == "test-session" - - # Call with the local kwargs (as the fix does) - mock_logger.log_event_on_langfuse( + # Create a Logging instance + logging_obj = Logging( + model="gpt-4", + messages=[{"role": "user", "content": "test"}], + stream=False, + call_type="completion", start_time=datetime.datetime.utcnow(), - end_time=datetime.datetime.utcnow(), - response_obj=None, - user_id=kwargs.get("user", None), - status_message="TestError: something failed", - level="ERROR", - kwargs=kwargs, + litellm_call_id="test-call-id-failure", + function_id="test-function-id", ) - # Verify original_response is NOT in the kwargs passed to Langfuse - assert "original_response" not in captured_kwargs.get("kwargs", {}) - # Verify session_id metadata is preserved in the kwargs passed to Langfuse - langfuse_metadata = captured_kwargs["kwargs"]["litellm_params"]["metadata"] - assert langfuse_metadata["session_id"] == "test-session" + # Set up model_call_details with original_response (simulates a coroutine) + mock_coroutine = MagicMock() + logging_obj.model_call_details["original_response"] = mock_coroutine + logging_obj.model_call_details["litellm_params"] = { + "metadata": {"session_id": "test-session-failure"}, + "litellm_session_id": None, + } + logging_obj.model_call_details["optional_params"] = {} + + # Capture what gets passed to log_event_on_langfuse + captured_kwargs = {} + mock_langfuse_logger = MagicMock() + + def capture_log_event(**log_kwargs): + captured_kwargs.update(log_kwargs) + return {"trace_id": "mock-trace-id", "generation_id": "mock-gen-id"} + + mock_langfuse_logger.log_event_on_langfuse.side_effect = capture_log_event + + # Set "langfuse" as a failure callback so the failure_handler processes it + original_failure_callback = litellm.failure_callback + litellm.failure_callback = ["langfuse"] + + try: + # Mock LangFuseHandler to return our capturing mock logger + with patch( + "litellm.litellm_core_utils.litellm_logging.LangFuseHandler" + ) as mock_handler_class: + mock_handler_class.get_langfuse_logger_for_request.return_value = ( + mock_langfuse_logger + ) + + # Call the actual failure_handler + test_exception = Exception("TestError: model not found") + logging_obj.failure_handler( + exception=test_exception, + traceback_exception="Traceback: test", + start_time=datetime.datetime.utcnow(), + end_time=datetime.datetime.utcnow(), + ) + + # Verify log_event_on_langfuse was actually called + assert mock_langfuse_logger.log_event_on_langfuse.called, ( + "log_event_on_langfuse was not called" + ) + + # Verify original_response is NOT in the kwargs passed to Langfuse + langfuse_kwargs = captured_kwargs.get("kwargs", {}) + assert "original_response" not in langfuse_kwargs, ( + "original_response should be excluded from kwargs passed to Langfuse" + ) + + # Verify session_id metadata is preserved in the kwargs + langfuse_metadata = langfuse_kwargs.get("litellm_params", {}).get( + "metadata", {} + ) + assert langfuse_metadata.get("session_id") == "test-session-failure", ( + "session_id should be preserved in kwargs passed to Langfuse" + ) + + # Verify level is ERROR + assert captured_kwargs.get("level") == "ERROR" + finally: + litellm.failure_callback = original_failure_callback def test_max_langfuse_clients_limit(): From 016a4fd6089b038e66843c11bd776ef82e7138aa Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Mon, 2 Mar 2026 20:20:27 +0530 Subject: [PATCH 009/418] Fix async failure path not logging to Langfuse (proxy bug) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The proxy uses async_failure_handler → LangfusePromptManagement.async_log_failure_event(), which silently returned when standard_logging_object was None. This meant failed LLM calls never created traces in Langfuse. Remove the early return and fall back to extracting the error message from kwargs["exception"] when standard_logging_object is unavailable. Co-Authored-By: Claude Opus 4.6 --- .../langfuse/langfuse_prompt_management.py | 9 +- .../integrations/test_langfuse.py | 130 ++++++++++++++++++ 2 files changed, 136 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse_prompt_management.py b/litellm/integrations/langfuse/langfuse_prompt_management.py index 3986fc6a6ef..d3de59c8e67 100644 --- a/litellm/integrations/langfuse/langfuse_prompt_management.py +++ b/litellm/integrations/langfuse/langfuse_prompt_management.py @@ -338,14 +338,17 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge Optional[StandardLoggingPayload], kwargs.get("standard_logging_object", None), ) - if standard_logging_object is None: - return + status_message = str(kwargs.get("exception", "Unknown error")) + if standard_logging_object is not None: + status_message = standard_logging_object.get( + "error_str", status_message + ) langfuse_logger_to_use.log_event_on_langfuse( start_time=start_time, end_time=end_time, response_obj=None, user_id=kwargs.get("user", None), - status_message=standard_logging_object["error_str"], + status_message=status_message, level="ERROR", kwargs=kwargs, ) diff --git a/tests/test_litellm/integrations/test_langfuse.py b/tests/test_litellm/integrations/test_langfuse.py index 15874113b41..084fd7d0480 100644 --- a/tests/test_litellm/integrations/test_langfuse.py +++ b/tests/test_litellm/integrations/test_langfuse.py @@ -669,6 +669,136 @@ def test_failure_handler_langfuse_kwargs_excludes_original_response(): litellm.failure_callback = original_failure_callback +@pytest.mark.asyncio +async def test_async_log_failure_event_logs_to_langfuse(): + """ + Test that LangfusePromptManagement.async_log_failure_event() calls + log_event_on_langfuse with level=ERROR even when standard_logging_object + is present. This is the code path the proxy uses for failed LLM calls. + """ + from litellm.integrations.langfuse.langfuse_prompt_management import ( + LangfusePromptManagement, + ) + + mock_langfuse_module = MagicMock() + mock_langfuse_module.version.__version__ = "3.0.0" + + with patch.dict( + "os.environ", + { + "LANGFUSE_SECRET_KEY": "test-secret", + "LANGFUSE_PUBLIC_KEY": "test-public", + "LANGFUSE_HOST": "https://test.langfuse.com", + }, + ), patch.dict("sys.modules", {"langfuse": mock_langfuse_module}): + prompt_mgmt = LangfusePromptManagement() + + # Mock the langfuse logger returned by get_langfuse_logger_for_request + mock_logger = MagicMock() + mock_logger.log_event_on_langfuse.return_value = { + "trace_id": "mock-trace", + "generation_id": "mock-gen", + } + + with patch( + "litellm.integrations.langfuse.langfuse_prompt_management.LangFuseHandler" + ) as mock_handler: + mock_handler.get_langfuse_logger_for_request.return_value = mock_logger + + kwargs = { + "litellm_params": { + "metadata": {"session_id": "test-session-fail"}, + }, + "litellm_call_id": "call-fail-123", + "user": "test-user", + "exception": Exception("API error: model not found"), + "standard_logging_object": { + "error_str": "API error: model not found", + "trace_id": "std-trace-fail", + "metadata": {}, + }, + } + + await prompt_mgmt.async_log_failure_event( + kwargs=kwargs, + response_obj=None, + start_time=datetime.datetime.utcnow(), + end_time=datetime.datetime.utcnow(), + ) + + # Verify log_event_on_langfuse was called + assert mock_logger.log_event_on_langfuse.called, ( + "log_event_on_langfuse was not called for failure event" + ) + call_kwargs = mock_logger.log_event_on_langfuse.call_args[1] + assert call_kwargs["level"] == "ERROR" + assert call_kwargs["status_message"] == "API error: model not found" + assert call_kwargs["response_obj"] is None + + +@pytest.mark.asyncio +async def test_async_log_failure_event_works_without_standard_logging_object(): + """ + Test that async_log_failure_event() still logs to Langfuse even when + standard_logging_object is None (e.g. when get_standard_logging_object_payload + threw an exception). This is the critical fix — before, it silently returned. + """ + from litellm.integrations.langfuse.langfuse_prompt_management import ( + LangfusePromptManagement, + ) + + mock_langfuse_module = MagicMock() + mock_langfuse_module.version.__version__ = "3.0.0" + + with patch.dict( + "os.environ", + { + "LANGFUSE_SECRET_KEY": "test-secret", + "LANGFUSE_PUBLIC_KEY": "test-public", + "LANGFUSE_HOST": "https://test.langfuse.com", + }, + ), patch.dict("sys.modules", {"langfuse": mock_langfuse_module}): + prompt_mgmt = LangfusePromptManagement() + + mock_logger = MagicMock() + mock_logger.log_event_on_langfuse.return_value = { + "trace_id": "mock-trace", + "generation_id": "mock-gen", + } + + with patch( + "litellm.integrations.langfuse.langfuse_prompt_management.LangFuseHandler" + ) as mock_handler: + mock_handler.get_langfuse_logger_for_request.return_value = mock_logger + + kwargs = { + "litellm_params": { + "metadata": {"session_id": "test-session-no-slo"}, + }, + "litellm_call_id": "call-no-slo-456", + "user": "test-user", + "exception": Exception("InternalServerError: something broke"), + "standard_logging_object": None, # This is the key — it's None + } + + await prompt_mgmt.async_log_failure_event( + kwargs=kwargs, + response_obj=None, + start_time=datetime.datetime.utcnow(), + end_time=datetime.datetime.utcnow(), + ) + + # CRITICAL: log_event_on_langfuse MUST still be called + assert mock_logger.log_event_on_langfuse.called, ( + "log_event_on_langfuse was NOT called when standard_logging_object " + "is None — failure trace would be silently dropped" + ) + call_kwargs = mock_logger.log_event_on_langfuse.call_args[1] + assert call_kwargs["level"] == "ERROR" + # Falls back to exception from kwargs + assert "InternalServerError" in call_kwargs["status_message"] + + def test_max_langfuse_clients_limit(): """ Test that the max langfuse clients limit is respected when initializing multiple clients From cac041c9447a798fe611ef4f433142ed90c63cbc Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Mon, 2 Mar 2026 20:55:31 +0530 Subject: [PATCH 010/418] Align Langfuse trace_id fallback with DB session_id for failed requests When standard_logging_object is None (failure case), Langfuse was falling back to litellm_call_id while the DB used litellm_trace_id as session_id. This caused the Session ID in LiteLLM logs to not match the trace in Langfuse. Now Langfuse checks litellm_trace_id first, matching the DB. Co-Authored-By: Claude Opus 4.6 --- litellm/integrations/langfuse/langfuse.py | 5 +- .../integrations/test_langfuse.py | 73 +++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 7bf97665fd2..afad3f50940 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -610,9 +610,10 @@ class LangFuseLogger: trace_id = cast( Optional[str], standard_logging_object.get("trace_id") ) - # Fallback to litellm_call_id if no trace_id found + # Fallback: use litellm_trace_id from kwargs (matches DB session_id), + # then litellm_call_id as last resort if trace_id is None: - trace_id = litellm_call_id + trace_id = kwargs.get("litellm_trace_id") or litellm_call_id existing_trace_id = clean_metadata.pop("existing_trace_id", None) # If existing_trace_id is provided, use it as the trace_id to return # This allows continuing an existing trace while still returning the correct trace_id diff --git a/tests/test_litellm/integrations/test_langfuse.py b/tests/test_litellm/integrations/test_langfuse.py index 084fd7d0480..b4028709218 100644 --- a/tests/test_litellm/integrations/test_langfuse.py +++ b/tests/test_litellm/integrations/test_langfuse.py @@ -467,6 +467,79 @@ class TestLangfuseUsageDetails(unittest.TestCase): assert self.last_trace_kwargs.get("id") == "call-id-xyz" + def test_log_langfuse_v2_uses_litellm_trace_id_fallback_over_call_id(self): + """ + When standard_logging_object has no trace_id, but kwargs contains + litellm_trace_id (the same ID the DB stores as Session ID), Langfuse + should use litellm_trace_id — NOT litellm_call_id. This ensures the + trace_id in Langfuse matches the Session ID shown in LiteLLM logs. + """ + payload = self._build_standard_logging_payload() # no trace_id + kwargs = self._build_langfuse_kwargs(payload) + kwargs["litellm_trace_id"] = "trace-id-from-kwargs" + self.last_trace_kwargs = {} + + with patch( + "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", + side_effect=lambda generation_params, **kwargs: generation_params, + create=True, + ): + self.logger._log_langfuse_v2( + user_id="user-1", + metadata={}, + litellm_params={"metadata": {}}, + output=None, + start_time=datetime.datetime.utcnow(), + end_time=datetime.datetime.utcnow(), + kwargs=kwargs, + optional_params={}, + input=None, + response_obj=None, + level="ERROR", + litellm_call_id="call-id-xyz", + ) + + # litellm_trace_id should be preferred over litellm_call_id + assert self.last_trace_kwargs.get("id") == "trace-id-from-kwargs" + + def test_log_langfuse_v2_uses_litellm_trace_id_when_standard_logging_object_none(self): + """ + When standard_logging_object is None (failure case where + get_standard_logging_object_payload threw), litellm_trace_id from kwargs + should be used as the Langfuse trace_id. This matches the DB Session ID. + """ + kwargs = { + "standard_logging_object": None, + "model": "gpt-4", + "call_type": "completion", + "cache_hit": False, + "messages": [], + "litellm_trace_id": "trace-id-failure", + } + self.last_trace_kwargs = {} + + with patch( + "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", + side_effect=lambda generation_params, **kwargs: generation_params, + create=True, + ): + self.logger._log_langfuse_v2( + user_id="user-1", + metadata={}, + litellm_params={"metadata": {}}, + output=None, + start_time=datetime.datetime.utcnow(), + end_time=datetime.datetime.utcnow(), + kwargs=kwargs, + optional_params={}, + input=None, + response_obj=None, + level="ERROR", + litellm_call_id="call-id-different", + ) + + # Must use litellm_trace_id, not litellm_call_id + assert self.last_trace_kwargs.get("id") == "trace-id-failure" def test_log_langfuse_v2_session_id_passed_as_trace_session_id(self): """ From 2f927fef3bcf662e221bb3b9b8cf8147291d4e6e Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Mon, 2 Mar 2026 21:15:52 +0530 Subject: [PATCH 011/418] Fix root cause: model_call_details stored None for litellm_trace_id Logging.__init__ stored the raw litellm_trace_id parameter (None when not explicitly provided) in model_call_details, while self.litellm_trace_id always held a valid UUID. When get_standard_logging_object_payload() failed, both the DB and Langfuse fell back to kwargs["litellm_trace_id"] which was None, causing each to generate different random UUIDs. Now model_call_details stores self.litellm_trace_id (always valid), so all fallback paths use the same ID. Co-Authored-By: Claude Opus 4.6 --- litellm/litellm_core_utils/litellm_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index d5419c98685..73512f7a12b 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -405,7 +405,7 @@ class Logging(LiteLLMLoggingBaseClass): self.passthrough_guardrails_config: Optional[Dict[str, Any]] = None self.model_call_details: Dict[str, Any] = { - "litellm_trace_id": litellm_trace_id, + "litellm_trace_id": self.litellm_trace_id, "litellm_call_id": litellm_call_id, "input": _input, "litellm_params": litellm_params, From 46c4d5b37dac5577a860c434900fd79985e6c3cf Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Mon, 2 Mar 2026 21:28:36 +0530 Subject: [PATCH 012/418] Update litellm/integrations/langfuse/langfuse_prompt_management.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/integrations/langfuse/langfuse_prompt_management.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse_prompt_management.py b/litellm/integrations/langfuse/langfuse_prompt_management.py index d3de59c8e67..8b0c64d563f 100644 --- a/litellm/integrations/langfuse/langfuse_prompt_management.py +++ b/litellm/integrations/langfuse/langfuse_prompt_management.py @@ -341,8 +341,8 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge status_message = str(kwargs.get("exception", "Unknown error")) if standard_logging_object is not None: status_message = standard_logging_object.get( - "error_str", status_message - ) + "error_str", None + ) or status_message langfuse_logger_to_use.log_event_on_langfuse( start_time=start_time, end_time=end_time, From 338a634762d7dac2801a169dae7dbe9c7098aa3b Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Mon, 2 Mar 2026 21:31:35 +0530 Subject: [PATCH 013/418] Fix root cause: DB spend log session_id didn't match Langfuse trace_id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The proxy has two separate failure paths: 1. async_failure_handler → Langfuse callback (uses model_call_details with standard_logging_object containing the correct trace_id) 2. post_call_failure_hook → _ProxyDBLogger → spend log (uses request_data which did NOT have standard_logging_object, so session_id fell to random uuid4()) These two paths used different data dicts, so the DB session_id was a random UUID unrelated to the Langfuse trace_id. Users could not search by the Session ID from LiteLLM logs in Langfuse for failed requests. Fix: In _ProxyDBLogger.async_post_call_failure_hook, propagate standard_logging_object and litellm_trace_id from the litellm_logging_obj (already present in request_data) before writing the spend log. Co-Authored-By: Claude Opus 4.6 --- .../proxy/hooks/proxy_track_cost_callback.py | 16 +++++ .../hooks/test_proxy_track_cost_callback.py | 62 +++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 0734756d8ed..9a806fa4f87 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -110,6 +110,22 @@ class _ProxyDBLogger(CustomLogger): "custom_llm_provider" ) or request_data.get("custom_llm_provider", "") + # Propagate standard_logging_object and litellm_trace_id from the + # Logging instance so that _get_session_id_for_spend_log uses the same + # trace_id that Langfuse received (via async_failure_handler). + # Without this, the DB session_id would be a random UUID that doesn't + # match the Langfuse trace_id, making failed requests unsearchable. + _litellm_logging_obj = request_data.get("litellm_logging_obj") + if _litellm_logging_obj is not None: + if "standard_logging_object" not in request_data: + request_data["standard_logging_object"] = getattr( + _litellm_logging_obj, "model_call_details", {} + ).get("standard_logging_object") + if request_data.get("litellm_trace_id") is None: + request_data["litellm_trace_id"] = getattr( + _litellm_logging_obj, "litellm_trace_id", None + ) + await proxy_logging_obj.db_spend_update_writer.update_database( token=user_api_key_dict.api_key, response_cost=0.0, diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index c46b8df5efc..d269a9531fd 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -169,6 +169,68 @@ async def test_track_cost_callback_skips_when_no_standard_logging_object(): mock_proxy_logging.failed_tracking_alert.assert_not_called() +@pytest.mark.asyncio +async def test_async_post_call_failure_hook_propagates_trace_id_from_logging_obj(): + """ + When an LLM call fails, the proxy calls post_call_failure_hook with + request_data that doesn't contain standard_logging_object. But the + litellm_logging_obj (set by function_setup) is in request_data and + holds the standard_logging_object with the correct trace_id. + + The failure hook should propagate this so the DB spend log's session_id + matches the Langfuse trace_id. + """ + logger = _ProxyDBLogger() + + user_api_key_dict = UserAPIKeyAuth( + api_key="test_api_key", + user_id="test_user_id", + team_id="test_team_id", + ) + + # Simulate a litellm_logging_obj with model_call_details containing + # the standard_logging_object (as set by _failure_handler_helper_fn) + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_trace_id = "trace-id-from-logging-obj" + mock_logging_obj.model_call_details = { + "standard_logging_object": { + "trace_id": "trace-id-from-logging-obj", + "error_str": "InternalServerError", + } + } + + request_data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "metadata": {}, + "litellm_params": {}, + "litellm_logging_obj": mock_logging_obj, + # Note: no "standard_logging_object" and no "litellm_trace_id" + } + + with patch( + "litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database", + new_callable=AsyncMock, + ) as mock_update_database: + await logger.async_post_call_failure_hook( + request_data=request_data, + original_exception=Exception("Provider error"), + user_api_key_dict=user_api_key_dict, + ) + + mock_update_database.assert_called_once() + call_kwargs = mock_update_database.call_args[1]["kwargs"] + + # standard_logging_object should have been propagated from logging obj + assert call_kwargs.get("standard_logging_object") is not None + assert ( + call_kwargs["standard_logging_object"]["trace_id"] + == "trace-id-from-logging-obj" + ) + # litellm_trace_id should also be propagated as a fallback + assert call_kwargs.get("litellm_trace_id") == "trace-id-from-logging-obj" + + @pytest.mark.asyncio async def test_enrich_failure_metadata_with_team_alias(): """ From fb69de98e5fe3552fc3d9c53a7a17b4fff781b25 Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Tue, 3 Mar 2026 00:33:22 +0530 Subject: [PATCH 014/418] Update litellm/proxy/hooks/proxy_track_cost_callback.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/proxy/hooks/proxy_track_cost_callback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 9a806fa4f87..8cddaed5be1 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -117,7 +117,7 @@ class _ProxyDBLogger(CustomLogger): # match the Langfuse trace_id, making failed requests unsearchable. _litellm_logging_obj = request_data.get("litellm_logging_obj") if _litellm_logging_obj is not None: - if "standard_logging_object" not in request_data: + if not request_data.get("standard_logging_object"): request_data["standard_logging_object"] = getattr( _litellm_logging_obj, "model_call_details", {} ).get("standard_logging_object") From a2f3beb26f15c7354257b2d4a84bf621a819f4f6 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Tue, 3 Mar 2026 21:47:55 -0300 Subject: [PATCH 015/418] Update tests/test_litellm/llms/base_llm/test_base_model_iterator.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- tests/test_litellm/llms/base_llm/test_base_model_iterator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/llms/base_llm/test_base_model_iterator.py b/tests/test_litellm/llms/base_llm/test_base_model_iterator.py index d5166c4690e..96cd299b2bc 100644 --- a/tests/test_litellm/llms/base_llm/test_base_model_iterator.py +++ b/tests/test_litellm/llms/base_llm/test_base_model_iterator.py @@ -4,7 +4,7 @@ Tests for BaseModelResponseIterator - specifically testing that empty SSE lines import pytest from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator -from litellm.types.utils import GenericStreamingChunk, ModelResponseStream +from litellm.types.utils import GenericStreamingChunk class TestBaseModelResponseIterator: From 20bf3aa8070a4dc150bb8edaddb6bd3306b83a53 Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Tue, 10 Mar 2026 17:16:46 +0530 Subject: [PATCH 016/418] fix: pop sensitive keys from langfuse --- litellm/litellm_core_utils/litellm_logging.py | 50 +++++++++---------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e450b233c7e..73a8b92c1bb 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1653,9 +1653,7 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details[ "standard_logging_object" - ] = self._build_standard_logging_payload( - logging_result, start_time, end_time - ) + ] = self._build_standard_logging_payload(logging_result, start_time, end_time) if ( standard_logging_payload := self.model_call_details.get( @@ -2518,9 +2516,7 @@ class Logging(LiteLLMLoggingBaseClass): ## STANDARDIZED LOGGING PAYLOAD self.model_call_details[ "standard_logging_object" - ] = self._build_standard_logging_payload( - result, start_time, end_time - ) + ] = self._build_standard_logging_payload(result, start_time, end_time) # print standard logging payload if ( @@ -4195,8 +4191,7 @@ def _maybe_auto_initialize_arize_phoenix(_in_memory_loggers: list) -> None: litellm.logging_callback_manager.add_litellm_callback(phoenix_logger) verbose_logger.info( - "Auto-initialized Arize Phoenix logger alongside otel " - "(endpoint=%s)", + "Auto-initialized Arize Phoenix logger alongside otel " "(endpoint=%s)", arize_phoenix_config.endpoint, ) except Exception as e: @@ -4755,9 +4750,11 @@ class StandardLoggingPayloadSetup: ).model_dump() if isinstance(_raw, dict): if ResponseAPILoggingUtils._is_response_api_usage(_raw): - return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( - _raw - ).model_dump() + return ( + ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + _raw + ).model_dump() + ) return _raw if isinstance(_raw, Usage): return _raw.model_dump() @@ -5482,21 +5479,23 @@ def scrub_sensitive_keys_in_metadata(litellm_params: Optional[dict]): litellm_params["_langfuse_masking_function"] = masking_fn litellm_params["metadata"] = metadata - ## check user_api_key_metadata for sensitive logging keys - cleaned_user_api_key_metadata = {} - if "user_api_key_metadata" in metadata and isinstance( - metadata["user_api_key_metadata"], dict - ): - for k, v in metadata["user_api_key_metadata"].items(): - if k == "logging": # prevent logging user logging keys - cleaned_user_api_key_metadata[ - k - ] = "scrubbed_by_litellm_for_sensitive_keys" - else: - cleaned_user_api_key_metadata[k] = v + ## remove sensitive logging/callback keys from metadata dicts + ## these contain credentials (langfuse_secret_key, langfuse_public_key, etc.) + _sensitive_keys = {"logging", "callback_settings"} - metadata["user_api_key_metadata"] = cleaned_user_api_key_metadata - litellm_params["metadata"] = metadata + for metadata_field in ( + "user_api_key_metadata", + "user_api_key_auth_metadata", + "user_api_key_team_metadata", + ): + if metadata_field in metadata and isinstance(metadata[metadata_field], dict): + for sensitive_key in _sensitive_keys: + metadata[metadata_field].pop(sensitive_key, None) + + ## remove user_api_key_auth entirely - contains full auth object with nested credentials + metadata.pop("user_api_key_auth", None) + + litellm_params["metadata"] = metadata return litellm_params @@ -5603,4 +5602,3 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload: model_parameters={"stream": True}, hidden_params=hidden_params, ) - From 57a48e352695d5d3343813907482711aec6f6f5d Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 10 Mar 2026 21:03:20 -0700 Subject: [PATCH 017/418] fix(agents.tsx): support granting agents access to subagents --- .../litellm_proxy_extras/schema.prisma | 1 + litellm/proxy/_types.py | 75 ++-- litellm/proxy/schema.prisma | 1 + litellm/types/agents.py | 2 + schema.prisma | 1 + .../src/components/agents/add_agent_form.tsx | 373 ++++++++++-------- 6 files changed, 252 insertions(+), 201 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 8d4bdffb2dd..939f1eb0f45 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -267,6 +267,7 @@ model LiteLLM_ObjectPermissionTable { vector_stores String[] @default([]) agents String[] @default([]) agent_access_groups String[] @default([]) + models String[] @default([]) blocked_tools String[] @default([]) // Tool names blocked for any key/team/user with this permission teams LiteLLM_TeamTable[] projects LiteLLM_ProjectTable[] diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 36790e9feae..add3ab4a1f6 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1,59 +1,40 @@ import enum import json from datetime import datetime -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional, Union +from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Literal, + Optional, Union) import httpx -from pydantic import ( - BaseModel, - ConfigDict, - Field, - Json, - field_validator, - model_validator, -) +from pydantic import (BaseModel, ConfigDict, Field, Json, field_validator, + model_validator) from typing_extensions import Required, TypedDict from litellm._uuid import uuid from litellm.types.integrations.slack_alerting import AlertType -from litellm.types.llms.openai import ( - AllMessageValues, - OpenAIFileObject, - ResponsesAPIResponse, -) -from litellm.types.mcp import ( - MCPAuthType, - MCPCredentials, - MCPTransport, - MCPTransportType, -) +from litellm.types.llms.openai import (AllMessageValues, OpenAIFileObject, + ResponsesAPIResponse) +from litellm.types.mcp import (MCPAuthType, MCPCredentials, MCPTransport, + MCPTransportType) from litellm.types.mcp_server.mcp_server_manager import MCPInfo from litellm.types.router import RouterErrors, UpdateRouterConfig from litellm.types.secret_managers.main import KeyManagementSystem -from litellm.types.utils import ( - CallTypes, - CostBreakdown, - EmbeddingResponse, - GenericBudgetConfigType, - ImageResponse, - LiteLLMBatch, - LiteLLMFineTuningJob, - LiteLLMPydanticObjectBase, - ModelResponse, - ProviderField, - StandardCallbackDynamicParams, - StandardLoggingGuardrailInformation, - StandardLoggingMCPToolCall, - StandardLoggingModelInformation, - StandardLoggingPayloadErrorInformation, - StandardLoggingPayloadStatus, - StandardLoggingVectorStoreRequest, - StandardPassThroughResponseObject, - TextCompletionResponse, -) +from litellm.types.utils import (CallTypes, CostBreakdown, EmbeddingResponse, + GenericBudgetConfigType, ImageResponse, + LiteLLMBatch, LiteLLMFineTuningJob, + LiteLLMPydanticObjectBase, ModelResponse, + ProviderField, StandardCallbackDynamicParams, + StandardLoggingGuardrailInformation, + StandardLoggingMCPToolCall, + StandardLoggingModelInformation, + StandardLoggingPayloadErrorInformation, + StandardLoggingPayloadStatus, + StandardLoggingVectorStoreRequest, + StandardPassThroughResponseObject, + TextCompletionResponse) from litellm.types.videos.main import VideoObject -from .types_utils.utils import get_instance_fn, validate_custom_validate_return_type +from .types_utils.utils import (get_instance_fn, + validate_custom_validate_return_type) if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -855,6 +836,7 @@ class LiteLLM_ObjectPermissionBase(LiteLLMPydanticObjectBase): vector_stores: Optional[List[str]] = None agents: Optional[List[str]] = None agent_access_groups: Optional[List[str]] = None + models: Optional[List[str]] = None class GenerateRequestBase(LiteLLMPydanticObjectBase): @@ -2470,7 +2452,8 @@ class UserAPIKeyAuth( This is used to track number of requests/spend for health check calls. """ - from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME + from litellm.constants import \ + LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME return cls( api_key=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, @@ -2502,7 +2485,8 @@ class UserAPIKeyAuth( This is used to track actions performed by automated system jobs. """ - from litellm.constants import LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME + from litellm.constants import \ + LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME return cls( api_key=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, @@ -2908,7 +2892,8 @@ class LiteLLM_AuditLogs(LiteLLMPydanticObjectBase): @model_validator(mode="after") def mask_api_keys(self): - from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker + from litellm.litellm_core_utils.sensitive_data_masker import \ + SensitiveDataMasker masker = SensitiveDataMasker(sensitive_patterns={"key"}) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 721c3e404d2..b68872e2ed8 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -267,6 +267,7 @@ model LiteLLM_ObjectPermissionTable { vector_stores String[] @default([]) agents String[] @default([]) agent_access_groups String[] @default([]) + models String[] @default([]) blocked_tools String[] @default([]) // Tool names blocked for any key/team/user with this permission teams LiteLLM_TeamTable[] projects LiteLLM_ProjectTable[] diff --git a/litellm/types/agents.py b/litellm/types/agents.py index 951fbfcabd1..efb2e73bfb5 100644 --- a/litellm/types/agents.py +++ b/litellm/types/agents.py @@ -172,6 +172,8 @@ class AgentObjectPermission(TypedDict, total=False): mcp_servers: Optional[List[str]] mcp_access_groups: Optional[List[str]] mcp_tool_permissions: Optional[Dict[str, List[str]]] + models: Optional[List[str]] + agents: Optional[List[str]] class AgentConfig(TypedDict, total=False): diff --git a/schema.prisma b/schema.prisma index 8d4bdffb2dd..939f1eb0f45 100644 --- a/schema.prisma +++ b/schema.prisma @@ -267,6 +267,7 @@ model LiteLLM_ObjectPermissionTable { vector_stores String[] @default([]) agents String[] @default([]) agent_access_groups String[] @default([]) + models String[] @default([]) blocked_tools String[] @default([]) // Tool names blocked for any key/team/user with this permission teams LiteLLM_TeamTable[] projects LiteLLM_ProjectTable[] diff --git a/ui/litellm-dashboard/src/components/agents/add_agent_form.tsx b/ui/litellm-dashboard/src/components/agents/add_agent_form.tsx index 0cec0331f43..5b739e5a16b 100644 --- a/ui/litellm-dashboard/src/components/agents/add_agent_form.tsx +++ b/ui/litellm-dashboard/src/components/agents/add_agent_form.tsx @@ -6,6 +6,7 @@ import CreatedKeyDisplay from "../shared/CreatedKeyDisplay"; import { createAgentCall, getAgentCreateMetadata, + getAgentsList, keyCreateForAgentCall, keyListCall, keyUpdateCall, @@ -45,7 +46,7 @@ const AddAgentForm: React.FC = ({ const [agentTypeMetadata, setAgentTypeMetadata] = useState([]); const [loadingMetadata, setLoadingMetadata] = useState(false); - // Step 1: key assignment state + // Step 3: key assignment state const [keyAssignOption, setKeyAssignOption] = useState<"create_new" | "existing_key" | "skip">("create_new"); const [newKeyName, setNewKeyName] = useState(""); const [newKeyModels, setNewKeyModels] = useState([]); @@ -54,8 +55,10 @@ const AddAgentForm: React.FC = ({ const [loadingKeys, setLoadingKeys] = useState(false); const [availableModels, setAvailableModels] = useState([]); const [loadingModels, setLoadingModels] = useState(false); + const [availableAgents, setAvailableAgents] = useState<{agent_id: string; agent_name: string}[]>([]); + const [loadingAgents, setLoadingAgents] = useState(false); - // Step 2: results + // Step 4: results const [createdAgentName, setCreatedAgentName] = useState(""); const [createdKeyValue, setCreatedKeyValue] = useState(null); const [assignedKeyAlias, setAssignedKeyAlias] = useState(null); @@ -82,9 +85,9 @@ const AddAgentForm: React.FC = ({ fetchMetadata(); }, []); - // Fetch existing keys when assign key step becomes active (step 2) + // Fetch existing keys when Agent Management step becomes active (step 3) useEffect(() => { - if (currentStep === 2 && accessToken && existingKeys.length === 0) { + if (currentStep === 3 && accessToken && existingKeys.length === 0) { const fetchKeys = async () => { setLoadingKeys(true); try { @@ -100,9 +103,9 @@ const AddAgentForm: React.FC = ({ } }, [currentStep, accessToken]); - // Fetch available models when Assign Key step is active (same list as key generation) + // Fetch available models when Agent Management step is active (same list as key generation) useEffect(() => { - if (currentStep !== 2 || !accessToken || !userId || !userRole) return; + if ((currentStep !== 1 && currentStep !== 3) || !accessToken || !userId || !userRole) return; let cancelled = false; setLoadingModels(true); modelAvailableCall(accessToken, userId, userRole) @@ -125,6 +128,25 @@ const AddAgentForm: React.FC = ({ }; }, [currentStep, accessToken, userId, userRole]); + useEffect(() => { + if (currentStep !== 1 || !accessToken) return; + let cancelled = false; + setLoadingAgents(true); + getAgentsList(accessToken) + .then((response) => { + if (cancelled) return; + const agents = response?.agents ?? []; + setAvailableAgents(agents.map((a: any) => ({ agent_id: a.agent_id, agent_name: a.agent_name }))); + }) + .catch((error) => { + if (!cancelled) console.error("Error fetching agents:", error); + }) + .finally(() => { + if (!cancelled) setLoadingAgents(false); + }); + return () => { cancelled = true; }; + }, [currentStep, accessToken]); + const selectedAgentTypeInfo = agentTypeMetadata.find( (info) => info.agent_type === agentType ); @@ -207,11 +229,14 @@ const AddAgentForm: React.FC = ({ // Build object_permission from MCP Tools step (allowed_mcp_servers_and_groups, mcp_tool_permissions) const mcpServersAndGroups = values.allowed_mcp_servers_and_groups; const mcpToolPermissions = values.mcp_tool_permissions || {}; - if ( - mcpServersAndGroups && - (mcpServersAndGroups.servers?.length > 0 || mcpServersAndGroups.accessGroups?.length > 0) || - Object.keys(mcpToolPermissions).length > 0 - ) { + const entitlementModels = values.entitlement_models || []; + const entitlementAgents = values.entitlement_agents || []; + const hasObjectPermission = + (mcpServersAndGroups?.servers?.length > 0 || mcpServersAndGroups?.accessGroups?.length > 0) || + Object.keys(mcpToolPermissions).length > 0 || + entitlementModels.length > 0 || + entitlementAgents.length > 0; + if (hasObjectPermission) { agentData.object_permission = {}; if (mcpServersAndGroups?.servers?.length > 0) { agentData.object_permission.mcp_servers = mcpServersAndGroups.servers; @@ -222,6 +247,12 @@ const AddAgentForm: React.FC = ({ if (Object.keys(mcpToolPermissions).length > 0) { agentData.object_permission.mcp_tool_permissions = mcpToolPermissions; } + if (entitlementModels.length > 0) { + agentData.object_permission.models = entitlementModels; + } + if (entitlementAgents.length > 0) { + agentData.object_permission.agents = entitlementAgents; + } } // Wire trace-id flags and budget controls into agent litellm_params (before create call) @@ -264,7 +295,7 @@ const AddAgentForm: React.FC = ({ setAssignedKeyAlias(keyInfo?.key_alias || selectedExistingKey.slice(0, 12) + "…"); } - setCurrentStep(3); + setCurrentStep(4); onSuccess(); } catch (error) { console.error("Error creating agent:", error); @@ -293,11 +324,54 @@ const AddAgentForm: React.FC = ({ onClose(); }; - const renderMCPToolsStep = () => ( + const renderEntitlementsStep = () => (

- Optionally restrict which MCP servers and tools this agent can use. Leave empty to allow all (subject to key/team permissions). + Configure which models, agents, and MCP tools this agent is allowed to use. Leave fields empty to allow all (subject to key/team permissions).

+ + Allowed Models} + name="entitlement_models" + tooltip="Restrict which models this agent can call. Leave empty to allow all." + > + + (option?.label as string ?? "").toLowerCase().includes(input.toLowerCase()) + } + options={availableAgents.map((a) => ({ + label: a.agent_name, + value: a.agent_id, + }))} + /> + + + + @@ -338,122 +412,121 @@ const AddAgentForm: React.FC = ({
)} + + ); - Tracing, - children: ( -
-
-
- - Require x-litellm-trace-id on calls TO this agent - -

- Only accept this agent being invoked with a trace-id (e.g. when used as a sub-agent). -

-
- -
- -
-
- - Require x-litellm-trace-id on calls BY this agent - -

- Requires LLM/MCP calls made by this agent to include x-litellm-trace-id for session tracking. -

-
- { - setRequireTraceIdOutbound(checked); - if (!checked) { - setMaxIterations(null); - setMaxBudgetPerSession(null); - } - }} - /> -
-
- ), - }, - { - key: "budgets_and_rate_limits", - label: Budgets & Rate Limits, - children: ( -
- {!requireTraceIdOutbound && ( -
- Enable "Require x-litellm-trace-id on calls BY this agent" in Tracing to configure budgets and rate limits. -
- )} - -
Session Budgets
-
-
- - setMaxIterations(val)} - /> -

Hard cap on LLM calls per session

-
-
- - setMaxBudgetPerSession(val)} - /> -

Max spend per trace before returning 429

-
-
- - - -
Agent Rate Limits
-

- Global rate limits applied across all callers of this agent. + const renderObservabilityStep = () => ( +

+
+

Tracing

+
+
+
+ + Require x-litellm-trace-id on calls TO this agent + +

+ Only accept this agent being invoked with a trace-id (e.g. when used as a sub-agent).

-
- - - - - - -
- -
Per-Session Rate Limits
-

- Rate limits per session (x-litellm-trace-id). Each session gets its own counters. -

-
- - - - - - -
- ), - }, - ]} /> + +
+ +
+
+ + Require x-litellm-trace-id on calls BY this agent + +

+ Requires LLM/MCP calls made by this agent to include x-litellm-trace-id for session tracking. +

+
+ { + setRequireTraceIdOutbound(checked); + if (!checked) { + setMaxIterations(null); + setMaxBudgetPerSession(null); + } + }} + /> +
+
+
+ + + +
+

Budgets & Rate Limits

+
+ {!requireTraceIdOutbound && ( +
+ Enable "Require x-litellm-trace-id on calls BY this agent" in Tracing to configure budgets and rate limits. +
+ )} + +
Session Budgets
+
+
+ + setMaxIterations(val)} + /> +

Hard cap on LLM calls per session

+
+
+ + setMaxBudgetPerSession(val)} + /> +

Max spend per trace before returning 429

+
+
+ + + +
Agent Rate Limits
+

+ Global rate limits applied across all callers of this agent. +

+
+ + + + + + +
+ +
Per-Session Rate Limits
+

+ Rate limits per session (x-litellm-trace-id). Each session gets its own counters. +

+
+ + + + + + +
+
+
); @@ -645,25 +718,6 @@ const AddAgentForm: React.FC = ({ placeholder="e.g. my-agent-key" />
-
- - } + placeholder="Or paste a custom logo URL..." + value={value && !WELL_KNOWN_LOGOS.some((l) => l.url === value) ? value : ""} + onChange={(e) => { + const v = e.target.value.trim(); + onChange?.(v || undefined); + }} + className="rounded-lg" + size="small" + /> +
+ ); +}; + +export default MCPLogoSelector; diff --git a/ui/litellm-dashboard/src/components/mcp_tools/OpenAPIFormSection.tsx b/ui/litellm-dashboard/src/components/mcp_tools/OpenAPIFormSection.tsx index 23aae6cb14f..b25e1dcadd4 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/OpenAPIFormSection.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/OpenAPIFormSection.tsx @@ -12,6 +12,8 @@ interface OpenAPIFormSectionProps { onValuesChange: (updates: Record) => void; /** Called when key tools change (from registry preset selection). */ onKeyToolsChange?: (tools: OpenAPIKeyTool[]) => void; + /** Called when a preset is selected so the parent can set the logo URL from icon_url. */ + onLogoUrlChange?: (url: string | undefined) => void; /** Called when the OAuth docs URL changes (e.g. link to create a GitHub OAuth App). */ onOAuthDocsUrlChange?: (url: string | null) => void; } @@ -26,6 +28,7 @@ const OpenAPIFormSection: React.FC = ({ accessToken, onValuesChange, onKeyToolsChange, + onLogoUrlChange, onOAuthDocsUrlChange, }) => { const [selectedPreset, setSelectedPreset] = useState(null); @@ -33,6 +36,7 @@ const OpenAPIFormSection: React.FC = ({ const handlePresetSelect = (entry: OpenAPIRegistryEntry) => { setSelectedPreset(entry.name); onKeyToolsChange?.(entry.key_tools ?? []); + onLogoUrlChange?.(entry.icon_url || undefined); const updates: Record = { spec_path: entry.spec_url, }; diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index 90ecd4731cf..74945731a24 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -11,6 +11,7 @@ import MCPToolConfiguration from "./mcp_tool_configuration"; import StdioConfiguration from "./StdioConfiguration"; import MCPPermissionManagement from "./MCPPermissionManagement"; import OpenAPIFormSection, { OpenAPIKeyTool } from "./OpenAPIFormSection"; +import MCPLogoSelector from "./MCPLogoSelector"; import { isAdminRole } from "@/utils/roles"; import { validateMCPServerUrl, validateMCPServerName } from "./utils"; import NotificationsManager from "../molecules/notifications_manager"; @@ -70,6 +71,7 @@ const CreateMCPServer: React.FC = ({ const [keyTools, setKeyTools] = useState([]); const [searchValue, setSearchValue] = useState(""); const [oauthAccessToken, setOauthAccessToken] = useState(null); + const [logoUrl, setLogoUrl] = useState(undefined); const [oauthDocsUrl, setOauthDocsUrl] = useState(null); // Single hook call shared by MCPConnectionStatus and MCPToolConfiguration to avoid duplicate requests. @@ -101,6 +103,7 @@ const CreateMCPServer: React.FC = ({ allowedTools, searchValue, aliasManuallyEdited, + logoUrl, }), ); } catch (err) { @@ -202,6 +205,9 @@ const CreateMCPServer: React.FC = ({ if (typeof parsed.aliasManuallyEdited === "boolean") { setAliasManuallyEdited(parsed.aliasManuallyEdited); } + if (parsed.logoUrl) { + setLogoUrl(parsed.logoUrl); + } } catch (err) { console.error("Failed to restore MCP create state", err); } finally { @@ -357,6 +363,7 @@ const CreateMCPServer: React.FC = ({ mcp_info: { server_name: restValues.server_name || restValues.url, description: restValues.description, + logo_url: logoUrl || undefined, mcp_server_cost_info: Object.keys(costConfig).length > 0 ? costConfig : null, }, mcp_access_groups: accessGroups, @@ -394,6 +401,7 @@ const CreateMCPServer: React.FC = ({ clearTools(); setAllowedTools([]); setAliasManuallyEdited(false); + setLogoUrl(undefined); setModalVisible(false); onCreateSuccess(response); } @@ -414,6 +422,7 @@ const CreateMCPServer: React.FC = ({ clearTools(); setAllowedTools([]); setAliasManuallyEdited(false); + setLogoUrl(undefined); setModalVisible(false); }; @@ -590,6 +599,8 @@ const CreateMCPServer: React.FC = ({ /> + + GitHub / Source URL} name="source_url" @@ -645,6 +656,7 @@ const CreateMCPServer: React.FC = ({ setFormValues((prev) => ({ ...prev, ...updates })) } onKeyToolsChange={setKeyTools} + onLogoUrlChange={setLogoUrl} onOAuthDocsUrlChange={setOauthDocsUrl} /> )} diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_columns.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_columns.tsx index 8130f96856f..ea5ccf1c847 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_columns.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_columns.tsx @@ -1,3 +1,4 @@ +import { useState } from "react"; import { ColumnDef } from "@tanstack/react-table"; import { MCPServer } from "./types"; import { Icon } from "@tremor/react"; @@ -6,6 +7,82 @@ import { getMaskedAndFullUrl } from "./utils"; import { Tooltip } from "antd"; import { CheckOutlined } from "@ant-design/icons"; +const HealthStatusBadge: React.FC<{ + server: MCPServer; + isLoadingHealth?: boolean; + isRechecking?: boolean; + onRecheck?: (serverId: string) => void; +}> = ({ server, isLoadingHealth, isRechecking, onRecheck }) => { + const [isHovered, setIsHovered] = useState(false); + const status = server.status || "unknown"; + const lastCheck = server.last_health_check; + const error = server.health_check_error; + + if (isLoadingHealth || isRechecking) { + return ( + + + Checking + + ); + } + + const getStatusColor = (status: string) => { + switch (status) { + case "healthy": + return "text-green-700 bg-green-50 border border-green-200"; + case "unhealthy": + return "text-red-700 bg-red-50 border border-red-200"; + default: + return "text-gray-600 bg-gray-50 border border-gray-200"; + } + }; + + const getStatusIcon = (status: string) => { + switch (status) { + case "healthy": + return "✓"; + case "unhealthy": + return "✗"; + default: + return "?"; + } + }; + + const isClickable = !!onRecheck; + + const tooltipContent = ( +
+
Health Status: {status}
+ {lastCheck &&
Last Check: {new Date(lastCheck).toLocaleString()}
} + {error && ( +
+
Error:
+
{error}
+
+ )} + {!lastCheck && !error &&
No health check data available
} + {isClickable &&
Click to recheck
} +
+ ); + + return ( + + setIsHovered(true)} + onMouseLeave={() => setIsHovered(false)} + onClick={isClickable ? () => onRecheck(server.server_id) : undefined} + > + {isHovered && isClickable ? "↻" : getStatusIcon(status)} + {isHovered && isClickable + ? "Recheck" + : status.charAt(0).toUpperCase() + status.slice(1)} + + + ); +}; + export const mcpServerColumns = ( userRole: string, onView: (serverId: string) => void, @@ -13,6 +90,8 @@ export const mcpServerColumns = ( onDelete: (serverId: string) => void, isLoadingHealth?: boolean, onByokConnect?: (server: MCPServer) => void, + onRecheckHealth?: (serverId: string) => void, + recheckingServerIds?: Set, ): ColumnDef[] => [ { accessorKey: "server_id", @@ -31,6 +110,23 @@ export const mcpServerColumns = ( accessorKey: "server_name", header: "Name", enableSorting: true, + cell: ({ row }) => { + const logoUrl = row.original.mcp_info?.logo_url; + const name = row.original.server_name; + return ( +
+ {logoUrl ? ( + {`${name { (e.target as HTMLImageElement).style.display = "none"; }} + /> + ) : null} + {name} +
+ ); + }, }, { accessorKey: "alias", @@ -81,68 +177,14 @@ export const mcpServerColumns = ( { id: "health_status", header: "Health Status", - cell: ({ row }) => { - const server = row.original; - const status = server.status || "unknown"; - const lastCheck = server.last_health_check; - const error = server.health_check_error; - - if (isLoadingHealth) { - return ( - - - Checking - - ); - } - - const getStatusColor = (status: string) => { - switch (status) { - case "healthy": - return "text-green-700 bg-green-50 border border-green-200"; - case "unhealthy": - return "text-red-700 bg-red-50 border border-red-200"; - default: - return "text-gray-600 bg-gray-50 border border-gray-200"; - } - }; - - const getStatusIcon = (status: string) => { - switch (status) { - case "healthy": - return "✓"; - case "unhealthy": - return "✗"; - default: - return "?"; - } - }; - - const tooltipContent = ( -
-
Health Status: {status}
- {lastCheck &&
Last Check: {new Date(lastCheck).toLocaleString()}
} - {error && ( -
-
Error:
-
{error}
-
- )} - {!lastCheck && !error &&
No health check data available
} -
- ); - - return ( - - - {getStatusIcon(status)} - {status.charAt(0).toUpperCase() + status.slice(1)} - - - ); - }, + cell: ({ row }) => ( + + ), }, { id: "mcp_access_groups", diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index fc55542a0c9..eadf93d8a96 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -8,6 +8,7 @@ import MCPServerCostConfig from "./mcp_server_cost_config"; import MCPPermissionManagement from "./MCPPermissionManagement"; import MCPToolConfiguration from "./mcp_tool_configuration"; import StdioConfiguration from "./StdioConfiguration"; +import MCPLogoSelector from "./MCPLogoSelector"; import { validateMCPServerUrl, validateMCPServerName } from "./utils"; import NotificationsManager from "../molecules/notifications_manager"; import { useMcpOAuthFlow } from "@/hooks/useMcpOAuthFlow"; @@ -41,6 +42,7 @@ const MCPServerEdit: React.FC = ({ const [toolNameToDisplayName, setToolNameToDisplayName] = useState>({}); const [toolNameToDescription, setToolNameToDescription] = useState>({}); const [pendingRestoredValues, setPendingRestoredValues] = useState | null>(null); + const [logoUrl, setLogoUrl] = useState(mcpServer.mcp_info?.logo_url || undefined); const authType = Form.useWatch("auth_type", form) as string | undefined; const transportType = Form.useWatch("transport", form) as string | undefined; const isStdioTransport = transportType === "stdio"; @@ -538,6 +540,7 @@ const MCPServerEdit: React.FC = ({ mcp_info: { server_name: mcpInfoServerName, description: restValues.description, + logo_url: logoUrl || undefined, mcp_server_cost_info: Object.keys(costConfig).length > 0 ? costConfig : null, }, mcp_access_groups: accessGroups, @@ -604,6 +607,7 @@ const MCPServerEdit: React.FC = ({ + @@ -818,6 +820,122 @@ const CreateMCPServer: React.FC = ({ /> )} + {transportType !== "stdio" && transportType !== "" && isAwsSigV4AuthType && ( + <> +

+ For MCP servers hosted on AWS Bedrock AgentCore.{" "} + + View docs → + +

+ + AWS Region + + + + + } + name={["credentials", "aws_region_name"]} + rules={[{ required: true, message: "AWS region is required for SigV4 auth" }]} + > + + + + AWS Service Name + + + + + } + name={["credentials", "aws_service_name"]} + > + + + + AWS Access Key ID + + + + + } + name={["credentials", "aws_access_key_id"]} + dependencies={[["credentials", "aws_secret_access_key"]]} + rules={[ + ({ getFieldValue }) => ({ + validator(_, value) { + const secretKey = getFieldValue(["credentials", "aws_secret_access_key"]); + if (secretKey && !value) { + return Promise.reject(new Error("Access Key ID is required when Secret Access Key is provided")); + } + return Promise.resolve(); + }, + }), + ]} + > + + + + AWS Secret Access Key + + + + + } + name={["credentials", "aws_secret_access_key"]} + dependencies={[["credentials", "aws_access_key_id"]]} + rules={[ + ({ getFieldValue }) => ({ + validator(_, value) { + const accessKeyId = getFieldValue(["credentials", "aws_access_key_id"]); + if (accessKeyId && !value) { + return Promise.reject(new Error("Secret Access Key is required when Access Key ID is provided")); + } + return Promise.resolve(); + }, + }), + ]} + > + + + + AWS Session Token + + + + + } + name={["credentials", "aws_session_token"]} + > + + + + )} + {/* Stdio Configuration - only show for stdio transport */} diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index eadf93d8a96..04cce343038 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -22,7 +22,7 @@ interface MCPServerEditProps { } const AUTH_TYPES_REQUIRING_AUTH_VALUE = [AUTH_TYPE.API_KEY, AUTH_TYPE.BEARER_TOKEN, AUTH_TYPE.TOKEN, AUTH_TYPE.BASIC]; -const AUTH_TYPES_REQUIRING_CREDENTIALS = [...AUTH_TYPES_REQUIRING_AUTH_VALUE, AUTH_TYPE.OAUTH2]; +const AUTH_TYPES_REQUIRING_CREDENTIALS = [...AUTH_TYPES_REQUIRING_AUTH_VALUE, AUTH_TYPE.OAUTH2, AUTH_TYPE.AWS_SIGV4]; const EDIT_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-edit-state"; const MCPServerEdit: React.FC = ({ @@ -50,6 +50,7 @@ const MCPServerEdit: React.FC = ({ const isMCPTransport = !isStdioTransport && !isOpenAPITransport; const shouldShowAuthValueField = authType ? AUTH_TYPES_REQUIRING_AUTH_VALUE.includes(authType) : false; const isOAuthAuthType = authType === AUTH_TYPE.OAUTH2; + const isAwsSigV4AuthType = authType === AUTH_TYPE.AWS_SIGV4; const oauthFlowTypeValue = Form.useWatch("oauth_flow_type", form) as string | undefined; const isM2MFlow = isOAuthAuthType && oauthFlowTypeValue === OAUTH_FLOW.M2M; @@ -665,6 +666,7 @@ const MCPServerEdit: React.FC = ({ Token Basic Auth OAuth + AWS SigV4 (Bedrock AgentCore MCPs) )} @@ -883,6 +885,100 @@ const MCPServerEdit: React.FC = ({ )} + {!isStdioTransport && isAwsSigV4AuthType && ( + <> +

+ For MCP servers hosted on AWS Bedrock AgentCore.{" "} + + View docs → + +

+ + AWS Region + + + + + } + name={["credentials", "aws_region_name"]} + rules={[]} + > + + + + AWS Service Name + + + + + } + name={["credentials", "aws_service_name"]} + > + + + + AWS Access Key ID + + + + + } + name={["credentials", "aws_access_key_id"]} + rules={[]} + > + + + + AWS Secret Access Key + + + + + } + name={["credentials", "aws_secret_access_key"]} + rules={[]} + > + + + + AWS Session Token + + + + + } + name={["credentials", "aws_session_token"]} + > + + + + )} + {/* Permission Management / Access Control Section */}
{ pagination?: PaginationState; onPaginationChange?: OnChangeFn; enablePagination?: boolean; + onRowClick?: (row: TData) => void; } export function AllModelsDataTable({ @@ -41,6 +42,7 @@ export function AllModelsDataTable({ pagination, onPaginationChange, enablePagination = false, + onRowClick, }: AllModelsDataTableProps) { const [columnResizeMode] = React.useState("onChange"); const [columnSizing, setColumnSizing] = React.useState({}); @@ -174,7 +176,11 @@ export function AllModelsDataTable({ ) : tableInstance.getRowModel().rows.length > 0 ? ( tableInstance.getRowModel().rows.map((row) => ( - + onRowClick?.(row.original)} + > {row.getVisibleCells().map((cell) => ( { pagination?: PaginationState; onPaginationChange?: OnChangeFn; enablePagination?: boolean; + onRowClick?: (row: TData) => void; } export function ModelDataTable({ @@ -40,6 +41,7 @@ export function ModelDataTable({ pagination, onPaginationChange, enablePagination = false, + onRowClick, }: ModelDataTableProps) { const [sorting, setSorting] = React.useState(defaultSorting); const [columnResizeMode] = React.useState("onChange"); @@ -164,7 +166,11 @@ export function ModelDataTable({ ) : tableInstance.getRowModel().rows.length > 0 ? ( tableInstance.getRowModel().rows.map((row) => ( - + onRowClick?.(row.original)} + className={onRowClick ? "cursor-pointer hover:bg-gray-50" : ""} + > {row.getVisibleCells().map((cell) => ( void, expandedRows: Set, setExpandedRows: (expandedRows: Set) => void, + onDeleteClick?: (modelId: string) => void, ): ColumnDef[] => [ { header: () => Model ID, @@ -67,7 +68,10 @@ export const columns = ( ellipsis className="text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs cursor-pointer w-full block" style={{ fontSize: 14, padding: '1px 8px' }} - onClick={() => setSelectedModelId(model.model_info.id)} + onClick={(e) => { + e.stopPropagation(); + setSelectedModelId(model.model_info.id); + }} > {model.model_info.id} @@ -297,7 +301,10 @@ export const columns = ( size="xs" variant="light" className="font-mono text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs font-normal px-2 py-0.5 text-left overflow-hidden truncate w-full" - onClick={() => setSelectedTeamId(model.model_info.team_id)} + onClick={(e: React.MouseEvent) => { + e.stopPropagation(); + setSelectedTeamId(model.model_info.team_id); + }} > {model.model_info.team_id.slice(0, 7)}... @@ -409,9 +416,10 @@ export const columns = ( { - if (canEditModel) { - setSelectedModelId(model.model_info.id); + onClick={(e) => { + e.stopPropagation(); + if (canEditModel && onDeleteClick) { + onDeleteClick(model.model_info.id); } }} className={!canEditModel ? "opacity-50 cursor-not-allowed" : "cursor-pointer hover:text-red-600"} diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index dcbdd2f73e1..ea3d5a16221 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -9038,9 +9038,7 @@ export const updateUiSettings = async (accessToken: string, settings: Record = ({ accessToken, isEmbedded setLoading(true); const _modelHubData = await modelHubPublicModelsCall(); console.log("ModelHubData:", _modelHubData); - setModelHubData(_modelHubData); + setModelHubData(Array.isArray(_modelHubData) ? _modelHubData : []); } catch (error) { console.error("There was an error fetching the public model data", error); setServiceStatus("Service unavailable"); @@ -150,7 +150,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded setAgentLoading(true); const _agentHubData = await agentHubPublicModelsCall(); console.log("AgentHubData:", _agentHubData); - setAgentHubData(_agentHubData); + setAgentHubData(Array.isArray(_agentHubData) ? _agentHubData : []); } catch (error) { console.error("There was an error fetching the public agent data", error); } finally { @@ -163,7 +163,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded setMcpLoading(true); const _mcpHubData = await mcpHubPublicServersCall(); console.log("MCPHubData:", _mcpHubData); - setMcpHubData(_mcpHubData); + setMcpHubData(Array.isArray(_mcpHubData) ? _mcpHubData : []); } catch (error) { console.error("There was an error fetching the public MCP server data", error); } finally { @@ -199,7 +199,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded const getUniqueProviders = (data: ModelGroupInfo[]) => { const providers = new Set(); data.forEach((model) => { - model.providers.forEach((provider) => providers.add(provider)); + (model.providers ?? []).forEach((provider) => providers.add(provider)); }); return Array.from(providers); }; @@ -532,7 +532,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded accessorKey: "providers", enableSorting: true, cell: ({ row }) => { - const providers = row.original.providers; + const providers = row.original.providers ?? []; return (
@@ -760,7 +760,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded accessorKey: "description", enableSorting: false, cell: ({ row }) => { - const description = row.original.description; + const description = row.original.description ?? ""; const truncated = description.length > 80 ? description.substring(0, 80) + "..." : description; return ( @@ -897,7 +897,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded accessorKey: "mcp_info.description", enableSorting: false, cell: ({ row }) => { - const description = row.original.mcp_info?.description || "-"; + const description = String(row.original.mcp_info?.description ?? "-"); const truncated = description.length > 80 ? description.substring(0, 80) + "..." : description; return ( @@ -912,7 +912,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded accessorKey: "url", enableSorting: false, cell: ({ row }) => { - const url = row.original.url; + const url = row.original.url ?? ""; const truncated = url.length > 40 ? url.substring(0, 40) + "..." : url; return ( @@ -1336,7 +1336,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded
Providers:
- {selectedModel.providers.map((provider) => { + {(selectedModel.providers ?? []).map((provider) => { const { logo } = getProviderLogoAndName(provider); return ( @@ -1460,7 +1460,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded )} {/* Supported OpenAI Parameters */} - {selectedModel.supported_openai_params && ( + {selectedModel.supported_openai_params && selectedModel.supported_openai_params.length > 0 && (
Supported OpenAI Parameters
@@ -1634,7 +1634,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded
Input Modes:
- {selectedAgent.defaultInputModes?.map((mode) => ( + {(selectedAgent.defaultInputModes ?? []).map((mode) => ( {mode} @@ -1644,7 +1644,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded
Output Modes:
- {selectedAgent.defaultOutputModes?.map((mode) => ( + {(selectedAgent.defaultOutputModes ?? []).map((mode) => ( {mode} From 72c98489d12e9709d6450078a2215d1001c5ae73 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 12 Mar 2026 18:26:11 +0530 Subject: [PATCH 079/418] Revert "fix(vertex): shallow copy parameters before mutating in _build_vertex_schema_for_gemini_2" This reverts commit 08d81f5d7c7239cbf065e8de45506184ba917742. --- litellm/llms/vertex_ai/common_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 078fce63cc1..ad1e70f2ce2 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -534,7 +534,6 @@ def _build_vertex_schema_for_gemini_2(parameters: dict) -> dict: """ valid_schema_fields = set(get_type_hints(Schema).keys()) - parameters = dict(parameters) # shallow copy to avoid mutating caller's dict defs = parameters.pop("$defs", {}) unpack_defs(parameters, defs) From 412a283569d575425e5eca3a62eef8dbecdc4b90 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 12 Mar 2026 18:26:11 +0530 Subject: [PATCH 080/418] Revert "fix(vertex): skip harmful schema transforms for Gemini 2.0+ tool parameters" This reverts commit a9c3095cc539c446884649fe6855b44861182c96. --- litellm/llms/vertex_ai/common_utils.py | 22 ----- .../vertex_and_google_ai_studio_gemini.py | 24 ++--- .../vertex_ai/test_vertex_ai_common_utils.py | 91 ------------------- 3 files changed, 6 insertions(+), 131 deletions(-) diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index ad1e70f2ce2..c02d63414c5 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -520,28 +520,6 @@ def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False): return parameters -def _build_vertex_schema_for_gemini_2(parameters: dict) -> dict: - """ - Minimal schema builder for Gemini 2.0+ tool parameters. - - Gemini 2.0+ accepts standard JSON Schema natively in tool parameters, - including lowercase types, anyOf with null, and bare {} (TYPE_UNSPECIFIED). - The only transformation needed is resolving $ref/$defs, which Gemini does - NOT support in tool parameters (returns 400). - - This avoids the harmful transforms in _build_vertex_schema that break - JsonValue/Any semantics by coercing {} to {"type": "object"}. - """ - valid_schema_fields = set(get_type_hints(Schema).keys()) - - defs = parameters.pop("$defs", {}) - unpack_defs(parameters, defs) - - parameters = filter_schema_fields(parameters, valid_schema_fields) - - return parameters - - def _build_json_schema(parameters: dict) -> dict: """ Build a JSON Schema for use with Gemini's responseJsonSchema parameter. diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index df7a4a6511d..6cd430d6cba 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -97,7 +97,6 @@ from ..common_utils import ( VertexAIError, _build_json_schema, _build_vertex_schema, - _build_vertex_schema_for_gemini_2, supports_response_json_schema, ) from ..vertex_llm_base import VertexBase @@ -468,7 +467,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return None def _map_function( # noqa: PLR0915 - self, value: List[dict], optional_params: dict, model: str = "" + self, value: List[dict], optional_params: dict ) -> List[Tools]: """ Map OpenAI-style tools/functions to Vertex AI format. @@ -511,21 +510,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "parameters" in _openai_function_object and _openai_function_object["parameters"] is not None and isinstance(_openai_function_object["parameters"], dict) - ): - if supports_response_json_schema(model): - # Gemini 2.0+: minimal transform (resolve $ref only) - _openai_function_object["parameters"] = ( - _build_vertex_schema_for_gemini_2( - _openai_function_object["parameters"] - ) - ) - else: - # Gemini 1.5: full OpenAPI-style transform - _openai_function_object["parameters"] = ( - _build_vertex_schema( - _openai_function_object["parameters"] - ) - ) + ): # OPENAI accepts JSON Schema, Google accepts OpenAPI schema. + _openai_function_object["parameters"] = _build_vertex_schema( + _openai_function_object["parameters"] + ) openai_function_object = _openai_function_object @@ -1063,7 +1051,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ): # Pass optional_params so _map_function can add toolConfig if needed mapped_tools = self._map_function( - value=value, optional_params=optional_params, model=model + value=value, optional_params=optional_params ) optional_params = self._add_tools_to_optional_params( optional_params, mapped_tools diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index a39c7da2c71..94323e06901 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -11,7 +11,6 @@ sys.path.insert( ) # Adds the parent directory to the system path from litellm.llms.vertex_ai.common_utils import ( - _build_vertex_schema_for_gemini_2, _get_vertex_url, convert_anyof_null_to_nullable, get_vertex_location_from_url, @@ -1403,93 +1402,3 @@ def test_add_object_type_does_not_add_type_when_anyof_present(): # Verify type was not added (anyOf handles the type) assert "type" not in input_schema, "type should not be added when anyOf is present" - - -class TestBuildVertexSchemaForGemini2: - """Tests for _build_vertex_schema_for_gemini_2 — minimal transform for Gemini 2.0+ tools.""" - - def test_jsonvalue_standalone_preserved(self): - """JsonValue (bare {}) should NOT be coerced to {"type": "object"}.""" - schema = { - "type": "object", - "properties": { - "name": {"type": "string"}, - "value": {}, - }, - "required": ["name", "value"], - } - result = _build_vertex_schema_for_gemini_2(schema) - assert result["properties"]["value"] == {} - - def test_optional_jsonvalue_anyof_preserved(self): - """Optional[JsonValue] anyOf with null should be preserved, not converted to nullable.""" - schema = { - "type": "object", - "properties": { - "name": {"type": "string"}, - "value": { - "anyOf": [ - {"type": "array", "items": {}}, - {}, - {"type": "null"}, - ] - }, - }, - "required": ["name"], - } - result = _build_vertex_schema_for_gemini_2(schema) - value_schema = result["properties"]["value"] - assert "anyOf" in value_schema - assert len(value_schema["anyOf"]) == 3 - assert {"type": "null"} in value_schema["anyOf"] - assert {} in value_schema["anyOf"] - - def test_ref_defs_resolved(self): - """$ref/$defs should be resolved since Gemini doesn't support them in tool params.""" - schema = { - "type": "object", - "properties": { - "value": {"$ref": "#/$defs/JsonValue"}, - }, - "$defs": {"JsonValue": {}}, - } - result = _build_vertex_schema_for_gemini_2(schema) - assert "$ref" not in result["properties"]["value"] - assert "$defs" not in result - assert result["properties"]["value"] == {} - - def test_unsupported_fields_stripped(self): - """Fields not in Vertex Schema TypedDict should be removed.""" - schema = { - "type": "object", - "properties": { - "name": {"type": "string", "additionalProperties": False}, - }, - "additionalProperties": False, - "$schema": "http://json-schema.org/draft-07/schema#", - } - result = _build_vertex_schema_for_gemini_2(schema) - assert "additionalProperties" not in result - assert "$schema" not in result - - def test_no_type_coercion(self): - """Schemas without type should NOT have type: object added.""" - schema = { - "type": "object", - "properties": { - "data": {"description": "Any data"}, - }, - } - result = _build_vertex_schema_for_gemini_2(schema) - assert "type" not in result["properties"]["data"] - - def test_items_empty_preserved(self): - """items: {} should NOT be coerced to items: {"type": "object"}.""" - schema = { - "type": "object", - "properties": { - "values": {"type": "array", "items": {}}, - }, - } - result = _build_vertex_schema_for_gemini_2(schema) - assert result["properties"]["values"]["items"] == {} From 0f91a4f9da25e0984e4799dc0d81c3892f005dd3 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 12 Mar 2026 18:33:14 +0530 Subject: [PATCH 081/418] Fix test_get_tools_for_single_server --- tests/mcp_tests/test_mcp_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index 5a0a42d6f77..2544e06598e 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -1826,6 +1826,7 @@ async def test_get_tools_for_single_server(): mock_manager._get_tools_from_server.assert_called_once_with( server=mock_server, mcp_auth_header="Bearer test_token", + extra_headers=None, add_prefix=False, raw_headers=None, ) From 18df137021ced849957d19dd4762641bcef36a51 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 12 Mar 2026 18:33:40 +0530 Subject: [PATCH 082/418] Fix mypy error --- .../guardrails/guardrail_hooks/presidio.py | 142 ++++++++++-------- 1 file changed, 77 insertions(+), 65 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 4ce0f3ef5e8..b84c74bee4c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -1122,87 +1122,73 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): ) return response - async def async_post_call_streaming_iterator_hook( + async def _stream_apply_output_masking( self, - user_api_key_dict: UserAPIKeyAuth, response: Any, request_data: dict, ) -> AsyncGenerator[Union[ModelResponseStream, bytes], None]: - """ - Process streaming response chunks to unmask PII tokens when needed. - """ + """Apply Presidio masking to streaming output (apply_to_output=True path).""" from litellm.llms.base_llm.base_model_iterator import ( convert_model_response_to_streaming, ) from litellm.main import stream_chunk_builder from litellm.types.utils import ModelResponse - # --- Output masking path (apply_to_output=True) --- - if self.apply_to_output: - all_chunks: List[ModelResponseStream] = [] - try: - async for chunk in response: - if isinstance(chunk, ModelResponseStream): - all_chunks.append(chunk) - elif isinstance(chunk, bytes): - # Anthropic native SSE: pass through as-is - yield chunk # type: ignore[misc] - continue + all_chunks: List[ModelResponseStream] = [] + try: + async for chunk in response: + if isinstance(chunk, ModelResponseStream): + all_chunks.append(chunk) + elif isinstance(chunk, bytes): + yield chunk # type: ignore[misc] + continue - if not all_chunks: - # All chunks were Anthropic native SSE bytes — output - # masking cannot be applied to raw bytes. Log a warning - # so operators know PII masking was skipped for this stream. - verbose_proxy_logger.warning( - "Presidio apply_to_output: streaming response contained only " - "bytes chunks (Anthropic native SSE). Output PII masking was " - "skipped for this response." - ) - return - - assembled_model_response = stream_chunk_builder( - chunks=all_chunks, messages=request_data.get("messages") + if not all_chunks: + verbose_proxy_logger.warning( + "Presidio apply_to_output: streaming response contained only " + "bytes chunks (Anthropic native SSE). Output PII masking was " + "skipped for this response." ) - - if not isinstance(assembled_model_response, ModelResponse): - for chunk in all_chunks: - yield chunk - return - - # Apply Presidio masking on the assembled response - await self._process_response_for_pii( - response=assembled_model_response, - request_data=request_data, - mode="mask", - ) - - mock_response_stream = convert_model_response_to_streaming( - assembled_model_response - ) - yield mock_response_stream return - except Exception as e: - verbose_proxy_logger.error( - f"Error masking streaming PII output: {str(e)}" - ) - # Cannot re-iterate `response` — it's already consumed. - # If we collected chunks before the error, replay those. + assembled_model_response = stream_chunk_builder( + chunks=all_chunks, messages=request_data.get("messages") + ) + + if not isinstance(assembled_model_response, ModelResponse): for chunk in all_chunks: yield chunk return - # --- PII unmasking path (output_parse_pii=True) --- - metadata = (request_data.get("metadata") or {}) if request_data else {} - pii_tokens = metadata.get("pii_tokens", {}) - if not pii_tokens and request_data: - verbose_proxy_logger.debug( - "No pii_tokens in request_data['metadata'] for streaming unmask path" + await self._process_response_for_pii( + response=assembled_model_response, + request_data=request_data, + mode="mask", ) - if not (self.output_parse_pii and pii_tokens): - async for chunk in response: + + mock_response_stream = convert_model_response_to_streaming( + assembled_model_response + ) + yield mock_response_stream + + except Exception as e: + verbose_proxy_logger.error( + f"Error masking streaming PII output: {str(e)}" + ) + for chunk in all_chunks: yield chunk - return + + async def _stream_pii_unmasking( + self, + response: Any, + request_data: dict, + ) -> AsyncGenerator[Union[ModelResponseStream, bytes], None]: + """Apply PII unmasking to streaming output (output_parse_pii=True path).""" + from litellm.llms.base_llm.base_model_iterator import ( + convert_model_response_to_streaming, + ) + from litellm.main import stream_chunk_builder + from litellm.types.utils import ModelResponse remaining_chunks: List[ModelResponseStream] = [] try: @@ -1210,7 +1196,6 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if isinstance(chunk, ModelResponseStream): remaining_chunks.append(chunk) elif isinstance(chunk, bytes): - # Anthropic native SSE: pass through as-is yield chunk # type: ignore[misc] continue @@ -1226,13 +1211,10 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): yield chunk return - # --- PRESERVE USAGE METADATA --- - # stream_chunk_builder might miss usage if it's only in the last chunk self._preserve_usage_from_last_chunk( assembled_model_response, remaining_chunks ) - # Apply PII unmasking to assembled content (unmasking tokens back to original text) await self._process_response_for_pii( response=assembled_model_response, request_data=request_data, @@ -1249,6 +1231,36 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): for chunk in remaining_chunks: yield chunk + async def async_post_call_streaming_iterator_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + response: Any, + request_data: dict, + ) -> AsyncGenerator[Union[ModelResponseStream, bytes], None]: + """ + Process streaming response chunks to unmask PII tokens when needed. + """ + if self.apply_to_output: + async for chunk in self._stream_apply_output_masking( + response, request_data + ): + yield chunk + return + + metadata = (request_data.get("metadata") or {}) if request_data else {} + pii_tokens = metadata.get("pii_tokens", {}) + if not pii_tokens and request_data: + verbose_proxy_logger.debug( + "No pii_tokens in request_data['metadata'] for streaming unmask path" + ) + if not (self.output_parse_pii and pii_tokens): + async for chunk in response: + yield chunk + return + + async for chunk in self._stream_pii_unmasking(response, request_data): + yield chunk + @staticmethod def _preserve_usage_from_last_chunk( assembled_model_response: Any, From 7c70015a5fc4c3064cb67970835f1b58f23e1667 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 12 Mar 2026 18:33:58 +0530 Subject: [PATCH 083/418] Fix mcp error --- litellm/responses/main.py | 58 +++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 789d3b20af3..3f2065fe346 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -24,6 +24,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i LiteLLMResponsesTransformationHandler, ) from litellm.constants import request_timeout +from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.prompt_templates.common_utils import ( update_responses_input_with_model_file_ids, @@ -652,34 +653,37 @@ def responses( # Native MCP Responses API ######################################################### if LiteLLM_Proxy_MCP_Handler._should_use_litellm_mcp_gateway(tools=tools): - return aresponses_api_with_mcp( - input=input, - model=model, - include=include, - instructions=instructions, - max_output_tokens=max_output_tokens, - prompt=prompt, - metadata=metadata, - parallel_tool_calls=parallel_tool_calls, - previous_response_id=previous_response_id, - reasoning=reasoning, - store=store, - background=background, - stream=stream, - temperature=temperature, - text=text, - tool_choice=tool_choice, - tools=tools, - top_p=top_p, - truncation=truncation, - user=user, - extra_headers=extra_headers, - extra_query=extra_query, - extra_body=extra_body, - timeout=timeout, - custom_llm_provider=custom_llm_provider, + mcp_call_kwargs = { + "input": input, + "model": model, + "include": include, + "instructions": instructions, + "max_output_tokens": max_output_tokens, + "prompt": prompt, + "metadata": metadata, + "parallel_tool_calls": parallel_tool_calls, + "previous_response_id": previous_response_id, + "reasoning": reasoning, + "store": store, + "background": background, + "stream": stream, + "temperature": temperature, + "text": text, + "tool_choice": tool_choice, + "tools": tools, + "top_p": top_p, + "truncation": truncation, + "user": user, + "extra_headers": extra_headers, + "extra_query": extra_query, + "extra_body": extra_body, + "timeout": timeout, + "custom_llm_provider": custom_llm_provider, **kwargs, - ) + } + if _is_async: + return aresponses_api_with_mcp(**mcp_call_kwargs) + return run_async_function(aresponses_api_with_mcp, **mcp_call_kwargs) # get provider config responses_api_provider_config: Optional[ From 374c35a6b795de3bd99619641900ac5cccf0ab43 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 12 Mar 2026 18:34:15 +0530 Subject: [PATCH 084/418] Fix update deprecated model test --- tests/llm_translation/test_gemini.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index c9ee3625395..796b35b436e 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -271,7 +271,7 @@ def test_gemini_context_caching_separate_messages(): def test_gemini_image_generation(): # litellm._turn_on_debug() response = completion( - model="gemini/gemini-2.0-flash-exp-image-generation", + model="gemini/gemini-2.5-flash-image-preview", messages=[{"role": "user", "content": "Generate an image of a cat"}], modalities=["image", "text"], ) From 15d873e2049795ff2075965600d3c34dba0b2a35 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 12 Mar 2026 18:34:20 +0530 Subject: [PATCH 085/418] Fix update deprecated model test --- tests/local_testing/test_exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py index 4cc2723ace8..2c950d79067 100644 --- a/tests/local_testing/test_exceptions.py +++ b/tests/local_testing/test_exceptions.py @@ -927,7 +927,7 @@ def test_anthropic_tool_calling_exception(): ] try: litellm.completion( - model="claude-3-5-sonnet-20240620", + model="claude-haiku-4-5-20251001", messages=[{"role": "user", "content": "Hey, how's it going?"}], tools=tools, ) From 982f3917c527d99854981e88507ad4b864a376ea Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 12 Mar 2026 18:35:01 +0530 Subject: [PATCH 086/418] Fix test_standard_logging_payload --- .../test_custom_callback_input.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/local_testing/test_custom_callback_input.py b/tests/local_testing/test_custom_callback_input.py index fcdfcfe6e70..a28151d47a8 100644 --- a/tests/local_testing/test_custom_callback_input.py +++ b/tests/local_testing/test_custom_callback_input.py @@ -1085,10 +1085,15 @@ def test_standard_logging_payload(model, turn_off_message_logging): if turn_off_message_logging: print("checks redacted-by-litellm") assert "redacted-by-litellm" == slobject["messages"][0]["content"] - # response is a full ModelResponse dict (choices format) since d84e5e381acf response = slobject["response"] - assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" - assert response["choices"][0]["message"].get("audio") is None + if "choices" in response: + assert ( + response["choices"][0]["message"]["content"] + == "redacted-by-litellm" + ) + assert response["choices"][0]["message"].get("audio") is None + else: + assert response["text"] == "redacted-by-litellm" @pytest.mark.parametrize( @@ -1188,10 +1193,15 @@ def test_standard_logging_payload_audio(turn_off_message_logging, stream): if turn_off_message_logging: print("checks redacted-by-litellm") assert "redacted-by-litellm" == slobject["messages"][0]["content"] - # response is a full ModelResponse dict (choices format) since d84e5e381acf response = slobject["response"] - assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" - assert response["choices"][0]["message"].get("audio") is None + if "choices" in response: + assert ( + response["choices"][0]["message"]["content"] + == "redacted-by-litellm" + ) + assert response["choices"][0]["message"].get("audio") is None + else: + assert response["text"] == "redacted-by-litellm" @pytest.mark.skip(reason="Works locally. Flaky on ci/cd") From f6238e781eaf5ca19005c06a05bac2f2c548705e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 12 Mar 2026 18:35:17 +0530 Subject: [PATCH 087/418] Fix mypy --- litellm/llms/openai/chat/gpt_5_transformation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index beb76f3d80a..f7d7c437cbe 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -194,7 +194,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if has_tools and reasoning_effort not in (None, "none"): non_default_params.pop("reasoning_effort", None) optional_params.pop("reasoning_effort", None) - reasoning_effort = None + reasoning_effort = None # noqa: F841 # gpt-5.1/5.2 support logprobs, top_p, top_logprobs only when reasoning_effort="none" supports_none = self._supports_reasoning_effort_level(model, "none") From f5be79419c7c5e4869d4fb76770c50c782551d6f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 12 Mar 2026 18:36:04 +0530 Subject: [PATCH 088/418] Fix test_claude_agent_sdk_streaming --- .../bedrock/chat/converse_transformation.py | 3 ++ .../chat/test_converse_transformation.py | 34 +++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index d210f294c64..7dd32b99bc0 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1199,6 +1199,9 @@ class AmazonConverseConfig(BaseConfig): + supported_config_params ) inference_params.pop("json_mode", None) # used for handling json_schema + # Anthropic-only key. Bedrock expects `outputConfig` (camelCase) and + # will reject `output_config` if it leaks through pass-through routes. + inference_params.pop("output_config", None) # Extract requestMetadata before processing other parameters request_metadata = inference_params.pop("requestMetadata", None) diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 345f3ae7c5d..317faa5457a 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -2616,11 +2616,11 @@ def test_empty_assistant_message_handling(): empty or whitespace-only content with a placeholder to prevent AWS Bedrock Converse API 400 Bad Request errors. """ + # Import the litellm module that factory.py uses to ensure we patch the correct reference + import litellm.litellm_core_utils.prompt_templates.factory as factory_module from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_converse_messages_pt, ) - # Import the litellm module that factory.py uses to ensure we patch the correct reference - import litellm.litellm_core_utils.prompt_templates.factory as factory_module # Test case 1: Empty string content - test with modify_params=True to prevent merging messages = [ @@ -3135,7 +3135,12 @@ def test_native_structured_output_no_fake_stream(): def test_transform_request_with_output_config(): """Test that outputConfig flows through _transform_request_helper into the final request.""" - from litellm.types.llms.bedrock import OutputConfigBlock, OutputFormat, OutputFormatStructure, JsonSchemaDefinition + from litellm.types.llms.bedrock import ( + JsonSchemaDefinition, + OutputConfigBlock, + OutputFormat, + OutputFormatStructure, + ) config = AmazonConverseConfig() @@ -3170,6 +3175,29 @@ def test_transform_request_with_output_config(): assert result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] == "TestSchema" +def test_transform_request_strips_anthropic_output_config(): + """ + output_config is Anthropic-specific and must never be forwarded to Bedrock. + """ + config = AmazonConverseConfig() + messages = [{"role": "user", "content": "hello"}] + + result = config._transform_request( + model="us.amazon.nova-pro-v1:0", + messages=messages, + optional_params={ + "maxTokens": 64, + "output_config": {"effort": "low"}, + }, + litellm_params={}, + headers={}, + ) + + assert "outputConfig" not in result + additional_fields = result.get("additionalModelRequestFields", {}) + assert "output_config" not in additional_fields + + def test_transform_response_native_structured_output(): """Test response handling when model returns JSON as text content (native structured output).""" response_json = { From f4103c51a6b314855f7bc94759ce2f6401edb68d Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 12 Mar 2026 18:40:37 +0530 Subject: [PATCH 089/418] address greptile review feedback (greploop iteration 1) - Add api-version query param to Azure realtime URLs - Remove Content-Type from Azure realtime_calls headers (httpx sets it) - Add token expiry validation in proxy_realtime_calls endpoint - Fix type annotations for upstream_resp Made-with: Cursor --- .../llms/azure/realtime/http_transformation.py | 11 ++++++++--- litellm/proxy/realtime_endpoints/endpoints.py | 16 ++++++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/litellm/llms/azure/realtime/http_transformation.py b/litellm/llms/azure/realtime/http_transformation.py index 069b924d691..ef9a2d92d48 100644 --- a/litellm/llms/azure/realtime/http_transformation.py +++ b/litellm/llms/azure/realtime/http_transformation.py @@ -24,9 +24,10 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): or "" ) - def get_complete_url(self, api_base: Optional[str], model: str) -> str: + def get_complete_url(self, api_base: Optional[str], model: str, api_version: Optional[str] = None) -> str: base = self.get_api_base(api_base).rstrip("/") - return f"{base}/v1/realtime/client_secrets" + version = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" + return f"{base}/openai/realtime/client_secrets?api-version={version}" def validate_environment( self, @@ -40,8 +41,12 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): "Content-Type": "application/json", } + def get_realtime_calls_url(self, api_base: Optional[str], model: str, api_version: Optional[str] = None) -> str: + base = self.get_api_base(api_base).rstrip("/") + version = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" + return f"{base}/openai/realtime/calls?api-version={version}" + def get_realtime_calls_headers(self, ephemeral_key: str) -> dict: return { "api-key": ephemeral_key, - "Content-Type": "application/sdp", } diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index 70fb897c14c..75587f08289 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -1,8 +1,10 @@ #### Realtime WebRTC Endpoints ##### import json +import time from typing import Any, Dict, Optional +import httpx from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi import status as http_status @@ -148,7 +150,7 @@ async def create_realtime_client_secret( llm_router=llm_router, user_model=user_model, ) - upstream_resp = await llm_call + upstream_resp: httpx.Response = await llm_call # type: ignore except Exception as e: await proxy_logging_obj.post_call_failure_hook( @@ -264,6 +266,16 @@ async def proxy_realtime_calls( sdp_body: bytes = await request.body() decoded_payload = _decode_realtime_token_payload(decrypted_token_value) if decoded_payload is not None: + # Check token expiry + expires_at = decoded_payload.get("expires_at") + if expires_at is not None and isinstance(expires_at, int): + if time.time() > expires_at: + return Response( + content=json.dumps({"error": "Token has expired"}), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) + openai_ephemeral_key = decoded_payload.get("ephemeral_key", "") model = ( decoded_payload.get("model_id") @@ -319,7 +331,7 @@ async def proxy_realtime_calls( llm_router=llm_router, user_model=user_model, ) - upstream_resp = await llm_call + upstream_resp: httpx.Response = await llm_call # type: ignore except Exception as e: await proxy_logging_obj.post_call_failure_hook( From bb451cfcb061c58e85180255dcdb7719b2d2e2ec Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 12 Mar 2026 18:53:22 +0530 Subject: [PATCH 090/418] address greptile review feedback (greploop iteration 2) - Thread api_version through HTTP handlers to Azure realtime endpoints - Make expires_at optional in RealtimeClientSecretResponse - Fix test token expiry times to be in the future - Populate user_id and team_id in minimal_auth for spend tracking Made-with: Cursor --- .../base_llm/realtime/http_transformation.py | 4 ++-- litellm/llms/custom_httpx/llm_http_handler.py | 6 ++++-- .../openai/realtime/http_transformation.py | 4 ++-- litellm/proxy/realtime_endpoints/endpoints.py | 13 ++++++++++--- litellm/realtime_api/main.py | 9 ++++++--- litellm/types/realtime.py | 2 +- .../test_realtime_webrtc_endpoints.py | 19 ++++++++++++------- 7 files changed, 37 insertions(+), 20 deletions(-) diff --git a/litellm/llms/base_llm/realtime/http_transformation.py b/litellm/llms/base_llm/realtime/http_transformation.py index ccac7b0c688..7aadd49ffd3 100644 --- a/litellm/llms/base_llm/realtime/http_transformation.py +++ b/litellm/llms/base_llm/realtime/http_transformation.py @@ -54,7 +54,7 @@ class BaseRealtimeHTTPConfig(ABC): # ------------------------------------------------------------------ # @abstractmethod - def get_complete_url(self, api_base: Optional[str], model: str) -> str: + def get_complete_url(self, api_base: Optional[str], model: str, api_version: Optional[str] = None) -> str: """Return the full URL for POST /realtime/client_secrets.""" @abstractmethod @@ -76,7 +76,7 @@ class BaseRealtimeHTTPConfig(ABC): # ------------------------------------------------------------------ # def get_realtime_calls_url( - self, api_base: Optional[str], model: str + self, api_base: Optional[str], model: str, api_version: Optional[str] = None ) -> str: """Return the full URL for POST /realtime/calls (SDP exchange).""" base = (api_base or "").rstrip("/") diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 8f49e79a72c..2c0a9a4f6f3 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -4746,6 +4746,7 @@ class BaseLLMHTTPHandler: model: Optional[str] = None, extra_headers: Optional[Dict[str, Any]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + api_version: Optional[str] = None, ) -> httpx.Response: """ Forward POST /v1/realtime/client_secrets to upstream provider. @@ -4761,7 +4762,7 @@ class BaseLLMHTTPHandler: async_httpx_client = client if provider_config is not None: - url = provider_config.get_complete_url(api_base=api_base, model=model or "") + url = provider_config.get_complete_url(api_base=api_base, model=model or "", api_version=api_version) headers: Dict[str, Any] = provider_config.validate_environment( headers={}, model=model or "", api_key=api_key ) @@ -4811,6 +4812,7 @@ class BaseLLMHTTPHandler: session_config: Optional[Dict[str, Any]] = None, extra_headers: Optional[Dict[str, Any]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + api_version: Optional[str] = None, ) -> httpx.Response: """ Forward POST /v1/realtime/calls (SDP exchange) to upstream provider. @@ -4830,7 +4832,7 @@ class BaseLLMHTTPHandler: async_httpx_client = client if provider_config is not None: - url = provider_config.get_realtime_calls_url(api_base=api_base, model=model or "") + url = provider_config.get_realtime_calls_url(api_base=api_base, model=model or "", api_version=api_version) headers: Dict[str, Any] = provider_config.get_realtime_calls_headers( ephemeral_key=openai_ephemeral_key ) diff --git a/litellm/llms/openai/realtime/http_transformation.py b/litellm/llms/openai/realtime/http_transformation.py index 33d1cdf322b..ff69ef987db 100644 --- a/litellm/llms/openai/realtime/http_transformation.py +++ b/litellm/llms/openai/realtime/http_transformation.py @@ -25,13 +25,13 @@ class OpenAIRealtimeHTTPConfig(BaseRealtimeHTTPConfig): or "" ) - def get_complete_url(self, api_base: Optional[str], model: str) -> str: + def get_complete_url(self, api_base: Optional[str], model: str, api_version: Optional[str] = None) -> str: base = self.get_api_base(api_base).rstrip("/") if base.endswith("/v1"): base = base[:-3] return f"{base}/v1/realtime/client_secrets" - def get_realtime_calls_url(self, api_base: Optional[str], model: str) -> str: + def get_realtime_calls_url(self, api_base: Optional[str], model: str, api_version: Optional[str] = None) -> str: base = self.get_api_base(api_base).rstrip("/") if base.endswith("/v1"): base = base[:-3] diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index 75587f08289..bb286d1fd0d 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -282,14 +282,21 @@ async def proxy_realtime_calls( or request.query_params.get("model") or "gpt-4o-realtime-preview" ) + user_id = decoded_payload.get("user_id") or None + team_id = decoded_payload.get("team_id") or None else: # Backward compatibility: older tokens contained only encrypted upstream key. openai_ephemeral_key = decrypted_token_value model = request.query_params.get("model", "gpt-4o-realtime-preview") + user_id = None + team_id = None - # Build a minimal UserAPIKeyAuth so we can pass through the logging pipeline - # even though this endpoint uses the provider ephemeral key for auth. - minimal_auth = UserAPIKeyAuth() + # Build a minimal UserAPIKeyAuth with user/team IDs from the token + # so spend tracking and budget enforcement work correctly. + minimal_auth = UserAPIKeyAuth( + user_id=user_id, + team_id=team_id, + ) data: dict = {} try: diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 01b76ad805c..81f29ca6e30 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -4,8 +4,7 @@ import os from typing import Any, Dict, Optional, cast import litellm -from litellm.constants import request_timeout -from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES +from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, request_timeout from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler @@ -56,7 +55,9 @@ def _get_realtime_http_provider_config( Uses ProviderConfigManager so each provider keeps its credential-resolution and URL-construction logic in its own transformation class. """ - from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig + from litellm.llms.base_llm.realtime.http_transformation import ( + BaseRealtimeHTTPConfig, + ) provider_config: Optional[BaseRealtimeHTTPConfig] = None if custom_llm_provider in LlmProviders._member_map_.values(): @@ -138,6 +139,7 @@ async def acreate_realtime_client_secret( model=model_name, extra_headers=kwargs.get("extra_headers"), client=kwargs.get("client"), + api_version=litellm_params.api_version, ) @@ -182,6 +184,7 @@ async def arealtime_calls( session_config=session, extra_headers=kwargs.get("extra_headers"), client=kwargs.get("client"), + api_version=litellm_params.api_version, ) diff --git a/litellm/types/realtime.py b/litellm/types/realtime.py index d341a32654d..62e4044061b 100644 --- a/litellm/types/realtime.py +++ b/litellm/types/realtime.py @@ -112,6 +112,6 @@ class RealtimeClientSecretResponse(BaseModel): The `session` field is kept as a raw dict so unknown fields pass through. """ - expires_at: int + expires_at: Optional[int] = None value: str session: Optional[Dict[str, Any]] = None diff --git a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index 1ab876e7ff7..3d82e4177a5 100644 --- a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -7,6 +7,7 @@ Tests for LiteLLM proxy realtime WebRTC HTTP endpoints: import json import os import sys +import time from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -59,19 +60,20 @@ def test_encode_realtime_token_payload_none_optional_fields(): def test_decode_realtime_token_payload_valid(): + future_expires_at = int(time.time()) + 3600 payload = _encode_realtime_token_payload( ephemeral_key="epk_abc", model_id="gpt-4o", user_id=None, team_id=None, - expires_at=999, + expires_at=future_expires_at, ) decrypted = json.loads(payload) # simulate decrypted value result = _decode_realtime_token_payload(json.dumps(decrypted)) assert result is not None assert result["ephemeral_key"] == "epk_abc" assert result["model_id"] == "gpt-4o" - assert result["expires_at"] == 999 + assert result["expires_at"] == future_expires_at def test_decode_realtime_token_payload_invalid_version(): @@ -115,14 +117,15 @@ def proxy_app(): @pytest.fixture def mock_route_request_client_secrets(): """Mock route_request to return a fake upstream client_secrets response.""" + future_expires_at = int(time.time()) + 3600 mock_resp = MagicMock(spec=httpx.Response) mock_resp.status_code = 200 - mock_resp.text = '{"value":"upstream_ephemeral_key","expires_at":999}' - mock_resp.content = b'{"value":"upstream_ephemeral_key","expires_at":999}' + mock_resp.text = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' + mock_resp.content = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}'.encode() mock_resp.headers = {} mock_resp.json.return_value = { "value": "upstream_ephemeral_key", - "expires_at": 999, + "expires_at": future_expires_at, } async def _mock_route(*args, **kwargs): @@ -215,7 +218,8 @@ async def test_client_secrets_success_with_mock( assert response.status_code == 200 data = response.json() assert "value" in data - assert data["expires_at"] == 999 + assert data["expires_at"] is not None + assert data["expires_at"] > int(time.time()) # Should be in the future # Proxy encrypts the upstream value, so returned value should differ assert data["value"] != "upstream_ephemeral_key" @@ -259,12 +263,13 @@ async def test_realtime_calls_success_with_valid_encrypted_token( proxy_server.master_key = "sk-test-master-key" # Build a valid encrypted token (same format as client_secrets returns) + future_expires_at = int(time.time()) + 3600 token_payload = _encode_realtime_token_payload( ephemeral_key="fake_upstream_epk", model_id="gpt-4o-realtime-preview", user_id=None, team_id=None, - expires_at=999, + expires_at=future_expires_at, ) encrypted_token = encrypt_value_helper(token_payload) From fa68d69bcfedb04e0d54e32f498069d0c4c32c32 Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 12 Mar 2026 10:28:27 -0300 Subject: [PATCH 091/418] fix: restore _get_effort_level and is_model_gpt_5_4_plus_model (PR #23151) Independent fix (base: main) collaterally removed by PR #23276. Restores: - _get_effort_level() for extracting effort from string or dict - is_model_gpt_5_4_plus_model() classmethod - effective_effort usage in xhigh/tool-drop/sampling/temperature guards - Azure: _get_effort_level import and usage for dict reasoning_effort - Azure: gpt-5.4+ tool+reasoning drop logic --- .../llms/azure/chat/gpt_5_transformation.py | 24 ++++-- .../llms/openai/chat/gpt_5_transformation.py | 77 +++++++++++++++---- 2 files changed, 79 insertions(+), 22 deletions(-) diff --git a/litellm/llms/azure/chat/gpt_5_transformation.py b/litellm/llms/azure/chat/gpt_5_transformation.py index a8c5a14ea58..81c3dfded71 100644 --- a/litellm/llms/azure/chat/gpt_5_transformation.py +++ b/litellm/llms/azure/chat/gpt_5_transformation.py @@ -4,7 +4,10 @@ from typing import List import litellm from litellm.exceptions import UnsupportedParamsError -from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config +from litellm.llms.openai.chat.gpt_5_transformation import ( + OpenAIGPT5Config, + _get_effort_level, +) from litellm.types.llms.openai import AllMessageValues from .gpt_transformation import AzureOpenAIConfig @@ -85,20 +88,21 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") ) + effective_effort = _get_effort_level(reasoning_effort_value) # gpt-5.1/5.2/5.4 support reasoning_effort='none', but other gpt-5 models don't # See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/reasoning supports_none = self._supports_reasoning_effort_level(model, "none") - if reasoning_effort_value == "none" and not supports_none: + if effective_effort == "none" and not supports_none: if litellm.drop_params is True or ( drop_params is not None and drop_params is True ): non_default_params = non_default_params.copy() optional_params = optional_params.copy() - if non_default_params.get("reasoning_effort") == "none": + if _get_effort_level(non_default_params.get("reasoning_effort")) == "none": non_default_params.pop("reasoning_effort") - if optional_params.get("reasoning_effort") == "none": + if _get_effort_level(optional_params.get("reasoning_effort")) == "none": optional_params.pop("reasoning_effort") else: raise UnsupportedParamsError( @@ -121,9 +125,19 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): ) # Only drop reasoning_effort='none' for models that don't support it - if result.get("reasoning_effort") == "none" and not supports_none: + result_effort = _get_effort_level(result.get("reasoning_effort")) + if result_effort == "none" and not supports_none: result.pop("reasoning_effort") + # Azure Chat Completions: gpt-5.4+ does not support tools + reasoning together. + # Drop reasoning_effort when both are present (OpenAI routes to Responses API; Azure does not). + if self.is_model_gpt_5_4_plus_model(model): + has_tools = bool( + non_default_params.get("tools") or optional_params.get("tools") + ) + if has_tools and result_effort not in (None, "none"): + result.pop("reasoning_effort", None) + return result def transform_request( diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index beb76f3d80a..f186bc60859 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -25,6 +25,22 @@ def _normalize_reasoning_effort_for_chat_completion( return None +def _get_effort_level(value: Union[str, dict, None]) -> Optional[str]: + """Extract the effective effort level from reasoning_effort (string or dict). + + Use this for guards that compare effort level (e.g. xhigh validation, "none" checks). + Ensures dict inputs like {"effort": "none", "summary": "detailed"} are correctly + treated as effort="none" for validation purposes. + """ + if value is None: + return None + if isinstance(value, str): + return value + if isinstance(value, dict) and "effort" in value: + return value["effort"] + return None + + class OpenAIGPT5Config(OpenAIGPTConfig): """Configuration for gpt-5 models including GPT-5-Codex variants. @@ -70,6 +86,19 @@ class OpenAIGPT5Config(OpenAIGPTConfig): model_name = model.split("/")[-1] return model_name.startswith("gpt-5.4") + @classmethod + def is_model_gpt_5_4_plus_model(cls, model: str) -> bool: + """Check if the model is gpt-5.4 or newer (5.4, 5.5, 5.6, etc., including pro).""" + model_name = model.split("/")[-1] + if not model_name.startswith("gpt-5."): + return False + try: + version_str = model_name.replace("gpt-5.", "").split("-")[0] + major = version_str.split(".")[0] + return int(major) >= 4 + except (ValueError, IndexError): + return False + @classmethod def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool: """Check if the model supports a specific reasoning_effort level. @@ -150,21 +179,32 @@ class OpenAIGPT5Config(OpenAIGPTConfig): drop_params=drop_params, ) - # Normalize reasoning_effort: chat completion API expects a string, not a dict - # (e.g. {'effort': 'high', 'summary': 'detailed'} -> 'high') + # Get raw reasoning_effort and effective effort level for all guards. + # Use effective_effort (extracted string) for xhigh validation, "none" checks, and + # tool/sampling guards — dict inputs like {"effort": "none", "summary": "detailed"} + # must be treated as effort="none" to avoid incorrect tool-drop or sampling errors. raw_reasoning_effort = ( non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") ) - normalized = _normalize_reasoning_effort_for_chat_completion(raw_reasoning_effort) - if raw_reasoning_effort is not None and normalized is not None: - if "reasoning_effort" in non_default_params: - non_default_params["reasoning_effort"] = normalized - if "reasoning_effort" in optional_params: - optional_params["reasoning_effort"] = normalized + effective_effort = _get_effort_level(raw_reasoning_effort) - reasoning_effort = normalized or raw_reasoning_effort - if reasoning_effort is not None and reasoning_effort == "xhigh": + # Normalize to string for Chat Completions API when dict has only "effort". + # Preserve full dict (e.g. {"effort": "high", "summary": "detailed"}) for Responses API. + if isinstance(raw_reasoning_effort, dict) and set(raw_reasoning_effort.keys()) <= {"effort"}: + normalized = _normalize_reasoning_effort_for_chat_completion(raw_reasoning_effort) + if normalized is not None: + if "reasoning_effort" in non_default_params: + non_default_params["reasoning_effort"] = normalized + if "reasoning_effort" in optional_params: + optional_params["reasoning_effort"] = normalized + + reasoning_effort = ( + non_default_params.get("reasoning_effort") + or optional_params.get("reasoning_effort") + or raw_reasoning_effort + ) + if effective_effort is not None and effective_effort == "xhigh": if not self._supports_reasoning_effort_level(model, "xhigh"): if litellm.drop_params or drop_params: non_default_params.pop("reasoning_effort", None) @@ -191,17 +231,20 @@ class OpenAIGPT5Config(OpenAIGPTConfig): has_tools = bool( non_default_params.get("tools") or optional_params.get("tools") ) - if has_tools and reasoning_effort not in (None, "none"): - non_default_params.pop("reasoning_effort", None) - optional_params.pop("reasoning_effort", None) - reasoning_effort = None + if has_tools and effective_effort not in (None, "none"): + # Check if this will be routed to Responses API + # If so, keep reasoning_effort; otherwise drop it for chat completions API + if not self.is_model_gpt_5_4_plus_model(model): + non_default_params.pop("reasoning_effort", None) + optional_params.pop("reasoning_effort", None) + reasoning_effort = None # gpt-5.1/5.2 support logprobs, top_p, top_logprobs only when reasoning_effort="none" supports_none = self._supports_reasoning_effort_level(model, "none") if supports_none: sampling_params = ["logprobs", "top_logprobs", "top_p"] has_sampling = any(p in non_default_params for p in sampling_params) - if has_sampling and reasoning_effort not in (None, "none"): + if has_sampling and effective_effort not in (None, "none"): if litellm.drop_params or drop_params: for p in sampling_params: non_default_params.pop(p, None) @@ -211,7 +254,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): "gpt-5.1/5.2/5.4 only support logprobs, top_p, top_logprobs when " "reasoning_effort='none'. Current reasoning_effort='{}'. " "To drop unsupported params set `litellm.drop_params = True`" - ).format(reasoning_effort), + ).format(effective_effort), status_code=400, ) @@ -219,7 +262,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): temperature_value: Optional[float] = non_default_params.pop("temperature") if temperature_value is not None: # models supporting reasoning_effort="none" also support flexible temperature - if supports_none and (reasoning_effort == "none" or reasoning_effort is None): + if supports_none and (effective_effort == "none" or effective_effort is None): optional_params["temperature"] = temperature_value elif temperature_value == 1: optional_params["temperature"] = temperature_value From b7cfcdd35d49597e126a4bdcfb2123e585eea482 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 12 Mar 2026 19:06:57 +0530 Subject: [PATCH 092/418] Add docs --- .../realtime_webrtc_http_endpoints/index.md | 236 ++++++++ docs/my-website/docs/proxy/realtime_webrtc.md | 163 +++++ .../src/components/WebRTCTester.jsx | 571 ++++++++++++++++++ 3 files changed, 970 insertions(+) create mode 100644 docs/my-website/blog/realtime_webrtc_http_endpoints/index.md create mode 100644 docs/my-website/docs/proxy/realtime_webrtc.md create mode 100644 ui/litellm-dashboard/src/components/WebRTCTester.jsx diff --git a/docs/my-website/blog/realtime_webrtc_http_endpoints/index.md b/docs/my-website/blog/realtime_webrtc_http_endpoints/index.md new file mode 100644 index 00000000000..8907f3cd404 --- /dev/null +++ b/docs/my-website/blog/realtime_webrtc_http_endpoints/index.md @@ -0,0 +1,236 @@ +--- +slug: realtime_webrtc_http_endpoints +title: "Realtime WebRTC HTTP Endpoints on LiteLLM Proxy" +date: 2026-03-12T10:00:00 +authors: + - name: Sameer Kankute + title: SWE @ LiteLLM (LLM Translation) + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg + - name: Krrish Dholakia + title: "CEO, LiteLLM" + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +description: "Use the LiteLLM proxy to route OpenAI-style WebRTC realtime via HTTP: client_secrets and SDP exchange." +tags: [realtime, webrtc, proxy, openai] +hide_table_of_contents: false +--- +--- +id: webrtc +title: "/realtime - WebRTC Support" +sidebar_label: "/realtime WebRTC" +--- + +import WebRTCTester from '@site/src/components/WebRTCTester'; + +Use this to connect to the Realtime API via WebRTC from browser/mobile clients, with LiteLLM handling authentication and key management. + +**Supported Providers:** +- OpenAI +- Azure OpenAI + +:::info When to use WebRTC vs WebSocket? +- Use **WebSocket** (`/v1/realtime`) for server-to-server connections +- Use **WebRTC** (`/v1/realtime/client_secrets` + `/v1/realtime/calls`) for browser/mobile clients where lower latency matters +::: + +## How it works + +WebRTC keeps your provider API keys secure while allowing the browser to stream audio directly to OpenAI/Azure — without routing audio through LiteLLM. + +``` +Browser LiteLLM Proxy OpenAI/Azure + | | | + |-- POST /v1/realtime/ | | + | client_secrets -------->| | + | [LiteLLM API key] |-- POST /v1/realtime/ | + | | sessions [Real key] -->| + | |<-- { ek_... } -----------| + | | encrypt(ek_...) | + |<-- { encrypted_token } ---| | + | | | + |-- POST /v1/realtime/calls | | + | [SDP + encrypted_token]>| | + | | decrypt → ek_... | + | |-- POST /v1/realtime/ | + | | calls [SDP + ek_...] ->| + | |<-- SDP answer -----------| + |<-- SDP answer ------------| | + | | | + |===== audio P2P direct to OpenAI/Azure =============>| +``` + +LiteLLM **never touches the audio stream** — it only handles token issuance and the SDP exchange. All audio flows directly browser ↔ provider. + +--- + +## Proxy Setup + +### Add model to config + +```yaml +model_list: + - model_name: gpt-4o-realtime + litellm_params: + model: openai/gpt-4o-realtime-preview-2024-12-17 + api_key: os.environ/OPENAI_API_KEY + model_info: + mode: realtime +``` + +For Azure: + +```yaml +model_list: + - model_name: gpt-4o-realtime + litellm_params: + model: azure/gpt-4o-realtime-preview + api_key: os.environ/AZURE_API_KEY + api_base: os.environ/AZURE_API_BASE + model_info: + mode: realtime +``` + +### Start the proxy + +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +--- + +## Client Usage + +### Step 1 — Get an encrypted session token + +Call `POST /v1/realtime/client_secrets` from your browser using your LiteLLM API key. LiteLLM fetches a real ephemeral key from OpenAI, encrypts it, and returns the encrypted token — so your provider key never reaches the browser. + +```javascript +const tokenResponse = await fetch("http://your-litellm-proxy:4000/v1/realtime/client_secrets", { + method: "POST", + headers: { + "Authorization": "Bearer sk-litellm-your-key", + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: "gpt-4o-realtime", // model_name from your config + }), +}); + +const { client_secret } = await tokenResponse.json(); +const ENCRYPTED_TOKEN = client_secret.value; // encrypted by LiteLLM, not the real ek_... +``` + +### Step 2 — Establish WebRTC connection via LiteLLM + +Use the standard WebRTC APIs to set up the peer connection, then send your SDP offer to LiteLLM's `/v1/realtime/calls` endpoint. LiteLLM decrypts the token (which encodes the model) and forwards the SDP to OpenAI — no need to pass `model` again. + +```javascript +const pc = new RTCPeerConnection(); + +// Set up to play remote audio from the model +const audioEl = document.createElement("audio"); +audioEl.autoplay = true; +pc.ontrack = (e) => (audioEl.srcObject = e.streams[0]); + +// Add local audio track for microphone input +const ms = await navigator.mediaDevices.getUserMedia({ audio: true }); +pc.addTrack(ms.getTracks()[0]); + +// Set up data channel for sending and receiving events +const dc = pc.createDataChannel("oai-events"); + +// Create SDP offer +const offer = await pc.createOffer(); +await pc.setLocalDescription(offer); + +// Send SDP to LiteLLM — model is decoded from the token, no ?model= needed +const sdpResponse = await fetch("http://your-litellm-proxy:4000/v1/realtime/calls", { + method: "POST", + headers: { + "Authorization": `Bearer ${ENCRYPTED_TOKEN}`, + "Content-Type": "application/sdp", + }, + body: offer.sdp, +}); + +const answer = { type: "answer", sdp: await sdpResponse.text() }; +await pc.setRemoteDescription(answer); + +// Audio now flows directly browser <-> OpenAI/Azure (P2P) +``` + +### Step 3 — Send and receive events + +Use the WebRTC data channel to send and receive session events: + +```javascript +// Listen for server events +dc.addEventListener("message", (e) => { + const event = JSON.parse(e.data); + console.log(event); +}); + +// Send a client event +dc.send(JSON.stringify({ + type: "session.update", + session: { + instructions: "You are a helpful assistant.", + }, +})); +``` + +--- + +## Try it live + +Paste your LiteLLM proxy URL and API key to run a real end-to-end WebRTC session right here. + + + +--- + +## FAQ + +### Why do I get `401 Token has expired` on `/v1/realtime/calls`? + +The encrypted token returned by `/v1/realtime/client_secrets` is short-lived. +Generate a fresh token right before creating your WebRTC offer, and avoid reusing old tokens across page refreshes or long idle periods. + +### Do I send my LiteLLM key or provider key to `/v1/realtime/calls`? + +Use the **encrypted token** from `/v1/realtime/client_secrets` as: + +```http +Authorization: Bearer +``` + +Do not send your raw OpenAI/Azure key from the client. + +### Do I need to pass `model` again on `/v1/realtime/calls`? + +Usually no. The encrypted token encodes routing metadata (including model), so LiteLLM can route the SDP exchange without `?model=...`. + +### Azure call failing with `api-version` errors - what should I check? + +Make sure your Azure deployment config includes a valid `api_version` in `litellm_params` (or set `AZURE_API_VERSION`), plus correct `api_base` and deployment/model mapping. + +### Why does the SDP request need `Content-Type: application/sdp` on the client? + +Your browser sends raw SDP text to LiteLLM, so `application/sdp` is correct for the client-to-proxy request. +LiteLLM then transforms and forwards provider-specific payloads upstream. + +### The browser asks for microphone permission but I hear no audio. What can I check? + +- Confirm microphone permission is granted for your site. +- Ensure `pc.ontrack` sets an autoplay-enabled audio element. +- Verify your network allows WebRTC (no restrictive firewall or enterprise policy). +- Check browser console logs for ICE and SDP negotiation errors. + +--- diff --git a/docs/my-website/docs/proxy/realtime_webrtc.md b/docs/my-website/docs/proxy/realtime_webrtc.md new file mode 100644 index 00000000000..26770b6685b --- /dev/null +++ b/docs/my-website/docs/proxy/realtime_webrtc.md @@ -0,0 +1,163 @@ +# /realtime - WebRTC Support + +Use this to connect to the Realtime API via WebRTC from browser/mobile clients, with LiteLLM handling authentication and key management. + +Supported Providers: +- OpenAI +- Azure + +:::info +**When to use WebRTC vs WebSocket?** + +- Use **WebSocket** (`/v1/realtime`) for server-to-server connections +- Use **WebRTC** (`/v1/realtime/client_secrets` + `/v1/realtime/calls`) for browser/mobile clients where lower latency matters +::: + +## How it works + +WebRTC keeps your provider API keys secure while allowing the browser to stream audio directly to OpenAI/Azure — without routing audio through LiteLLM. + +``` +Browser LiteLLM Proxy OpenAI/Azure + | | | + |-- POST /v1/realtime/ | | + | client_secrets -------->| | + | [LiteLLM API key] |-- POST /v1/realtime/ | + | | sessions [Real key] -->| + | |<-- { ek_... } -----------| + | | encrypt(ek_...) | + |<-- { encrypted_token } ---| | + | | | + |-- POST /v1/realtime/calls | | + | [SDP + encrypted_token]>| | + | | decrypt → ek_... | + | |-- POST /v1/realtime/ | + | | calls [SDP + ek_...] ->| + | |<-- SDP answer -----------| + |<-- SDP answer ------------| | + | | | + |===== audio P2P direct to OpenAI/Azure =============>| +``` + +LiteLLM **never touches the audio stream** — it only handles token issuance and the SDP exchange. All audio flows directly browser ↔ provider. + +## Proxy Usage + +### Add model to config + +```yaml +model_list: + - model_name: gpt-4o-realtime + litellm_params: + model: openai/gpt-4o-realtime-preview-2024-12-17 + api_key: os.environ/OPENAI_API_KEY + model_info: + mode: realtime +``` + +For Azure: + +```yaml +model_list: + - model_name: gpt-4o-realtime + litellm_params: + model: azure/gpt-4o-realtime-preview + api_key: os.environ/AZURE_API_KEY + api_base: os.environ/AZURE_API_BASE + model_info: + mode: realtime +``` + +### Start proxy + +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +## Client Usage + +### Step 1 — Get an encrypted session token + +Call `POST /v1/realtime/client_secrets` from your browser using your LiteLLM API key. LiteLLM will fetch a real ephemeral key from OpenAI, encrypt it, and return the encrypted token — so the real provider key never reaches your browser. + +```javascript +const tokenResponse = await fetch("http://your-litellm-proxy:4000/v1/realtime/client_secrets", { + method: "POST", + headers: { + "Authorization": "Bearer sk-litellm-your-key", + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: "gpt-4o-realtime", // your model name from config + }), +}); + +const { client_secret } = await tokenResponse.json(); +const ENCRYPTED_TOKEN = client_secret.value; // encrypted by LiteLLM, not the real ek_... +``` + +### Step 2 — Establish WebRTC connection via LiteLLM + +Use standard WebRTC APIs to set up the peer connection, then send your SDP offer to LiteLLM's `/v1/realtime/calls` endpoint. LiteLLM decrypts the token and forwards the SDP to OpenAI using the real ephemeral key. + +```javascript +// Create a peer connection +const pc = new RTCPeerConnection(); + +// Set up to play remote audio from the model +const audioEl = document.createElement("audio"); +audioEl.autoplay = true; +pc.ontrack = (e) => (audioEl.srcObject = e.streams[0]); + +// Add local audio track for microphone input +const ms = await navigator.mediaDevices.getUserMedia({ audio: true }); +pc.addTrack(ms.getTracks()[0]); + +// Set up data channel for sending and receiving events +const dc = pc.createDataChannel("oai-events"); + +// Create SDP offer +const offer = await pc.createOffer(); +await pc.setLocalDescription(offer); + +// Send SDP to LiteLLM — it decrypts the token and forwards to OpenAI +const sdpResponse = await fetch("http://your-litellm-proxy:4000/v1/realtime/calls", { + method: "POST", + headers: { + "Authorization": `Bearer ${ENCRYPTED_TOKEN}`, + "Content-Type": "application/sdp", + }, + body: offer.sdp, +}); + +// Set the SDP answer from OpenAI (returned via LiteLLM) +const answer = { + type: "answer", + sdp: await sdpResponse.text(), +}; +await pc.setRemoteDescription(answer); + +// Audio now flows directly browser <-> OpenAI/Azure (P2P) +``` + +### Step 3 — Send and receive events + +Use the WebRTC data channel to send and receive session events: + +```javascript +// Listen for server events +dc.addEventListener("message", (e) => { + const event = JSON.parse(e.data); + console.log(event); +}); + +// Send a client event +dc.send(JSON.stringify({ + type: "session.update", + session: { + instructions: "You are a helpful assistant.", + }, +})); +``` \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/WebRTCTester.jsx b/ui/litellm-dashboard/src/components/WebRTCTester.jsx new file mode 100644 index 00000000000..e8b439014fc --- /dev/null +++ b/ui/litellm-dashboard/src/components/WebRTCTester.jsx @@ -0,0 +1,571 @@ +import { useState, useRef, useEffect, useCallback } from 'react'; + +const STYLES = ` +.wrt-wrap { + font-family: 'JetBrains Mono', 'Fira Code', monospace; + background: #0d0d14; + border: 1px solid #1e1e2e; + border-radius: 10px; + overflow: hidden; + margin: 24px 0; +} + +.wrt-toggle { + display: flex; + align-items: center; + justify-content: space-between; + padding: 14px 20px; + cursor: pointer; + user-select: none; + background: #0d0d14; + transition: background 0.15s; +} +.wrt-toggle:hover { background: #111120; } + +.wrt-toggle-left { display: flex; align-items: center; gap: 10px; } + +.wrt-live-dot { + width: 8px; height: 8px; border-radius: 50%; + background: #00ff88; + box-shadow: 0 0 8px #00ff88; + animation: wrt-blink 2s infinite; +} +@keyframes wrt-blink { 0%,100%{opacity:1} 50%{opacity:0.4} } + +.wrt-toggle-title { font-size: 12px; font-weight: 600; color: #e2e8f0; letter-spacing: 0.06em; } +.wrt-toggle-sub { font-size: 10px; color: #4a5568; margin-top: 1px; } +.wrt-chevron { font-size: 11px; color: #4a5568; transition: transform 0.2s; } +.wrt-chevron.open { transform: rotate(180deg); } + +.wrt-body { + border-top: 1px solid #1e1e2e; + display: grid; + grid-template-columns: 280px 1fr; + height: 460px; +} + +.wrt-sidebar { + border-right: 1px solid #1e1e2e; + padding: 14px; + display: flex; + flex-direction: column; + gap: 12px; + overflow-y: auto; +} + +.wrt-label { + font-size: 9px; + letter-spacing: 0.15em; + color: #4a5568; + text-transform: uppercase; + margin-bottom: 5px; +} + +.wrt-field { display: flex; flex-direction: column; gap: 4px; margin-bottom: 6px; } +.wrt-field label { font-size: 10px; color: #4a5568; } +.wrt-field input { + background: #0a0a0f; + border: 1px solid #1e1e2e; + border-radius: 5px; + color: #e2e8f0; + font-family: inherit; + font-size: 11px; + padding: 7px 9px; + outline: none; + width: 100%; + transition: border-color 0.2s; +} +.wrt-field input:focus { border-color: #7c3aed; } + +.wrt-divider { height: 1px; background: #1e1e2e; } + +.wrt-btn { + display: flex; align-items: center; justify-content: center; + border: none; border-radius: 5px; cursor: pointer; + font-family: inherit; font-size: 11px; font-weight: 600; + padding: 8px; width: 100%; + transition: all 0.15s; letter-spacing: 0.04em; +} +.wrt-btn + .wrt-btn { margin-top: 5px; } +.wrt-btn-primary { background: #00ff88; color: #000; } +.wrt-btn-primary:hover:not(:disabled) { filter: brightness(1.1); } +.wrt-btn-primary:disabled { opacity: 0.35; cursor: not-allowed; } +.wrt-btn-danger { background: transparent; color: #ff4466; border: 1px solid #ff4466; } +.wrt-btn-danger:hover:not(:disabled) { background: rgba(255,68,102,0.08); } +.wrt-btn-danger:disabled { opacity: 0.3; cursor: not-allowed; } +.wrt-btn-ghost { background: #111118; color: #e2e8f0; border: 1px solid #1e1e2e; } +.wrt-btn-ghost:hover { border-color: #7c3aed; } + +.wrt-flow { display: flex; align-items: center; padding: 4px 0; gap: 0; } +.wrt-flow-box { + padding: 4px 7px; border-radius: 4px; font-size: 9px; + border: 1px solid #1e1e2e; color: #4a5568; + transition: all 0.3s; white-space: nowrap; +} +.wrt-flow-box.active { border-color: #00ff88; color: #00ff88; box-shadow: 0 0 8px rgba(0,255,136,0.15); } +.wrt-flow-arrow { font-size: 10px; color: #4a5568; padding: 0 4px; transition: color 0.3s; } +.wrt-flow-arrow.active { color: #00ff88; } + +.wrt-meta { display: flex; flex-direction: column; gap: 4px; } +.wrt-meta-row { display: flex; justify-content: space-between; font-size: 10px; } +.wrt-meta-row span:first-child { color: #4a5568; } +.wrt-meta-row span:last-child { color: #e2e8f0; } + +.wrt-status-pill { + display: flex; align-items: center; gap: 6px; + font-size: 10px; color: #4a5568; + background: #111118; border: 1px solid #1e1e2e; + border-radius: 100px; padding: 3px 10px; +} +.wrt-status-dot { + width: 6px; height: 6px; border-radius: 50%; + background: #4a5568; transition: all 0.3s; +} +.wrt-status-dot.connected { background: #00ff88; box-shadow: 0 0 6px #00ff88; } +.wrt-status-dot.connecting { background: #ffaa00; animation: wrt-blink 1s infinite; } +.wrt-status-dot.error { background: #ff4466; } + +.wrt-main { display: flex; flex-direction: column; overflow: hidden; } + +.wrt-header { + display: flex; align-items: center; justify-content: space-between; + padding: 8px 14px; border-bottom: 1px solid #1e1e2e; background: #111118; +} +.wrt-header-title { font-size: 10px; color: #4a5568; letter-spacing: 0.08em; } + +.wrt-tabs { display: flex; padding: 0 14px; border-bottom: 1px solid #1e1e2e; } +.wrt-tab { + font-size: 9px; letter-spacing: 0.08em; padding: 10px 12px; cursor: pointer; + color: #4a5568; border-bottom: 2px solid transparent; transition: all 0.15s; + user-select: none; +} +.wrt-tab.active { color: #00ff88; border-bottom-color: #00ff88; } +.wrt-tab:hover:not(.active) { color: #e2e8f0; } + +.wrt-tab-content { flex: 1; overflow: hidden; display: none; flex-direction: column; } +.wrt-tab-content.active { display: flex; } + +.wrt-log { + flex: 1; overflow-y: auto; padding: 8px 12px; + display: flex; flex-direction: column; gap: 2px; +} +.wrt-log::-webkit-scrollbar { width: 3px; } +.wrt-log::-webkit-scrollbar-thumb { background: #1e1e2e; border-radius: 2px; } + +.wrt-entry { + display: grid; grid-template-columns: 58px 56px 1fr; gap: 8px; + padding: 3px 7px; border-radius: 3px; + border-left: 2px solid transparent; + font-size: 10px; line-height: 1.5; + animation: wrt-fadein 0.15s ease; +} +@keyframes wrt-fadein { from { opacity:0; transform:translateY(2px); } to { opacity:1; transform:none; } } + +.wrt-entry.info { border-left-color: #7c3aed; } +.wrt-entry.info .we-tag { color: #7c3aed; } +.wrt-entry.success { border-left-color: #00ff88; } +.wrt-entry.success .we-tag { color: #00ff88; } +.wrt-entry.error { border-left-color: #ff4466; } +.wrt-entry.error .we-tag { color: #ff4466; } +.wrt-entry.warn { border-left-color: #ffaa00; } +.wrt-entry.warn .we-tag { color: #ffaa00; } +.wrt-entry.step { border-left-color: #60a5fa; } +.wrt-entry.step .we-tag { color: #60a5fa; } + +.we-time { color: #4a5568; font-size: 9px; padding-top: 1px; } +.we-tag { font-size: 9px; font-weight: 700; padding-top: 1px; } +.we-msg { color: #e2e8f0; word-break: break-all; white-space: pre-wrap; } + +.wrt-empty { + display: flex; flex-direction: column; align-items: center; justify-content: center; + flex: 1; gap: 6px; color: #4a5568; font-size: 11px; +} + +.wrt-sdp-pane { flex: 1; display: grid; grid-template-columns: 1fr 1fr; overflow: hidden; } +.wrt-sdp-box { display: flex; flex-direction: column; border-right: 1px solid #1e1e2e; overflow: hidden; } +.wrt-sdp-box:last-child { border-right: none; } +.wrt-sdp-hdr { + padding: 7px 12px; border-bottom: 1px solid #1e1e2e; + font-size: 9px; color: #4a5568; letter-spacing: 0.08em; + display: flex; align-items: center; gap: 6px; +} +.wrt-sdp-dot { width: 5px; height: 5px; border-radius: 50%; background: #1e1e2e; } +.wrt-sdp-dot.active { background: #00ff88; } +.wrt-sdp-pane textarea { + flex: 1; background: transparent; border: none; color: #e2e8f0; + font-family: inherit; font-size: 10px; padding: 10px 12px; + resize: none; outline: none; line-height: 1.5; +} + +.wrt-audio-pane { + flex: 1; display: flex; flex-direction: column; + align-items: center; justify-content: center; gap: 14px; +} +.wrt-viz { display: flex; align-items: center; gap: 2px; height: 44px; } +.wrt-bar { width: 3px; border-radius: 2px; min-height: 2px; background: #00ff88; transition: height 0.05s; } +.wrt-mic-btn { + width: 52px; height: 52px; border-radius: 50%; + background: #111118; border: 1.5px solid #1e1e2e; + font-size: 18px; cursor: pointer; + display: flex; align-items: center; justify-content: center; transition: all 0.2s; +} +.wrt-mic-btn.active { border-color: #00ff88; box-shadow: 0 0 16px rgba(0,255,136,0.2); } +.wrt-audio-status { font-size: 10px; color: #4a5568; text-align: center; } +`; + +function useLog() { + const [entries, setEntries] = useState([]); + const add = useCallback((level, tag, msg) => { + const time = new Date().toTimeString().slice(0, 8); + setEntries(prev => [...prev, { level, tag, msg, time, id: Date.now() + Math.random() }]); + }, []); + const clear = useCallback(() => setEntries([]), []); + return { entries, add, clear }; +} + +export default function WebRTCTester() { + const [open, setOpen] = useState(false); + const [activeTab, setActiveTab] = useState('logs'); + const [proxyUrl, setProxyUrl] = useState('http://localhost:4000'); + const [apiKey, setApiKey] = useState('sk-1234'); + const [model, setModel] = useState('gpt-4o-realtime'); + const [status, setStatus] = useState('idle'); + const [flowStep, setFlowStep] = useState(0); + const [tokenPreview, setTokenPreview] = useState('—'); + const [iceState, setIceState] = useState('—'); + const [connState, setConnState] = useState('—'); + const [dcState, setDcState] = useState('—'); + const [sdpOffer, setSdpOffer] = useState(''); + const [sdpAnswer, setSdpAnswer] = useState(''); + const [offerActive, setOfferActive] = useState(false); + const [answerActive, setAnswerActive] = useState(false); + const [audioStatus, setAudioStatus] = useState('Start a session first'); + const [micActive, setMicActive] = useState(false); + const [bars, setBars] = useState(Array(28).fill(2)); + const [connected, setConnected] = useState(false); + + const { entries, add: log, clear: clearLogs } = useLog(); + const logRef = useRef(null); + + const pcRef = useRef(null); + const dcRef = useRef(null); + const streamRef = useRef(null); + const audioCtxRef = useRef(null); + const analyserRef = useRef(null); + const animRef = useRef(null); + const tokenRef = useRef(null); + const micRef = useRef(false); + const remoteAudioRef = useRef(null); + + useEffect(() => { + if (logRef.current) logRef.current.scrollTop = logRef.current.scrollHeight; + }, [entries]); + + function drawBars() { + animRef.current = requestAnimationFrame(drawBars); + if (!analyserRef.current) return; + const data = new Uint8Array(analyserRef.current.frequencyBinCount); + analyserRef.current.getByteFrequencyData(data); + setBars(Array.from({ length: 28 }, (_, i) => Math.max(2, ((data[i] || 0) / 255) * 42))); + } + + function setupAnalyser(stream) { + audioCtxRef.current = new AudioContext(); + const src = audioCtxRef.current.createMediaStreamSource(stream); + analyserRef.current = audioCtxRef.current.createAnalyser(); + analyserRef.current.fftSize = 64; + src.connect(analyserRef.current); + drawBars(); + } + + async function startSession() { + const url = proxyUrl.trim().replace(/\/$/, ''); + const key = apiKey.trim(); + const mdl = model.trim(); + + setConnected(true); + setStatus('connecting'); + setFlowStep(1); + + // Step 1: ephemeral token + log('step', 'STEP 1', `POST ${url}/v1/realtime/client_secrets`); + let tokenResp; + try { + const r = await fetch(`${url}/v1/realtime/client_secrets`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${key}` }, + body: JSON.stringify({ model: mdl }), + }); + log('info', 'HTTP', `${r.status} ${r.statusText}`); + const raw = await r.text(); + if (!r.ok) { log('error', 'ERR', raw); stopSession(); return; } + tokenResp = JSON.parse(raw); + log('success', 'TOKEN', 'Received encrypted ephemeral token'); + } catch (e) { + log('error', 'ERR', `client_secrets failed: ${e.message}`); + stopSession(); return; + } + + const token = tokenResp?.client_secret?.value ?? tokenResp?.value; + if (!token) { log('error', 'ERR', `Cannot extract token: ${JSON.stringify(tokenResp)}`); stopSession(); return; } + tokenRef.current = token; + setTokenPreview(token.slice(0, 10) + '…'); + log('info', 'TOKEN', `Preview: ${token.slice(0, 10)}…`); + + // Step 2: PeerConnection + log('step', 'STEP 2', 'Creating RTCPeerConnection'); + const pc = new RTCPeerConnection(); + pcRef.current = pc; + + pc.oniceconnectionstatechange = () => { + setIceState(pc.iceConnectionState); + log('info', 'ICE', pc.iceConnectionState); + if (pc.iceConnectionState === 'connected' || pc.iceConnectionState === 'completed') { + setStatus('connected'); setFlowStep(3); + } + if (pc.iceConnectionState === 'failed' || pc.iceConnectionState === 'disconnected') { + setStatus('error'); + } + }; + + pc.onconnectionstatechange = () => { + setConnState(pc.connectionState); + log('info', 'CONN', pc.connectionState); + }; + + pc.ontrack = (e) => { + log('success', 'AUDIO', 'Remote audio track received from OpenAI'); + if (remoteAudioRef.current) remoteAudioRef.current.srcObject = e.streams[0]; + setupAnalyser(e.streams[0]); + setAudioStatus('Receiving audio from OpenAI ✓'); + }; + + const dc = pc.createDataChannel('oai-events'); + dcRef.current = dc; + dc.onopen = () => { setDcState('open'); log('success', 'DC', 'Data channel open — ready!'); setStatus('connected'); }; + dc.onclose = () => { setDcState('closed'); log('warn', 'DC', 'Closed'); }; + dc.onmessage = (e) => { + try { log('info', 'EVENT', JSON.parse(e.data).type ?? 'unknown'); } + catch { log('info', 'EVENT', e.data.slice(0, 100)); } + }; + + // Mic + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + streamRef.current = stream; + stream.getTracks().forEach(t => pc.addTrack(t, stream)); + log('success', 'MIC', 'Microphone access granted'); + setAudioStatus('Mic active — waiting for remote audio'); + micRef.current = true; + setMicActive(true); + } catch (e) { + log('warn', 'MIC', `Mic denied: ${e.message}`); + const ctx = new AudioContext(); + const dest = ctx.createMediaStreamDestination(); + dest.stream.getTracks().forEach(t => pc.addTrack(t, dest.stream)); + } + + // Step 3: SDP offer + log('step', 'STEP 3', 'Creating SDP offer'); + const offer = await pc.createOffer(); + await pc.setLocalDescription(offer); + setSdpOffer(offer.sdp); + setOfferActive(true); + log('info', 'SDP', `Offer created (${offer.sdp.split('\n').length} lines)`); + + // Step 4: SDP exchange + setFlowStep(2); + log('step', 'STEP 4', `POST ${url}/v1/realtime/calls`); + try { + const r = await fetch(`${url}/v1/realtime/calls`, { + method: 'POST', + headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/sdp' }, + body: offer.sdp, + }); + log('info', 'HTTP', `${r.status} ${r.statusText}`); + if (!r.ok) { log('error', 'ERR', await r.text()); stopSession(); return; } + const ans = await r.text(); + log('success', 'SDP', `Answer received (${ans.split('\n').length} lines)`); + + // Step 5: remote description + log('step', 'STEP 5', 'Setting remote description'); + await pc.setRemoteDescription({ type: 'answer', sdp: ans }); + setSdpAnswer(ans); + setAnswerActive(true); + log('success', 'CONN', '✓ Session established — Browser ↔ LiteLLM ↔ OpenAI'); + } catch (e) { + log('error', 'ERR', `calls failed: ${e.message}`); + stopSession(); + } + } + + function stopSession() { + if (pcRef.current) { pcRef.current.close(); pcRef.current = null; } + if (streamRef.current) { streamRef.current.getTracks().forEach(t => t.stop()); streamRef.current = null; } + if (animRef.current) { cancelAnimationFrame(animRef.current); animRef.current = null; } + tokenRef.current = null; + micRef.current = false; + setConnected(false); + setStatus('idle'); + setFlowStep(0); + setTokenPreview('—'); + setIceState('—'); + setConnState('—'); + setDcState('—'); + setMicActive(false); + setOfferActive(false); + setAnswerActive(false); + setBars(Array(28).fill(2)); + setAudioStatus('Start a session first'); + log('warn', 'SESSION', 'Session stopped'); + } + + function toggleMic() { + if (!streamRef.current) { log('warn', 'MIC', 'No active session'); return; } + const next = !micRef.current; + micRef.current = next; + streamRef.current.getAudioTracks().forEach(t => { t.enabled = next; }); + setMicActive(next); + log('info', 'MIC', next ? 'Unmuted' : 'Muted'); + } + + const f = (n) => flowStep >= n; + + return ( + <> + +
+ {/* Toggle header */} +
setOpen(o => !o)}> +
+
+
+
INTERACTIVE TESTER
+
Browser → LiteLLM → OpenAI · WebRTC
+
+
+ ▼ +
+ + {open && ( +
+ {/* Sidebar */} +
+
+
Proxy Config
+
+ + setProxyUrl(e.target.value)} placeholder="http://localhost:4000" /> +
+
+ + setApiKey(e.target.value)} placeholder="sk-1234" /> +
+
+ + setModel(e.target.value)} /> +
+
+ +
+ +
+
Flow
+
+
Browser
+
→
+
LiteLLM
+
→
+
OpenAI
+
+
+ +
+ +
+
Controls
+ + + +
+ +
+ +
+
Session Info
+
+ {[['token', tokenPreview], ['ice', iceState], ['conn', connState], ['data ch.', dcState]].map(([k, v]) => ( +
{k}{v}
+ ))} +
+
+
+ + {/* Right panel */} +
+
+ WEBRTC REALTIME TESTER +
+
+ {status} +
+
+ +
+ {['logs','sdp','audio'].map(t => ( +
setActiveTab(t)}> + {t.toUpperCase()} +
+ ))} +
+ + {/* Logs */} +
+
+ {entries.length === 0 + ?
📡
Hit "Start Session" to begin
+ : entries.map(e => ( +
+ {e.time} + [{e.tag}] + {e.msg} +
+ )) + } +
+
+ + {/* SDP */} +
+
+
+
SDP OFFER
+