From 085a7a0c31fb39d6d7fed0f7e74140856b286db4 Mon Sep 17 00:00:00 2001 From: Chesars Date: Sun, 28 Dec 2025 11:20:25 -0300 Subject: [PATCH 01/52] Prevent auto_update_price_and_context_window workflow from running in forks Add repository check to ensure the scheduled workflow only runs in the main BerriAI/litellm repository, preventing unnecessary executions in forks. --- .github/workflows/auto_update_price_and_context_window.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml index e7d65242c19..98b9d868e68 100644 --- a/.github/workflows/auto_update_price_and_context_window.yml +++ b/.github/workflows/auto_update_price_and_context_window.yml @@ -7,6 +7,7 @@ on: jobs: auto_update_price_and_context_window: + if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 48b25a00c742ef78190508dfa72973d544f0a2ae Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Sat, 7 Feb 2026 09:46:17 +0530 Subject: [PATCH 02/52] fix tts metrics issues --- litellm/proxy/proxy_server.py | 99 +++++++----- .../test_audio_speech_prometheus_hooks.py | 141 ++++++++++++++++++ 2 files changed, 206 insertions(+), 34 deletions(-) create mode 100644 tests/test_litellm/proxy/test_audio_speech_prometheus_hooks.py diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 637893872d9..8926dfe8405 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -804,7 +804,9 @@ async def proxy_startup_event(app: FastAPI): # noqa: PLR0915 verbose_proxy_logger.debug("About to initialize semantic tool filter") _config = proxy_config.get_config_state() _litellm_settings = _config.get("litellm_settings", {}) - verbose_proxy_logger.debug(f"litellm_settings keys = {list(_litellm_settings.keys())}") + verbose_proxy_logger.debug( + f"litellm_settings keys = {list(_litellm_settings.keys())}" + ) await ProxyStartupEvent._initialize_semantic_tool_filter( llm_router=llm_router, litellm_settings=_litellm_settings, @@ -1292,7 +1294,9 @@ redis_usage_cache: Optional[ RedisCache ] = None # redis cache used for tracking spend, tpm/rpm limits polling_via_cache_enabled: Union[Literal["all"], List[str], bool] = False -native_background_mode: List[str] = [] # Models that should use native provider background mode instead of polling +native_background_mode: List[ + str +] = [] # Models that should use native provider background mode instead of polling polling_cache_ttl: int = 3600 # Default 1 hour TTL for polling cache user_custom_auth = None user_custom_key_generate = None @@ -3924,9 +3928,7 @@ class ProxyConfig: ) if self._should_load_db_object(object_type="semantic_filter_settings"): - await self._init_semantic_filter_settings_in_db( - prisma_client=prisma_client - ) + await self._init_semantic_filter_settings_in_db(prisma_client=prisma_client) async def _init_semantic_filter_settings_in_db(self, prisma_client: PrismaClient): """ @@ -4859,20 +4861,24 @@ class ProxyStartupEvent: ): """Initialize MCP semantic tool filter if configured""" from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook - + verbose_proxy_logger.info( f"Initializing semantic tool filter: llm_router={llm_router is not None}, " f"litellm_settings keys={list(litellm_settings.keys())}" ) - - mcp_semantic_filter_config = litellm_settings.get("mcp_semantic_tool_filter", None) - verbose_proxy_logger.debug(f"Semantic filter config: {mcp_semantic_filter_config}") - + + mcp_semantic_filter_config = litellm_settings.get( + "mcp_semantic_tool_filter", None + ) + verbose_proxy_logger.debug( + f"Semantic filter config: {mcp_semantic_filter_config}" + ) + hook = await SemanticToolFilterHook.initialize_from_config( config=mcp_semantic_filter_config, llm_router=llm_router, ) - + if hook: verbose_proxy_logger.debug("✅ Semantic tool filter hook registered") litellm.logging_callback_manager.add_litellm_callback(hook) @@ -6413,6 +6419,13 @@ async def audio_speech( "audio/wav" # Gemini TTS returns WAV format after conversion ) + # Proxy-level success hook (e.g. Prometheus litellm_proxy_total_requests_metric) + await proxy_logging_obj.post_call_success_hook( + data=data, + response=response, # type: ignore[arg-type] + user_api_key_dict=user_api_key_dict, + ) + return StreamingResponse( _audio_speech_chunk_generator(response), # type: ignore[arg-type] media_type=media_type, @@ -6420,6 +6433,11 @@ async def audio_speech( ) except Exception as e: + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=e, + request_data=data, + ) verbose_proxy_logger.error( "litellm.proxy.proxy_server.audio_speech(): Exception occured - {}".format( str(e) @@ -8208,7 +8226,8 @@ async def _apply_search_filter_to_models( # Fetch database models if we need more for the current page if router_models_count < models_needed_for_page: models_to_fetch = min( - models_needed_for_page - router_models_count, db_models_total_count + models_needed_for_page - router_models_count, + db_models_total_count, ) if models_to_fetch > 0: @@ -8244,21 +8263,21 @@ async def _apply_search_filter_to_models( def _normalize_datetime_for_sorting(dt: Any) -> Optional[datetime]: """ Normalize a datetime value to a timezone-aware UTC datetime for sorting. - + This function handles: - None values: returns None - String values: parses ISO format strings and converts to UTC-aware datetime - Datetime objects: converts naive datetimes to UTC-aware, and aware datetimes to UTC - + Args: dt: Datetime value (None, str, or datetime object) - + Returns: UTC-aware datetime object, or None if input is None or cannot be parsed """ if dt is None: return None - + if isinstance(dt, str): try: # Handle ISO format strings, including 'Z' suffix @@ -8272,14 +8291,14 @@ def _normalize_datetime_for_sorting(dt: Any) -> Optional[datetime]: return parsed_dt except (ValueError, AttributeError): return None - + if isinstance(dt, datetime): # If naive, assume UTC and make it aware if dt.tzinfo is None: return dt.replace(tzinfo=timezone.utc) # If aware, convert to UTC return dt.astimezone(timezone.utc) - + return None @@ -8299,46 +8318,60 @@ def _sort_models( Returns: Sorted list of models """ - if not sort_by or sort_by not in ["model_name", "created_at", "updated_at", "costs", "status"]: + if not sort_by or sort_by not in [ + "model_name", + "created_at", + "updated_at", + "costs", + "status", + ]: return all_models reverse = sort_order.lower() == "desc" def get_sort_key(model: Dict[str, Any]) -> Any: model_info = model.get("model_info", {}) - + if sort_by == "model_name": return model.get("model_name", "").lower() - + elif sort_by == "created_at": created_at = model_info.get("created_at") normalized_dt = _normalize_datetime_for_sorting(created_at) if normalized_dt is None: # Put None values at the end for asc, at the start for desc - return (datetime.max.replace(tzinfo=timezone.utc) if not reverse else datetime.min.replace(tzinfo=timezone.utc)) + return ( + datetime.max.replace(tzinfo=timezone.utc) + if not reverse + else datetime.min.replace(tzinfo=timezone.utc) + ) return normalized_dt - + elif sort_by == "updated_at": updated_at = model_info.get("updated_at") normalized_dt = _normalize_datetime_for_sorting(updated_at) if normalized_dt is None: - return (datetime.max.replace(tzinfo=timezone.utc) if not reverse else datetime.min.replace(tzinfo=timezone.utc)) + return ( + datetime.max.replace(tzinfo=timezone.utc) + if not reverse + else datetime.min.replace(tzinfo=timezone.utc) + ) return normalized_dt - + elif sort_by == "costs": input_cost = model_info.get("input_cost_per_token", 0) or 0 output_cost = model_info.get("output_cost_per_token", 0) or 0 total_cost = input_cost + output_cost # Put 0 or None costs at the end for asc, at the start for desc if total_cost == 0: - return (float("inf") if not reverse else float("-inf")) + return float("inf") if not reverse else float("-inf") return total_cost - + elif sort_by == "status": # False (config) comes before True (db) for asc db_model = model_info.get("db_model", False) return db_model - + return None try: @@ -8534,9 +8567,7 @@ async def _find_model_by_id( ) if db_model: # Convert database model to router format - decrypted_models = proxy_config.decrypt_model_list_from_db( - [db_model] - ) + decrypted_models = proxy_config.decrypt_model_list_from_db([db_model]) if decrypted_models: found_model = decrypted_models[0] except Exception as e: @@ -8710,13 +8741,13 @@ async def model_info_v2( ) verbose_proxy_logger.debug("all_models: %s", all_models) - + # Append A2A agents to models list all_models = await append_agents_to_model_info( models=all_models, user_api_key_dict=user_api_key_dict, ) - + # Update total count to include agents search_total_count = len(all_models) @@ -9559,7 +9590,7 @@ async def model_group_info( model_groups: List[ModelGroupInfoProxy] = _get_model_group_info( llm_router=llm_router, all_models_str=all_models_str, model_group=model_group ) - + # Append A2A agents to model groups model_groups = await append_agents_to_model_group( model_groups=model_groups, diff --git a/tests/test_litellm/proxy/test_audio_speech_prometheus_hooks.py b/tests/test_litellm/proxy/test_audio_speech_prometheus_hooks.py new file mode 100644 index 00000000000..00ca386a6ac --- /dev/null +++ b/tests/test_litellm/proxy/test_audio_speech_prometheus_hooks.py @@ -0,0 +1,141 @@ +""" +Regression tests: proxy /v1/audio/speech (TTS) must call proxy-level success/failure +hooks so Prometheus metrics (litellm_proxy_total_requests_metric, litellm_proxy_failed_requests_metric) +and other callbacks see TTS requests. +""" +import asyncio +import os +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +# Import after path setup so proxy_server is loadable +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.proxy_server import app, initialize + + +def _mock_user_api_key_auth(): + """Bypass auth for tests so /v1/audio/speech doesn't require a real key.""" + return MagicMock() + + +def _make_mock_tts_response(): + """Mock response for handler: llm_call = await route_request(), response = await llm_call, then _audio_speech_chunk_generator does await response.aiter_bytes() and async for chunk in it.""" + + async def _chunks(): + yield b"\xff\xfb" + + def _aiter_bytes(chunk_size=8192): + async def _wrapper(): + return _chunks() + + return _wrapper() + + inner = MagicMock() + inner.aiter_bytes = _aiter_bytes + inner._hidden_params = {} + + async def _resolver(): + return inner + + return _resolver() + + +@pytest.fixture +def client_no_auth(): + from litellm.proxy.proxy_server import cleanup_router_config_variables + + cleanup_router_config_variables() + filepath = os.path.dirname(os.path.abspath(__file__)) + config_fp = os.path.join(filepath, "test_configs", "test_config_no_auth.yaml") + asyncio.run(initialize(config=config_fp, debug=True)) + return TestClient(app) + + +@pytest.mark.asyncio +@pytest.mark.retry(retries=0) +async def test_audio_speech_success_calls_post_call_success_hook(client_no_auth): + """TTS success path must call proxy_logging_obj.post_call_success_hook (Prometheus total requests).""" + mock_success_hook = AsyncMock() + mock_failure_hook = AsyncMock() + mock_pre_call = AsyncMock(side_effect=lambda *, data, **kw: data) + mock_update_status = AsyncMock() + + mock_logging = MagicMock() + mock_logging.post_call_success_hook = mock_success_hook + mock_logging.post_call_failure_hook = mock_failure_hook + mock_logging.pre_call_hook = mock_pre_call + mock_logging.update_request_status = mock_update_status + + async def _mock_route_request(*, data, route_type, llm_router, user_model): + assert route_type == "aspeech" + return _make_mock_tts_response() + + original_overrides = app.dependency_overrides.copy() + app.dependency_overrides[user_api_key_auth] = _mock_user_api_key_auth + try: + with ( + patch("litellm.proxy.proxy_server.proxy_logging_obj", mock_logging), + patch( + "litellm.proxy.proxy_server.route_request", + side_effect=_mock_route_request, + ), + ): + response = client_no_auth.post( + "/v1/audio/speech", + json={"model": "tts-1", "input": "hello"}, + headers={"Content-Type": "application/json"}, + ) + finally: + app.dependency_overrides = original_overrides + + assert response.status_code == 200 + mock_success_hook.assert_awaited_once() + mock_failure_hook.assert_not_called() + # Ensure we passed through the right call type + call_kw = mock_success_hook.call_args.kwargs + assert "data" in call_kw and "user_api_key_dict" in call_kw + + +@pytest.mark.asyncio +@pytest.mark.retry(retries=0) +async def test_audio_speech_failure_calls_post_call_failure_hook(client_no_auth): + """TTS failure path must call proxy_logging_obj.post_call_failure_hook (Prometheus failed requests).""" + mock_success_hook = AsyncMock() + mock_failure_hook = AsyncMock() + mock_pre_call = AsyncMock(side_effect=lambda *, data, **kw: data) + + mock_logging = MagicMock() + mock_logging.post_call_success_hook = mock_success_hook + mock_logging.post_call_failure_hook = mock_failure_hook + mock_logging.pre_call_hook = mock_pre_call + + async def _mock_route_request_raise(*, data, route_type, llm_router, user_model): + raise ValueError("mock rate limit") + + original_overrides = app.dependency_overrides.copy() + app.dependency_overrides[user_api_key_auth] = _mock_user_api_key_auth + # Don't re-raise server exceptions so we get the 500 response instead of ValueError + client = TestClient(app, raise_server_exceptions=False) + try: + with ( + patch("litellm.proxy.proxy_server.proxy_logging_obj", mock_logging), + patch( + "litellm.proxy.proxy_server.route_request", + side_effect=_mock_route_request_raise, + ), + ): + response = client.post( + "/v1/audio/speech", + json={"model": "tts-1", "input": "hello"}, + headers={"Content-Type": "application/json"}, + ) + finally: + app.dependency_overrides = original_overrides + + assert response.status_code == 500 + mock_failure_hook.assert_awaited_once() + mock_success_hook.assert_not_called() + call_kw = mock_failure_hook.call_args.kwargs + assert "user_api_key_dict" in call_kw and "original_exception" in call_kw From 61d35cac3476cd59d41a289c9393c9a13424c931 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Mon, 23 Feb 2026 19:10:54 -0300 Subject: [PATCH 03/52] fix: use atomic increment-first pattern for model RPM rate limiting Replace racy check-then-increment RPM logic with atomic increment-first pattern to prevent concurrent requests from bypassing the rate limit. Co-Authored-By: Claude Opus 4.6 --- .../pre_call_checks/model_rate_limit_check.py | 45 +---------------- .../test_enforce_model_rate_limits.py | 50 +++++++++++++++++-- 2 files changed, 47 insertions(+), 48 deletions(-) diff --git a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py index e5be61690ba..836f9858744 100644 --- a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py @@ -129,27 +129,8 @@ class ModelRateLimitingCheck(CustomLogger): ), ) - # Check RPM limit + # Check RPM limit (atomic increment-first to avoid race conditions) if rpm_limit is not None: - # First check local cache - current_rpm = self.dual_cache.get_cache(key=rpm_key, local_only=True) - if current_rpm >= rpm_limit: - raise litellm.RateLimitError( - message=f"Model rate limit exceeded. RPM limit={rpm_limit}, current usage={current_rpm}", - llm_provider="", - model=model_name, - response=httpx.Response( - status_code=429, - content=f"{RouterErrors.user_defined_ratelimit_error.value} rpm limit={rpm_limit}. current usage={current_rpm}. id={model_id}, model_group={model_group}", - headers={"retry-after": str(60)}, - request=httpx.Request( - method="model_rate_limit_check", - url="https://github.com/BerriAI/litellm", - ), - ), - ) - - # Check redis cache and increment current_rpm = self.dual_cache.increment_cache( key=rpm_key, value=1, ttl=RoutingArgs.ttl ) @@ -226,30 +207,8 @@ class ModelRateLimitingCheck(CustomLogger): num_retries=0, # Don't retry - return 429 immediately ) - # Check RPM limit + # Check RPM limit (atomic increment-first to avoid race conditions) if rpm_limit is not None: - # First check local cache - current_rpm = await self.dual_cache.async_get_cache( - key=rpm_key, local_only=True - ) - if current_rpm is not None and current_rpm >= rpm_limit: - raise litellm.RateLimitError( - message=f"Model rate limit exceeded. RPM limit={rpm_limit}, current usage={current_rpm}", - llm_provider="", - model=model_name, - response=httpx.Response( - status_code=429, - content=f"{RouterErrors.user_defined_ratelimit_error.value} rpm limit={rpm_limit}. current usage={current_rpm}. id={model_id}, model_group={model_group}", - headers={"retry-after": str(60)}, - request=httpx.Request( - method="model_rate_limit_check", - url="https://github.com/BerriAI/litellm", - ), - ), - num_retries=0, # Don't retry - return 429 immediately - ) - - # Check redis cache and increment current_rpm = await self.dual_cache.async_increment_cache( key=rpm_key, value=1, diff --git a/tests/test_litellm/test_router/test_enforce_model_rate_limits.py b/tests/test_litellm/test_router/test_enforce_model_rate_limits.py index 3bca3df4e1d..1def253ac93 100644 --- a/tests/test_litellm/test_router/test_enforce_model_rate_limits.py +++ b/tests/test_litellm/test_router/test_enforce_model_rate_limits.py @@ -5,12 +5,14 @@ This feature allows users to enforce TPM/RPM limits set on model deployments regardless of the routing strategy being used. """ +import asyncio from unittest.mock import AsyncMock, MagicMock import pytest import litellm from litellm import Router +from litellm.caching.dual_cache import DualCache from litellm.router_utils.pre_call_checks.model_rate_limit_check import ( ModelRateLimitingCheck, ) @@ -88,7 +90,7 @@ class TestModelRateLimitingCheck: def test_pre_call_check_raises_rate_limit_error_when_over_rpm(self): """Test that RateLimitError is raised when RPM limit is exceeded.""" mock_cache = MagicMock() - mock_cache.get_cache.return_value = 10 # Already at limit + mock_cache.increment_cache.return_value = 11 # Over limit after increment check = ModelRateLimitingCheck(dual_cache=mock_cache) @@ -103,12 +105,11 @@ class TestModelRateLimitingCheck: check.pre_call_check(deployment) assert "RPM limit=10" in str(exc_info.value) - assert "current usage=10" in str(exc_info.value) + assert "current usage=11" in str(exc_info.value) def test_pre_call_check_allows_request_under_limit(self): """Test that requests are allowed when under the limit.""" mock_cache = MagicMock() - mock_cache.get_cache.return_value = 5 mock_cache.increment_cache.return_value = 6 check = ModelRateLimitingCheck(dual_cache=mock_cache) @@ -188,7 +189,8 @@ class TestModelRateLimitingCheckAsync: async def test_async_pre_call_check_raises_rate_limit_error_when_over_rpm(self): """Test that RateLimitError is raised when RPM limit is exceeded (async).""" mock_cache = MagicMock() - mock_cache.async_get_cache = AsyncMock(return_value=10) # Already at limit + mock_cache.async_get_cache = AsyncMock(return_value=None) + mock_cache.async_increment_cache = AsyncMock(return_value=11) # Over limit check = ModelRateLimitingCheck(dual_cache=mock_cache) @@ -208,7 +210,7 @@ class TestModelRateLimitingCheckAsync: async def test_async_pre_call_check_allows_request_under_limit(self): """Test that requests are allowed when under the limit (async).""" mock_cache = MagicMock() - mock_cache.async_get_cache = AsyncMock(return_value=5) + mock_cache.async_get_cache = AsyncMock(return_value=None) mock_cache.async_increment_cache = AsyncMock(return_value=6) check = ModelRateLimitingCheck(dual_cache=mock_cache) @@ -313,3 +315,41 @@ class TestRouterWithEnforceModelRateLimits: break assert found, "ModelRateLimitingCheck should be in litellm.callbacks" + + +class TestModelRateLimitConcurrency: + """Test that RPM rate limiting is atomic under concurrent requests.""" + + @pytest.mark.asyncio + async def test_concurrent_requests_respect_rpm_limit(self): + """ + Fire 4 concurrent async requests with RPM limit of 2. + Exactly 2 should succeed and 2 should raise RateLimitError. + + This test validates the atomic increment-first pattern: + the old check-then-increment pattern would let 3+ through + due to a race condition on the local cache read. + """ + dual_cache = DualCache() + check = ModelRateLimitingCheck(dual_cache=dual_cache) + + deployment = { + "rpm": 2, + "litellm_params": {"model": "gpt-4"}, + "model_info": {"id": "concurrent-test-id"}, + "model_name": "test-model", + } + + async def attempt_request(): + return await check.async_pre_call_check(deployment) + + results = await asyncio.gather( + *[attempt_request() for _ in range(4)], + return_exceptions=True, + ) + + successes = [r for r in results if not isinstance(r, Exception)] + failures = [r for r in results if isinstance(r, litellm.RateLimitError)] + + assert len(successes) == 2, f"Expected 2 successes, got {len(successes)}" + assert len(failures) == 2, f"Expected 2 rate limit errors, got {len(failures)}" From db3d61f43372cd7cc18ad8c557321b5e9e59f19d Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Tue, 24 Feb 2026 16:49:06 -0300 Subject: [PATCH 04/52] feat(ci): add duplicate issue detection and auto-close bot Add a Python script that detects duplicate issues using title similarity (difflib.SequenceMatcher) and closes them via the gh CLI. Two-tier system: - 0.6 threshold: informational comment via existing wow-actions step - 0.85 threshold: auto-close with comment, label, and not_planned reason Includes a workflow_dispatch workflow for one-time batch scans and integrates auto-close into the existing check_duplicate_issues workflow for newly opened issues. Co-Authored-By: Claude Opus 4.6 --- .github/scripts/close_duplicate_issues.py | 211 +++++++++++++++++++ .github/workflows/check_duplicate_issues.yml | 17 ++ .github/workflows/scan_duplicate_issues.yml | 45 ++++ 3 files changed, 273 insertions(+) create mode 100755 .github/scripts/close_duplicate_issues.py create mode 100644 .github/workflows/scan_duplicate_issues.yml diff --git a/.github/scripts/close_duplicate_issues.py b/.github/scripts/close_duplicate_issues.py new file mode 100755 index 00000000000..127bad31acd --- /dev/null +++ b/.github/scripts/close_duplicate_issues.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +""" +Detect and close duplicate GitHub issues using title similarity. + +Modes: + --scan Compare all open issues against each other (batch) + --issue-number N Check a single issue against older open issues + +Requires the `gh` CLI to be authenticated. +""" + +import argparse +import difflib +import json +import re +import subprocess +import sys + + +def normalize_title(title: str) -> str: + """Strip common prefixes, lowercase, and collapse whitespace.""" + title = re.sub( + r"^\[?(bug|feature request|enhancement|question|docs)[:\]]?\s*", + "", + title, + flags=re.IGNORECASE, + ) + return " ".join(title.lower().split()) + + +def gh(*args: str) -> str: + """Run a gh CLI command and return stdout.""" + result = subprocess.run( + ["gh", *args], + capture_output=True, + text=True, + check=True, + ) + return result.stdout + + +def fetch_open_issues(repo: str | None) -> list[dict]: + """Fetch all open issues (excluding PRs) via gh api --paginate.""" + endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc" + cmd = ["api", "--paginate", endpoint] + if repo: + cmd.extend(["-f", f"owner={repo.split('/')[0]}", "-f", f"repo={repo.split('/')[1]}"]) + endpoint = f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc" + cmd = ["api", "--paginate", endpoint] + else: + cmd = ["api", "--paginate", "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"] + + raw = gh(*cmd) + # gh --paginate concatenates JSON arrays, so we may get multiple arrays + issues = [] + for line in raw.strip().splitlines(): + line = line.strip() + if not line: + continue + parsed = json.loads(line) + if isinstance(parsed, list): + issues.extend(parsed) + else: + issues.append(parsed) + + # Filter out pull requests (they also appear in the issues endpoint) + return [i for i in issues if "pull_request" not in i] + + +def close_as_duplicate( + issue_number: int, duplicate_of: int, repo: str | None, dry_run: bool +) -> None: + """Close an issue as duplicate of another, adding a comment and label.""" + repo_args = ["--repo", repo] if repo else [] + + if dry_run: + print(f" [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}") + return + + # Add comment + comment_body = ( + f"Closing as duplicate of #{duplicate_of}.\n\n" + "If you believe this is not a duplicate, please reopen and add context " + "explaining how this differs." + ) + gh("issue", "comment", str(issue_number), "--body", comment_body, *repo_args) + + # Add label + gh("issue", "edit", str(issue_number), "--add-label", "duplicate", *repo_args) + + # Close with not_planned reason + gh( + "api", + f"repos/{repo or '{owner}/{repo}'}/issues/{issue_number}", + "-X", + "PATCH", + "-f", + "state=closed", + "-f", + "state_reason=not_planned", + ) + + print(f" Closed #{issue_number} as duplicate of #{duplicate_of}") + + +def find_duplicate( + issue: dict, candidates: list[dict], threshold: float +) -> dict | None: + """Return the first candidate whose normalized title is above threshold.""" + norm = normalize_title(issue["title"]) + for candidate in candidates: + if candidate["number"] == issue["number"]: + continue + cand_norm = normalize_title(candidate["title"]) + ratio = difflib.SequenceMatcher(None, norm, cand_norm).ratio() + if ratio >= threshold: + return candidate + return None + + +def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bool) -> int: + """Compare every issue against all older issues. Returns count of duplicates found.""" + # Sort oldest first + issues.sort(key=lambda i: i["number"]) + closed_count = 0 + + for idx, issue in enumerate(issues): + older = issues[:idx] + if not older: + continue + dup = find_duplicate(issue, older, threshold) + if dup: + ratio = difflib.SequenceMatcher( + None, + normalize_title(issue["title"]), + normalize_title(dup["title"]), + ).ratio() + print( + f"#{issue['number']}: \"{issue['title']}\"\n" + f" -> duplicate of #{dup['number']}: \"{dup['title']}\" " + f"({ratio:.0%} similar)" + ) + close_as_duplicate(issue["number"], dup["number"], repo, dry_run) + closed_count += 1 + + return closed_count + + +def check_single( + issue_number: int, issues: list[dict], threshold: float, repo: str | None, dry_run: bool +) -> bool: + """Check a single issue against all older open issues. Returns True if duplicate found.""" + target = None + for i in issues: + if i["number"] == issue_number: + target = i + break + + if target is None: + print(f"Issue #{issue_number} not found among open issues.") + return False + + older = [i for i in issues if i["number"] < issue_number] + dup = find_duplicate(target, older, threshold) + if dup: + ratio = difflib.SequenceMatcher( + None, + normalize_title(target["title"]), + normalize_title(dup["title"]), + ).ratio() + print( + f"#{target['number']}: \"{target['title']}\"\n" + f" -> duplicate of #{dup['number']}: \"{dup['title']}\" " + f"({ratio:.0%} similar)" + ) + close_as_duplicate(issue_number, dup["number"], repo, dry_run) + return True + + print(f"#{issue_number}: no duplicate found above threshold {threshold}") + return False + + +def main() -> None: + parser = argparse.ArgumentParser(description="Detect and close duplicate GitHub issues") + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--scan", action="store_true", help="Scan all open issues") + mode.add_argument("--issue-number", type=int, help="Check a single issue number") + parser.add_argument("--threshold", type=float, default=0.85, help="Similarity threshold (0-1)") + parser.add_argument("--close", action="store_true", help="Actually close duplicates (default is dry-run)") + parser.add_argument("--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted.") + args = parser.parse_args() + + dry_run = not args.close + + if dry_run: + print("=== DRY RUN MODE (pass --close to actually close issues) ===\n") + + print("Fetching open issues...") + issues = fetch_open_issues(args.repo) + print(f"Found {len(issues)} open issues.\n") + + if args.scan: + count = scan_all(issues, args.threshold, args.repo, dry_run) + print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}") + else: + found = check_single(args.issue_number, issues, args.threshold, args.repo, dry_run) + sys.exit(0 if found else 0) # Always exit 0; finding no dup is not an error + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/check_duplicate_issues.yml b/.github/workflows/check_duplicate_issues.yml index 9477dd2f8e2..ff252e37d9a 100644 --- a/.github/workflows/check_duplicate_issues.yml +++ b/.github/workflows/check_duplicate_issues.yml @@ -27,3 +27,20 @@ jobs: {{/issues}} Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference. + + - name: Checkout close script + if: github.event.action == 'opened' + uses: actions/checkout@v4 + with: + sparse-checkout: .github/scripts + + - name: Auto-close if high-confidence duplicate + if: github.event.action == 'opened' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 .github/scripts/close_duplicate_issues.py \ + --issue-number ${{ github.event.issue.number }} \ + --repo ${{ github.repository }} \ + --threshold 0.85 \ + --close diff --git a/.github/workflows/scan_duplicate_issues.yml b/.github/workflows/scan_duplicate_issues.yml new file mode 100644 index 00000000000..fce3e5f7954 --- /dev/null +++ b/.github/workflows/scan_duplicate_issues.yml @@ -0,0 +1,45 @@ +name: Scan Duplicate Issues (One-Time) + +on: + workflow_dispatch: + inputs: + threshold: + description: "Similarity threshold (0-1)" + required: false + default: "0.85" + close: + description: "Actually close duplicates (false = dry run)" + required: false + type: boolean + default: false + +jobs: + scan: + runs-on: ubuntu-latest + permissions: + issues: write + contents: read + steps: + - name: Checkout scripts + uses: actions/checkout@v4 + with: + sparse-checkout: .github/scripts + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Scan for duplicate issues + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + CLOSE_FLAG="" + if [ "${{ inputs.close }}" = "true" ]; then + CLOSE_FLAG="--close" + fi + python3 .github/scripts/close_duplicate_issues.py \ + --scan \ + --repo ${{ github.repository }} \ + --threshold ${{ inputs.threshold }} \ + $CLOSE_FLAG From 10e769a5e4108b669fb1a39ee01c4bb660d524e2 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Wed, 25 Feb 2026 22:03:51 +0530 Subject: [PATCH 05/52] feat: ability to trace metrics --- docs/my-website/docs/observability/datadog.md | 60 ++ litellm/__init__.py | 756 +++++++++++++----- litellm/integrations/callback_configs.json | 23 +- .../integrations/datadog/datadog_metrics.py | 288 +++++++ .../custom_logger_registry.py | 2 + litellm/litellm_core_utils/litellm_logging.py | 30 +- litellm/proxy/common_utils/callback_utils.py | 8 +- .../health_endpoints/_health_endpoints.py | 16 + litellm/types/integrations/datadog_metrics.py | 19 + .../datadog/test_datadog_metrics.py | 272 +++++++ 10 files changed, 1245 insertions(+), 229 deletions(-) create mode 100644 litellm/integrations/datadog/datadog_metrics.py create mode 100644 litellm/types/integrations/datadog_metrics.py create mode 100644 tests/test_litellm/integrations/datadog/test_datadog_metrics.py diff --git a/docs/my-website/docs/observability/datadog.md b/docs/my-website/docs/observability/datadog.md index 9385b0020cf..ef48594cbfa 100644 --- a/docs/my-website/docs/observability/datadog.md +++ b/docs/my-website/docs/observability/datadog.md @@ -7,6 +7,7 @@ import TabItem from '@theme/TabItem'; LiteLLM Supports logging to the following Datdog Integrations: - `datadog` [Datadog Logs](https://docs.datadoghq.com/logs/) - `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/) +- `datadog_metrics` [Datadog Custom Metrics](#datadog-custom-metrics) - `datadog_cost_management` [Datadog Cloud Cost Management](#datadog-cloud-cost-management) - `ddtrace-run` [Datadog Tracing](#datadog-tracing) @@ -168,6 +169,65 @@ On the Datadog LLM Observability page, you should see that both input messages a +## Datadog Custom Metrics + +| Feature | Details | +|---------|---------| +| **What is logged** | Latency metrics, request counts by status code | +| **Events** | Success + Failure | +| **Product Link** | [Datadog Metrics](https://docs.datadoghq.com/metrics/) | + +Publishes the following metrics to Datadog via the `/api/v2/series` endpoint: + +| Metric | Type | Description | +|--------|------|-------------| +| `litellm.request.total_latency` | Gauge | End-to-end request latency (seconds) | +| `litellm.llm_api.latency` | Gauge | Time spent waiting for the LLM provider response (seconds) | +| `litellm.llm_api.request_count` | Count | Request count, tagged with status code | + +Using `total_latency` and `llm_api.latency`, you can derive **internal latency** = `total_latency - llm_api.latency`. + +All metrics include the following tags: `env`, `service`, `version`, `HOSTNAME`, `POD_NAME`, `provider`, `model_name`, `model_group`, `team`, `status_code`. + +**Step 1**: Create a `config.yaml` file + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo +litellm_settings: + success_callback: ["datadog_metrics"] + failure_callback: ["datadog_metrics"] +``` + +**Step 2**: Set required env variables + +```shell +DD_API_KEY="your-api-key" +DD_SITE="us5.datadoghq.com" # your datadog site +``` + +**Step 3**: Start the proxy and make a test request + +```shell +litellm --config config.yaml +``` + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer sk-1234' \ + --data '{ + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hello"}] +}' +``` + +**Step 4**: View metrics in Datadog Metrics Explorer + +Navigate to **Metrics > Explorer** in Datadog and search for `litellm.request.total_latency`, `litellm.llm_api.latency`, or `litellm.llm_api.request_count`. + ## Datadog Cloud Cost Management | Feature | Details | diff --git a/litellm/__init__.py b/litellm/__init__.py index 6e42f2c1ea5..41f5b0ff42d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -75,6 +75,7 @@ from litellm.constants import ( ) import httpx import dotenv + # register_async_client_cleanup is lazy-loaded and called on first access litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV" @@ -106,6 +107,7 @@ _custom_logger_compatible_callbacks_literal = Literal[ "otel", "datadog", "datadog_llm_observability", + "datadog_metrics", "galileo", "braintrust", "arize", @@ -147,7 +149,9 @@ _known_custom_logger_compatible_callbacks: List = list( get_args(_custom_logger_compatible_callbacks_literal) ) callbacks: List[ - Union[Callable, _custom_logger_compatible_callbacks_literal, "CustomLogger"] # CustomLogger is lazy-loaded + Union[ + Callable, _custom_logger_compatible_callbacks_literal, "CustomLogger" + ] # CustomLogger is lazy-loaded ] = [] callback_settings: Dict[str, Dict[str, Any]] = {} initialized_langfuse_clients: int = 0 @@ -157,42 +161,50 @@ prometheus_initialize_budget_metrics: Optional[bool] = False require_auth_for_metrics_endpoint: Optional[bool] = False argilla_batch_size: Optional[int] = None datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload. -gcs_pub_sub_use_v1: Optional[bool] = ( - False # if you want to use v1 gcs pubsub logged payload -) -generic_api_use_v1: Optional[bool] = ( - False # if you want to use v1 generic api logged payload -) +gcs_pub_sub_use_v1: Optional[ + bool +] = False # if you want to use v1 gcs pubsub logged payload +generic_api_use_v1: Optional[ + bool +] = False # if you want to use v1 generic api logged payload argilla_transformation_object: Optional[Dict[str, Any]] = None -_async_input_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded +_async_input_callback: List[ + Union[str, Callable, "CustomLogger"] +] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. -_async_success_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded +_async_success_callback: List[ + Union[str, Callable, "CustomLogger"] +] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. -_async_failure_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded +_async_failure_callback: List[ + Union[str, Callable, "CustomLogger"] +] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. pre_call_rules: List[Callable] = [] post_call_rules: List[Callable] = [] turn_off_message_logging: Optional[bool] = False -standard_logging_payload_excluded_fields: Optional[List[str]] = None # Fields to exclude from StandardLoggingPayload before callbacks receive it +standard_logging_payload_excluded_fields: Optional[ + List[str] +] = None # Fields to exclude from StandardLoggingPayload before callbacks receive it log_raw_request_response: bool = False redact_messages_in_exceptions: Optional[bool] = False redact_user_api_key_info: Optional[bool] = False filter_invalid_headers: Optional[bool] = False -add_user_information_to_llm_headers: Optional[bool] = ( - None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers -) +add_user_information_to_llm_headers: Optional[ + bool +] = None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers store_audit_logs = False # Enterprise feature, allow users to see audit logs ### end of callbacks ############# -email: Optional[str] = ( - None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -) -token: Optional[str] = ( - None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -) +email: Optional[ + str +] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +token: Optional[ + str +] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 telemetry = True max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False)) @@ -251,9 +263,9 @@ use_client: bool = False ssl_verify: Union[str, bool] = True ssl_security_level: Optional[str] = None ssl_certificate: Optional[str] = None -ssl_ecdh_curve: Optional[str] = ( - None # Set to 'X25519' to disable PQC and improve performance -) +ssl_ecdh_curve: Optional[ + str +] = None # Set to 'X25519' to disable PQC and improve performance disable_streaming_logging: bool = False disable_token_counter: bool = False disable_add_transform_inline_image_block: bool = False @@ -303,24 +315,20 @@ enable_loadbalancing_on_batch_endpoints: Optional[bool] = None enable_caching_on_provider_specific_optional_params: bool = ( False # feature-flag for caching on optional params - e.g. 'top_k' ) -caching: bool = ( - False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -) -caching_with_models: bool = ( - False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -) -cache: Optional["Cache"] = ( - None # cache object <- use this - https://docs.litellm.ai/docs/caching -) +caching: bool = False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +caching_with_models: bool = False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +cache: Optional[ + "Cache" +] = None # cache object <- use this - https://docs.litellm.ai/docs/caching default_in_memory_ttl: Optional[float] = None default_redis_ttl: Optional[float] = None default_redis_batch_cache_expiry: Optional[float] = None model_alias_map: Dict[str, str] = {} model_group_settings: Optional["ModelGroupSettings"] = None max_budget: float = 0.0 # set the max budget across all providers -budget_duration: Optional[str] = ( - None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). -) +budget_duration: Optional[ + str +] = None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). default_soft_budget: float = ( DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0 ) @@ -329,9 +337,7 @@ forward_traceparent_to_llm_provider: bool = False _current_cost = 0.0 # private variable, used if max budget is set error_logs: Dict = {} -add_function_to_prompt: bool = ( - False # if function calling not supported by api, append function call details to system prompt -) +add_function_to_prompt: bool = False # if function calling not supported by api, append function call details to system prompt client_session: Optional[httpx.Client] = None aclient_session: Optional[httpx.AsyncClient] = None model_fallbacks: Optional[List] = None # Deprecated for 'litellm.fallbacks' @@ -378,9 +384,7 @@ prometheus_emit_stream_label: bool = False disable_add_prefix_to_prompt: bool = ( False # used by anthropic, to disable adding prefix to prompt ) -disable_copilot_system_to_assistant: bool = ( - False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. -) +disable_copilot_system_to_assistant: bool = False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. public_mcp_servers: Optional[List[str]] = None public_model_groups: Optional[List[str]] = None public_agent_groups: Optional[List[str]] = None @@ -399,17 +403,13 @@ if TYPE_CHECKING: ######## Networking Settings ######## -use_aiohttp_transport: bool = ( - True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead. -) +use_aiohttp_transport: bool = True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead. aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings disable_aiohttp_transport: bool = False # Set this to true to use httpx instead disable_aiohttp_trust_env: bool = ( False # When False, aiohttp will respect HTTP(S)_PROXY env vars ) -force_ipv4: bool = ( - False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. -) +force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. network_mock: bool = False # When True, use mock transport — no real network calls ####### STOP SEQUENCE LIMIT ####### @@ -424,13 +424,13 @@ context_window_fallbacks: Optional[List] = None content_policy_fallbacks: Optional[List] = None allowed_fails: int = 3 allow_dynamic_callback_disabling: bool = True -num_retries_per_request: Optional[int] = ( - None # for the request overall (incl. fallbacks + model retries) -) +num_retries_per_request: Optional[ + int +] = None # for the request overall (incl. fallbacks + model retries) ####### SECRET MANAGERS ##################### -secret_manager_client: Optional[Any] = ( - None # list of instantiated key management clients - e.g. azure kv, infisical, etc. -) +secret_manager_client: Optional[ + Any +] = None # list of instantiated key management clients - e.g. azure kv, infisical, etc. _google_kms_resource_name: Optional[str] = None _key_management_system: Optional["KeyManagementSystem"] = None # Note: KeyManagementSettings must be eagerly imported because _key_management_settings @@ -443,12 +443,12 @@ output_parse_pii: bool = False from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map model_cost = get_model_cost_map(url=model_cost_map_url) -cost_discount_config: Dict[str, float] = ( - {} -) # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount -cost_margin_config: Dict[str, Union[float, Dict[str, float]]] = ( - {} -) # Provider-specific or global cost margins. Examples: +cost_discount_config: Dict[ + str, float +] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount +cost_margin_config: Dict[ + str, Union[float, Dict[str, float]] +] = {} # Provider-specific or global cost margins. Examples: # Percentage: {"openai": 0.10} = 10% margin # Fixed: {"openai": {"fixed_amount": 0.001}} = $0.001 per request # Global: {"global": 0.05} = 5% global margin on all providers @@ -1107,10 +1107,12 @@ openai_video_generation_models = ["sora-2"] # Import KeyManagementSettings here (before utils import) because _key_management_settings # is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils) from litellm.types.secret_managers.main import KeyManagementSettings + _key_management_settings: KeyManagementSettings = KeyManagementSettings() # client must be imported immediately as it's used as a decorator at function definition time from .utils import client + # Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py # (which imports tiktoken) at import time @@ -1139,6 +1141,7 @@ from .llms.topaz.common_utils import TopazModelInfo # OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access # OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access from .llms.xai.common_utils import XAIModelInfo + # PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json) # All remaining configs are now lazy loaded - see _lazy_imports_registry.py @@ -1220,6 +1223,7 @@ from .batch_completion.main import * # type: ignore from .rerank_api.main import * from .llms.anthropic.experimental_pass_through.messages.handler import * from .responses.main import * + # Interactions API is available as litellm.interactions module # Usage: litellm.interactions.create(), litellm.interactions.get(), etc. from . import interactions @@ -1278,12 +1282,12 @@ from . import rag from .types.llms.custom_llm import CustomLLMItem custom_provider_map: List[CustomLLMItem] = [] -_custom_providers: List[str] = ( - [] -) # internal helper util, used to track names of custom providers -disable_hf_tokenizer_download: Optional[bool] = ( - None # disable huggingface tokenizer download. Defaults to openai clk100 -) +_custom_providers: List[ + str +] = [] # internal helper util, used to track names of custom providers +disable_hf_tokenizer_download: Optional[ + bool +] = None # disable huggingface tokenizer download. Defaults to openai clk100 global_disable_no_log_param: bool = False ### CLI UTILITIES ### @@ -1322,128 +1326,318 @@ if TYPE_CHECKING: from litellm.caching.caching import Cache # Type stubs for lazy-loaded configs to help mypy - from .llms.bedrock.chat.converse_transformation import AmazonConverseConfig as AmazonConverseConfig - from .llms.openai_like.chat.handler import OpenAILikeChatConfig as OpenAILikeChatConfig - from .llms.galadriel.chat.transformation import GaladrielChatConfig as GaladrielChatConfig + from .llms.bedrock.chat.converse_transformation import ( + AmazonConverseConfig as AmazonConverseConfig, + ) + from .llms.openai_like.chat.handler import ( + OpenAILikeChatConfig as OpenAILikeChatConfig, + ) + from .llms.galadriel.chat.transformation import ( + GaladrielChatConfig as GaladrielChatConfig, + ) from .llms.github.chat.transformation import GithubChatConfig as GithubChatConfig - from .llms.azure_ai.anthropic.transformation import AzureAnthropicConfig as AzureAnthropicConfig + from .llms.azure_ai.anthropic.transformation import ( + AzureAnthropicConfig as AzureAnthropicConfig, + ) from .llms.bytez.chat.transformation import BytezChatConfig as BytezChatConfig - from .llms.compactifai.chat.transformation import CompactifAIChatConfig as CompactifAIChatConfig + from .llms.compactifai.chat.transformation import ( + CompactifAIChatConfig as CompactifAIChatConfig, + ) from .llms.empower.chat.transformation import EmpowerChatConfig as EmpowerChatConfig from .llms.minimax.chat.transformation import MinimaxChatConfig as MinimaxChatConfig - from .llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig as AiohttpOpenAIChatConfig - from .llms.huggingface.chat.transformation import HuggingFaceChatConfig as HuggingFaceChatConfig - from .llms.huggingface.embedding.transformation import HuggingFaceEmbeddingConfig as HuggingFaceEmbeddingConfig + from .llms.aiohttp_openai.chat.transformation import ( + AiohttpOpenAIChatConfig as AiohttpOpenAIChatConfig, + ) + from .llms.huggingface.chat.transformation import ( + HuggingFaceChatConfig as HuggingFaceChatConfig, + ) + from .llms.huggingface.embedding.transformation import ( + HuggingFaceEmbeddingConfig as HuggingFaceEmbeddingConfig, + ) from .llms.oobabooga.chat.transformation import OobaboogaConfig as OobaboogaConfig from .llms.maritalk import MaritalkConfig as MaritalkConfig - from .llms.openrouter.chat.transformation import OpenrouterConfig as OpenrouterConfig + from .llms.openrouter.chat.transformation import ( + OpenrouterConfig as OpenrouterConfig, + ) from .llms.datarobot.chat.transformation import DataRobotConfig as DataRobotConfig from .llms.anthropic.chat.transformation import AnthropicConfig as AnthropicConfig - from .llms.anthropic.completion.transformation import AnthropicTextConfig as AnthropicTextConfig + from .llms.anthropic.completion.transformation import ( + AnthropicTextConfig as AnthropicTextConfig, + ) from .llms.groq.stt.transformation import GroqSTTConfig as GroqSTTConfig from .llms.triton.completion.transformation import TritonConfig as TritonConfig - from .llms.triton.completion.transformation import TritonGenerateConfig as TritonGenerateConfig - from .llms.triton.completion.transformation import TritonInferConfig as TritonInferConfig - from .llms.triton.embedding.transformation import TritonEmbeddingConfig as TritonEmbeddingConfig - from .llms.huggingface.rerank.transformation import HuggingFaceRerankConfig as HuggingFaceRerankConfig - from .llms.databricks.chat.transformation import DatabricksConfig as DatabricksConfig - from .llms.databricks.embed.transformation import DatabricksEmbeddingConfig as DatabricksEmbeddingConfig + from .llms.triton.completion.transformation import ( + TritonGenerateConfig as TritonGenerateConfig, + ) + from .llms.triton.completion.transformation import ( + TritonInferConfig as TritonInferConfig, + ) + from .llms.triton.embedding.transformation import ( + TritonEmbeddingConfig as TritonEmbeddingConfig, + ) + from .llms.huggingface.rerank.transformation import ( + HuggingFaceRerankConfig as HuggingFaceRerankConfig, + ) + from .llms.databricks.chat.transformation import ( + DatabricksConfig as DatabricksConfig, + ) + from .llms.databricks.embed.transformation import ( + DatabricksEmbeddingConfig as DatabricksEmbeddingConfig, + ) from .llms.predibase.chat.transformation import PredibaseConfig as PredibaseConfig from .llms.replicate.chat.transformation import ReplicateConfig as ReplicateConfig from .llms.snowflake.chat.transformation import SnowflakeConfig as SnowflakeConfig - from .llms.cohere.rerank.transformation import CohereRerankConfig as CohereRerankConfig - from .llms.cohere.rerank_v2.transformation import CohereRerankV2Config as CohereRerankV2Config - from .llms.azure_ai.rerank.transformation import AzureAIRerankConfig as AzureAIRerankConfig - from .llms.infinity.rerank.transformation import InfinityRerankConfig as InfinityRerankConfig - from .llms.jina_ai.rerank.transformation import JinaAIRerankConfig as JinaAIRerankConfig - from .llms.deepinfra.rerank.transformation import DeepinfraRerankConfig as DeepinfraRerankConfig - from .llms.hosted_vllm.rerank.transformation import HostedVLLMRerankConfig as HostedVLLMRerankConfig - from .llms.nvidia_nim.rerank.transformation import NvidiaNimRerankConfig as NvidiaNimRerankConfig - from .llms.nvidia_nim.rerank.ranking_transformation import NvidiaNimRankingConfig as NvidiaNimRankingConfig - from .llms.vertex_ai.rerank.transformation import VertexAIRerankConfig as VertexAIRerankConfig - from .llms.fireworks_ai.rerank.transformation import FireworksAIRerankConfig as FireworksAIRerankConfig - from .llms.voyage.rerank.transformation import VoyageRerankConfig as VoyageRerankConfig - from .llms.watsonx.rerank.transformation import IBMWatsonXRerankConfig as IBMWatsonXRerankConfig + from .llms.cohere.rerank.transformation import ( + CohereRerankConfig as CohereRerankConfig, + ) + from .llms.cohere.rerank_v2.transformation import ( + CohereRerankV2Config as CohereRerankV2Config, + ) + from .llms.azure_ai.rerank.transformation import ( + AzureAIRerankConfig as AzureAIRerankConfig, + ) + from .llms.infinity.rerank.transformation import ( + InfinityRerankConfig as InfinityRerankConfig, + ) + from .llms.jina_ai.rerank.transformation import ( + JinaAIRerankConfig as JinaAIRerankConfig, + ) + from .llms.deepinfra.rerank.transformation import ( + DeepinfraRerankConfig as DeepinfraRerankConfig, + ) + from .llms.hosted_vllm.rerank.transformation import ( + HostedVLLMRerankConfig as HostedVLLMRerankConfig, + ) + from .llms.nvidia_nim.rerank.transformation import ( + NvidiaNimRerankConfig as NvidiaNimRerankConfig, + ) + from .llms.nvidia_nim.rerank.ranking_transformation import ( + NvidiaNimRankingConfig as NvidiaNimRankingConfig, + ) + from .llms.vertex_ai.rerank.transformation import ( + VertexAIRerankConfig as VertexAIRerankConfig, + ) + from .llms.fireworks_ai.rerank.transformation import ( + FireworksAIRerankConfig as FireworksAIRerankConfig, + ) + from .llms.voyage.rerank.transformation import ( + VoyageRerankConfig as VoyageRerankConfig, + ) + from .llms.watsonx.rerank.transformation import ( + IBMWatsonXRerankConfig as IBMWatsonXRerankConfig, + ) from .llms.clarifai.chat.transformation import ClarifaiConfig as ClarifaiConfig from .llms.ai21.chat.transformation import AI21ChatConfig as AI21ChatConfig from .llms.meta_llama.chat.transformation import LlamaAPIConfig as LlamaAPIConfig - from .llms.together_ai.completion.transformation import TogetherAITextCompletionConfig as TogetherAITextCompletionConfig - from .llms.cloudflare.chat.transformation import CloudflareChatConfig as CloudflareChatConfig + from .llms.together_ai.completion.transformation import ( + TogetherAITextCompletionConfig as TogetherAITextCompletionConfig, + ) + from .llms.cloudflare.chat.transformation import ( + CloudflareChatConfig as CloudflareChatConfig, + ) from .llms.novita.chat.transformation import NovitaConfig as NovitaConfig from .llms.petals.completion.transformation import PetalsConfig as PetalsConfig from .llms.ollama.chat.transformation import OllamaChatConfig as OllamaChatConfig from .llms.ollama.completion.transformation import OllamaConfig as OllamaConfig - from .llms.sagemaker.completion.transformation import SagemakerConfig as SagemakerConfig - from .llms.sagemaker.chat.transformation import SagemakerChatConfig as SagemakerChatConfig + from .llms.sagemaker.completion.transformation import ( + SagemakerConfig as SagemakerConfig, + ) + from .llms.sagemaker.chat.transformation import ( + SagemakerChatConfig as SagemakerChatConfig, + ) from .llms.cohere.chat.transformation import CohereChatConfig as CohereChatConfig - from .llms.anthropic.experimental_pass_through.messages.transformation import AnthropicMessagesConfig as AnthropicMessagesConfig - from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeMessagesConfig as AmazonAnthropicClaudeMessagesConfig + from .llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig as AnthropicMessagesConfig, + ) + from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeMessagesConfig as AmazonAnthropicClaudeMessagesConfig, + ) from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig - from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig as VertexGeminiConfig - from .llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig as GoogleAIStudioGeminiConfig - from .llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig as VertexAIAnthropicConfig - from .llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import VertexAILlama3Config as VertexAILlama3Config - from .llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import VertexAIAi21Config as VertexAIAi21Config - from .llms.bedrock.chat.invoke_handler import AmazonCohereChatConfig as AmazonCohereChatConfig - from .llms.bedrock.common_utils import AmazonBedrockGlobalConfig as AmazonBedrockGlobalConfig - from .llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation import AmazonAI21Config as AmazonAI21Config - from .llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import AmazonInvokeNovaConfig as AmazonInvokeNovaConfig - from .llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation import AmazonQwen2Config as AmazonQwen2Config - from .llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import AmazonQwen3Config as AmazonQwen3Config - from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation import AmazonAnthropicConfig as AmazonAnthropicConfig - from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeConfig as AmazonAnthropicClaudeConfig - from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import AmazonCohereConfig as AmazonCohereConfig - from .llms.bedrock.chat.invoke_transformations.amazon_llama_transformation import AmazonLlamaConfig as AmazonLlamaConfig - from .llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import AmazonDeepSeekR1Config as AmazonDeepSeekR1Config - from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation import AmazonMistralConfig as AmazonMistralConfig - from .llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import AmazonMoonshotConfig as AmazonMoonshotConfig - from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import AmazonTitanConfig as AmazonTitanConfig - from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import AmazonTwelveLabsPegasusConfig as AmazonTwelveLabsPegasusConfig - from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import AmazonInvokeConfig as AmazonInvokeConfig - from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import AmazonBedrockOpenAIConfig as AmazonBedrockOpenAIConfig - from .llms.bedrock.image_generation.amazon_stability1_transformation import AmazonStabilityConfig as AmazonStabilityConfig - from .llms.bedrock.image_generation.amazon_stability3_transformation import AmazonStability3Config as AmazonStability3Config - from .llms.bedrock.image_generation.amazon_nova_canvas_transformation import AmazonNovaCanvasConfig as AmazonNovaCanvasConfig - from .llms.bedrock.embed.amazon_titan_g1_transformation import AmazonTitanG1Config as AmazonTitanG1Config - from .llms.bedrock.embed.amazon_titan_multimodal_transformation import AmazonTitanMultimodalEmbeddingG1Config as AmazonTitanMultimodalEmbeddingG1Config - from .llms.cohere.chat.v2_transformation import CohereV2ChatConfig as CohereV2ChatConfig - from .llms.bedrock.embed.cohere_transformation import BedrockCohereEmbeddingConfig as BedrockCohereEmbeddingConfig - from .llms.bedrock.embed.twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig as TwelveLabsMarengoEmbeddingConfig - from .llms.bedrock.embed.amazon_nova_transformation import AmazonNovaEmbeddingConfig as AmazonNovaEmbeddingConfig - from .llms.openai.openai import OpenAIConfig as OpenAIConfig, MistralEmbeddingConfig as MistralEmbeddingConfig - from .llms.openai.image_variations.transformation import OpenAIImageVariationConfig as OpenAIImageVariationConfig - from .llms.deepgram.audio_transcription.transformation import DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig - from .llms.topaz.image_variations.transformation import TopazImageVariationConfig as TopazImageVariationConfig - from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig as OpenAITextCompletionConfig + from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig as VertexGeminiConfig, + ) + from .llms.gemini.chat.transformation import ( + GoogleAIStudioGeminiConfig as GoogleAIStudioGeminiConfig, + ) + from .llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import ( + VertexAIAnthropicConfig as VertexAIAnthropicConfig, + ) + from .llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import ( + VertexAILlama3Config as VertexAILlama3Config, + ) + from .llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import ( + VertexAIAi21Config as VertexAIAi21Config, + ) + from .llms.bedrock.chat.invoke_handler import ( + AmazonCohereChatConfig as AmazonCohereChatConfig, + ) + from .llms.bedrock.common_utils import ( + AmazonBedrockGlobalConfig as AmazonBedrockGlobalConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation import ( + AmazonAI21Config as AmazonAI21Config, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import ( + AmazonInvokeNovaConfig as AmazonInvokeNovaConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation import ( + AmazonQwen2Config as AmazonQwen2Config, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import ( + AmazonQwen3Config as AmazonQwen3Config, + ) + from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation import ( + AmazonAnthropicConfig as AmazonAnthropicConfig, + ) + from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeConfig as AmazonAnthropicClaudeConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import ( + AmazonCohereConfig as AmazonCohereConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_llama_transformation import ( + AmazonLlamaConfig as AmazonLlamaConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import ( + AmazonDeepSeekR1Config as AmazonDeepSeekR1Config, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation import ( + AmazonMistralConfig as AmazonMistralConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import ( + AmazonMoonshotConfig as AmazonMoonshotConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import ( + AmazonTitanConfig as AmazonTitanConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import ( + AmazonTwelveLabsPegasusConfig as AmazonTwelveLabsPegasusConfig, + ) + from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + AmazonInvokeConfig as AmazonInvokeConfig, + ) + from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import ( + AmazonBedrockOpenAIConfig as AmazonBedrockOpenAIConfig, + ) + from .llms.bedrock.image_generation.amazon_stability1_transformation import ( + AmazonStabilityConfig as AmazonStabilityConfig, + ) + from .llms.bedrock.image_generation.amazon_stability3_transformation import ( + AmazonStability3Config as AmazonStability3Config, + ) + from .llms.bedrock.image_generation.amazon_nova_canvas_transformation import ( + AmazonNovaCanvasConfig as AmazonNovaCanvasConfig, + ) + from .llms.bedrock.embed.amazon_titan_g1_transformation import ( + AmazonTitanG1Config as AmazonTitanG1Config, + ) + from .llms.bedrock.embed.amazon_titan_multimodal_transformation import ( + AmazonTitanMultimodalEmbeddingG1Config as AmazonTitanMultimodalEmbeddingG1Config, + ) + from .llms.cohere.chat.v2_transformation import ( + CohereV2ChatConfig as CohereV2ChatConfig, + ) + from .llms.bedrock.embed.cohere_transformation import ( + BedrockCohereEmbeddingConfig as BedrockCohereEmbeddingConfig, + ) + from .llms.bedrock.embed.twelvelabs_marengo_transformation import ( + TwelveLabsMarengoEmbeddingConfig as TwelveLabsMarengoEmbeddingConfig, + ) + from .llms.bedrock.embed.amazon_nova_transformation import ( + AmazonNovaEmbeddingConfig as AmazonNovaEmbeddingConfig, + ) + from .llms.openai.openai import ( + OpenAIConfig as OpenAIConfig, + MistralEmbeddingConfig as MistralEmbeddingConfig, + ) + from .llms.openai.image_variations.transformation import ( + OpenAIImageVariationConfig as OpenAIImageVariationConfig, + ) + from .llms.deepgram.audio_transcription.transformation import ( + DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig, + ) + from .llms.topaz.image_variations.transformation import ( + TopazImageVariationConfig as TopazImageVariationConfig, + ) + from litellm.llms.openai.completion.transformation import ( + OpenAITextCompletionConfig as OpenAITextCompletionConfig, + ) from .llms.groq.chat.transformation import GroqChatConfig as GroqChatConfig from .llms.a2a.chat.transformation import A2AConfig as A2AConfig - from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig as VoyageEmbeddingConfig - from .llms.voyage.embedding.transformation_contextual import VoyageContextualEmbeddingConfig as VoyageContextualEmbeddingConfig - from .llms.infinity.embedding.transformation import InfinityEmbeddingConfig as InfinityEmbeddingConfig - from .llms.azure_ai.chat.transformation import AzureAIStudioConfig as AzureAIStudioConfig + from .llms.voyage.embedding.transformation import ( + VoyageEmbeddingConfig as VoyageEmbeddingConfig, + ) + from .llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig as VoyageContextualEmbeddingConfig, + ) + from .llms.infinity.embedding.transformation import ( + InfinityEmbeddingConfig as InfinityEmbeddingConfig, + ) + from .llms.azure_ai.chat.transformation import ( + AzureAIStudioConfig as AzureAIStudioConfig, + ) from .llms.mistral.chat.transformation import MistralConfig as MistralConfig - from .llms.openai.responses.transformation import OpenAIResponsesAPIConfig as OpenAIResponsesAPIConfig - from .llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig as AzureOpenAIResponsesAPIConfig - from .llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig as AzureOpenAIOSeriesResponsesAPIConfig - from .llms.xai.responses.transformation import XAIResponsesAPIConfig as XAIResponsesAPIConfig - from .llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig - from .llms.volcengine.responses.transformation import VolcEngineResponsesAPIConfig as VolcEngineResponsesAPIConfig - from .llms.manus.responses.transformation import ManusResponsesAPIConfig as ManusResponsesAPIConfig - from .llms.perplexity.responses.transformation import PerplexityResponsesConfig as PerplexityResponsesConfig - from .llms.databricks.responses.transformation import DatabricksResponsesAPIConfig as DatabricksResponsesAPIConfig - from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig - from .llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config - from .llms.anthropic.skills.transformation import AnthropicSkillsConfig as AnthropicSkillsConfig - from .llms.base_llm.skills.transformation import BaseSkillsAPIConfig as BaseSkillsAPIConfig - from .llms.gradient_ai.chat.transformation import GradientAIConfig as GradientAIConfig + from .llms.openai.responses.transformation import ( + OpenAIResponsesAPIConfig as OpenAIResponsesAPIConfig, + ) + from .llms.azure.responses.transformation import ( + AzureOpenAIResponsesAPIConfig as AzureOpenAIResponsesAPIConfig, + ) + from .llms.azure.responses.o_series_transformation import ( + AzureOpenAIOSeriesResponsesAPIConfig as AzureOpenAIOSeriesResponsesAPIConfig, + ) + from .llms.xai.responses.transformation import ( + XAIResponsesAPIConfig as XAIResponsesAPIConfig, + ) + from .llms.litellm_proxy.responses.transformation import ( + LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig, + ) + from .llms.volcengine.responses.transformation import ( + VolcEngineResponsesAPIConfig as VolcEngineResponsesAPIConfig, + ) + from .llms.manus.responses.transformation import ( + ManusResponsesAPIConfig as ManusResponsesAPIConfig, + ) + from .llms.perplexity.responses.transformation import ( + PerplexityResponsesConfig as PerplexityResponsesConfig, + ) + from .llms.databricks.responses.transformation import ( + DatabricksResponsesAPIConfig as DatabricksResponsesAPIConfig, + ) + from .llms.gemini.interactions.transformation import ( + GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig, + ) + from .llms.openai.chat.o_series_transformation import ( + OpenAIOSeriesConfig as OpenAIOSeriesConfig, + OpenAIOSeriesConfig as OpenAIO1Config, + ) + from .llms.anthropic.skills.transformation import ( + AnthropicSkillsConfig as AnthropicSkillsConfig, + ) + from .llms.base_llm.skills.transformation import ( + BaseSkillsAPIConfig as BaseSkillsAPIConfig, + ) + from .llms.gradient_ai.chat.transformation import ( + GradientAIConfig as GradientAIConfig, + ) from .llms.openai.chat.gpt_transformation import OpenAIGPTConfig as OpenAIGPTConfig - from .llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config as OpenAIGPT5Config - from .llms.openai.transcriptions.whisper_transformation import OpenAIWhisperAudioTranscriptionConfig as OpenAIWhisperAudioTranscriptionConfig - from .llms.openai.transcriptions.gpt_transformation import OpenAIGPTAudioTranscriptionConfig as OpenAIGPTAudioTranscriptionConfig - from .llms.openai.chat.gpt_audio_transformation import OpenAIGPTAudioConfig as OpenAIGPTAudioConfig + from .llms.openai.chat.gpt_5_transformation import ( + OpenAIGPT5Config as OpenAIGPT5Config, + ) + from .llms.openai.transcriptions.whisper_transformation import ( + OpenAIWhisperAudioTranscriptionConfig as OpenAIWhisperAudioTranscriptionConfig, + ) + from .llms.openai.transcriptions.gpt_transformation import ( + OpenAIGPTAudioTranscriptionConfig as OpenAIGPTAudioTranscriptionConfig, + ) + from .llms.openai.chat.gpt_audio_transformation import ( + OpenAIGPTAudioConfig as OpenAIGPTAudioConfig, + ) from .llms.nvidia_nim.chat.transformation import NvidiaNimConfig as NvidiaNimConfig - from .llms.nvidia_nim.embed import NvidiaNimEmbeddingConfig as NvidiaNimEmbeddingConfig + from .llms.nvidia_nim.embed import ( + NvidiaNimEmbeddingConfig as NvidiaNimEmbeddingConfig, + ) # Type stubs for lazy-loaded config instances openaiOSeriesConfig: OpenAIOSeriesConfig @@ -1455,21 +1649,47 @@ if TYPE_CHECKING: # Import config classes that need type stubs (for mypy) - import with _ prefix to avoid circular reference from .llms.vllm.completion.transformation import VLLMConfig as _VLLMConfig - from .llms.deepseek.chat.transformation import DeepSeekChatConfig as _DeepSeekChatConfig - from .llms.sap.chat.transformation import GenAIHubOrchestrationConfig as _GenAIHubOrchestrationConfig - from .llms.sap.embed.transformation import GenAIHubEmbeddingConfig as _GenAIHubEmbeddingConfig - from .llms.azure.chat.o_series_transformation import AzureOpenAIO1Config as _AzureOpenAIO1Config - from .llms.perplexity.chat.transformation import PerplexityChatConfig as _PerplexityChatConfig + from .llms.deepseek.chat.transformation import ( + DeepSeekChatConfig as _DeepSeekChatConfig, + ) + from .llms.sap.chat.transformation import ( + GenAIHubOrchestrationConfig as _GenAIHubOrchestrationConfig, + ) + from .llms.sap.embed.transformation import ( + GenAIHubEmbeddingConfig as _GenAIHubEmbeddingConfig, + ) + from .llms.azure.chat.o_series_transformation import ( + AzureOpenAIO1Config as _AzureOpenAIO1Config, + ) + from .llms.perplexity.chat.transformation import ( + PerplexityChatConfig as _PerplexityChatConfig, + ) from .llms.nscale.chat.transformation import NscaleConfig as _NscaleConfig - from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig as _IBMWatsonXChatConfig - from .llms.watsonx.completion.transformation import IBMWatsonXAIConfig as _IBMWatsonXAIConfig - from .llms.litellm_proxy.chat.transformation import LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig + from .llms.watsonx.chat.transformation import ( + IBMWatsonXChatConfig as _IBMWatsonXChatConfig, + ) + from .llms.watsonx.completion.transformation import ( + IBMWatsonXAIConfig as _IBMWatsonXAIConfig, + ) + from .llms.litellm_proxy.chat.transformation import ( + LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig, + ) from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig - from .llms.llamafile.chat.transformation import LlamafileChatConfig as _LlamafileChatConfig - from .llms.lm_studio.chat.transformation import LMStudioChatConfig as _LMStudioChatConfig - from .llms.lm_studio.embed.transformation import LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig - from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig as _IBMWatsonXEmbeddingConfig - from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig as _VertexGeminiConfig + from .llms.llamafile.chat.transformation import ( + LlamafileChatConfig as _LlamafileChatConfig, + ) + from .llms.lm_studio.chat.transformation import ( + LMStudioChatConfig as _LMStudioChatConfig, + ) + from .llms.lm_studio.embed.transformation import ( + LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig, + ) + from .llms.watsonx.embed.transformation import ( + IBMWatsonXEmbeddingConfig as _IBMWatsonXEmbeddingConfig, + ) + from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig as _VertexGeminiConfig, + ) # Type stubs for lazy-loaded config classes (to help mypy understand types) VLLMConfig: Type[_VLLMConfig] @@ -1489,55 +1709,122 @@ if TYPE_CHECKING: IBMWatsonXEmbeddingConfig: Type[_IBMWatsonXEmbeddingConfig] VertexAIConfig: Type[_VertexGeminiConfig] # Alias for VertexGeminiConfig - from .llms.featherless_ai.chat.transformation import FeatherlessAIConfig as FeatherlessAIConfig + from .llms.featherless_ai.chat.transformation import ( + FeatherlessAIConfig as FeatherlessAIConfig, + ) from .llms.cerebras.chat import CerebrasConfig as CerebrasConfig from .llms.baseten.chat import BasetenConfig as BasetenConfig from .llms.sambanova.chat import SambanovaConfig as SambanovaConfig - from .llms.sambanova.embedding.transformation import SambaNovaEmbeddingConfig as SambaNovaEmbeddingConfig - from .llms.fireworks_ai.chat.transformation import FireworksAIConfig as FireworksAIConfig - from .llms.fireworks_ai.completion.transformation import FireworksAITextCompletionConfig as FireworksAITextCompletionConfig - from .llms.fireworks_ai.audio_transcription.transformation import FireworksAIAudioTranscriptionConfig as FireworksAIAudioTranscriptionConfig - from .llms.fireworks_ai.embed.fireworks_ai_transformation import FireworksAIEmbeddingConfig as FireworksAIEmbeddingConfig - from .llms.friendliai.chat.transformation import FriendliaiChatConfig as FriendliaiChatConfig - from .llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig as JinaAIEmbeddingConfig + from .llms.sambanova.embedding.transformation import ( + SambaNovaEmbeddingConfig as SambaNovaEmbeddingConfig, + ) + from .llms.fireworks_ai.chat.transformation import ( + FireworksAIConfig as FireworksAIConfig, + ) + from .llms.fireworks_ai.completion.transformation import ( + FireworksAITextCompletionConfig as FireworksAITextCompletionConfig, + ) + from .llms.fireworks_ai.audio_transcription.transformation import ( + FireworksAIAudioTranscriptionConfig as FireworksAIAudioTranscriptionConfig, + ) + from .llms.fireworks_ai.embed.fireworks_ai_transformation import ( + FireworksAIEmbeddingConfig as FireworksAIEmbeddingConfig, + ) + from .llms.friendliai.chat.transformation import ( + FriendliaiChatConfig as FriendliaiChatConfig, + ) + from .llms.jina_ai.embedding.transformation import ( + JinaAIEmbeddingConfig as JinaAIEmbeddingConfig, + ) from .llms.xai.chat.transformation import XAIChatConfig as XAIChatConfig from .llms.zai.chat.transformation import ZAIChatConfig as ZAIChatConfig from .llms.aiml.chat.transformation import AIMLChatConfig as AIMLChatConfig - from .llms.volcengine.chat.transformation import VolcEngineChatConfig as VolcEngineChatConfig, VolcEngineChatConfig as VolcEngineConfig - from .llms.codestral.completion.transformation import CodestralTextCompletionConfig as CodestralTextCompletionConfig - from .llms.azure.azure import AzureOpenAIAssistantsAPIConfig as AzureOpenAIAssistantsAPIConfig + from .llms.volcengine.chat.transformation import ( + VolcEngineChatConfig as VolcEngineChatConfig, + VolcEngineChatConfig as VolcEngineConfig, + ) + from .llms.codestral.completion.transformation import ( + CodestralTextCompletionConfig as CodestralTextCompletionConfig, + ) + from .llms.azure.azure import ( + AzureOpenAIAssistantsAPIConfig as AzureOpenAIAssistantsAPIConfig, + ) from .llms.heroku.chat.transformation import HerokuChatConfig as HerokuChatConfig from .llms.cometapi.chat.transformation import CometAPIConfig as CometAPIConfig - from .llms.azure.chat.gpt_transformation import AzureOpenAIConfig as AzureOpenAIConfig - from .llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config as AzureOpenAIGPT5Config - from .llms.azure.completion.transformation import AzureOpenAITextConfig as AzureOpenAITextConfig - from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig as HostedVLLMChatConfig - from .llms.hosted_vllm.embedding.transformation import HostedVLLMEmbeddingConfig as HostedVLLMEmbeddingConfig - from .llms.github_copilot.chat.transformation import GithubCopilotConfig as GithubCopilotConfig - from .llms.github_copilot.responses.transformation import GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig - from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig + from .llms.azure.chat.gpt_transformation import ( + AzureOpenAIConfig as AzureOpenAIConfig, + ) + from .llms.azure.chat.gpt_5_transformation import ( + AzureOpenAIGPT5Config as AzureOpenAIGPT5Config, + ) + from .llms.azure.completion.transformation import ( + AzureOpenAITextConfig as AzureOpenAITextConfig, + ) + from .llms.hosted_vllm.chat.transformation import ( + HostedVLLMChatConfig as HostedVLLMChatConfig, + ) + from .llms.hosted_vllm.embedding.transformation import ( + HostedVLLMEmbeddingConfig as HostedVLLMEmbeddingConfig, + ) + from .llms.github_copilot.chat.transformation import ( + GithubCopilotConfig as GithubCopilotConfig, + ) + from .llms.github_copilot.responses.transformation import ( + GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig, + ) + from .llms.github_copilot.embedding.transformation import ( + GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig, + ) from .llms.chatgpt.chat.transformation import ChatGPTConfig as ChatGPTConfig - from .llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig as ChatGPTResponsesAPIConfig + from .llms.chatgpt.responses.transformation import ( + ChatGPTResponsesAPIConfig as ChatGPTResponsesAPIConfig, + ) from .llms.gigachat.chat.transformation import GigaChatConfig as GigaChatConfig - from .llms.gigachat.embedding.transformation import GigaChatEmbeddingConfig as GigaChatEmbeddingConfig + from .llms.gigachat.embedding.transformation import ( + GigaChatEmbeddingConfig as GigaChatEmbeddingConfig, + ) from .llms.nebius.chat.transformation import NebiusConfig as NebiusConfig from .llms.wandb.chat.transformation import WandbConfig as WandbConfig - from .llms.dashscope.chat.transformation import DashScopeChatConfig as DashScopeChatConfig - from .llms.moonshot.chat.transformation import MoonshotChatConfig as MoonshotChatConfig - from .llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig as DockerModelRunnerChatConfig + from .llms.dashscope.chat.transformation import ( + DashScopeChatConfig as DashScopeChatConfig, + ) + from .llms.moonshot.chat.transformation import ( + MoonshotChatConfig as MoonshotChatConfig, + ) + from .llms.docker_model_runner.chat.transformation import ( + DockerModelRunnerChatConfig as DockerModelRunnerChatConfig, + ) from .llms.v0.chat.transformation import V0ChatConfig as V0ChatConfig from .llms.oci.chat.transformation import OCIChatConfig as OCIChatConfig from .llms.morph.chat.transformation import MorphChatConfig as MorphChatConfig from .llms.ragflow.chat.transformation import RAGFlowConfig as RAGFlowConfig - from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig as LambdaAIChatConfig - from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig as HyperbolicChatConfig - from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig as VercelAIGatewayConfig - from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig as OVHCloudChatConfig - from .llms.ovhcloud.embedding.transformation import OVHCloudEmbeddingConfig as OVHCloudEmbeddingConfig - from .llms.cometapi.embed.transformation import CometAPIEmbeddingConfig as CometAPIEmbeddingConfig - from .llms.lemonade.chat.transformation import LemonadeChatConfig as LemonadeChatConfig - from .llms.snowflake.embedding.transformation import SnowflakeEmbeddingConfig as SnowflakeEmbeddingConfig - from .llms.amazon_nova.chat.transformation import AmazonNovaChatConfig as AmazonNovaChatConfig + from .llms.lambda_ai.chat.transformation import ( + LambdaAIChatConfig as LambdaAIChatConfig, + ) + from .llms.hyperbolic.chat.transformation import ( + HyperbolicChatConfig as HyperbolicChatConfig, + ) + from .llms.vercel_ai_gateway.chat.transformation import ( + VercelAIGatewayConfig as VercelAIGatewayConfig, + ) + from .llms.ovhcloud.chat.transformation import ( + OVHCloudChatConfig as OVHCloudChatConfig, + ) + from .llms.ovhcloud.embedding.transformation import ( + OVHCloudEmbeddingConfig as OVHCloudEmbeddingConfig, + ) + from .llms.cometapi.embed.transformation import ( + CometAPIEmbeddingConfig as CometAPIEmbeddingConfig, + ) + from .llms.lemonade.chat.transformation import ( + LemonadeChatConfig as LemonadeChatConfig, + ) + from .llms.snowflake.embedding.transformation import ( + SnowflakeEmbeddingConfig as SnowflakeEmbeddingConfig, + ) + from .llms.amazon_nova.chat.transformation import ( + AmazonNovaChatConfig as AmazonNovaChatConfig, + ) from litellm.caching.llm_caching_handler import LLMClientCache from litellm.types.llms.bedrock import COHERE_EMBEDDING_INPUT_TYPES from litellm.types.utils import ( @@ -1598,6 +1885,7 @@ if TYPE_CHECKING: # Bedrock tool name mappings instance (lazy-loaded) from litellm.caching.caching import InMemoryCache + bedrock_tool_name_mappings: InMemoryCache # Azure exception class (lazy-loaded) @@ -1616,11 +1904,15 @@ if TYPE_CHECKING: from litellm.types.integrations.datadog_llm_obs import DatadogLLMObsInitParams # Logging callback manager class and instance (lazy-loaded) - from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager + from litellm.litellm_core_utils.logging_callback_manager import ( + LoggingCallbackManager, + ) + logging_callback_manager: LoggingCallbackManager # provider_list is lazy-loaded from litellm.types.utils import LlmProviders + provider_list: List[Union[LlmProviders, str]] # Note: AmazonConverseConfig and OpenAILikeChatConfig are imported above in TYPE_CHECKING block @@ -1645,7 +1937,10 @@ def __getattr__(name: str) -> Any: global _async_client_cleanup_registered # Register async client cleanup on first access (only once) if not _async_client_cleanup_registered: - from litellm.llms.custom_httpx.async_client_cleanup import register_async_client_cleanup + from litellm.llms.custom_httpx.async_client_cleanup import ( + register_async_client_cleanup, + ) + register_async_client_cleanup() _async_client_cleanup_registered = True @@ -1662,36 +1957,45 @@ def __getattr__(name: str) -> Any: # Lazy load encoding from main.py to avoid heavy tiktoken import if name == "encoding": from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() # Check if already cached if "encoding" not in _globals: from .main import encoding as _encoding + _globals["encoding"] = _encoding return _globals["encoding"] # Lazy load bedrock_tool_name_mappings instance if name == "bedrock_tool_name_mappings": from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() # Check if already cached if "bedrock_tool_name_mappings" not in _globals: - from .llms.bedrock.chat.invoke_handler import bedrock_tool_name_mappings as _bedrock_tool_name_mappings + from .llms.bedrock.chat.invoke_handler import ( + bedrock_tool_name_mappings as _bedrock_tool_name_mappings, + ) + _globals["bedrock_tool_name_mappings"] = _bedrock_tool_name_mappings return _globals["bedrock_tool_name_mappings"] # Lazy load AzureOpenAIError exception class if name == "AzureOpenAIError": from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() # Check if already cached if "AzureOpenAIError" not in _globals: from .llms.azure.common_utils import AzureOpenAIError as _AzureOpenAIError + _globals["AzureOpenAIError"] = _AzureOpenAIError return _globals["AzureOpenAIError"] # Lazy load openaiOSeriesConfig instance if name == "openaiOSeriesConfig": from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() if "openaiOSeriesConfig" not in _globals: # Import the config class and instantiate it @@ -1709,6 +2013,7 @@ def __getattr__(name: str) -> Any: } if name in _config_instances: from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() if name not in _globals: # Import the config class and instantiate it @@ -1723,17 +2028,20 @@ def __getattr__(name: str) -> Any: # Lazy load provider_list if name == "provider_list": from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() # Check if already cached if "provider_list" not in _globals: # LlmProviders is eagerly imported above, so we can import it directly from litellm.types.utils import LlmProviders + _globals["provider_list"] = list(LlmProviders) return _globals["provider_list"] # Lazy load priority_reservation_settings instance if name == "priority_reservation_settings": from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() # Check if already cached if "priority_reservation_settings" not in _globals: @@ -1745,6 +2053,7 @@ def __getattr__(name: str) -> Any: # Lazy load logging_callback_manager instance if name == "logging_callback_manager": from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() # Check if already cached if "logging_callback_manager" not in _globals: @@ -1756,19 +2065,41 @@ def __getattr__(name: str) -> Any: # Lazy load _service_logger module if name == "_service_logger": from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() # Check if already cached if "_service_logger" not in _globals: # Import the module lazily import litellm._service_logger + _globals["_service_logger"] = litellm._service_logger return _globals["_service_logger"] # Lazy load evals module functions - if name in ["acreate_eval", "alist_evals", "aget_eval", "aupdate_eval", "adelete_eval", "acancel_eval", - "create_eval", "list_evals", "get_eval", "update_eval", "delete_eval", "cancel_eval", - "acreate_run", "alist_runs", "aget_run", "acancel_run", "adelete_run", - "create_run", "list_runs", "get_run", "cancel_run", "delete_run"]: + if name in [ + "acreate_eval", + "alist_evals", + "aget_eval", + "aupdate_eval", + "adelete_eval", + "acancel_eval", + "create_eval", + "list_evals", + "get_eval", + "update_eval", + "delete_eval", + "cancel_eval", + "acreate_run", + "alist_runs", + "aget_run", + "acancel_run", + "adelete_run", + "create_run", + "list_runs", + "get_run", + "cancel_run", + "delete_run", + ]: from litellm.evals.main import ( acreate_eval, alist_evals, @@ -1793,6 +2124,7 @@ def __getattr__(name: str) -> Any: cancel_run, delete_run, ) + return locals()[name] raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/litellm/integrations/callback_configs.json b/litellm/integrations/callback_configs.json index 6a003b8c499..c2b0c4ddce9 100644 --- a/litellm/integrations/callback_configs.json +++ b/litellm/integrations/callback_configs.json @@ -83,6 +83,27 @@ }, "description": "Datadog Logging Integration" }, + { + "id": "datadog_metrics", + "displayName": "Datadog Metrics", + "logo": "datadog.png", + "supports_key_team_logging": false, + "dynamic_params": { + "dd_api_key": { + "type": "password", + "ui_name": "API Key", + "description": "Datadog API key for authentication", + "required": true + }, + "dd_site": { + "type": "text", + "ui_name": "Site", + "description": "Datadog site URL (e.g., us5.datadoghq.com)", + "required": true + } + }, + "description": "Datadog Custom Metrics Integration" + }, { "id": "datadog_cost_management", "displayName": "Datadog Cost Management", @@ -434,4 +455,4 @@ }, "description": "SQS Queue (AWS) Logging Integration" } -] \ No newline at end of file +] diff --git a/litellm/integrations/datadog/datadog_metrics.py b/litellm/integrations/datadog/datadog_metrics.py new file mode 100644 index 00000000000..ca3e4aa5a1f --- /dev/null +++ b/litellm/integrations/datadog/datadog_metrics.py @@ -0,0 +1,288 @@ +import asyncio +import os +import time +from datetime import datetime +from typing import List, Optional, Union + +from litellm._logging import verbose_logger +from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, +) +from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus +from litellm.types.integrations.datadog_metrics import ( + DatadogMetricPoint, + DatadogMetricSeries, + DatadogMetricsPayload, +) +from litellm.types.utils import StandardLoggingPayload + + +class DatadogMetricsLogger(CustomBatchLogger): + def __init__(self, **kwargs): + self.dd_api_key = os.getenv("DD_API_KEY") + self.dd_app_key = os.getenv("DD_APP_KEY") + self.dd_site = os.getenv("DD_SITE", "datadoghq.com") + + if not self.dd_api_key: + verbose_logger.warning( + "Datadog Metrics: DD_API_KEY is required. Integration will not work." + ) + + self.upload_url = f"https://api.{self.dd_site}/api/v2/series" + + self.async_client = get_async_httpx_client( + llm_provider=httpxSpecialProvider.LoggingCallback + ) + + # Initialize lock + self.flush_lock = asyncio.Lock() + + # Check if flush_lock is already in kwargs to avoid double passing + kwargs["flush_lock"] = self.flush_lock + + # Send metrics more quickly to datadog (every 5 seconds) + if "flush_interval" not in kwargs: + kwargs["flush_interval"] = 5 + + super().__init__(**kwargs) + + # Start periodic flush task + asyncio.create_task(self.periodic_flush()) + + def _extract_tags( + self, + log: StandardLoggingPayload, + status_code: Optional[Union[str, int]] = None, + ) -> List[str]: + """ + Builds the list of tags for a Datadog metric point + """ + from litellm.integrations.datadog.datadog_handler import ( + get_datadog_env, + get_datadog_hostname, + get_datadog_pod_name, + get_datadog_service, + ) + + # Base tags + tags = [ + f"env:{get_datadog_env()}", + f"service:{get_datadog_service()}", + f"version:{os.getenv('DD_VERSION', 'unknown')}", + f"HOSTNAME:{get_datadog_hostname()}", + f"POD_NAME:{get_datadog_pod_name()}", + ] + + # Add metric-specific tags + if provider := log.get("custom_llm_provider"): + tags.append(f"provider:{provider}") + + if model := log.get("model"): + tags.append(f"model_name:{model}") + + if model_group := log.get("model_group"): + tags.append(f"model_group:{model_group}") + + if status_code is not None: + tags.append(f"status_code:{status_code}") + + # Extract team tag + metadata = log.get("metadata", {}) or {} + team_tag = ( + metadata.get("user_api_key_team_alias") + or metadata.get("team_alias") # type: ignore + or metadata.get("user_api_key_team_id") + or metadata.get("team_id") # type: ignore + ) + + if team_tag: + tags.append(f"team:{team_tag}") + + return tags + + def _add_metrics_from_log( + self, + log: StandardLoggingPayload, + kwargs: dict, + status_code: Union[str, int] = "200", + ): + """ + Extracts latencies and appends Datadog metric series to the queue + """ + tags = self._extract_tags(log, status_code=status_code) + + # We record metrics with the end_time as the timestamp for the point + end_time_dt = kwargs.get("end_time") or datetime.now() + timestamp = int(end_time_dt.timestamp()) + + # 1. Total Request Latency Metric (End to End) + start_time_dt = kwargs.get("start_time") + if start_time_dt and end_time_dt: + total_duration = (end_time_dt - start_time_dt).total_seconds() + series_total_latency: DatadogMetricSeries = { + "metric": "litellm.request.total_latency", + "type": 3, # gauge + "points": [{"timestamp": timestamp, "value": total_duration}], + "tags": tags, + } + self.log_queue.append(series_total_latency) + + # 2. LLM API Latency Metric (Provider alone) + api_call_start_time = kwargs.get("api_call_start_time") + if api_call_start_time and end_time_dt: + llm_api_duration = (end_time_dt - api_call_start_time).total_seconds() + series_llm_latency: DatadogMetricSeries = { + "metric": "litellm.llm_api.latency", + "type": 3, # gauge + "points": [{"timestamp": timestamp, "value": llm_api_duration}], + "tags": tags, + } + self.log_queue.append(series_llm_latency) + + # 3. Request Count / Status Code + series_count: DatadogMetricSeries = { + "metric": "litellm.llm_api.request_count", + "type": 1, # count + "points": [{"timestamp": timestamp, "value": 1.0}], + "tags": tags, + } + self.log_queue.append(series_count) + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + try: + standard_logging_object: Optional[StandardLoggingPayload] = kwargs.get( + "standard_logging_object", None + ) + + if standard_logging_object is None: + return + + self._add_metrics_from_log( + log=standard_logging_object, kwargs=kwargs, status_code="200" + ) + + if len(self.log_queue) >= self.batch_size: + await self.flush_queue() + + except Exception as e: + verbose_logger.exception( + f"Datadog Metrics: Error in async_log_success_event: {str(e)}" + ) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + try: + standard_logging_object: Optional[StandardLoggingPayload] = kwargs.get( + "standard_logging_object", None + ) + + if standard_logging_object is None: + return + + # Extract status code from error information + status_code = "500" # default + error_information = ( + standard_logging_object.get("error_information", {}) or {} + ) + if "error_code" in error_information and error_information["error_code"] is not None: # type: ignore + status_code = str(error_information["error_code"]) # type: ignore + + self._add_metrics_from_log( + log=standard_logging_object, kwargs=kwargs, status_code=status_code + ) + + if len(self.log_queue) >= self.batch_size: + await self.flush_queue() + + except Exception as e: + verbose_logger.exception( + f"Datadog Metrics: Error in async_log_failure_event: {str(e)}" + ) + + async def async_send_batch(self): + if not self.log_queue: + return + + try: + # We must only send the current batch, so copy and clear log queue + batch = self.log_queue.copy() + # Note: CustomBatchLogger clears queue in flush_queue, but we'll manually copy what we need + + payload_data: DatadogMetricsPayload = {"series": batch} + + await self._upload_to_datadog(payload_data) + + except Exception as e: + verbose_logger.exception( + f"Datadog Metrics: Error in async_send_batch: {str(e)}" + ) + + async def _upload_to_datadog(self, payload: DatadogMetricsPayload): + if not self.dd_api_key: + return + + headers = { + "Content-Type": "application/json", + "DD-API-KEY": self.dd_api_key, + } + + if self.dd_app_key: + headers["DD-APPLICATION-KEY"] = self.dd_app_key + + import gzip + + from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + + json_data = safe_dumps(payload) + compressed_data = gzip.compress(json_data.encode("utf-8")) + headers["Content-Encoding"] = "gzip" + + response = await self.async_client.post( + self.upload_url, content=compressed_data, headers=headers # type: ignore + ) + + response.raise_for_status() + + verbose_logger.debug( + f"Datadog Metrics: Uploaded {len(payload['series'])} metric points. Status: {response.status_code}" + ) + + async def async_health_check(self) -> IntegrationHealthCheckStatus: + """ + Check if the service is healthy + """ + try: + # Send a test metric point to Datadog + test_metric_point: DatadogMetricPoint = { + "timestamp": int(time.time()), + "value": 1.0, + } + test_metric_series: DatadogMetricSeries = { + "metric": "litellm.health_check", + "type": 3, # Gauge + "points": [test_metric_point], + "tags": ["env:health_check"], + } + + payload_data: DatadogMetricsPayload = {"series": [test_metric_series]} + + await self._upload_to_datadog(payload_data) + + return IntegrationHealthCheckStatus( + status="healthy", + error_message=None, + ) + except Exception as e: + return IntegrationHealthCheckStatus( + status="unhealthy", + error_message=str(e), + ) + + async def get_request_response_payload( + self, + request_id: str, + start_time_utc: Optional[datetime], + end_time_utc: Optional[datetime], + ) -> Optional[dict]: + pass diff --git a/litellm/litellm_core_utils/custom_logger_registry.py b/litellm/litellm_core_utils/custom_logger_registry.py index fc73701ea9d..2d483f78613 100644 --- a/litellm/litellm_core_utils/custom_logger_registry.py +++ b/litellm/litellm_core_utils/custom_logger_registry.py @@ -20,6 +20,7 @@ from litellm.integrations.braintrust_logging import BraintrustLogger from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger from litellm.integrations.datadog.datadog import DataDogLogger from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger +from litellm.integrations.datadog.datadog_metrics import DatadogMetricsLogger from litellm.integrations.deepeval import DeepEvalLogger from litellm.integrations.dotprompt import DotpromptManager from litellm.integrations.focus.focus_logger import FocusLogger @@ -66,6 +67,7 @@ class CustomLoggerRegistry: "prometheus": PrometheusLogger, "datadog": DataDogLogger, "datadog_llm_observability": DataDogLLMObsLogger, + "datadog_metrics": DatadogMetricsLogger, "gcs_bucket": GCSBucketLogger, "opik": OpikLogger, "argilla": ArgillaLogger, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 0601e7e8455..98f573d471b 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -134,6 +134,7 @@ from ..integrations.azure_storage.azure_storage import AzureBlobStorageLogger from ..integrations.custom_prompt_management import CustomPromptManagement from ..integrations.datadog.datadog import DataDogLogger from ..integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger +from ..integrations.datadog.datadog_metrics import DatadogMetricsLogger from ..integrations.dotprompt import DotpromptManager from ..integrations.dynamodb import DyanmoDBLogger from ..integrations.galileo import GalileoObserve @@ -1653,9 +1654,7 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details[ "standard_logging_object" - ] = self._build_standard_logging_payload( - logging_result, start_time, end_time - ) + ] = self._build_standard_logging_payload(logging_result, start_time, end_time) if ( standard_logging_payload := self.model_call_details.get( @@ -2518,9 +2517,7 @@ class Logging(LiteLLMLoggingBaseClass): ## STANDARDIZED LOGGING PAYLOAD self.model_call_details[ "standard_logging_object" - ] = self._build_standard_logging_payload( - result, start_time, end_time - ) + ] = self._build_standard_logging_payload(result, start_time, end_time) # print standard logging payload if ( @@ -3665,6 +3662,14 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _datadog_llm_obs_logger = DataDogLLMObsLogger() _in_memory_loggers.append(_datadog_llm_obs_logger) return _datadog_llm_obs_logger # type: ignore + elif logging_integration == "datadog_metrics": + for callback in _in_memory_loggers: + if isinstance(callback, DatadogMetricsLogger): + return callback # type: ignore + + _datadog_metrics_logger = DatadogMetricsLogger() + _in_memory_loggers.append(_datadog_metrics_logger) + return _datadog_metrics_logger # type: ignore elif logging_integration == "azure_sentinel": for callback in _in_memory_loggers: if isinstance(callback, AzureSentinelLogger): @@ -4214,6 +4219,10 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 for callback in _in_memory_loggers: if isinstance(callback, DataDogLLMObsLogger): return callback + elif logging_integration == "datadog_metrics": + for callback in _in_memory_loggers: + if isinstance(callback, DatadogMetricsLogger): + return callback elif logging_integration == "azure_sentinel": for callback in _in_memory_loggers: if isinstance(callback, AzureSentinelLogger): @@ -4695,9 +4704,11 @@ class StandardLoggingPayloadSetup: ).model_dump() if isinstance(_raw, dict): if ResponseAPILoggingUtils._is_response_api_usage(_raw): - return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( - _raw - ).model_dump() + return ( + ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + _raw + ).model_dump() + ) return _raw if isinstance(_raw, Usage): return _raw.model_dump() @@ -5543,4 +5554,3 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload: model_parameters={"stream": True}, hidden_params=hidden_params, ) - diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 62ca6dc2ae2..9ecae363ed7 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -390,9 +390,7 @@ def get_logging_caching_headers(request_data: Dict) -> Optional[Dict]: ) if "applied_policies" in _metadata: - headers["x-litellm-applied-policies"] = ",".join( - _metadata["applied_policies"] - ) + headers["x-litellm-applied-policies"] = ",".join(_metadata["applied_policies"]) if "policy_sources" in _metadata: sources = _metadata["policy_sources"] @@ -449,9 +447,7 @@ def add_policy_to_applied_policies_header( request_data["metadata"] = _metadata -def add_policy_sources_to_metadata( - request_data: Dict, policy_sources: Dict[str, str] -): +def add_policy_sources_to_metadata(request_data: Dict, policy_sources: Dict[str, str]): """ Store policy match reasons in metadata for x-litellm-policy-sources header. diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index f3bed3656f6..48e3e8c0ad0 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -226,6 +226,7 @@ async def health_services_endpoint( # noqa: PLR0915 "custom_callback_api", "langsmith", "datadog", + "datadog_metrics", "datadog_llm_observability", "generic_api", "arize", @@ -280,6 +281,21 @@ async def health_services_endpoint( # noqa: PLR0915 else "Datadog is healthy" ), } + elif service == "datadog_metrics": + from litellm.integrations.datadog.datadog_metrics import ( + DatadogMetricsLogger, + ) + + datadog_metrics_logger = DatadogMetricsLogger() + response = await datadog_metrics_logger.async_health_check() + return { + "status": response["status"], + "message": ( + response["error_message"] + if response["status"] == "unhealthy" + else "Datadog Metrics is healthy" + ), + } elif service == "arize": from litellm.integrations.arize.arize import ArizeLogger diff --git a/litellm/types/integrations/datadog_metrics.py b/litellm/types/integrations/datadog_metrics.py new file mode 100644 index 00000000000..4b21881ed5f --- /dev/null +++ b/litellm/types/integrations/datadog_metrics.py @@ -0,0 +1,19 @@ +from typing import List + +from typing_extensions import TypedDict + + +class DatadogMetricPoint(TypedDict): + timestamp: int # Unix epoch seconds + value: float # The metric value + + +class DatadogMetricSeries(TypedDict): + metric: str + type: int # 1=count, 2=rate, 3=gauge, distribution is submitted as type=3, but distributions use a different endpoint /api/v1/distribution_points, wait actually according to DD /api/v2/series: 0=unspecified, 1=count, 2=rate, 3=gauge. For histogram/distribution we use type 3 or 1. + points: List[DatadogMetricPoint] + tags: List[str] + + +class DatadogMetricsPayload(TypedDict): + series: List[DatadogMetricSeries] diff --git a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py new file mode 100644 index 00000000000..1008a4f78cf --- /dev/null +++ b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py @@ -0,0 +1,272 @@ +import os +import time +from datetime import datetime, timedelta +from unittest.mock import AsyncMock + +import pytest +from httpx import Response + +from litellm.integrations.datadog.datadog_metrics import DatadogMetricsLogger +from litellm.types.utils import StandardLoggingPayload + + +@pytest.fixture +def clean_env(): + """Set test env vars and restore originals after test.""" + keys = ["DD_API_KEY", "DD_APP_KEY", "DD_SITE", "DD_ENV", "DD_SERVICE", "DD_VERSION"] + originals = {k: os.environ.get(k) for k in keys} + + os.environ["DD_API_KEY"] = "test_api_key" + os.environ["DD_APP_KEY"] = "test_app_key" + os.environ["DD_SITE"] = "test.datadoghq.com" + os.environ["DD_ENV"] = "test-env" + os.environ["DD_SERVICE"] = "test-service" + os.environ["DD_VERSION"] = "1.0.0" + + yield + + for k, v in originals.items(): + if v is not None: + os.environ[k] = v + elif k in os.environ: + del os.environ[k] + + +@pytest.mark.asyncio +async def test_init(clean_env): + """Test initialization sets up clients and url correctly.""" + logger = DatadogMetricsLogger() + assert logger.dd_api_key == "test_api_key" + assert logger.dd_site == "test.datadoghq.com" + assert logger.upload_url == "https://api.test.datadoghq.com/api/v2/series" + + +@pytest.mark.asyncio +async def test_extract_tags(clean_env): + """Test tag extraction from a StandardLoggingPayload.""" + logger = DatadogMetricsLogger() + + payload = StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4o", + model_group="gpt-4", + metadata={"user_api_key_team_alias": "test-team"}, + ) + + tags = logger._extract_tags(log=payload, status_code="200") + + assert "env:test-env" in tags + assert "service:test-service" in tags + assert "version:1.0.0" in tags + assert "provider:openai" in tags + assert "model_name:gpt-4o" in tags + assert "model_group:gpt-4" in tags + assert "status_code:200" in tags + assert "team:test-team" in tags + + +@pytest.mark.asyncio +async def test_extract_tags_no_team(clean_env): + """Test tag extraction when no team info is present.""" + logger = DatadogMetricsLogger() + + payload = StandardLoggingPayload( + custom_llm_provider="anthropic", + model="claude-3-sonnet", + ) + + tags = logger._extract_tags(log=payload, status_code="500") + + assert "provider:anthropic" in tags + assert "model_name:claude-3-sonnet" in tags + assert "status_code:500" in tags + assert not any(tag.startswith("team:") for tag in tags) + + +@pytest.mark.asyncio +async def test_add_metrics_from_log(clean_env): + """Test that _add_metrics_from_log appends the correct metric series to the queue.""" + logger = DatadogMetricsLogger(batch_size=100) + + now = datetime.now() + start_time = now - timedelta(seconds=2) + api_call_start_time = now - timedelta(seconds=1) + + payload = StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4o", + ) + + kwargs = { + "start_time": start_time, + "api_call_start_time": api_call_start_time, + "end_time": now, + } + + logger._add_metrics_from_log(log=payload, kwargs=kwargs, status_code="200") + + # Should have 3 series: total_latency, llm_api_latency, request_count + assert len(logger.log_queue) == 3 + + metrics = {s["metric"]: s for s in logger.log_queue} + + # Total latency ~2s + total = metrics["litellm.request.total_latency"] + assert total["type"] == 3 # gauge + assert abs(total["points"][0]["value"] - 2.0) < 0.1 + + # LLM API latency ~1s + llm = metrics["litellm.llm_api.latency"] + assert llm["type"] == 3 # gauge + assert abs(llm["points"][0]["value"] - 1.0) < 0.1 + + # Request count + count = metrics["litellm.llm_api.request_count"] + assert count["type"] == 1 # count + assert count["points"][0]["value"] == 1.0 + assert "status_code:200" in count["tags"] + + +@pytest.mark.asyncio +async def test_async_log_success_event(clean_env): + """Test that success events are added to the queue.""" + logger = DatadogMetricsLogger(batch_size=100) + + now = datetime.now() + start_time = now - timedelta(seconds=1) + + await logger.async_log_success_event( + kwargs={ + "standard_logging_object": StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4o", + ), + "start_time": start_time, + "end_time": now, + }, + response_obj=None, + start_time=start_time, + end_time=now, + ) + + # At least request_count and total_latency + assert len(logger.log_queue) >= 2 + + +@pytest.mark.asyncio +async def test_async_log_success_event_no_standard_logging_object(clean_env): + """Test that events without standard_logging_object are skipped.""" + logger = DatadogMetricsLogger(batch_size=100) + + await logger.async_log_success_event( + kwargs={}, + response_obj=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + assert len(logger.log_queue) == 0 + + +@pytest.mark.asyncio +async def test_async_log_failure_event_extracts_status_code(clean_env): + """Test that failure events extract the error status code.""" + logger = DatadogMetricsLogger(batch_size=100) + + now = datetime.now() + start_time = now - timedelta(seconds=1) + + await logger.async_log_failure_event( + kwargs={ + "standard_logging_object": StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4o", + error_information={"error_code": "429"}, + ), + "start_time": start_time, + "end_time": now, + }, + response_obj=None, + start_time=start_time, + end_time=now, + ) + + count_series = next( + (s for s in logger.log_queue if s["metric"] == "litellm.llm_api.request_count"), + None, + ) + assert count_series is not None + assert "status_code:429" in count_series["tags"] + + +@pytest.mark.asyncio +async def test_async_log_failure_event_default_status_code(clean_env): + """Test that failure events default to 500 when no error_code is present.""" + logger = DatadogMetricsLogger(batch_size=100) + + now = datetime.now() + + await logger.async_log_failure_event( + kwargs={ + "standard_logging_object": StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4o", + ), + "start_time": now, + "end_time": now, + }, + response_obj=None, + start_time=now, + end_time=now, + ) + + count_series = next( + (s for s in logger.log_queue if s["metric"] == "litellm.llm_api.request_count"), + None, + ) + assert count_series is not None + assert "status_code:500" in count_series["tags"] + + +@pytest.mark.asyncio +async def test_async_send_batch(clean_env): + """Test that async_send_batch uploads metrics to Datadog.""" + logger = DatadogMetricsLogger() + logger.async_client = AsyncMock() + logger.async_client.post.return_value = Response(202, json={"status": "ok"}) + + # Manually add a metric series to the queue + logger.log_queue = [ + { + "metric": "litellm.request.total_latency", + "type": 3, + "points": [{"timestamp": int(time.time()), "value": 1.5}], + "tags": ["env:test"], + } + ] + + await logger.async_send_batch() + + assert logger.async_client.post.called + call_args = logger.async_client.post.call_args + assert call_args[0][0] == "https://api.test.datadoghq.com/api/v2/series" + + # Verify gzip + JSON payload + import gzip + import json + + compressed = call_args[1]["content"] + payload = json.loads(gzip.decompress(compressed).decode("utf-8")) + assert len(payload["series"]) == 1 + assert payload["series"][0]["metric"] == "litellm.request.total_latency" + + +@pytest.mark.asyncio +async def test_async_send_batch_empty_queue(clean_env): + """Test that async_send_batch does nothing when queue is empty.""" + logger = DatadogMetricsLogger() + logger.async_client = AsyncMock() + + await logger.async_send_batch() + + assert not logger.async_client.post.called From cd60e3d4e0add4e3a84ce6c5f284a4863b8a9cad Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Wed, 25 Feb 2026 22:27:51 +0530 Subject: [PATCH 06/52] fix: req changes --- docs/my-website/docs/observability/datadog.md | 2 +- litellm/integrations/datadog/datadog_metrics.py | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/my-website/docs/observability/datadog.md b/docs/my-website/docs/observability/datadog.md index ef48594cbfa..e83cfcbafe0 100644 --- a/docs/my-website/docs/observability/datadog.md +++ b/docs/my-website/docs/observability/datadog.md @@ -193,7 +193,7 @@ All metrics include the following tags: `env`, `service`, `version`, `HOSTNAME`, ```yaml model_list: - - model_name: gpt-3.5-turbo + - model_name: gpt-3.5-turbo litellm_params: model: gpt-3.5-turbo litellm_settings: diff --git a/litellm/integrations/datadog/datadog_metrics.py b/litellm/integrations/datadog/datadog_metrics.py index ca3e4aa5a1f..43dd0662604 100644 --- a/litellm/integrations/datadog/datadog_metrics.py +++ b/litellm/integrations/datadog/datadog_metrics.py @@ -1,4 +1,5 @@ import asyncio +import gzip import os import time from datetime import datetime @@ -6,6 +7,7 @@ from typing import List, Optional, Union from litellm._logging import verbose_logger from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -39,8 +41,9 @@ class DatadogMetricsLogger(CustomBatchLogger): # Initialize lock self.flush_lock = asyncio.Lock() - # Check if flush_lock is already in kwargs to avoid double passing - kwargs["flush_lock"] = self.flush_lock + # Only set flush_lock if not already provided by caller + if "flush_lock" not in kwargs: + kwargs["flush_lock"] = self.flush_lock # Send metrics more quickly to datadog (every 5 seconds) if "flush_interval" not in kwargs: @@ -230,10 +233,6 @@ class DatadogMetricsLogger(CustomBatchLogger): if self.dd_app_key: headers["DD-APPLICATION-KEY"] = self.dd_app_key - import gzip - - from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - json_data = safe_dumps(payload) compressed_data = gzip.compress(json_data.encode("utf-8")) headers["Content-Encoding"] = "gzip" From c08ef3f96a8441ec79c0d4fa61ffa02702000885 Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Wed, 25 Feb 2026 22:28:59 +0530 Subject: [PATCH 07/52] Update docs/my-website/docs/observability/datadog.md Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> From 94be31a8169496c96e378cfc70bc2d7a7f89efd5 Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Thu, 26 Feb 2026 11:30:33 +0530 Subject: [PATCH 08/52] fix req changes --- litellm/__init__.py | 757 +++++------------- .../integrations/datadog/datadog_metrics.py | 25 +- litellm/litellm_core_utils/litellm_logging.py | 35 +- .../health_endpoints/_health_endpoints.py | 2 +- litellm/types/integrations/datadog_metrics.py | 2 +- 5 files changed, 247 insertions(+), 574 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 41f5b0ff42d..4531e017606 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -75,7 +75,6 @@ from litellm.constants import ( ) import httpx import dotenv - # register_async_client_cleanup is lazy-loaded and called on first access litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV" @@ -106,8 +105,8 @@ _custom_logger_compatible_callbacks_literal = Literal[ "prometheus", "otel", "datadog", - "datadog_llm_observability", "datadog_metrics", + "datadog_llm_observability", "galileo", "braintrust", "arize", @@ -149,9 +148,7 @@ _known_custom_logger_compatible_callbacks: List = list( get_args(_custom_logger_compatible_callbacks_literal) ) callbacks: List[ - Union[ - Callable, _custom_logger_compatible_callbacks_literal, "CustomLogger" - ] # CustomLogger is lazy-loaded + Union[Callable, _custom_logger_compatible_callbacks_literal, "CustomLogger"] # CustomLogger is lazy-loaded ] = [] callback_settings: Dict[str, Dict[str, Any]] = {} initialized_langfuse_clients: int = 0 @@ -161,50 +158,42 @@ prometheus_initialize_budget_metrics: Optional[bool] = False require_auth_for_metrics_endpoint: Optional[bool] = False argilla_batch_size: Optional[int] = None datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload. -gcs_pub_sub_use_v1: Optional[ - bool -] = False # if you want to use v1 gcs pubsub logged payload -generic_api_use_v1: Optional[ - bool -] = False # if you want to use v1 generic api logged payload +gcs_pub_sub_use_v1: Optional[bool] = ( + False # if you want to use v1 gcs pubsub logged payload +) +generic_api_use_v1: Optional[bool] = ( + False # if you want to use v1 generic api logged payload +) argilla_transformation_object: Optional[Dict[str, Any]] = None -_async_input_callback: List[ - Union[str, Callable, "CustomLogger"] -] = ( # CustomLogger is lazy-loaded +_async_input_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. -_async_success_callback: List[ - Union[str, Callable, "CustomLogger"] -] = ( # CustomLogger is lazy-loaded +_async_success_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. -_async_failure_callback: List[ - Union[str, Callable, "CustomLogger"] -] = ( # CustomLogger is lazy-loaded +_async_failure_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. pre_call_rules: List[Callable] = [] post_call_rules: List[Callable] = [] turn_off_message_logging: Optional[bool] = False -standard_logging_payload_excluded_fields: Optional[ - List[str] -] = None # Fields to exclude from StandardLoggingPayload before callbacks receive it +standard_logging_payload_excluded_fields: Optional[List[str]] = None # Fields to exclude from StandardLoggingPayload before callbacks receive it log_raw_request_response: bool = False redact_messages_in_exceptions: Optional[bool] = False redact_user_api_key_info: Optional[bool] = False filter_invalid_headers: Optional[bool] = False -add_user_information_to_llm_headers: Optional[ - bool -] = None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers +add_user_information_to_llm_headers: Optional[bool] = ( + None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers +) store_audit_logs = False # Enterprise feature, allow users to see audit logs ### end of callbacks ############# -email: Optional[ - str -] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -token: Optional[ - str -] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +email: Optional[str] = ( + None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +) +token: Optional[str] = ( + None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +) telemetry = True max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False)) @@ -263,9 +252,9 @@ use_client: bool = False ssl_verify: Union[str, bool] = True ssl_security_level: Optional[str] = None ssl_certificate: Optional[str] = None -ssl_ecdh_curve: Optional[ - str -] = None # Set to 'X25519' to disable PQC and improve performance +ssl_ecdh_curve: Optional[str] = ( + None # Set to 'X25519' to disable PQC and improve performance +) disable_streaming_logging: bool = False disable_token_counter: bool = False disable_add_transform_inline_image_block: bool = False @@ -315,20 +304,24 @@ enable_loadbalancing_on_batch_endpoints: Optional[bool] = None enable_caching_on_provider_specific_optional_params: bool = ( False # feature-flag for caching on optional params - e.g. 'top_k' ) -caching: bool = False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -caching_with_models: bool = False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 -cache: Optional[ - "Cache" -] = None # cache object <- use this - https://docs.litellm.ai/docs/caching +caching: bool = ( + False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +) +caching_with_models: bool = ( + False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 +) +cache: Optional["Cache"] = ( + None # cache object <- use this - https://docs.litellm.ai/docs/caching +) default_in_memory_ttl: Optional[float] = None default_redis_ttl: Optional[float] = None default_redis_batch_cache_expiry: Optional[float] = None model_alias_map: Dict[str, str] = {} model_group_settings: Optional["ModelGroupSettings"] = None max_budget: float = 0.0 # set the max budget across all providers -budget_duration: Optional[ - str -] = None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). +budget_duration: Optional[str] = ( + None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). +) default_soft_budget: float = ( DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0 ) @@ -337,7 +330,9 @@ forward_traceparent_to_llm_provider: bool = False _current_cost = 0.0 # private variable, used if max budget is set error_logs: Dict = {} -add_function_to_prompt: bool = False # if function calling not supported by api, append function call details to system prompt +add_function_to_prompt: bool = ( + False # if function calling not supported by api, append function call details to system prompt +) client_session: Optional[httpx.Client] = None aclient_session: Optional[httpx.AsyncClient] = None model_fallbacks: Optional[List] = None # Deprecated for 'litellm.fallbacks' @@ -384,7 +379,9 @@ prometheus_emit_stream_label: bool = False disable_add_prefix_to_prompt: bool = ( False # used by anthropic, to disable adding prefix to prompt ) -disable_copilot_system_to_assistant: bool = False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. +disable_copilot_system_to_assistant: bool = ( + False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. +) public_mcp_servers: Optional[List[str]] = None public_model_groups: Optional[List[str]] = None public_agent_groups: Optional[List[str]] = None @@ -403,13 +400,17 @@ if TYPE_CHECKING: ######## Networking Settings ######## -use_aiohttp_transport: bool = True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead. +use_aiohttp_transport: bool = ( + True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead. +) aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings disable_aiohttp_transport: bool = False # Set this to true to use httpx instead disable_aiohttp_trust_env: bool = ( False # When False, aiohttp will respect HTTP(S)_PROXY env vars ) -force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. +force_ipv4: bool = ( + False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. +) network_mock: bool = False # When True, use mock transport — no real network calls ####### STOP SEQUENCE LIMIT ####### @@ -424,13 +425,13 @@ context_window_fallbacks: Optional[List] = None content_policy_fallbacks: Optional[List] = None allowed_fails: int = 3 allow_dynamic_callback_disabling: bool = True -num_retries_per_request: Optional[ - int -] = None # for the request overall (incl. fallbacks + model retries) +num_retries_per_request: Optional[int] = ( + None # for the request overall (incl. fallbacks + model retries) +) ####### SECRET MANAGERS ##################### -secret_manager_client: Optional[ - Any -] = None # list of instantiated key management clients - e.g. azure kv, infisical, etc. +secret_manager_client: Optional[Any] = ( + None # list of instantiated key management clients - e.g. azure kv, infisical, etc. +) _google_kms_resource_name: Optional[str] = None _key_management_system: Optional["KeyManagementSystem"] = None # Note: KeyManagementSettings must be eagerly imported because _key_management_settings @@ -443,12 +444,12 @@ output_parse_pii: bool = False from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map model_cost = get_model_cost_map(url=model_cost_map_url) -cost_discount_config: Dict[ - str, float -] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount -cost_margin_config: Dict[ - str, Union[float, Dict[str, float]] -] = {} # Provider-specific or global cost margins. Examples: +cost_discount_config: Dict[str, float] = ( + {} +) # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount +cost_margin_config: Dict[str, Union[float, Dict[str, float]]] = ( + {} +) # Provider-specific or global cost margins. Examples: # Percentage: {"openai": 0.10} = 10% margin # Fixed: {"openai": {"fixed_amount": 0.001}} = $0.001 per request # Global: {"global": 0.05} = 5% global margin on all providers @@ -1107,12 +1108,10 @@ openai_video_generation_models = ["sora-2"] # Import KeyManagementSettings here (before utils import) because _key_management_settings # is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils) from litellm.types.secret_managers.main import KeyManagementSettings - _key_management_settings: KeyManagementSettings = KeyManagementSettings() # client must be imported immediately as it's used as a decorator at function definition time from .utils import client - # Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py # (which imports tiktoken) at import time @@ -1141,7 +1140,6 @@ from .llms.topaz.common_utils import TopazModelInfo # OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access # OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access from .llms.xai.common_utils import XAIModelInfo - # PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json) # All remaining configs are now lazy loaded - see _lazy_imports_registry.py @@ -1223,7 +1221,6 @@ from .batch_completion.main import * # type: ignore from .rerank_api.main import * from .llms.anthropic.experimental_pass_through.messages.handler import * from .responses.main import * - # Interactions API is available as litellm.interactions module # Usage: litellm.interactions.create(), litellm.interactions.get(), etc. from . import interactions @@ -1282,12 +1279,12 @@ from . import rag from .types.llms.custom_llm import CustomLLMItem custom_provider_map: List[CustomLLMItem] = [] -_custom_providers: List[ - str -] = [] # internal helper util, used to track names of custom providers -disable_hf_tokenizer_download: Optional[ - bool -] = None # disable huggingface tokenizer download. Defaults to openai clk100 +_custom_providers: List[str] = ( + [] +) # internal helper util, used to track names of custom providers +disable_hf_tokenizer_download: Optional[bool] = ( + None # disable huggingface tokenizer download. Defaults to openai clk100 +) global_disable_no_log_param: bool = False ### CLI UTILITIES ### @@ -1326,318 +1323,128 @@ if TYPE_CHECKING: from litellm.caching.caching import Cache # Type stubs for lazy-loaded configs to help mypy - from .llms.bedrock.chat.converse_transformation import ( - AmazonConverseConfig as AmazonConverseConfig, - ) - from .llms.openai_like.chat.handler import ( - OpenAILikeChatConfig as OpenAILikeChatConfig, - ) - from .llms.galadriel.chat.transformation import ( - GaladrielChatConfig as GaladrielChatConfig, - ) + from .llms.bedrock.chat.converse_transformation import AmazonConverseConfig as AmazonConverseConfig + from .llms.openai_like.chat.handler import OpenAILikeChatConfig as OpenAILikeChatConfig + from .llms.galadriel.chat.transformation import GaladrielChatConfig as GaladrielChatConfig from .llms.github.chat.transformation import GithubChatConfig as GithubChatConfig - from .llms.azure_ai.anthropic.transformation import ( - AzureAnthropicConfig as AzureAnthropicConfig, - ) + from .llms.azure_ai.anthropic.transformation import AzureAnthropicConfig as AzureAnthropicConfig from .llms.bytez.chat.transformation import BytezChatConfig as BytezChatConfig - from .llms.compactifai.chat.transformation import ( - CompactifAIChatConfig as CompactifAIChatConfig, - ) + from .llms.compactifai.chat.transformation import CompactifAIChatConfig as CompactifAIChatConfig from .llms.empower.chat.transformation import EmpowerChatConfig as EmpowerChatConfig from .llms.minimax.chat.transformation import MinimaxChatConfig as MinimaxChatConfig - from .llms.aiohttp_openai.chat.transformation import ( - AiohttpOpenAIChatConfig as AiohttpOpenAIChatConfig, - ) - from .llms.huggingface.chat.transformation import ( - HuggingFaceChatConfig as HuggingFaceChatConfig, - ) - from .llms.huggingface.embedding.transformation import ( - HuggingFaceEmbeddingConfig as HuggingFaceEmbeddingConfig, - ) + from .llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig as AiohttpOpenAIChatConfig + from .llms.huggingface.chat.transformation import HuggingFaceChatConfig as HuggingFaceChatConfig + from .llms.huggingface.embedding.transformation import HuggingFaceEmbeddingConfig as HuggingFaceEmbeddingConfig from .llms.oobabooga.chat.transformation import OobaboogaConfig as OobaboogaConfig from .llms.maritalk import MaritalkConfig as MaritalkConfig - from .llms.openrouter.chat.transformation import ( - OpenrouterConfig as OpenrouterConfig, - ) + from .llms.openrouter.chat.transformation import OpenrouterConfig as OpenrouterConfig from .llms.datarobot.chat.transformation import DataRobotConfig as DataRobotConfig from .llms.anthropic.chat.transformation import AnthropicConfig as AnthropicConfig - from .llms.anthropic.completion.transformation import ( - AnthropicTextConfig as AnthropicTextConfig, - ) + from .llms.anthropic.completion.transformation import AnthropicTextConfig as AnthropicTextConfig from .llms.groq.stt.transformation import GroqSTTConfig as GroqSTTConfig from .llms.triton.completion.transformation import TritonConfig as TritonConfig - from .llms.triton.completion.transformation import ( - TritonGenerateConfig as TritonGenerateConfig, - ) - from .llms.triton.completion.transformation import ( - TritonInferConfig as TritonInferConfig, - ) - from .llms.triton.embedding.transformation import ( - TritonEmbeddingConfig as TritonEmbeddingConfig, - ) - from .llms.huggingface.rerank.transformation import ( - HuggingFaceRerankConfig as HuggingFaceRerankConfig, - ) - from .llms.databricks.chat.transformation import ( - DatabricksConfig as DatabricksConfig, - ) - from .llms.databricks.embed.transformation import ( - DatabricksEmbeddingConfig as DatabricksEmbeddingConfig, - ) + from .llms.triton.completion.transformation import TritonGenerateConfig as TritonGenerateConfig + from .llms.triton.completion.transformation import TritonInferConfig as TritonInferConfig + from .llms.triton.embedding.transformation import TritonEmbeddingConfig as TritonEmbeddingConfig + from .llms.huggingface.rerank.transformation import HuggingFaceRerankConfig as HuggingFaceRerankConfig + from .llms.databricks.chat.transformation import DatabricksConfig as DatabricksConfig + from .llms.databricks.embed.transformation import DatabricksEmbeddingConfig as DatabricksEmbeddingConfig from .llms.predibase.chat.transformation import PredibaseConfig as PredibaseConfig from .llms.replicate.chat.transformation import ReplicateConfig as ReplicateConfig from .llms.snowflake.chat.transformation import SnowflakeConfig as SnowflakeConfig - from .llms.cohere.rerank.transformation import ( - CohereRerankConfig as CohereRerankConfig, - ) - from .llms.cohere.rerank_v2.transformation import ( - CohereRerankV2Config as CohereRerankV2Config, - ) - from .llms.azure_ai.rerank.transformation import ( - AzureAIRerankConfig as AzureAIRerankConfig, - ) - from .llms.infinity.rerank.transformation import ( - InfinityRerankConfig as InfinityRerankConfig, - ) - from .llms.jina_ai.rerank.transformation import ( - JinaAIRerankConfig as JinaAIRerankConfig, - ) - from .llms.deepinfra.rerank.transformation import ( - DeepinfraRerankConfig as DeepinfraRerankConfig, - ) - from .llms.hosted_vllm.rerank.transformation import ( - HostedVLLMRerankConfig as HostedVLLMRerankConfig, - ) - from .llms.nvidia_nim.rerank.transformation import ( - NvidiaNimRerankConfig as NvidiaNimRerankConfig, - ) - from .llms.nvidia_nim.rerank.ranking_transformation import ( - NvidiaNimRankingConfig as NvidiaNimRankingConfig, - ) - from .llms.vertex_ai.rerank.transformation import ( - VertexAIRerankConfig as VertexAIRerankConfig, - ) - from .llms.fireworks_ai.rerank.transformation import ( - FireworksAIRerankConfig as FireworksAIRerankConfig, - ) - from .llms.voyage.rerank.transformation import ( - VoyageRerankConfig as VoyageRerankConfig, - ) - from .llms.watsonx.rerank.transformation import ( - IBMWatsonXRerankConfig as IBMWatsonXRerankConfig, - ) + from .llms.cohere.rerank.transformation import CohereRerankConfig as CohereRerankConfig + from .llms.cohere.rerank_v2.transformation import CohereRerankV2Config as CohereRerankV2Config + from .llms.azure_ai.rerank.transformation import AzureAIRerankConfig as AzureAIRerankConfig + from .llms.infinity.rerank.transformation import InfinityRerankConfig as InfinityRerankConfig + from .llms.jina_ai.rerank.transformation import JinaAIRerankConfig as JinaAIRerankConfig + from .llms.deepinfra.rerank.transformation import DeepinfraRerankConfig as DeepinfraRerankConfig + from .llms.hosted_vllm.rerank.transformation import HostedVLLMRerankConfig as HostedVLLMRerankConfig + from .llms.nvidia_nim.rerank.transformation import NvidiaNimRerankConfig as NvidiaNimRerankConfig + from .llms.nvidia_nim.rerank.ranking_transformation import NvidiaNimRankingConfig as NvidiaNimRankingConfig + from .llms.vertex_ai.rerank.transformation import VertexAIRerankConfig as VertexAIRerankConfig + from .llms.fireworks_ai.rerank.transformation import FireworksAIRerankConfig as FireworksAIRerankConfig + from .llms.voyage.rerank.transformation import VoyageRerankConfig as VoyageRerankConfig + from .llms.watsonx.rerank.transformation import IBMWatsonXRerankConfig as IBMWatsonXRerankConfig from .llms.clarifai.chat.transformation import ClarifaiConfig as ClarifaiConfig from .llms.ai21.chat.transformation import AI21ChatConfig as AI21ChatConfig from .llms.meta_llama.chat.transformation import LlamaAPIConfig as LlamaAPIConfig - from .llms.together_ai.completion.transformation import ( - TogetherAITextCompletionConfig as TogetherAITextCompletionConfig, - ) - from .llms.cloudflare.chat.transformation import ( - CloudflareChatConfig as CloudflareChatConfig, - ) + from .llms.together_ai.completion.transformation import TogetherAITextCompletionConfig as TogetherAITextCompletionConfig + from .llms.cloudflare.chat.transformation import CloudflareChatConfig as CloudflareChatConfig from .llms.novita.chat.transformation import NovitaConfig as NovitaConfig from .llms.petals.completion.transformation import PetalsConfig as PetalsConfig from .llms.ollama.chat.transformation import OllamaChatConfig as OllamaChatConfig from .llms.ollama.completion.transformation import OllamaConfig as OllamaConfig - from .llms.sagemaker.completion.transformation import ( - SagemakerConfig as SagemakerConfig, - ) - from .llms.sagemaker.chat.transformation import ( - SagemakerChatConfig as SagemakerChatConfig, - ) + from .llms.sagemaker.completion.transformation import SagemakerConfig as SagemakerConfig + from .llms.sagemaker.chat.transformation import SagemakerChatConfig as SagemakerChatConfig from .llms.cohere.chat.transformation import CohereChatConfig as CohereChatConfig - from .llms.anthropic.experimental_pass_through.messages.transformation import ( - AnthropicMessagesConfig as AnthropicMessagesConfig, - ) - from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( - AmazonAnthropicClaudeMessagesConfig as AmazonAnthropicClaudeMessagesConfig, - ) + from .llms.anthropic.experimental_pass_through.messages.transformation import AnthropicMessagesConfig as AnthropicMessagesConfig + from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeMessagesConfig as AmazonAnthropicClaudeMessagesConfig from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig - from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig as VertexGeminiConfig, - ) - from .llms.gemini.chat.transformation import ( - GoogleAIStudioGeminiConfig as GoogleAIStudioGeminiConfig, - ) - from .llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import ( - VertexAIAnthropicConfig as VertexAIAnthropicConfig, - ) - from .llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import ( - VertexAILlama3Config as VertexAILlama3Config, - ) - from .llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import ( - VertexAIAi21Config as VertexAIAi21Config, - ) - from .llms.bedrock.chat.invoke_handler import ( - AmazonCohereChatConfig as AmazonCohereChatConfig, - ) - from .llms.bedrock.common_utils import ( - AmazonBedrockGlobalConfig as AmazonBedrockGlobalConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation import ( - AmazonAI21Config as AmazonAI21Config, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import ( - AmazonInvokeNovaConfig as AmazonInvokeNovaConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation import ( - AmazonQwen2Config as AmazonQwen2Config, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import ( - AmazonQwen3Config as AmazonQwen3Config, - ) - from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation import ( - AmazonAnthropicConfig as AmazonAnthropicConfig, - ) - from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( - AmazonAnthropicClaudeConfig as AmazonAnthropicClaudeConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import ( - AmazonCohereConfig as AmazonCohereConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_llama_transformation import ( - AmazonLlamaConfig as AmazonLlamaConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import ( - AmazonDeepSeekR1Config as AmazonDeepSeekR1Config, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation import ( - AmazonMistralConfig as AmazonMistralConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import ( - AmazonMoonshotConfig as AmazonMoonshotConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import ( - AmazonTitanConfig as AmazonTitanConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import ( - AmazonTwelveLabsPegasusConfig as AmazonTwelveLabsPegasusConfig, - ) - from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( - AmazonInvokeConfig as AmazonInvokeConfig, - ) - from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import ( - AmazonBedrockOpenAIConfig as AmazonBedrockOpenAIConfig, - ) - from .llms.bedrock.image_generation.amazon_stability1_transformation import ( - AmazonStabilityConfig as AmazonStabilityConfig, - ) - from .llms.bedrock.image_generation.amazon_stability3_transformation import ( - AmazonStability3Config as AmazonStability3Config, - ) - from .llms.bedrock.image_generation.amazon_nova_canvas_transformation import ( - AmazonNovaCanvasConfig as AmazonNovaCanvasConfig, - ) - from .llms.bedrock.embed.amazon_titan_g1_transformation import ( - AmazonTitanG1Config as AmazonTitanG1Config, - ) - from .llms.bedrock.embed.amazon_titan_multimodal_transformation import ( - AmazonTitanMultimodalEmbeddingG1Config as AmazonTitanMultimodalEmbeddingG1Config, - ) - from .llms.cohere.chat.v2_transformation import ( - CohereV2ChatConfig as CohereV2ChatConfig, - ) - from .llms.bedrock.embed.cohere_transformation import ( - BedrockCohereEmbeddingConfig as BedrockCohereEmbeddingConfig, - ) - from .llms.bedrock.embed.twelvelabs_marengo_transformation import ( - TwelveLabsMarengoEmbeddingConfig as TwelveLabsMarengoEmbeddingConfig, - ) - from .llms.bedrock.embed.amazon_nova_transformation import ( - AmazonNovaEmbeddingConfig as AmazonNovaEmbeddingConfig, - ) - from .llms.openai.openai import ( - OpenAIConfig as OpenAIConfig, - MistralEmbeddingConfig as MistralEmbeddingConfig, - ) - from .llms.openai.image_variations.transformation import ( - OpenAIImageVariationConfig as OpenAIImageVariationConfig, - ) - from .llms.deepgram.audio_transcription.transformation import ( - DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig, - ) - from .llms.topaz.image_variations.transformation import ( - TopazImageVariationConfig as TopazImageVariationConfig, - ) - from litellm.llms.openai.completion.transformation import ( - OpenAITextCompletionConfig as OpenAITextCompletionConfig, - ) + from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig as VertexGeminiConfig + from .llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig as GoogleAIStudioGeminiConfig + from .llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig as VertexAIAnthropicConfig + from .llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import VertexAILlama3Config as VertexAILlama3Config + from .llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import VertexAIAi21Config as VertexAIAi21Config + from .llms.bedrock.chat.invoke_handler import AmazonCohereChatConfig as AmazonCohereChatConfig + from .llms.bedrock.common_utils import AmazonBedrockGlobalConfig as AmazonBedrockGlobalConfig + from .llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation import AmazonAI21Config as AmazonAI21Config + from .llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import AmazonInvokeNovaConfig as AmazonInvokeNovaConfig + from .llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation import AmazonQwen2Config as AmazonQwen2Config + from .llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import AmazonQwen3Config as AmazonQwen3Config + from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation import AmazonAnthropicConfig as AmazonAnthropicConfig + from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeConfig as AmazonAnthropicClaudeConfig + from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import AmazonCohereConfig as AmazonCohereConfig + from .llms.bedrock.chat.invoke_transformations.amazon_llama_transformation import AmazonLlamaConfig as AmazonLlamaConfig + from .llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import AmazonDeepSeekR1Config as AmazonDeepSeekR1Config + from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation import AmazonMistralConfig as AmazonMistralConfig + from .llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import AmazonMoonshotConfig as AmazonMoonshotConfig + from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import AmazonTitanConfig as AmazonTitanConfig + from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import AmazonTwelveLabsPegasusConfig as AmazonTwelveLabsPegasusConfig + from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import AmazonInvokeConfig as AmazonInvokeConfig + from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import AmazonBedrockOpenAIConfig as AmazonBedrockOpenAIConfig + from .llms.bedrock.image_generation.amazon_stability1_transformation import AmazonStabilityConfig as AmazonStabilityConfig + from .llms.bedrock.image_generation.amazon_stability3_transformation import AmazonStability3Config as AmazonStability3Config + from .llms.bedrock.image_generation.amazon_nova_canvas_transformation import AmazonNovaCanvasConfig as AmazonNovaCanvasConfig + from .llms.bedrock.embed.amazon_titan_g1_transformation import AmazonTitanG1Config as AmazonTitanG1Config + from .llms.bedrock.embed.amazon_titan_multimodal_transformation import AmazonTitanMultimodalEmbeddingG1Config as AmazonTitanMultimodalEmbeddingG1Config + from .llms.cohere.chat.v2_transformation import CohereV2ChatConfig as CohereV2ChatConfig + from .llms.bedrock.embed.cohere_transformation import BedrockCohereEmbeddingConfig as BedrockCohereEmbeddingConfig + from .llms.bedrock.embed.twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig as TwelveLabsMarengoEmbeddingConfig + from .llms.bedrock.embed.amazon_nova_transformation import AmazonNovaEmbeddingConfig as AmazonNovaEmbeddingConfig + from .llms.openai.openai import OpenAIConfig as OpenAIConfig, MistralEmbeddingConfig as MistralEmbeddingConfig + from .llms.openai.image_variations.transformation import OpenAIImageVariationConfig as OpenAIImageVariationConfig + from .llms.deepgram.audio_transcription.transformation import DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig + from .llms.topaz.image_variations.transformation import TopazImageVariationConfig as TopazImageVariationConfig + from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig as OpenAITextCompletionConfig from .llms.groq.chat.transformation import GroqChatConfig as GroqChatConfig from .llms.a2a.chat.transformation import A2AConfig as A2AConfig - from .llms.voyage.embedding.transformation import ( - VoyageEmbeddingConfig as VoyageEmbeddingConfig, - ) - from .llms.voyage.embedding.transformation_contextual import ( - VoyageContextualEmbeddingConfig as VoyageContextualEmbeddingConfig, - ) - from .llms.infinity.embedding.transformation import ( - InfinityEmbeddingConfig as InfinityEmbeddingConfig, - ) - from .llms.azure_ai.chat.transformation import ( - AzureAIStudioConfig as AzureAIStudioConfig, - ) + from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig as VoyageEmbeddingConfig + from .llms.voyage.embedding.transformation_contextual import VoyageContextualEmbeddingConfig as VoyageContextualEmbeddingConfig + from .llms.infinity.embedding.transformation import InfinityEmbeddingConfig as InfinityEmbeddingConfig + from .llms.azure_ai.chat.transformation import AzureAIStudioConfig as AzureAIStudioConfig from .llms.mistral.chat.transformation import MistralConfig as MistralConfig - from .llms.openai.responses.transformation import ( - OpenAIResponsesAPIConfig as OpenAIResponsesAPIConfig, - ) - from .llms.azure.responses.transformation import ( - AzureOpenAIResponsesAPIConfig as AzureOpenAIResponsesAPIConfig, - ) - from .llms.azure.responses.o_series_transformation import ( - AzureOpenAIOSeriesResponsesAPIConfig as AzureOpenAIOSeriesResponsesAPIConfig, - ) - from .llms.xai.responses.transformation import ( - XAIResponsesAPIConfig as XAIResponsesAPIConfig, - ) - from .llms.litellm_proxy.responses.transformation import ( - LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig, - ) - from .llms.volcengine.responses.transformation import ( - VolcEngineResponsesAPIConfig as VolcEngineResponsesAPIConfig, - ) - from .llms.manus.responses.transformation import ( - ManusResponsesAPIConfig as ManusResponsesAPIConfig, - ) - from .llms.perplexity.responses.transformation import ( - PerplexityResponsesConfig as PerplexityResponsesConfig, - ) - from .llms.databricks.responses.transformation import ( - DatabricksResponsesAPIConfig as DatabricksResponsesAPIConfig, - ) - from .llms.gemini.interactions.transformation import ( - GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig, - ) - from .llms.openai.chat.o_series_transformation import ( - OpenAIOSeriesConfig as OpenAIOSeriesConfig, - OpenAIOSeriesConfig as OpenAIO1Config, - ) - from .llms.anthropic.skills.transformation import ( - AnthropicSkillsConfig as AnthropicSkillsConfig, - ) - from .llms.base_llm.skills.transformation import ( - BaseSkillsAPIConfig as BaseSkillsAPIConfig, - ) - from .llms.gradient_ai.chat.transformation import ( - GradientAIConfig as GradientAIConfig, - ) + from .llms.openai.responses.transformation import OpenAIResponsesAPIConfig as OpenAIResponsesAPIConfig + from .llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig as AzureOpenAIResponsesAPIConfig + from .llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig as AzureOpenAIOSeriesResponsesAPIConfig + from .llms.xai.responses.transformation import XAIResponsesAPIConfig as XAIResponsesAPIConfig + from .llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig + from .llms.volcengine.responses.transformation import VolcEngineResponsesAPIConfig as VolcEngineResponsesAPIConfig + from .llms.manus.responses.transformation import ManusResponsesAPIConfig as ManusResponsesAPIConfig + from .llms.perplexity.responses.transformation import PerplexityResponsesConfig as PerplexityResponsesConfig + from .llms.databricks.responses.transformation import DatabricksResponsesAPIConfig as DatabricksResponsesAPIConfig + from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig + from .llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config + from .llms.anthropic.skills.transformation import AnthropicSkillsConfig as AnthropicSkillsConfig + from .llms.base_llm.skills.transformation import BaseSkillsAPIConfig as BaseSkillsAPIConfig + from .llms.gradient_ai.chat.transformation import GradientAIConfig as GradientAIConfig from .llms.openai.chat.gpt_transformation import OpenAIGPTConfig as OpenAIGPTConfig - from .llms.openai.chat.gpt_5_transformation import ( - OpenAIGPT5Config as OpenAIGPT5Config, - ) - from .llms.openai.transcriptions.whisper_transformation import ( - OpenAIWhisperAudioTranscriptionConfig as OpenAIWhisperAudioTranscriptionConfig, - ) - from .llms.openai.transcriptions.gpt_transformation import ( - OpenAIGPTAudioTranscriptionConfig as OpenAIGPTAudioTranscriptionConfig, - ) - from .llms.openai.chat.gpt_audio_transformation import ( - OpenAIGPTAudioConfig as OpenAIGPTAudioConfig, - ) + from .llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config as OpenAIGPT5Config + from .llms.openai.transcriptions.whisper_transformation import OpenAIWhisperAudioTranscriptionConfig as OpenAIWhisperAudioTranscriptionConfig + from .llms.openai.transcriptions.gpt_transformation import OpenAIGPTAudioTranscriptionConfig as OpenAIGPTAudioTranscriptionConfig + from .llms.openai.chat.gpt_audio_transformation import OpenAIGPTAudioConfig as OpenAIGPTAudioConfig from .llms.nvidia_nim.chat.transformation import NvidiaNimConfig as NvidiaNimConfig - from .llms.nvidia_nim.embed import ( - NvidiaNimEmbeddingConfig as NvidiaNimEmbeddingConfig, - ) + from .llms.nvidia_nim.embed import NvidiaNimEmbeddingConfig as NvidiaNimEmbeddingConfig # Type stubs for lazy-loaded config instances openaiOSeriesConfig: OpenAIOSeriesConfig @@ -1649,47 +1456,21 @@ if TYPE_CHECKING: # Import config classes that need type stubs (for mypy) - import with _ prefix to avoid circular reference from .llms.vllm.completion.transformation import VLLMConfig as _VLLMConfig - from .llms.deepseek.chat.transformation import ( - DeepSeekChatConfig as _DeepSeekChatConfig, - ) - from .llms.sap.chat.transformation import ( - GenAIHubOrchestrationConfig as _GenAIHubOrchestrationConfig, - ) - from .llms.sap.embed.transformation import ( - GenAIHubEmbeddingConfig as _GenAIHubEmbeddingConfig, - ) - from .llms.azure.chat.o_series_transformation import ( - AzureOpenAIO1Config as _AzureOpenAIO1Config, - ) - from .llms.perplexity.chat.transformation import ( - PerplexityChatConfig as _PerplexityChatConfig, - ) + from .llms.deepseek.chat.transformation import DeepSeekChatConfig as _DeepSeekChatConfig + from .llms.sap.chat.transformation import GenAIHubOrchestrationConfig as _GenAIHubOrchestrationConfig + from .llms.sap.embed.transformation import GenAIHubEmbeddingConfig as _GenAIHubEmbeddingConfig + from .llms.azure.chat.o_series_transformation import AzureOpenAIO1Config as _AzureOpenAIO1Config + from .llms.perplexity.chat.transformation import PerplexityChatConfig as _PerplexityChatConfig from .llms.nscale.chat.transformation import NscaleConfig as _NscaleConfig - from .llms.watsonx.chat.transformation import ( - IBMWatsonXChatConfig as _IBMWatsonXChatConfig, - ) - from .llms.watsonx.completion.transformation import ( - IBMWatsonXAIConfig as _IBMWatsonXAIConfig, - ) - from .llms.litellm_proxy.chat.transformation import ( - LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig, - ) + from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig as _IBMWatsonXChatConfig + from .llms.watsonx.completion.transformation import IBMWatsonXAIConfig as _IBMWatsonXAIConfig + from .llms.litellm_proxy.chat.transformation import LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig - from .llms.llamafile.chat.transformation import ( - LlamafileChatConfig as _LlamafileChatConfig, - ) - from .llms.lm_studio.chat.transformation import ( - LMStudioChatConfig as _LMStudioChatConfig, - ) - from .llms.lm_studio.embed.transformation import ( - LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig, - ) - from .llms.watsonx.embed.transformation import ( - IBMWatsonXEmbeddingConfig as _IBMWatsonXEmbeddingConfig, - ) - from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig as _VertexGeminiConfig, - ) + from .llms.llamafile.chat.transformation import LlamafileChatConfig as _LlamafileChatConfig + from .llms.lm_studio.chat.transformation import LMStudioChatConfig as _LMStudioChatConfig + from .llms.lm_studio.embed.transformation import LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig + from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig as _IBMWatsonXEmbeddingConfig + from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig as _VertexGeminiConfig # Type stubs for lazy-loaded config classes (to help mypy understand types) VLLMConfig: Type[_VLLMConfig] @@ -1709,122 +1490,55 @@ if TYPE_CHECKING: IBMWatsonXEmbeddingConfig: Type[_IBMWatsonXEmbeddingConfig] VertexAIConfig: Type[_VertexGeminiConfig] # Alias for VertexGeminiConfig - from .llms.featherless_ai.chat.transformation import ( - FeatherlessAIConfig as FeatherlessAIConfig, - ) + from .llms.featherless_ai.chat.transformation import FeatherlessAIConfig as FeatherlessAIConfig from .llms.cerebras.chat import CerebrasConfig as CerebrasConfig from .llms.baseten.chat import BasetenConfig as BasetenConfig from .llms.sambanova.chat import SambanovaConfig as SambanovaConfig - from .llms.sambanova.embedding.transformation import ( - SambaNovaEmbeddingConfig as SambaNovaEmbeddingConfig, - ) - from .llms.fireworks_ai.chat.transformation import ( - FireworksAIConfig as FireworksAIConfig, - ) - from .llms.fireworks_ai.completion.transformation import ( - FireworksAITextCompletionConfig as FireworksAITextCompletionConfig, - ) - from .llms.fireworks_ai.audio_transcription.transformation import ( - FireworksAIAudioTranscriptionConfig as FireworksAIAudioTranscriptionConfig, - ) - from .llms.fireworks_ai.embed.fireworks_ai_transformation import ( - FireworksAIEmbeddingConfig as FireworksAIEmbeddingConfig, - ) - from .llms.friendliai.chat.transformation import ( - FriendliaiChatConfig as FriendliaiChatConfig, - ) - from .llms.jina_ai.embedding.transformation import ( - JinaAIEmbeddingConfig as JinaAIEmbeddingConfig, - ) + from .llms.sambanova.embedding.transformation import SambaNovaEmbeddingConfig as SambaNovaEmbeddingConfig + from .llms.fireworks_ai.chat.transformation import FireworksAIConfig as FireworksAIConfig + from .llms.fireworks_ai.completion.transformation import FireworksAITextCompletionConfig as FireworksAITextCompletionConfig + from .llms.fireworks_ai.audio_transcription.transformation import FireworksAIAudioTranscriptionConfig as FireworksAIAudioTranscriptionConfig + from .llms.fireworks_ai.embed.fireworks_ai_transformation import FireworksAIEmbeddingConfig as FireworksAIEmbeddingConfig + from .llms.friendliai.chat.transformation import FriendliaiChatConfig as FriendliaiChatConfig + from .llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig as JinaAIEmbeddingConfig from .llms.xai.chat.transformation import XAIChatConfig as XAIChatConfig from .llms.zai.chat.transformation import ZAIChatConfig as ZAIChatConfig from .llms.aiml.chat.transformation import AIMLChatConfig as AIMLChatConfig - from .llms.volcengine.chat.transformation import ( - VolcEngineChatConfig as VolcEngineChatConfig, - VolcEngineChatConfig as VolcEngineConfig, - ) - from .llms.codestral.completion.transformation import ( - CodestralTextCompletionConfig as CodestralTextCompletionConfig, - ) - from .llms.azure.azure import ( - AzureOpenAIAssistantsAPIConfig as AzureOpenAIAssistantsAPIConfig, - ) + from .llms.volcengine.chat.transformation import VolcEngineChatConfig as VolcEngineChatConfig, VolcEngineChatConfig as VolcEngineConfig + from .llms.codestral.completion.transformation import CodestralTextCompletionConfig as CodestralTextCompletionConfig + from .llms.azure.azure import AzureOpenAIAssistantsAPIConfig as AzureOpenAIAssistantsAPIConfig from .llms.heroku.chat.transformation import HerokuChatConfig as HerokuChatConfig from .llms.cometapi.chat.transformation import CometAPIConfig as CometAPIConfig - from .llms.azure.chat.gpt_transformation import ( - AzureOpenAIConfig as AzureOpenAIConfig, - ) - from .llms.azure.chat.gpt_5_transformation import ( - AzureOpenAIGPT5Config as AzureOpenAIGPT5Config, - ) - from .llms.azure.completion.transformation import ( - AzureOpenAITextConfig as AzureOpenAITextConfig, - ) - from .llms.hosted_vllm.chat.transformation import ( - HostedVLLMChatConfig as HostedVLLMChatConfig, - ) - from .llms.hosted_vllm.embedding.transformation import ( - HostedVLLMEmbeddingConfig as HostedVLLMEmbeddingConfig, - ) - from .llms.github_copilot.chat.transformation import ( - GithubCopilotConfig as GithubCopilotConfig, - ) - from .llms.github_copilot.responses.transformation import ( - GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig, - ) - from .llms.github_copilot.embedding.transformation import ( - GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig, - ) + from .llms.azure.chat.gpt_transformation import AzureOpenAIConfig as AzureOpenAIConfig + from .llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config as AzureOpenAIGPT5Config + from .llms.azure.completion.transformation import AzureOpenAITextConfig as AzureOpenAITextConfig + from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig as HostedVLLMChatConfig + from .llms.hosted_vllm.embedding.transformation import HostedVLLMEmbeddingConfig as HostedVLLMEmbeddingConfig + from .llms.github_copilot.chat.transformation import GithubCopilotConfig as GithubCopilotConfig + from .llms.github_copilot.responses.transformation import GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig + from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig from .llms.chatgpt.chat.transformation import ChatGPTConfig as ChatGPTConfig - from .llms.chatgpt.responses.transformation import ( - ChatGPTResponsesAPIConfig as ChatGPTResponsesAPIConfig, - ) + from .llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig as ChatGPTResponsesAPIConfig from .llms.gigachat.chat.transformation import GigaChatConfig as GigaChatConfig - from .llms.gigachat.embedding.transformation import ( - GigaChatEmbeddingConfig as GigaChatEmbeddingConfig, - ) + from .llms.gigachat.embedding.transformation import GigaChatEmbeddingConfig as GigaChatEmbeddingConfig from .llms.nebius.chat.transformation import NebiusConfig as NebiusConfig from .llms.wandb.chat.transformation import WandbConfig as WandbConfig - from .llms.dashscope.chat.transformation import ( - DashScopeChatConfig as DashScopeChatConfig, - ) - from .llms.moonshot.chat.transformation import ( - MoonshotChatConfig as MoonshotChatConfig, - ) - from .llms.docker_model_runner.chat.transformation import ( - DockerModelRunnerChatConfig as DockerModelRunnerChatConfig, - ) + from .llms.dashscope.chat.transformation import DashScopeChatConfig as DashScopeChatConfig + from .llms.moonshot.chat.transformation import MoonshotChatConfig as MoonshotChatConfig + from .llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig as DockerModelRunnerChatConfig from .llms.v0.chat.transformation import V0ChatConfig as V0ChatConfig from .llms.oci.chat.transformation import OCIChatConfig as OCIChatConfig from .llms.morph.chat.transformation import MorphChatConfig as MorphChatConfig from .llms.ragflow.chat.transformation import RAGFlowConfig as RAGFlowConfig - from .llms.lambda_ai.chat.transformation import ( - LambdaAIChatConfig as LambdaAIChatConfig, - ) - from .llms.hyperbolic.chat.transformation import ( - HyperbolicChatConfig as HyperbolicChatConfig, - ) - from .llms.vercel_ai_gateway.chat.transformation import ( - VercelAIGatewayConfig as VercelAIGatewayConfig, - ) - from .llms.ovhcloud.chat.transformation import ( - OVHCloudChatConfig as OVHCloudChatConfig, - ) - from .llms.ovhcloud.embedding.transformation import ( - OVHCloudEmbeddingConfig as OVHCloudEmbeddingConfig, - ) - from .llms.cometapi.embed.transformation import ( - CometAPIEmbeddingConfig as CometAPIEmbeddingConfig, - ) - from .llms.lemonade.chat.transformation import ( - LemonadeChatConfig as LemonadeChatConfig, - ) - from .llms.snowflake.embedding.transformation import ( - SnowflakeEmbeddingConfig as SnowflakeEmbeddingConfig, - ) - from .llms.amazon_nova.chat.transformation import ( - AmazonNovaChatConfig as AmazonNovaChatConfig, - ) + from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig as LambdaAIChatConfig + from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig as HyperbolicChatConfig + from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig as VercelAIGatewayConfig + from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig as OVHCloudChatConfig + from .llms.ovhcloud.embedding.transformation import OVHCloudEmbeddingConfig as OVHCloudEmbeddingConfig + from .llms.cometapi.embed.transformation import CometAPIEmbeddingConfig as CometAPIEmbeddingConfig + from .llms.lemonade.chat.transformation import LemonadeChatConfig as LemonadeChatConfig + from .llms.snowflake.embedding.transformation import SnowflakeEmbeddingConfig as SnowflakeEmbeddingConfig + from .llms.amazon_nova.chat.transformation import AmazonNovaChatConfig as AmazonNovaChatConfig from litellm.caching.llm_caching_handler import LLMClientCache from litellm.types.llms.bedrock import COHERE_EMBEDDING_INPUT_TYPES from litellm.types.utils import ( @@ -1885,7 +1599,6 @@ if TYPE_CHECKING: # Bedrock tool name mappings instance (lazy-loaded) from litellm.caching.caching import InMemoryCache - bedrock_tool_name_mappings: InMemoryCache # Azure exception class (lazy-loaded) @@ -1904,15 +1617,11 @@ if TYPE_CHECKING: from litellm.types.integrations.datadog_llm_obs import DatadogLLMObsInitParams # Logging callback manager class and instance (lazy-loaded) - from litellm.litellm_core_utils.logging_callback_manager import ( - LoggingCallbackManager, - ) - + from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager logging_callback_manager: LoggingCallbackManager # provider_list is lazy-loaded from litellm.types.utils import LlmProviders - provider_list: List[Union[LlmProviders, str]] # Note: AmazonConverseConfig and OpenAILikeChatConfig are imported above in TYPE_CHECKING block @@ -1937,10 +1646,7 @@ def __getattr__(name: str) -> Any: global _async_client_cleanup_registered # Register async client cleanup on first access (only once) if not _async_client_cleanup_registered: - from litellm.llms.custom_httpx.async_client_cleanup import ( - register_async_client_cleanup, - ) - + from litellm.llms.custom_httpx.async_client_cleanup import register_async_client_cleanup register_async_client_cleanup() _async_client_cleanup_registered = True @@ -1957,45 +1663,36 @@ def __getattr__(name: str) -> Any: # Lazy load encoding from main.py to avoid heavy tiktoken import if name == "encoding": from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() # Check if already cached if "encoding" not in _globals: from .main import encoding as _encoding - _globals["encoding"] = _encoding return _globals["encoding"] # Lazy load bedrock_tool_name_mappings instance if name == "bedrock_tool_name_mappings": from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() # Check if already cached if "bedrock_tool_name_mappings" not in _globals: - from .llms.bedrock.chat.invoke_handler import ( - bedrock_tool_name_mappings as _bedrock_tool_name_mappings, - ) - + from .llms.bedrock.chat.invoke_handler import bedrock_tool_name_mappings as _bedrock_tool_name_mappings _globals["bedrock_tool_name_mappings"] = _bedrock_tool_name_mappings return _globals["bedrock_tool_name_mappings"] # Lazy load AzureOpenAIError exception class if name == "AzureOpenAIError": from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() # Check if already cached if "AzureOpenAIError" not in _globals: from .llms.azure.common_utils import AzureOpenAIError as _AzureOpenAIError - _globals["AzureOpenAIError"] = _AzureOpenAIError return _globals["AzureOpenAIError"] # Lazy load openaiOSeriesConfig instance if name == "openaiOSeriesConfig": from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() if "openaiOSeriesConfig" not in _globals: # Import the config class and instantiate it @@ -2013,7 +1710,6 @@ def __getattr__(name: str) -> Any: } if name in _config_instances: from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() if name not in _globals: # Import the config class and instantiate it @@ -2028,20 +1724,17 @@ def __getattr__(name: str) -> Any: # Lazy load provider_list if name == "provider_list": from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() # Check if already cached if "provider_list" not in _globals: # LlmProviders is eagerly imported above, so we can import it directly from litellm.types.utils import LlmProviders - _globals["provider_list"] = list(LlmProviders) return _globals["provider_list"] # Lazy load priority_reservation_settings instance if name == "priority_reservation_settings": from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() # Check if already cached if "priority_reservation_settings" not in _globals: @@ -2053,7 +1746,6 @@ def __getattr__(name: str) -> Any: # Lazy load logging_callback_manager instance if name == "logging_callback_manager": from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() # Check if already cached if "logging_callback_manager" not in _globals: @@ -2065,41 +1757,19 @@ def __getattr__(name: str) -> Any: # Lazy load _service_logger module if name == "_service_logger": from ._lazy_imports import _get_litellm_globals - _globals = _get_litellm_globals() # Check if already cached if "_service_logger" not in _globals: # Import the module lazily import litellm._service_logger - _globals["_service_logger"] = litellm._service_logger return _globals["_service_logger"] # Lazy load evals module functions - if name in [ - "acreate_eval", - "alist_evals", - "aget_eval", - "aupdate_eval", - "adelete_eval", - "acancel_eval", - "create_eval", - "list_evals", - "get_eval", - "update_eval", - "delete_eval", - "cancel_eval", - "acreate_run", - "alist_runs", - "aget_run", - "acancel_run", - "adelete_run", - "create_run", - "list_runs", - "get_run", - "cancel_run", - "delete_run", - ]: + if name in ["acreate_eval", "alist_evals", "aget_eval", "aupdate_eval", "adelete_eval", "acancel_eval", + "create_eval", "list_evals", "get_eval", "update_eval", "delete_eval", "cancel_eval", + "acreate_run", "alist_runs", "aget_run", "acancel_run", "adelete_run", + "create_run", "list_runs", "get_run", "cancel_run", "delete_run"]: from litellm.evals.main import ( acreate_eval, alist_evals, @@ -2124,7 +1794,6 @@ def __getattr__(name: str) -> Any: cancel_run, delete_run, ) - return locals()[name] raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/litellm/integrations/datadog/datadog_metrics.py b/litellm/integrations/datadog/datadog_metrics.py index 43dd0662604..a15918bc6df 100644 --- a/litellm/integrations/datadog/datadog_metrics.py +++ b/litellm/integrations/datadog/datadog_metrics.py @@ -7,6 +7,12 @@ from typing import List, Optional, Union from litellm._logging import verbose_logger from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.integrations.datadog.datadog_handler import ( + get_datadog_env, + get_datadog_hostname, + get_datadog_pod_name, + get_datadog_service, +) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, @@ -22,7 +28,7 @@ from litellm.types.utils import StandardLoggingPayload class DatadogMetricsLogger(CustomBatchLogger): - def __init__(self, **kwargs): + def __init__(self, start_periodic_flush: bool = True, **kwargs): self.dd_api_key = os.getenv("DD_API_KEY") self.dd_app_key = os.getenv("DD_APP_KEY") self.dd_site = os.getenv("DD_SITE", "datadoghq.com") @@ -51,8 +57,9 @@ class DatadogMetricsLogger(CustomBatchLogger): super().__init__(**kwargs) - # Start periodic flush task - asyncio.create_task(self.periodic_flush()) + # Start periodic flush task only if instructed + if start_periodic_flush: + asyncio.create_task(self.periodic_flush()) def _extract_tags( self, @@ -62,13 +69,6 @@ class DatadogMetricsLogger(CustomBatchLogger): """ Builds the list of tags for a Datadog metric point """ - from litellm.integrations.datadog.datadog_handler import ( - get_datadog_env, - get_datadog_hostname, - get_datadog_pod_name, - get_datadog_service, - ) - # Base tags tags = [ f"env:{get_datadog_env()}", @@ -188,8 +188,9 @@ class DatadogMetricsLogger(CustomBatchLogger): error_information = ( standard_logging_object.get("error_information", {}) or {} ) - if "error_code" in error_information and error_information["error_code"] is not None: # type: ignore - status_code = str(error_information["error_code"]) # type: ignore + error_code = error_information.get("error_code") # type: ignore + if error_code is not None: + status_code = str(error_code) self._add_metrics_from_log( log=standard_logging_object, kwargs=kwargs, status_code=status_code diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 98f573d471b..9a6c9c09f27 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -133,8 +133,8 @@ from ..integrations.azure_sentinel.azure_sentinel import AzureSentinelLogger from ..integrations.azure_storage.azure_storage import AzureBlobStorageLogger from ..integrations.custom_prompt_management import CustomPromptManagement from ..integrations.datadog.datadog import DataDogLogger -from ..integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger from ..integrations.datadog.datadog_metrics import DatadogMetricsLogger +from ..integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger from ..integrations.dotprompt import DotpromptManager from ..integrations.dynamodb import DyanmoDBLogger from ..integrations.galileo import GalileoObserve @@ -1654,7 +1654,9 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details[ "standard_logging_object" - ] = self._build_standard_logging_payload(logging_result, start_time, end_time) + ] = self._build_standard_logging_payload( + logging_result, start_time, end_time + ) if ( standard_logging_payload := self.model_call_details.get( @@ -2517,7 +2519,9 @@ class Logging(LiteLLMLoggingBaseClass): ## STANDARDIZED LOGGING PAYLOAD self.model_call_details[ "standard_logging_object" - ] = self._build_standard_logging_payload(result, start_time, end_time) + ] = self._build_standard_logging_payload( + result, start_time, end_time + ) # print standard logging payload if ( @@ -3658,10 +3662,6 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _datadog_logger = DataDogLogger() _in_memory_loggers.append(_datadog_logger) return _datadog_logger # type: ignore - elif logging_integration == "datadog_llm_observability": - _datadog_llm_obs_logger = DataDogLLMObsLogger() - _in_memory_loggers.append(_datadog_llm_obs_logger) - return _datadog_llm_obs_logger # type: ignore elif logging_integration == "datadog_metrics": for callback in _in_memory_loggers: if isinstance(callback, DatadogMetricsLogger): @@ -3670,6 +3670,10 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _datadog_metrics_logger = DatadogMetricsLogger() _in_memory_loggers.append(_datadog_metrics_logger) return _datadog_metrics_logger # type: ignore + elif logging_integration == "datadog_llm_observability": + _datadog_llm_obs_logger = DataDogLLMObsLogger() + _in_memory_loggers.append(_datadog_llm_obs_logger) + return _datadog_llm_obs_logger # type: ignore elif logging_integration == "azure_sentinel": for callback in _in_memory_loggers: if isinstance(callback, AzureSentinelLogger): @@ -4215,14 +4219,14 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 for callback in _in_memory_loggers: if isinstance(callback, DataDogLogger): return callback - elif logging_integration == "datadog_llm_observability": - for callback in _in_memory_loggers: - if isinstance(callback, DataDogLLMObsLogger): - return callback elif logging_integration == "datadog_metrics": for callback in _in_memory_loggers: if isinstance(callback, DatadogMetricsLogger): return callback + elif logging_integration == "datadog_llm_observability": + for callback in _in_memory_loggers: + if isinstance(callback, DataDogLLMObsLogger): + return callback elif logging_integration == "azure_sentinel": for callback in _in_memory_loggers: if isinstance(callback, AzureSentinelLogger): @@ -4704,11 +4708,9 @@ class StandardLoggingPayloadSetup: ).model_dump() if isinstance(_raw, dict): if ResponseAPILoggingUtils._is_response_api_usage(_raw): - return ( - ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( - _raw - ).model_dump() - ) + return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + _raw + ).model_dump() return _raw if isinstance(_raw, Usage): return _raw.model_dump() @@ -5554,3 +5556,4 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload: model_parameters={"stream": True}, hidden_params=hidden_params, ) + diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 48e3e8c0ad0..83891c90e14 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -286,7 +286,7 @@ async def health_services_endpoint( # noqa: PLR0915 DatadogMetricsLogger, ) - datadog_metrics_logger = DatadogMetricsLogger() + datadog_metrics_logger = DatadogMetricsLogger(start_periodic_flush=False) response = await datadog_metrics_logger.async_health_check() return { "status": response["status"], diff --git a/litellm/types/integrations/datadog_metrics.py b/litellm/types/integrations/datadog_metrics.py index 4b21881ed5f..7269a77cc25 100644 --- a/litellm/types/integrations/datadog_metrics.py +++ b/litellm/types/integrations/datadog_metrics.py @@ -10,7 +10,7 @@ class DatadogMetricPoint(TypedDict): class DatadogMetricSeries(TypedDict): metric: str - type: int # 1=count, 2=rate, 3=gauge, distribution is submitted as type=3, but distributions use a different endpoint /api/v1/distribution_points, wait actually according to DD /api/v2/series: 0=unspecified, 1=count, 2=rate, 3=gauge. For histogram/distribution we use type 3 or 1. + type: int # 0=unspecified, 1=count, 2=rate, 3=gauge points: List[DatadogMetricPoint] tags: List[str] From 218383f739e2654b9055d08b3638fcf90bbb499e Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Thu, 26 Feb 2026 11:45:28 +0530 Subject: [PATCH 09/52] fix: req changes --- litellm/integrations/datadog/datadog_metrics.py | 1 + litellm/proxy/health_endpoints/_health_endpoints.py | 11 ++++++++++- litellm/types/integrations/datadog_metrics.py | 5 +++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/datadog/datadog_metrics.py b/litellm/integrations/datadog/datadog_metrics.py index a15918bc6df..a22efd66000 100644 --- a/litellm/integrations/datadog/datadog_metrics.py +++ b/litellm/integrations/datadog/datadog_metrics.py @@ -150,6 +150,7 @@ class DatadogMetricsLogger(CustomBatchLogger): "type": 1, # count "points": [{"timestamp": timestamp, "value": 1.0}], "tags": tags, + "interval": self.flush_interval, } self.log_queue.append(series_count) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 83891c90e14..f37eb344d35 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -285,8 +285,17 @@ async def health_services_endpoint( # noqa: PLR0915 from litellm.integrations.datadog.datadog_metrics import ( DatadogMetricsLogger, ) + from litellm.litellm_core_utils.litellm_logging import ( + get_custom_logger_compatible_class, + ) - datadog_metrics_logger = DatadogMetricsLogger(start_periodic_flush=False) + datadog_metrics_logger = get_custom_logger_compatible_class( + "datadog_metrics" + ) + if datadog_metrics_logger is None: + datadog_metrics_logger = DatadogMetricsLogger( + start_periodic_flush=False + ) response = await datadog_metrics_logger.async_health_check() return { "status": response["status"], diff --git a/litellm/types/integrations/datadog_metrics.py b/litellm/types/integrations/datadog_metrics.py index 7269a77cc25..4c980cdee6d 100644 --- a/litellm/types/integrations/datadog_metrics.py +++ b/litellm/types/integrations/datadog_metrics.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional from typing_extensions import TypedDict @@ -8,11 +8,12 @@ class DatadogMetricPoint(TypedDict): value: float # The metric value -class DatadogMetricSeries(TypedDict): +class DatadogMetricSeries(TypedDict, total=False): metric: str type: int # 0=unspecified, 1=count, 2=rate, 3=gauge points: List[DatadogMetricPoint] tags: List[str] + interval: Optional[int] # Required for count (type=1) and rate (type=2) metrics class DatadogMetricsPayload(TypedDict): From 2fce35a1628af8b82275b6625c713213d18c7b0d Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Thu, 26 Feb 2026 12:45:49 -0300 Subject: [PATCH 10/52] test(ci): add secret scan test and CI job to prevent hardcoded credentials - Add unit test that scans Python source for Base64 Basic Auth patterns that would be flagged by secret scanners like GitGuardian/ggshield - Add secret-scan job to the linting CI workflow that runs the test on every PR and optionally runs ggshield if GITGUARDIAN_API_KEY is set Co-Authored-By: Claude Opus 4.6 --- .github/workflows/test-linting.yml | 27 ++++++++ tests/litellm/test_no_hardcoded_secrets.py | 74 ++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 tests/litellm/test_no_hardcoded_secrets.py diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 7c5c269f899..abd857ea2b0 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -74,3 +74,30 @@ jobs: - name: Check import safety run: | poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) + + secret-scan: + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Run secret scan test + run: | + pip install pytest + pytest tests/litellm/test_no_hardcoded_secrets.py -v + + - name: Run ggshield secret scan + if: env.GITGUARDIAN_API_KEY != '' + env: + GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} + run: | + pip install ggshield + ggshield secret scan repo . diff --git a/tests/litellm/test_no_hardcoded_secrets.py b/tests/litellm/test_no_hardcoded_secrets.py new file mode 100644 index 00000000000..f22eb1f3a72 --- /dev/null +++ b/tests/litellm/test_no_hardcoded_secrets.py @@ -0,0 +1,74 @@ +""" +Test to ensure no hardcoded secrets exist in the codebase. + +This catches Base64 Basic Authentication strings and other secret patterns +that would be flagged by secret scanners like GitGuardian/ggshield. +""" + +import base64 +import os +import re + +import pytest + +# Root of the litellm package +LITELLM_ROOT = os.path.join(os.path.dirname(__file__), "..", "..", "litellm") + +# Regex for Base64 Basic Auth patterns: 'Basic ' +# Matches strings like: Basic YW55dGhpbmc6YW55dGhpbmc= +BASIC_AUTH_PATTERN = re.compile( + r"""['"]Basic\s+([A-Za-z0-9+/]{16,}={0,2})['"]""" +) + +# Directories/files to skip +SKIP_DIRS = {"__pycache__", ".git", "node_modules", ".mypy_cache", ".ruff_cache"} + + +def _is_real_base64_credentials(match_str: str) -> bool: + """Check if a Base64 string decodes to something that looks like credentials (user:pass).""" + try: + # Add padding if needed - Base64 strings may omit trailing '=' + padded = match_str + "=" * (-len(match_str) % 4) + decoded = base64.b64decode(padded).decode("utf-8", errors="ignore") + return ":" in decoded + except Exception: + return False + + +def _collect_python_files(): + """Collect all Python files under the litellm package.""" + python_files = [] + for root, dirs, files in os.walk(LITELLM_ROOT): + dirs[:] = [d for d in dirs if d not in SKIP_DIRS] + for f in files: + if f.endswith(".py"): + python_files.append(os.path.join(root, f)) + return python_files + + +def test_no_hardcoded_basic_auth_secrets(): + """Ensure no hardcoded Base64 Basic Authentication credentials exist in source code. + + This test prevents regressions like the one caught by T-Mobile's GitGuardian + container scan, where a docstring contained a literal Base64-encoded + 'Basic YW55dGhpbmc6YW55dGhpbmc' string (anything:anything). + """ + violations = [] + + for filepath in _collect_python_files(): + with open(filepath, "r", errors="ignore") as f: + for line_num, line in enumerate(f, start=1): + for match in BASIC_AUTH_PATTERN.finditer(line): + b64_value = match.group(1) + if _is_real_base64_credentials(b64_value): + rel_path = os.path.relpath(filepath, LITELLM_ROOT) + violations.append( + f" {rel_path}:{line_num}: {match.group(0)}" + ) + + assert not violations, ( + "Found hardcoded Base64 Basic Auth credentials that will be flagged by " + "secret scanners (e.g. GitGuardian/ggshield):\n" + + "\n".join(violations) + + "\n\nUse placeholders like '' in comments/docs instead." + ) From 05c3a95da8c3daf293b5f5df8bafe0f60c6479a6 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Thu, 26 Feb 2026 12:48:43 -0300 Subject: [PATCH 11/52] fix(ci): add permissions block to secret-scan job Address github-advanced-security bot review comment by setting explicit minimal permissions (contents: read) for the GITHUB_TOKEN. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/test-linting.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index abd857ea2b0..bb400ccf525 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -78,6 +78,8 @@ jobs: secret-scan: runs-on: ubuntu-latest timeout-minutes: 5 + permissions: + contents: read steps: - uses: actions/checkout@v4 From 1c376afc855b944565d946614ccbca4b2854a2c3 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Thu, 26 Feb 2026 12:51:28 -0300 Subject: [PATCH 12/52] fix(ci): use secrets context in ggshield step condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step-level env is not visible to the if condition — reference secrets directly so ggshield actually runs when the key is configured. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/test-linting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index bb400ccf525..48bd21e0e3c 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -97,7 +97,7 @@ jobs: pytest tests/litellm/test_no_hardcoded_secrets.py -v - name: Run ggshield secret scan - if: env.GITGUARDIAN_API_KEY != '' + if: ${{ secrets.GITGUARDIAN_API_KEY != '' }} env: GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} run: | From 8a6a67bfcf62db1a2d7769a645452628732b220c Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Thu, 26 Feb 2026 17:49:44 -0300 Subject: [PATCH 13/52] fix(proxy): isolate get_config failures from model loading in sync loop A database timeout (httpcore.ReadTimeout) during get_config() in _update_llm_router would propagate and prevent ALL DB models from loading into the router. Now get_config() failures are caught separately so model add/delete operations still proceed. Similarly, _delete_deployment catches get_config failures and safely skips cleanup rather than crashing the entire sync cycle. Co-Authored-By: Claude Opus 4.6 --- litellm/proxy/proxy_server.py | 26 ++- .../test_update_llm_router_resilience.py | 173 ++++++++++++++++++ 2 files changed, 196 insertions(+), 3 deletions(-) create mode 100644 tests/test_litellm/proxy/test_update_llm_router_resilience.py diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index be76c2ac5fb..e41b603abc9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3502,7 +3502,15 @@ class ProxyConfig: combined_id_list.append(model_info.id) ## CONFIG MODELS ## - config = await self.get_config(config_file_path=user_config_file_path) + try: + config = await self.get_config(config_file_path=user_config_file_path) + except Exception as e: + verbose_proxy_logger.warning( + "Failed to load config in _delete_deployment: %s. " + "Skipping deployment cleanup to avoid removing valid models.", + str(e), + ) + return 0 model_list = config.get("model_list", None) if model_list: for model in model_list: @@ -3620,8 +3628,20 @@ class ProxyConfig: proxy_logging_obj: ProxyLogging, ): global llm_router, llm_model_list, master_key, general_settings - config_data = await proxy_config.get_config() - search_tools = self.parse_search_tools(config_data) + + # Load config separately so a timeout here doesn't block model loading + config_data: dict = {} + search_tools = None + try: + config_data = await proxy_config.get_config() + search_tools = self.parse_search_tools(config_data) + except Exception as e: + verbose_proxy_logger.warning( + "Failed to load config in _update_llm_router: %s. " + "Proceeding with model loading using cached/empty config.", + str(e), + ) + try: models_list: list = new_models if isinstance(new_models, list) else [] if llm_router is None and master_key is not None: diff --git a/tests/test_litellm/proxy/test_update_llm_router_resilience.py b/tests/test_litellm/proxy/test_update_llm_router_resilience.py new file mode 100644 index 00000000000..0ee865ab48c --- /dev/null +++ b/tests/test_litellm/proxy/test_update_llm_router_resilience.py @@ -0,0 +1,173 @@ +""" +Test that _update_llm_router and _delete_deployment are resilient to +config loading failures (e.g. database timeouts). + +This addresses a bug where httpcore.ReadTimeout from the Prisma client +during get_config() would prevent ALL DB models from loading into the +router, because the exception propagated up and was caught by the +catch-all handler in _update_llm_router. +""" + +import pytest +from unittest.mock import AsyncMock, MagicMock, patch + +from litellm.proxy.proxy_server import ProxyConfig + + +def _make_db_model(model_name: str, model_id: str): + """Helper to create a mock DB model record.""" + record = MagicMock() + record.model_id = model_id + record.model_name = model_name + record.litellm_params = {"model": model_name} + record.model_info = {"id": model_id} + record.created_by = "default_user_id" + record.created_at = None + record.updated_at = None + record.updated_by = None + return record + + +class TestUpdateLlmRouterResilience: + """Test _update_llm_router handles get_config failures gracefully.""" + + @pytest.mark.asyncio + async def test_models_loaded_when_get_config_times_out(self): + """DB models should still be added to the router when get_config() raises a timeout.""" + proxy_config = ProxyConfig() + + db_models = [_make_db_model("gpt-5.1", "db-id-1")] + + mock_router = MagicMock() + mock_router.get_model_list.return_value = [] + mock_router.get_model_ids.return_value = [] + + mock_proxy_logging = MagicMock() + + with ( + patch.object( + proxy_config, + "get_config", + new_callable=AsyncMock, + side_effect=Exception("httpcore.ReadTimeout"), + ), + patch.object(proxy_config, "_add_deployment", return_value=1) as mock_add, + patch.object( + proxy_config, + "_delete_deployment", + new_callable=AsyncMock, + return_value=0, + ), + patch("litellm.proxy.proxy_server.llm_router", mock_router), + patch("litellm.proxy.proxy_server.master_key", "sk-test"), + patch("litellm.proxy.proxy_server.llm_model_list", []), + patch("litellm.proxy.proxy_server.general_settings", {}), + ): + await proxy_config._update_llm_router( + new_models=db_models, + proxy_logging_obj=mock_proxy_logging, + ) + + # _add_deployment should still have been called despite get_config failure + mock_add.assert_called_once_with(db_models=db_models) + + @pytest.mark.asyncio + async def test_get_config_success_still_works(self): + """Normal flow should still work when get_config succeeds.""" + proxy_config = ProxyConfig() + + db_models = [_make_db_model("gpt-5.1", "db-id-1")] + + mock_router = MagicMock() + mock_router.get_model_list.return_value = [] + mock_router.get_model_ids.return_value = [] + + mock_proxy_logging = MagicMock() + + with ( + patch.object( + proxy_config, + "get_config", + new_callable=AsyncMock, + return_value={"model_list": []}, + ), + patch.object(proxy_config, "_add_deployment", return_value=1) as mock_add, + patch.object( + proxy_config, + "_delete_deployment", + new_callable=AsyncMock, + return_value=0, + ), + patch("litellm.proxy.proxy_server.llm_router", mock_router), + patch("litellm.proxy.proxy_server.master_key", "sk-test"), + patch("litellm.proxy.proxy_server.llm_model_list", []), + patch("litellm.proxy.proxy_server.general_settings", {}), + ): + await proxy_config._update_llm_router( + new_models=db_models, + proxy_logging_obj=mock_proxy_logging, + ) + + mock_add.assert_called_once_with(db_models=db_models) + + +class TestDeleteDeploymentResilience: + """Test _delete_deployment handles get_config failures gracefully.""" + + @pytest.mark.asyncio + async def test_returns_zero_when_get_config_times_out(self): + """Should return 0 (no deletions) when get_config fails, not raise.""" + proxy_config = ProxyConfig() + + db_models = [_make_db_model("gpt-5.1", "db-id-1")] + + mock_router = MagicMock() + mock_router.get_model_ids.return_value = ["db-id-1", "config-id-1"] + + with ( + patch.object( + proxy_config, + "get_config", + new_callable=AsyncMock, + side_effect=Exception("httpcore.ReadTimeout"), + ), + patch("litellm.proxy.proxy_server.llm_router", mock_router), + patch("litellm.proxy.proxy_server.premium_user", False), + ): + result = await proxy_config._delete_deployment(db_models=db_models) + + # Should safely return 0 instead of raising + assert result == 0 + # Should NOT have deleted any deployments + mock_router.delete_deployment.assert_not_called() + + @pytest.mark.asyncio + async def test_normal_delete_still_works(self): + """Normal deletion should work when get_config succeeds.""" + proxy_config = ProxyConfig() + + db_models = [_make_db_model("gpt-5.1", "db-id-1")] + + mock_router = MagicMock() + # Router has a model ID that's not in DB or config -> should be deleted + mock_router.get_model_ids.return_value = ["db-id-1", "stale-id"] + mock_router.delete_deployment.return_value = True + mock_router._generate_model_id = MagicMock(return_value="config-id-1") + + with ( + patch.object( + proxy_config, + "get_config", + new_callable=AsyncMock, + return_value={"model_list": [ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4"}, "model_info": {"id": "config-id-1"}} + ]}, + ), + patch("litellm.proxy.proxy_server.llm_router", mock_router), + patch("litellm.proxy.proxy_server.premium_user", False), + ): + result = await proxy_config._delete_deployment(db_models=db_models) + + # "stale-id" should have been deleted (not in db_models or config) + assert result == 1 + mock_router.delete_deployment.assert_called_once_with(id="stale-id") From 9b7b987b35902f4e9a26a1d6c57688f4504de47a Mon Sep 17 00:00:00 2001 From: Harshit Jain Date: Fri, 27 Feb 2026 15:15:57 +0530 Subject: [PATCH 14/52] fix: req changes --- litellm/integrations/datadog/datadog_metrics.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/litellm/integrations/datadog/datadog_metrics.py b/litellm/integrations/datadog/datadog_metrics.py index a22efd66000..70576d2d157 100644 --- a/litellm/integrations/datadog/datadog_metrics.py +++ b/litellm/integrations/datadog/datadog_metrics.py @@ -209,16 +209,14 @@ class DatadogMetricsLogger(CustomBatchLogger): if not self.log_queue: return + batch = self.log_queue.copy() + payload_data: DatadogMetricsPayload = {"series": batch} + try: - # We must only send the current batch, so copy and clear log queue - batch = self.log_queue.copy() - # Note: CustomBatchLogger clears queue in flush_queue, but we'll manually copy what we need - - payload_data: DatadogMetricsPayload = {"series": batch} - await self._upload_to_datadog(payload_data) - except Exception as e: + # Re-insert failed batch so next flush retries + self.log_queue.extend(batch) verbose_logger.exception( f"Datadog Metrics: Error in async_send_batch: {str(e)}" ) From 9e4cf0e4df174004b9f872fca9f3ac12157c3c4d Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Fri, 27 Feb 2026 15:59:36 +0530 Subject: [PATCH 15/52] Update tests/test_litellm/integrations/datadog/test_datadog_metrics.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../integrations/datadog/test_datadog_metrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py index 1008a4f78cf..c851f5fdead 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py @@ -35,9 +35,9 @@ def clean_env(): @pytest.mark.asyncio async def test_init(clean_env): """Test initialization sets up clients and url correctly.""" - logger = DatadogMetricsLogger() - assert logger.dd_api_key == "test_api_key" - assert logger.dd_site == "test.datadoghq.com" +async def test_init(clean_env): + """Test initialization sets up clients and url correctly.""" + logger = DatadogMetricsLogger(start_periodic_flush=False) assert logger.upload_url == "https://api.test.datadoghq.com/api/v2/series" From 9d3e97a2d8d33849ed6fead42a1e689e1f096f3f Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Fri, 27 Feb 2026 15:59:47 +0530 Subject: [PATCH 16/52] Update litellm/integrations/datadog/datadog_metrics.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/integrations/datadog/datadog_metrics.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/litellm/integrations/datadog/datadog_metrics.py b/litellm/integrations/datadog/datadog_metrics.py index 70576d2d157..fcf40701e28 100644 --- a/litellm/integrations/datadog/datadog_metrics.py +++ b/litellm/integrations/datadog/datadog_metrics.py @@ -215,11 +215,10 @@ class DatadogMetricsLogger(CustomBatchLogger): try: await self._upload_to_datadog(payload_data) except Exception as e: - # Re-insert failed batch so next flush retries - self.log_queue.extend(batch) verbose_logger.exception( f"Datadog Metrics: Error in async_send_batch: {str(e)}" ) + raise async def _upload_to_datadog(self, payload: DatadogMetricsPayload): if not self.dd_api_key: From c58aea4888465f5a7815288a0ed2aaada0cb4f9c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 27 Feb 2026 10:50:58 +0000 Subject: [PATCH 17/52] chore: regenerate poetry.lock to match pyproject.toml (#1) Co-authored-by: github-actions[bot] --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 34227a69ccb..0314a360542 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3222,15 +3222,15 @@ files = [ [[package]] name = "litellm-proxy-extras" -version = "0.4.48" +version = "0.4.49" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" groups = ["main"] markers = "extra == \"proxy\"" files = [ - {file = "litellm_proxy_extras-0.4.48-py3-none-any.whl", hash = "sha256:097001fccec5dbf4cffd902114898a9cfeba62673202447d55d2d0286cf93126"}, - {file = "litellm_proxy_extras-0.4.48.tar.gz", hash = "sha256:5d5d8acf31b92d0cd6738555fb4a2411819755155438de9fb23c724c356400a2"}, + {file = "litellm_proxy_extras-0.4.49-py3-none-any.whl", hash = "sha256:aeb0e08b4705c19fdc5b75a43c608a82fc36032f6d83be509dbf37baea62f2cd"}, + {file = "litellm_proxy_extras-0.4.49.tar.gz", hash = "sha256:d9bdae54d1e3398f2e2025c9d8b98a19e226874337d540d5415922d7dbbc97bb"}, ] [[package]] @@ -7989,4 +7989,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "b9b1e47b3b84748c0053be6a544c2399bf2601746a4f88dcb1be7c5e4eeab359" +content-hash = "bbc7d43f5484af4c8877fe66e34f8283069528379af49d573036ba144cc2eb7a" From 8dc8e826ba6d0dd97bf5c170be5d2d367faaab18 Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Fri, 27 Feb 2026 16:21:41 +0530 Subject: [PATCH 18/52] Update tests/test_litellm/integrations/datadog/test_datadog_metrics.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../test_litellm/integrations/datadog/test_datadog_metrics.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py index c851f5fdead..d5b6cce972e 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py @@ -35,8 +35,7 @@ def clean_env(): @pytest.mark.asyncio async def test_init(clean_env): """Test initialization sets up clients and url correctly.""" -async def test_init(clean_env): - """Test initialization sets up clients and url correctly.""" + logger = DatadogMetricsLogger(start_periodic_flush=False) logger = DatadogMetricsLogger(start_periodic_flush=False) assert logger.upload_url == "https://api.test.datadoghq.com/api/v2/series" From 44dd13994157d4ce479f68e7fa3a4084e202aabb Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Fri, 27 Feb 2026 16:21:58 +0530 Subject: [PATCH 19/52] Update tests/test_litellm/integrations/datadog/test_datadog_metrics.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- tests/test_litellm/integrations/datadog/test_datadog_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py index d5b6cce972e..3b396590e26 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py @@ -43,7 +43,7 @@ async def test_init(clean_env): @pytest.mark.asyncio async def test_extract_tags(clean_env): """Test tag extraction from a StandardLoggingPayload.""" - logger = DatadogMetricsLogger() + logger = DatadogMetricsLogger(start_periodic_flush=False) payload = StandardLoggingPayload( custom_llm_provider="openai", From 04fda9ec4ac9e28b6b73d0a3ee82a51cb6aaf98f Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Fri, 27 Feb 2026 16:28:33 +0530 Subject: [PATCH 20/52] Update tests/test_litellm/integrations/datadog/test_datadog_metrics.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- tests/test_litellm/integrations/datadog/test_datadog_metrics.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py index 3b396590e26..f9d79694619 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py @@ -36,7 +36,6 @@ def clean_env(): async def test_init(clean_env): """Test initialization sets up clients and url correctly.""" logger = DatadogMetricsLogger(start_periodic_flush=False) - logger = DatadogMetricsLogger(start_periodic_flush=False) assert logger.upload_url == "https://api.test.datadoghq.com/api/v2/series" From b6a53fb174f832c1dc0690a1350d77e158f5e793 Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Fri, 27 Feb 2026 16:28:48 +0530 Subject: [PATCH 21/52] Update tests/test_litellm/integrations/datadog/test_datadog_metrics.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- tests/test_litellm/integrations/datadog/test_datadog_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py index f9d79694619..35e2f537cc2 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py @@ -66,7 +66,7 @@ async def test_extract_tags(clean_env): @pytest.mark.asyncio async def test_extract_tags_no_team(clean_env): """Test tag extraction when no team info is present.""" - logger = DatadogMetricsLogger() + logger = DatadogMetricsLogger(start_periodic_flush=False) payload = StandardLoggingPayload( custom_llm_provider="anthropic", From 5d5409e77aa899887a4fc9c010100de4a61f4b56 Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Thu, 26 Feb 2026 20:07:46 +0530 Subject: [PATCH 22/52] fix presidio memory leak --- .../guardrails/guardrail_hooks/presidio.py | 289 ++++++++++++------ .../guardrails/guardrail_initializers.py | 1 + 2 files changed, 199 insertions(+), 91 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 34fbf47253b..d4ce2c371e0 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -427,6 +427,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): analyze_results: Any, output_parse_pii: bool, masked_entity_count: Dict[str, int], + request_data: Optional[Dict] = None, ) -> str: """ Send analysis results to the Presidio anonymizer endpoint to get redacted text @@ -482,13 +483,24 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if item["operator"] == "replace" and output_parse_pii is True: # check if token in dict # if exists, add a uuid to the replacement token for swapping back to the original text in llm response output parsing - if replacement in self.pii_tokens: - replacement = replacement + str(uuid.uuid4()) + pii_tokens = self.pii_tokens + if request_data is not None: + if "pii_tokens" not in request_data: + request_data["pii_tokens"] = {} + pii_tokens = request_data["pii_tokens"] - self.pii_tokens[replacement] = new_text[ + # Always append a UUID to ensure the replacement token is unique to this request and session. + # This prevents collisions where the LLM might hallucinate a generic token like [PHONE_NUMBER]. + replacement = f"{replacement}_{str(uuid.uuid4())[:12]}" + + pii_tokens[replacement] = new_text[ start:end ] # get text it'll replace + verbose_proxy_logger.info( + f"\033[92mPII Masking\033[0m: Created token {replacement} for original text: '{new_text[start:end]}'" + ) + new_text = new_text[:start] + replacement + new_text[end:] entity_type = item.get("entity_type", None) if entity_type is not None: @@ -525,10 +537,27 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): return analyze_results filtered_results: List[PresidioAnalyzeResponseItem] = [] + print( + "DEBUG filter input:", + analyze_results, + " deny_list:", + self.presidio_entities_deny_list, + ) for item in analyze_results: entity_type = item.get("entity_type") - if entity_type and entity_type in self.presidio_entities_deny_list: + deny_list_strings = [ + x.value if hasattr(x, "value") else str(x) + for x in self.presidio_entities_deny_list + ] + str_entity_type = str( + entity_type.value if hasattr(entity_type, "value") else entity_type + ) + print( + f"DEBUG entity_type: {entity_type}, str_entity_type: '{str_entity_type}', deny_strings: {deny_list_strings}" + ) + if entity_type and str_entity_type in deny_list_strings: + print(f"DEBUG Skipping {entity_type} due to deny list") continue if self.presidio_score_thresholds: @@ -621,6 +650,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): analyze_results=analyze_results, output_parse_pii=output_parse_pii, masked_entity_count=masked_entity_count, + request_data=request_data, ) return anonymized_text return redacted_text["text"] @@ -866,14 +896,130 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if isinstance(response, ModelResponse) and not isinstance( response.choices[0], StreamingChoices ): # /chat/completions requests - if isinstance(response.choices[0].message.content, str): - verbose_proxy_logger.debug( - f"self.pii_tokens: {self.pii_tokens}; initial response: {response.choices[0].message.content}" - ) - for key, value in self.pii_tokens.items(): - response.choices[0].message.content = response.choices[ - 0 - ].message.content.replace(key, value) + await self._process_response_for_pii( + response=response, + request_data=data, + mode="unmask", + ) + return response + + async def _process_response_for_pii( + self, + response: ModelResponse, + request_data: dict, + mode: Literal["mask", "unmask"], + ) -> ModelResponse: + """ + Helper to recursively process a ModelResponse for PII. + Handles all choices and tool calls. + """ + pii_tokens = ( + request_data.get("pii_tokens", self.pii_tokens) + if request_data + else self.pii_tokens + ) + presidio_config = self.get_presidio_settings_from_request_data( + request_data or {} + ) + + for choice in response.choices: + message = getattr(choice, "message", None) + if message is None: + continue + + # 1. Process content + content = getattr(message, "content", None) + if isinstance(content, str): + if mode == "unmask": + for token, original_text in pii_tokens.items(): + if token in content: + verbose_proxy_logger.info( + f"\033[94mPII Unmasking\033[0m: Found token {token} in response. Replacing with original text." + ) + content = content.replace(token, original_text) + # FALLBACK: Handle truncated tokens (token cut off by max_tokens) + elif any( + token.startswith(content[i:]) + for i in range( + max(0, len(content) - len(token)), len(content) + ) + if len(content[i:]) > 15 + ): + # If the end of content matches the start of a token, it's likely truncated + for i in range( + max(0, len(content) - len(token)), len(content) + ): + sub = content[i:] + if token.startswith(sub) and len(sub) > 15: + verbose_proxy_logger.info( + f"\033[93mPII Unmasking\033[0m: Found truncated token {sub}... in response. Replacing with original text." + ) + content = content[:i] + original_text + break + message.content = content + elif mode == "mask": + message.content = await self.check_pii( + text=content, + output_parse_pii=False, + presidio_config=presidio_config, + request_data=request_data, + ) + elif isinstance(content, list): + for item in content: + if not isinstance(item, dict): + continue + text_value = item.get("text") + if text_value is None: + continue + if mode == "unmask": + for token, original_text in pii_tokens.items(): + text_value = text_value.replace(token, original_text) + item["text"] = text_value + elif mode == "mask": + item["text"] = await self.check_pii( + text=text_value, + output_parse_pii=False, + presidio_config=presidio_config, + request_data=request_data, + ) + + # 2. Process tool calls + tool_calls = getattr(message, "tool_calls", None) + if tool_calls: + for tool_call in tool_calls: + function = getattr(tool_call, "function", None) + if function and hasattr(function, "arguments"): + args = function.arguments + if isinstance(args, str): + if mode == "unmask": + for token, original_text in pii_tokens.items(): + args = args.replace(token, original_text) + function.arguments = args + elif mode == "mask": + function.arguments = await self.check_pii( + text=args, + output_parse_pii=False, + presidio_config=presidio_config, + request_data=request_data, + ) + + # 3. Process legacy function calls + function_call = getattr(message, "function_call", None) + if function_call and hasattr(function_call, "arguments"): + args = function_call.arguments + if isinstance(args, str): + if mode == "unmask": + for token, original_text in pii_tokens.items(): + args = args.replace(token, original_text) + function_call.arguments = args + elif mode == "mask": + function_call.arguments = await self.check_pii( + text=args, + output_parse_pii=False, + presidio_config=presidio_config, + request_data=request_data, + ) + return response async def _mask_output_response( @@ -891,37 +1037,12 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if response.choices and isinstance(response.choices[0], StreamingChoices): return response - presidio_config = self.get_presidio_settings_from_request_data( - request_data or {} + await self._process_response_for_pii( + response=response, + request_data=request_data, + mode="mask", ) - - for choice in response.choices: - # Type narrowing: StreamingChoices doesn't have .message attribute - if not hasattr(choice, "message"): - continue - content = getattr(choice.message, "content", None) # type: ignore - if content is None: - continue - if isinstance(content, str): - choice.message.content = await self.check_pii( # type: ignore - text=content, - output_parse_pii=False, - presidio_config=presidio_config, - request_data=request_data, - ) - elif isinstance(content, list): - for item in content: - if not isinstance(item, dict): - continue - text_value = item.get("text") - if text_value is None: - continue - item["text"] = await self.check_pii( - text=text_value, - output_parse_pii=False, - presidio_config=presidio_config, - request_data=request_data, - ) + return response return response @@ -934,7 +1055,9 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): """ Process streaming response chunks to unmask PII tokens when needed. """ - from litellm.llms.base_llm.base_model_iterator import MockResponseIterator + from litellm.llms.base_llm.base_model_iterator import ( + convert_model_response_to_streaming, + ) from litellm.main import stream_chunk_builder from litellm.types.utils import ModelResponse @@ -959,45 +1082,16 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): return # Apply Presidio masking on the assembled response - presidio_config = self.get_presidio_settings_from_request_data( - request_data or {} - ) - - content_to_mask = "" - if ( - hasattr(assembled_model_response, "choices") - and len(assembled_model_response.choices) > 0 - ): - if hasattr( - assembled_model_response.choices[0], "message" - ) and hasattr( - assembled_model_response.choices[0].message, "content" - ): - content_to_mask = ( - assembled_model_response.choices[0].message.content or "" - ) - - masked_content = await self.check_pii( - text=content_to_mask, - output_parse_pii=False, - presidio_config=presidio_config, + await self._process_response_for_pii( + response=assembled_model_response, request_data=request_data, + mode="mask", ) - if ( - hasattr(assembled_model_response, "choices") - and len(assembled_model_response.choices) > 0 - ): - if hasattr(assembled_model_response.choices[0], "message"): - assembled_model_response.choices[ - 0 - ].message.content = masked_content - - mock_response = MockResponseIterator( - model_response=assembled_model_response + mock_response_stream = convert_model_response_to_streaming( + assembled_model_response ) - async for chunk in mock_response: - yield chunk + yield mock_response_stream return except Exception as e: @@ -1011,7 +1105,13 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): return # --- PII unmasking path (output_parse_pii=True) --- - if not (self.output_parse_pii and self.pii_tokens): + # --- PII unmasking path (output_parse_pii=True) --- + pii_tokens = ( + request_data.get("pii_tokens", self.pii_tokens) + if request_data + else self.pii_tokens + ) + if not (self.output_parse_pii and pii_tokens): async for chunk in response: yield chunk return @@ -1034,20 +1134,27 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): yield chunk return - # Apply PII unmasking to assembled content - for choice in assembled_model_response.choices: - if hasattr(choice, "message") and hasattr(choice.message, "content"): - content = choice.message.content - if isinstance(content, str): - for token, original_text in self.pii_tokens.items(): - content = content.replace(token, original_text) - choice.message.content = content + # --- PRESERVE USAGE METADATA --- + # stream_chunk_builder might miss usage if it's only in the last chunk + if ( + not hasattr(assembled_model_response, "usage") + or not assembled_model_response.usage + ) and all_chunks: + last_chunk = all_chunks[-1] + if hasattr(last_chunk, "usage") and last_chunk.usage: + assembled_model_response.usage = last_chunk.usage - mock_response = MockResponseIterator( - model_response=assembled_model_response + # Apply PII unmasking to assembled content (unmasking tokens back to original text) + await self._process_response_for_pii( + response=assembled_model_response, + request_data=request_data, + mode="unmask", ) - async for chunk in mock_response: - yield chunk + + mock_response_stream = convert_model_response_to_streaming( + assembled_model_response + ) + yield mock_response_stream except Exception as e: verbose_proxy_logger.error(f"Error in PII streaming processing: {str(e)}") diff --git a/litellm/proxy/guardrails/guardrail_initializers.py b/litellm/proxy/guardrails/guardrail_initializers.py index 639aebf45c9..109f2237165 100644 --- a/litellm/proxy/guardrails/guardrail_initializers.py +++ b/litellm/proxy/guardrails/guardrail_initializers.py @@ -93,6 +93,7 @@ def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail): presidio_analyzer_api_base=litellm_params.presidio_analyzer_api_base, presidio_anonymizer_api_base=litellm_params.presidio_anonymizer_api_base, presidio_language=litellm_params.presidio_language, + presidio_entities_deny_list=litellm_params.presidio_entities_deny_list, apply_to_output=False, ) params.update(overrides) From 1b632ed473852c0a3f028ec7fe62654bfcacd675 Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Thu, 26 Feb 2026 20:24:19 +0530 Subject: [PATCH 23/52] chore: remove debug logs from presidio guardrail --- .../guardrails/guardrail_hooks/presidio.py | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index d4ce2c371e0..d8bf2bc306a 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -497,10 +497,6 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): start:end ] # get text it'll replace - verbose_proxy_logger.info( - f"\033[92mPII Masking\033[0m: Created token {replacement} for original text: '{new_text[start:end]}'" - ) - new_text = new_text[:start] + replacement + new_text[end:] entity_type = item.get("entity_type", None) if entity_type is not None: @@ -537,12 +533,6 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): return analyze_results filtered_results: List[PresidioAnalyzeResponseItem] = [] - print( - "DEBUG filter input:", - analyze_results, - " deny_list:", - self.presidio_entities_deny_list, - ) for item in analyze_results: entity_type = item.get("entity_type") @@ -553,11 +543,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): str_entity_type = str( entity_type.value if hasattr(entity_type, "value") else entity_type ) - print( - f"DEBUG entity_type: {entity_type}, str_entity_type: '{str_entity_type}', deny_strings: {deny_list_strings}" - ) if entity_type and str_entity_type in deny_list_strings: - print(f"DEBUG Skipping {entity_type} due to deny list") continue if self.presidio_score_thresholds: @@ -933,9 +919,6 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if mode == "unmask": for token, original_text in pii_tokens.items(): if token in content: - verbose_proxy_logger.info( - f"\033[94mPII Unmasking\033[0m: Found token {token} in response. Replacing with original text." - ) content = content.replace(token, original_text) # FALLBACK: Handle truncated tokens (token cut off by max_tokens) elif any( @@ -951,9 +934,6 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): ): sub = content[i:] if token.startswith(sub) and len(sub) > 15: - verbose_proxy_logger.info( - f"\033[93mPII Unmasking\033[0m: Found truncated token {sub}... in response. Replacing with original text." - ) content = content[:i] + original_text break message.content = content From 3210da1e2a0bfcfabb6384766d5835875670a7a2 Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Fri, 27 Feb 2026 13:43:51 +0530 Subject: [PATCH 24/52] fix: req changes --- .../guardrails/guardrail_hooks/presidio.py | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index d8bf2bc306a..53d1c5f268a 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -533,13 +533,13 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): return analyze_results filtered_results: List[PresidioAnalyzeResponseItem] = [] + deny_list_strings = [ + x.value if hasattr(x, "value") else str(x) + for x in self.presidio_entities_deny_list + ] for item in analyze_results: entity_type = item.get("entity_type") - deny_list_strings = [ - x.value if hasattr(x, "value") else str(x) - for x in self.presidio_entities_deny_list - ] str_entity_type = str( entity_type.value if hasattr(entity_type, "value") else entity_type ) @@ -921,13 +921,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if token in content: content = content.replace(token, original_text) # FALLBACK: Handle truncated tokens (token cut off by max_tokens) - elif any( - token.startswith(content[i:]) - for i in range( - max(0, len(content) - len(token)), len(content) - ) - if len(content[i:]) > 15 - ): + else: # If the end of content matches the start of a token, it's likely truncated for i in range( max(0, len(content) - len(token)), len(content) @@ -1024,8 +1018,6 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): ) return response - return response - async def async_post_call_streaming_iterator_hook( self, user_api_key_dict: UserAPIKeyAuth, @@ -1084,7 +1076,6 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): yield chunk return - # --- PII unmasking path (output_parse_pii=True) --- # --- PII unmasking path (output_parse_pii=True) --- pii_tokens = ( request_data.get("pii_tokens", self.pii_tokens) @@ -1119,8 +1110,8 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if ( not hasattr(assembled_model_response, "usage") or not assembled_model_response.usage - ) and all_chunks: - last_chunk = all_chunks[-1] + ) and remaining_chunks: + last_chunk = remaining_chunks[-1] if hasattr(last_chunk, "usage") and last_chunk.usage: assembled_model_response.usage = last_chunk.usage From 1c0d07291270b231a69bc7dc35e75816b48a1698 Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Fri, 27 Feb 2026 18:42:09 +0530 Subject: [PATCH 25/52] Update litellm/proxy/proxy_server.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 00948d7f2ca..44f59c2cbbc 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4967,9 +4967,6 @@ class ProxyStartupEvent: f"config={mcp_semantic_filter_config}" ) - mcp_semantic_filter_config = litellm_settings.get( - "mcp_semantic_tool_filter", None - ) verbose_proxy_logger.debug( f"Semantic filter config: {mcp_semantic_filter_config}" ) From 2553698da530c624983ff00096a5841cb84ba098 Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Fri, 27 Feb 2026 23:39:42 +0530 Subject: [PATCH 26/52] feat: health check max tokens --- .../health_check_helpers.py | 4 +- litellm/proxy/health_check.py | 8 +++ .../proxy/test_health_check_max_tokens.py | 71 +++++++++++++++++++ 3 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 tests/test_litellm/proxy/test_health_check_max_tokens.py diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 47a27c8ef5b..9fb0c4126b3 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -44,7 +44,9 @@ class HealthCheckHelpers: model_params["model"] = cheapest_models[0] model_params["litellm_logging_obj"] = litellm_logging_obj model_params["fallbacks"] = fallback_models - model_params["max_tokens"] = 10 # gpt-5-nano throws errors for max_tokens=1 + model_params["max_tokens"] = model_params.get( + "max_tokens", 10 + ) # gpt-5-nano throws errors for max_tokens=1 await acompletion(**model_params) return {} diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index d228bdb2129..341ea4bd9e2 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -234,6 +234,14 @@ def _update_litellm_params_for_health_check( - for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID """ litellm_params["messages"] = _get_random_llm_message() + _health_check_max_tokens = model_info.get("health_check_max_tokens", None) + if _health_check_max_tokens is not None: + litellm_params["max_tokens"] = _health_check_max_tokens + elif "*" not in ( + model_info.get("health_check_model") or litellm_params.get("model") or "" + ): + litellm_params["max_tokens"] = 1 + _health_check_model = model_info.get("health_check_model", None) if _health_check_model is not None: litellm_params["model"] = _health_check_model diff --git a/tests/test_litellm/proxy/test_health_check_max_tokens.py b/tests/test_litellm/proxy/test_health_check_max_tokens.py new file mode 100644 index 00000000000..da2bb21abea --- /dev/null +++ b/tests/test_litellm/proxy/test_health_check_max_tokens.py @@ -0,0 +1,71 @@ + +import pytest +from litellm.proxy.health_check import _update_litellm_params_for_health_check +from litellm.litellm_core_utils.health_check_helpers import HealthCheckHelpers +from unittest.mock import AsyncMock, patch, MagicMock + +@pytest.mark.asyncio +async def test_update_litellm_params_max_tokens_default(): + """ + Test that max_tokens defaults to 1 for non-wildcard models. + """ + model_info = {} + litellm_params = {"model": "gpt-4"} + + updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) + + assert updated_params["max_tokens"] == 1 + +@pytest.mark.asyncio +async def test_update_litellm_params_max_tokens_custom(): + """ + Test that max_tokens respects health_check_max_tokens from model_info. + """ + model_info = {"health_check_max_tokens": 5} + litellm_params = {"model": "gpt-4"} + + updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) + + assert updated_params["max_tokens"] == 5 + +@pytest.mark.asyncio +async def test_update_litellm_params_max_tokens_wildcard(): + """ + Test that max_tokens does NOT default to 1 for wildcard models. + """ + model_info = {} + litellm_params = {"model": "openai/*"} + + updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) + + # Should not be set to 1 + assert "max_tokens" not in updated_params or updated_params["max_tokens"] != 1 + +@pytest.mark.asyncio +async def test_ahealth_check_wildcard_models_respects_max_tokens(): + """ + Test that ahealth_check_wildcard_models respects max_tokens if passed, + otherwise defaults to 10. + """ + with patch("litellm.litellm_core_utils.llm_request_utils.pick_cheapest_chat_models_from_llm_provider", return_value=["gpt-4o-mini"]), \ + patch("litellm.acompletion", new_callable=AsyncMock) as mock_acompletion: + + # Test Case 1: No max_tokens passed, should default to 10 + model_params = {} + await HealthCheckHelpers.ahealth_check_wildcard_models( + model="openai/*", + custom_llm_provider="openai", + model_params=model_params, + litellm_logging_obj=MagicMock() + ) + assert model_params["max_tokens"] == 10 + + # Test Case 2: Custom health_check_max_tokens passed via model_params, should be respected + model_params = {"max_tokens": 3} + await HealthCheckHelpers.ahealth_check_wildcard_models( + model="openai/*", + custom_llm_provider="openai", + model_params=model_params, + litellm_logging_obj=MagicMock() + ) + assert model_params["max_tokens"] == 3 From e3779a8fb3dd9f9295822519176596ffae39d219 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Fri, 27 Feb 2026 22:44:13 -0300 Subject: [PATCH 27/52] fix(lint): suppress PLR0915 too-many-statements in complex transform methods These three methods legitimately need many statements for their event/message transformation logic. Suppress the lint warning rather than artificially splitting the methods. Co-Authored-By: Claude Opus 4.6 --- .../responses_adapters/streaming_iterator.py | 2 +- .../responses_adapters/transformation.py | 2 +- litellm/llms/gemini/realtime/transformation.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 0e6268e82f3..2179e8005b8 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -64,7 +64,7 @@ class AnthropicResponsesStreamWrapper: self._current_block_index += 1 return self._current_block_index - def _process_event(self, event: Any) -> None: + def _process_event(self, event: Any) -> None: # noqa: PLR0915 """Convert one Responses API event into zero or more Anthropic chunks queued for emission.""" event_type = getattr(event, "type", None) if event_type is None and isinstance(event, dict): diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index c2752272905..fcef70cf255 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -48,7 +48,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: return source.get("url") return None - def translate_messages_to_responses_input( + def translate_messages_to_responses_input( # noqa: PLR0915 self, messages: List[ Union[ diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index d9465c95e3b..a3eedd36a64 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -829,7 +829,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): raise ValueError(f"Unknown openai event: {key}, value: {value}") return openai_event - def transform_realtime_response( + def transform_realtime_response( # noqa: PLR0915 self, message: Union[str, bytes], model: str, From 1ac3c230cb8fc96261a760734d101035d371ad20 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Sat, 28 Feb 2026 00:17:12 -0300 Subject: [PATCH 28/52] Update .github/scripts/close_duplicate_issues.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .github/scripts/close_duplicate_issues.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/scripts/close_duplicate_issues.py b/.github/scripts/close_duplicate_issues.py index 127bad31acd..4e17e1d6d8b 100755 --- a/.github/scripts/close_duplicate_issues.py +++ b/.github/scripts/close_duplicate_issues.py @@ -41,14 +41,11 @@ def gh(*args: str) -> str: def fetch_open_issues(repo: str | None) -> list[dict]: """Fetch all open issues (excluding PRs) via gh api --paginate.""" - endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc" - cmd = ["api", "--paginate", endpoint] if repo: - cmd.extend(["-f", f"owner={repo.split('/')[0]}", "-f", f"repo={repo.split('/')[1]}"]) endpoint = f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc" - cmd = ["api", "--paginate", endpoint] else: - cmd = ["api", "--paginate", "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"] + endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc" + cmd = ["api", "--paginate", endpoint] raw = gh(*cmd) # gh --paginate concatenates JSON arrays, so we may get multiple arrays From 94b7342da8806160a89a3755f786d734a1e71901 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Sat, 28 Feb 2026 00:17:22 -0300 Subject: [PATCH 29/52] Update .github/workflows/check_duplicate_issues.yml Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .github/workflows/check_duplicate_issues.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/check_duplicate_issues.yml b/.github/workflows/check_duplicate_issues.yml index ff252e37d9a..6d11ce573eb 100644 --- a/.github/workflows/check_duplicate_issues.yml +++ b/.github/workflows/check_duplicate_issues.yml @@ -34,6 +34,12 @@ jobs: with: sparse-checkout: .github/scripts + - name: Set up Python + if: github.event.action == 'opened' + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Auto-close if high-confidence duplicate if: github.event.action == 'opened' env: From 5a28ca985c1869c15d335fb0a98ec7a8d77d50f7 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Sat, 28 Feb 2026 00:17:31 -0300 Subject: [PATCH 30/52] Update .github/workflows/scan_duplicate_issues.yml Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .github/workflows/scan_duplicate_issues.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scan_duplicate_issues.yml b/.github/workflows/scan_duplicate_issues.yml index fce3e5f7954..eb001db33e1 100644 --- a/.github/workflows/scan_duplicate_issues.yml +++ b/.github/workflows/scan_duplicate_issues.yml @@ -33,13 +33,17 @@ jobs: - name: Scan for duplicate issues env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + INPUT_THRESHOLD: ${{ inputs.threshold }} + INPUT_CLOSE: ${{ inputs.close }} run: | CLOSE_FLAG="" - if [ "${{ inputs.close }}" = "true" ]; then + if [ "$INPUT_CLOSE" = "true" ]; then CLOSE_FLAG="--close" fi python3 .github/scripts/close_duplicate_issues.py \ --scan \ --repo ${{ github.repository }} \ - --threshold ${{ inputs.threshold }} \ + --threshold "$INPUT_THRESHOLD" \ $CLOSE_FLAG From fcabf9b602ade7a6986c46b360c2aa67b30fe27e Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Fri, 13 Feb 2026 06:24:41 -0300 Subject: [PATCH 31/52] fix(bedrock): filter internal json_tool_call when mixed with real tools Fixes #18381: When using both tools and response_format with Bedrock Converse API, LiteLLM internally adds json_tool_call to handle structured output. Bedrock may return both this internal tool AND real user-defined tools, breaking consumers like OpenAI Agents SDK. Changes: - Non-streaming: Added _filter_json_mode_tools() to handle 3 scenarios: only json_tool_call (convert to content), mixed (filter it out), or no json_tool_call (pass through) - Streaming: Added json_mode tracking to AWSEventStreamDecoder to suppress json_tool_call chunks and convert to text content - Fixed optional_params.pop() mutation issue Co-Authored-By: Claude Sonnet 4.5 --- litellm/llms/bedrock/chat/converse_handler.py | 2 +- .../bedrock/chat/converse_transformation.py | 115 +++++--- litellm/llms/bedrock/chat/invoke_handler.py | 64 +++-- .../chat/test_converse_transformation.py | 259 ++++++++++++++++++ 4 files changed, 391 insertions(+), 49 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 60a93b169c8..ec5b942ec1b 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -68,7 +68,7 @@ def make_sync_call( model_response=model_response, json_mode=json_mode ) else: - decoder = AWSEventStreamDecoder(model=model) + decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode) completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size)) # LOGGING diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index a0f2f65fb7f..1f48c290043 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1779,6 +1779,81 @@ class AmazonConverseConfig(BaseConfig): return content_str, tools, reasoningContentBlocks, citationsContentBlocks + @staticmethod + def _filter_json_mode_tools( + json_mode: Optional[bool], + tools: List[ChatCompletionToolCallChunk], + chat_completion_message: ChatCompletionResponseMessage, + ) -> Optional[List[ChatCompletionToolCallChunk]]: + """ + When json_mode is True, Bedrock may return the internal `json_tool_call` + tool alongside real user-defined tools. This method handles 3 scenarios: + + 1. Only json_tool_call present -> convert to text content, return None + 2. Mixed json_tool_call + real -> filter out json_tool_call, return real tools + 3. No json_tool_call / no json_mode -> return tools as-is + """ + if not json_mode or not tools: + return tools if tools else None + + json_tool_indices = [ + i + for i, t in enumerate(tools) + if t["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME + ] + + if not json_tool_indices: + # No json_tool_call found, return tools unchanged + return tools + + if len(json_tool_indices) == len(tools): + # All tools are json_tool_call — convert first one to content + verbose_logger.debug( + "Processing JSON tool call response for response_format" + ) + json_mode_content_str: Optional[str] = tools[0]["function"].get( + "arguments" + ) + if json_mode_content_str is not None: + try: + response_data = json.loads(json_mode_content_str) + if ( + isinstance(response_data, dict) + and "properties" in response_data + and len(response_data) == 1 + ): + response_data = response_data["properties"] + json_mode_content_str = json.dumps(response_data) + except json.JSONDecodeError: + pass + chat_completion_message["content"] = json_mode_content_str + return None + + # Mixed: filter out json_tool_call, keep real tools. + # Preserve the json_tool_call content as message text so the structured + # output from response_format is not silently lost. + first_idx = json_tool_indices[0] + json_mode_args = tools[first_idx]["function"].get("arguments") + if json_mode_args is not None: + try: + response_data = json.loads(json_mode_args) + if ( + isinstance(response_data, dict) + and "properties" in response_data + and len(response_data) == 1 + ): + response_data = response_data["properties"] + json_mode_args = json.dumps(response_data) + except json.JSONDecodeError: + pass + existing = chat_completion_message.get("content") or "" + chat_completion_message["content"] = ( + existing + json_mode_args if existing else json_mode_args + ) + + real_tools = [t for i, t in enumerate(tools) if i not in json_tool_indices] + return real_tools if real_tools else None + def _transform_response( # noqa: PLR0915 self, model: str, @@ -1801,7 +1876,7 @@ class AmazonConverseConfig(BaseConfig): additional_args={"complete_input_dict": data}, ) - json_mode: Optional[bool] = optional_params.pop("json_mode", None) + json_mode: Optional[bool] = optional_params.get("json_mode", None) ## RESPONSE OBJECT try: completion_response = ConverseResponseBlock(**response.json()) # type: ignore @@ -1885,37 +1960,13 @@ class AmazonConverseConfig(BaseConfig): self._transform_thinking_blocks(reasoningContentBlocks) ) chat_completion_message["content"] = content_str - if ( - json_mode is True - and tools is not None - and len(tools) == 1 - and tools[0]["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME - ): - verbose_logger.debug( - "Processing JSON tool call response for response_format" - ) - json_mode_content_str: Optional[str] = tools[0]["function"].get("arguments") - if json_mode_content_str is not None: - # Bedrock returns the response wrapped in a "properties" object - # We need to extract the actual content from this wrapper - try: - response_data = json.loads(json_mode_content_str) - - # If Bedrock wrapped the response in "properties", extract the content - if ( - isinstance(response_data, dict) - and "properties" in response_data - and len(response_data) == 1 - ): - response_data = response_data["properties"] - json_mode_content_str = json.dumps(response_data) - except json.JSONDecodeError: - # If parsing fails, use the original response - pass - - chat_completion_message["content"] = json_mode_content_str - elif tools: - chat_completion_message["tool_calls"] = tools + filtered_tools = self._filter_json_mode_tools( + json_mode=json_mode, + tools=tools, + chat_completion_message=chat_completion_message, + ) + if filtered_tools: + chat_completion_message["tool_calls"] = filtered_tools ## CALCULATING USAGE - bedrock returns usage in the headers usage = self._transform_usage(completion_response["usage"]) diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 1c58a11eebe..88f7341ed08 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -22,6 +22,7 @@ import litellm from litellm import verbose_logger from litellm._uuid import uuid from litellm.caching.caching import InMemoryCache +from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.logging_utils import track_llm_api_timing @@ -252,7 +253,7 @@ async def make_call( response.aiter_bytes(chunk_size=stream_chunk_size) ) else: - decoder = AWSEventStreamDecoder(model=model) + decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode) completion_stream = decoder.aiter_bytes( response.aiter_bytes(chunk_size=stream_chunk_size) ) @@ -346,7 +347,7 @@ def make_sync_call( response.iter_bytes(chunk_size=stream_chunk_size) ) else: - decoder = AWSEventStreamDecoder(model=model) + decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode) completion_stream = decoder.iter_bytes( response.iter_bytes(chunk_size=stream_chunk_size) ) @@ -1282,7 +1283,7 @@ def get_response_stream_shape(): class AWSEventStreamDecoder: - def __init__(self, model: str) -> None: + def __init__(self, model: str, json_mode: Optional[bool] = False) -> None: from botocore.parsers import EventStreamJSONParser self.model = model @@ -1290,6 +1291,8 @@ class AWSEventStreamDecoder: self.content_blocks: List[ContentBlockDeltaEvent] = [] self.tool_calls_index: Optional[int] = None self.response_id: Optional[str] = None + self.json_mode = json_mode + self._current_tool_name: Optional[str] = None def check_empty_tool_call_args(self) -> bool: """ @@ -1391,6 +1394,16 @@ class AWSEventStreamDecoder: response_tool_name = get_bedrock_tool_name( response_tool_name=_response_tool_name ) + self._current_tool_name = response_tool_name + + # When json_mode is True, suppress the internal json_tool_call + # and convert its content to text in delta events instead + if ( + self.json_mode is True + and response_tool_name == RESPONSE_FORMAT_TOOL_NAME + ): + return tool_use, provider_specific_fields, thinking_blocks + self.tool_calls_index = ( 0 if self.tool_calls_index is None else self.tool_calls_index + 1 ) @@ -1445,19 +1458,27 @@ class AWSEventStreamDecoder: if "text" in delta_obj: text = delta_obj["text"] elif "toolUse" in delta_obj: - tool_use = { - "id": None, - "type": "function", - "function": { - "name": None, - "arguments": delta_obj["toolUse"]["input"], - }, - "index": ( - self.tool_calls_index - if self.tool_calls_index is not None - else index - ), - } + # When json_mode is True and this is the internal json_tool_call, + # convert tool input to text content instead of tool call arguments + if ( + self.json_mode is True + and self._current_tool_name == RESPONSE_FORMAT_TOOL_NAME + ): + text = delta_obj["toolUse"]["input"] + else: + tool_use = { + "id": None, + "type": "function", + "function": { + "name": None, + "arguments": delta_obj["toolUse"]["input"], + }, + "index": ( + self.tool_calls_index + if self.tool_calls_index is not None + else index + ), + } elif "reasoningContent" in delta_obj: provider_specific_fields = { "reasoningContent": delta_obj["reasoningContent"], @@ -1494,6 +1515,17 @@ class AWSEventStreamDecoder: ) -> Optional[ChatCompletionToolCallChunk]: """Handle stop/contentBlockIndex event in converse chunk parsing.""" tool_use: Optional[ChatCompletionToolCallChunk] = None + + # If the ending block was the internal json_tool_call, skip emitting + # the empty-args tool chunk and reset tracking state + if ( + self.json_mode is True + and self._current_tool_name == RESPONSE_FORMAT_TOOL_NAME + ): + self._current_tool_name = None + return tool_use + + self._current_tool_name = None is_empty = self.check_empty_tool_call_args() if is_empty: tool_use = { diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index f6d3d3c12f7..345f3ae7c5d 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -3493,3 +3493,262 @@ class TestBedrockMinThinkingBudgetTokens: drop_params=False, ) assert "thinking" not in result or result.get("thinking") is None + +def test_transform_response_with_both_json_tool_call_and_real_tool(): + """ + When Bedrock returns BOTH json_tool_call AND a real tool (get_weather), + only the real tool should remain in tool_calls. The json_tool_call should be filtered out. + Fixes https://github.com/BerriAI/litellm/issues/18381 + """ + from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig + from litellm.types.utils import ModelResponse + + response_json = { + "metrics": {"latencyMs": 200}, + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "tooluse_json_001", + "name": "json_tool_call", + "input": { + "Current_Temperature": 62, + "Weather_Explanation": "Mild and cool.", + }, + } + }, + { + "toolUse": { + "toolUseId": "tooluse_weather_001", + "name": "get_weather", + "input": { + "location": "San Francisco, CA", + "unit": "fahrenheit", + }, + } + }, + ], + } + }, + "stopReason": "tool_use", + "usage": { + "inputTokens": 100, + "outputTokens": 50, + "totalTokens": 150, + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + }, + } + + class MockResponse: + def json(self): + return response_json + + @property + def text(self): + return json.dumps(response_json) + + config = AmazonConverseConfig() + model_response = ModelResponse() + optional_params = {"json_mode": True} + + result = config._transform_response( + model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + response=MockResponse(), + model_response=model_response, + stream=False, + logging_obj=None, + optional_params=optional_params, + api_key=None, + data=None, + messages=[], + encoding=None, + ) + + # Only real tool should remain + assert result.choices[0].message.tool_calls is not None + assert len(result.choices[0].message.tool_calls) == 1 + assert result.choices[0].message.tool_calls[0].function.name == "get_weather" + assert ( + result.choices[0].message.tool_calls[0].function.arguments + == '{"location": "San Francisco, CA", "unit": "fahrenheit"}' + ) + + # json_tool_call content should be preserved as message text + content = result.choices[0].message.content + assert content is not None + parsed = json.loads(content) + assert parsed["Current_Temperature"] == 62 + assert parsed["Weather_Explanation"] == "Mild and cool." + + +def test_transform_response_does_not_mutate_optional_params(): + """ + Verify that optional_params still contains json_mode after _transform_response. + Previously, .pop() was used which mutated the caller's dict. + """ + from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig + from litellm.types.utils import ModelResponse + + response_json = { + "metrics": {"latencyMs": 50}, + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "tooluse_001", + "name": "json_tool_call", + "input": {"result": "ok"}, + } + } + ], + } + }, + "stopReason": "tool_use", + "usage": { + "inputTokens": 10, + "outputTokens": 5, + "totalTokens": 15, + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + }, + } + + class MockResponse: + def json(self): + return response_json + + @property + def text(self): + return json.dumps(response_json) + + config = AmazonConverseConfig() + model_response = ModelResponse() + optional_params = {"json_mode": True, "other_key": "value"} + + config._transform_response( + model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + response=MockResponse(), + model_response=model_response, + stream=False, + logging_obj=None, + optional_params=optional_params, + api_key=None, + data=None, + messages=[], + encoding=None, + ) + + # json_mode should still be in optional_params (not popped) + assert "json_mode" in optional_params + assert optional_params["json_mode"] is True + assert optional_params["other_key"] == "value" + + +def test_streaming_filters_json_tool_call_with_real_tools(): + """ + Simulate streaming chunks where both json_tool_call and a real tool arrive. + Verify json_tool_call chunks are converted to text content while real tool + chunks pass through normally. + """ + from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder + from litellm.types.llms.bedrock import ( + ContentBlockDeltaEvent, + ContentBlockStartEvent, + ) + + decoder = AWSEventStreamDecoder(model="test-model", json_mode=True) + + # Chunk 1: json_tool_call start + json_start = ContentBlockStartEvent( + toolUse={ + "toolUseId": "tooluse_json_001", + "name": "json_tool_call", + } + ) + tool_use_1, _, _ = decoder._handle_converse_start_event(json_start) + # json_tool_call start should be suppressed (return None tool_use) + assert tool_use_1 is None + # tool_calls_index should NOT have been incremented + assert decoder.tool_calls_index is None + + # Chunk 2: json_tool_call delta — should become text, not tool_use + json_delta = ContentBlockDeltaEvent(toolUse={"input": '{"temp": 62}'}) + text_2, tool_use_2, _, _, _ = decoder._handle_converse_delta_event( + json_delta, index=0 + ) + assert text_2 == '{"temp": 62}' + assert tool_use_2 is None + + # Chunk 3: json_tool_call stop + stop_tool = decoder._handle_converse_stop_event(index=0) + assert stop_tool is None + # _current_tool_name should be reset + assert decoder._current_tool_name is None + + # Chunk 4: real tool start + real_start = ContentBlockStartEvent( + toolUse={ + "toolUseId": "tooluse_weather_001", + "name": "get_weather", + } + ) + tool_use_4, _, _ = decoder._handle_converse_start_event(real_start) + assert tool_use_4 is not None + assert tool_use_4["function"]["name"] == "get_weather" + assert decoder.tool_calls_index == 0 + + # Chunk 5: real tool delta + real_delta = ContentBlockDeltaEvent( + toolUse={"input": '{"location": "SF"}'} + ) + text_5, tool_use_5, _, _, _ = decoder._handle_converse_delta_event( + real_delta, index=1 + ) + assert text_5 == "" + assert tool_use_5 is not None + assert tool_use_5["function"]["arguments"] == '{"location": "SF"}' + + +def test_streaming_without_json_mode_passes_all_tools(): + """ + Verify backward compatibility: when json_mode=False, all tools + (including json_tool_call if present) pass through unchanged. + """ + from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder + from litellm.types.llms.bedrock import ( + ContentBlockDeltaEvent, + ContentBlockStartEvent, + ) + + decoder = AWSEventStreamDecoder(model="test-model", json_mode=False) + + # json_tool_call start — should pass through when json_mode=False + json_start = ContentBlockStartEvent( + toolUse={ + "toolUseId": "tooluse_json_001", + "name": "json_tool_call", + } + ) + tool_use, _, _ = decoder._handle_converse_start_event(json_start) + assert tool_use is not None + assert tool_use["function"]["name"] == "json_tool_call" + assert decoder.tool_calls_index == 0 + + # json_tool_call delta — should be a tool_use, not text + json_delta = ContentBlockDeltaEvent(toolUse={"input": '{"data": 1}'}) + text, tool_use_delta, _, _, _ = decoder._handle_converse_delta_event( + json_delta, index=0 + ) + assert text == "" + assert tool_use_delta is not None + assert tool_use_delta["function"]["arguments"] == '{"data": 1}' + From b983352d11bd9c88de6765c492abdb1a88a312ed Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Sun, 15 Feb 2026 11:34:38 -0300 Subject: [PATCH 32/52] refactor: extract duplicated JSON unwrapping into helper method Addresses review comment from greptile-apps: https://github.com/BerriAI/litellm/pull/21107#pullrequestreview-3796085353 Changes: - Added `_unwrap_bedrock_properties()` helper method to eliminate code duplication - Replaced two identical JSON unwrapping blocks (lines 1592-1601 and 1612-1620) with calls to the new helper method - Improves maintainability - single source of truth for Bedrock properties unwrapping logic The helper method: - Parses JSON string - Checks for single "properties" key structure - Unwraps and returns the properties value - Returns original string if unwrapping not needed or parsing fails No functional changes - pure refactoring. Co-Authored-By: Claude Sonnet 4.5 --- .../bedrock/chat/converse_transformation.py | 55 +++++++++++-------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 1f48c290043..905c8453ef2 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1779,6 +1779,33 @@ class AmazonConverseConfig(BaseConfig): return content_str, tools, reasoningContentBlocks, citationsContentBlocks + @staticmethod + def _unwrap_bedrock_properties(json_str: str) -> str: + """ + Unwrap Bedrock's response_format JSON structure. + + If the JSON has a single "properties" key, extract its value. + Otherwise, return the original string. + + Args: + json_str: JSON string to unwrap + + Returns: + Unwrapped JSON string or original if unwrapping not needed + """ + try: + response_data = json.loads(json_str) + if ( + isinstance(response_data, dict) + and "properties" in response_data + and len(response_data) == 1 + ): + response_data = response_data["properties"] + return json.dumps(response_data) + except json.JSONDecodeError: + pass + return json_str + @staticmethod def _filter_json_mode_tools( json_mode: Optional[bool], @@ -1815,17 +1842,9 @@ class AmazonConverseConfig(BaseConfig): "arguments" ) if json_mode_content_str is not None: - try: - response_data = json.loads(json_mode_content_str) - if ( - isinstance(response_data, dict) - and "properties" in response_data - and len(response_data) == 1 - ): - response_data = response_data["properties"] - json_mode_content_str = json.dumps(response_data) - except json.JSONDecodeError: - pass + json_mode_content_str = BedrockConverseConfig._unwrap_bedrock_properties( + json_mode_content_str + ) chat_completion_message["content"] = json_mode_content_str return None @@ -1835,17 +1854,9 @@ class AmazonConverseConfig(BaseConfig): first_idx = json_tool_indices[0] json_mode_args = tools[first_idx]["function"].get("arguments") if json_mode_args is not None: - try: - response_data = json.loads(json_mode_args) - if ( - isinstance(response_data, dict) - and "properties" in response_data - and len(response_data) == 1 - ): - response_data = response_data["properties"] - json_mode_args = json.dumps(response_data) - except json.JSONDecodeError: - pass + json_mode_args = BedrockConverseConfig._unwrap_bedrock_properties( + json_mode_args + ) existing = chat_completion_message.get("content") or "" chat_completion_message["content"] = ( existing + json_mode_args if existing else json_mode_args From fdf4c9dbdcb91cf7c6aae19f9eab82955c47ac85 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Sun, 15 Feb 2026 11:43:43 -0300 Subject: [PATCH 33/52] fix: use correct class name AmazonConverseConfig in helper method calls Fixed MyPy errors where BedrockConverseConfig was used instead of AmazonConverseConfig in the _unwrap_bedrock_properties() calls. Errors: - Line 1619: BedrockConverseConfig -> AmazonConverseConfig - Line 1631: BedrockConverseConfig -> AmazonConverseConfig Co-Authored-By: Claude Sonnet 4.5 --- litellm/llms/bedrock/chat/converse_transformation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 905c8453ef2..d210f294c64 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1842,7 +1842,7 @@ class AmazonConverseConfig(BaseConfig): "arguments" ) if json_mode_content_str is not None: - json_mode_content_str = BedrockConverseConfig._unwrap_bedrock_properties( + json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties( json_mode_content_str ) chat_completion_message["content"] = json_mode_content_str @@ -1854,7 +1854,7 @@ class AmazonConverseConfig(BaseConfig): first_idx = json_tool_indices[0] json_mode_args = tools[first_idx]["function"].get("arguments") if json_mode_args is not None: - json_mode_args = BedrockConverseConfig._unwrap_bedrock_properties( + json_mode_args = AmazonConverseConfig._unwrap_bedrock_properties( json_mode_args ) existing = chat_completion_message.get("content") or "" From ee703cea99252c0e45b2a74b9f376e2ecda9ea65 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 27 Feb 2026 20:30:47 -0800 Subject: [PATCH 34/52] fix(jwt): OIDC discovery URLs, roles array handling, dot-notation error hints (#22336) * fix(jwt): support OIDC discovery URLs, handle roles array, improve error hints Three fixes for Azure AD JWT auth: 1. OIDC discovery URL support - JWT_PUBLIC_KEY_URL can now be set to .well-known/openid-configuration endpoints. The proxy fetches the discovery doc, extracts jwks_uri, and caches it. 2. Handle roles claim as array - when team_id_jwt_field points to a list (e.g. AAD's "roles": ["team1"]), auto-unwrap the first element instead of crashing with 'unhashable type: list'. 3. Better error hint for dot-notation indexing - when team_id_jwt_field is set to "roles.0" or "roles[0]", the 401 error now explains to use "roles" instead and that LiteLLM auto-unwraps lists. * Add integration demo script for JWT auth fixes (OIDC discovery, array roles, dot-notation hints) Co-authored-by: Ishaan Jaff * Add demo_servers.py for manual JWT auth testing with mock JWKS/OIDC endpoints Co-authored-by: Ishaan Jaff * Add demo screenshots for PR comment Co-authored-by: Ishaan Jaff * Add integration test results with screenshots for PR review Co-authored-by: Ishaan Jaff * address greptile review feedback (greploop iteration 1) - fix: add HTTP status code check in _resolve_jwks_url before parsing JSON - fix: remove misleading bracket-notation hint from debug log (get_nested_value does not support it) * Update tests/test_litellm/proxy/auth/test_handle_jwt.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * remove demo scripts and assets --------- Co-authored-by: Cursor Agent Co-authored-by: Ishaan Jaff Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/proxy/auth/handle_jwt.py | 84 +++++- .../proxy/auth/test_handle_jwt.py | 269 ++++++++++++++++++ 2 files changed, 351 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 9921b74b561..553ba4d6c49 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -8,6 +8,7 @@ JWT token must have 'litellm_proxy_admin' in scope. import fnmatch import os +import re from typing import Any, List, Literal, Optional, Set, Tuple, cast from cryptography import x509 @@ -235,7 +236,17 @@ class JWTHandler: return self.litellm_jwtauth.team_id_default else: return default_value - # At this point, team_id is not the sentinel, so it should be a string + # AAD and other IdPs often send roles/groups as a list of strings. + # team_id_jwt_field is singular, so take the first element when a list + # is returned. This avoids "unhashable type: 'list'" errors downstream. + if isinstance(team_id, list): + if not team_id: + return default_value + verbose_proxy_logger.debug( + f"JWT Auth: team_id_jwt_field '{self.litellm_jwtauth.team_id_jwt_field}' " + f"returned a list {team_id}; using first element '{team_id[0]}' automatically." + ) + team_id = team_id[0] return team_id # type: ignore[return-value] elif self.litellm_jwtauth.team_id_default is not None: team_id = self.litellm_jwtauth.team_id_default @@ -453,6 +464,52 @@ class JWTHandler: scopes = [] return scopes + async def _resolve_jwks_url(self, url: str) -> str: + """ + If url points to an OIDC discovery document (*.well-known/openid-configuration), + fetch it and return the jwks_uri contained within. Otherwise return url unchanged. + This lets JWT_PUBLIC_KEY_URL be set to a well-known discovery endpoint instead of + requiring operators to manually find the JWKS URL. + """ + if ".well-known/openid-configuration" not in url: + return url + + cache_key = f"litellm_oidc_discovery_{url}" + cached_jwks_uri = await self.user_api_key_cache.async_get_cache(cache_key) + if cached_jwks_uri is not None: + return cached_jwks_uri + + verbose_proxy_logger.debug( + f"JWT Auth: Fetching OIDC discovery document from {url}" + ) + response = await self.http_handler.get(url) + if response.status_code != 200: + raise Exception( + f"JWT Auth: OIDC discovery endpoint {url} returned status {response.status_code}: {response.text}" + ) + try: + discovery = response.json() + except Exception as e: + raise Exception( + f"JWT Auth: Failed to parse OIDC discovery document at {url}: {e}" + ) + + jwks_uri = discovery.get("jwks_uri") + if not jwks_uri: + raise Exception( + f"JWT Auth: OIDC discovery document at {url} does not contain a 'jwks_uri' field." + ) + + verbose_proxy_logger.debug( + f"JWT Auth: Resolved OIDC discovery {url} -> jwks_uri={jwks_uri}" + ) + await self.user_api_key_cache.async_set_cache( + key=cache_key, + value=jwks_uri, + ttl=self.litellm_jwtauth.public_key_ttl, + ) + return jwks_uri + async def get_public_key(self, kid: Optional[str]) -> dict: keys_url = os.getenv("JWT_PUBLIC_KEY_URL") @@ -462,6 +519,7 @@ class JWTHandler: keys_url_list = [url.strip() for url in keys_url.split(",")] for key_url in keys_url_list: + key_url = await self._resolve_jwks_url(key_url) cache_key = f"litellm_jwt_auth_keys_{key_url}" cached_keys = await self.user_api_key_cache.async_get_cache(cache_key) @@ -913,8 +971,30 @@ class JWTAuthManager: if jwt_handler.is_required_team_id() is True: team_id_field = jwt_handler.litellm_jwtauth.team_id_jwt_field team_alias_field = jwt_handler.litellm_jwtauth.team_alias_jwt_field + hint = "" + if team_id_field: + # "roles.0" — dot-notation numeric indexing is not supported + if "." in team_id_field: + parts = team_id_field.rsplit(".", 1) + if parts[-1].isdigit(): + base_field = parts[0] + hint = ( + f" Hint: dot-notation array indexing (e.g. '{team_id_field}') is not " + f"supported. Use '{base_field}' instead — LiteLLM automatically " + f"uses the first element when the field value is a list." + ) + # "roles[0]" — bracket-notation indexing is also not supported in get_nested_value + elif "[" in team_id_field and team_id_field.endswith("]"): + m = re.match(r"^(\w+)\[(\d+)\]$", team_id_field) + if m: + base_field = m.group(1) + hint = ( + f" Hint: array indexing (e.g. '{team_id_field}') is not supported " + f"in team_id_jwt_field. Use '{base_field}' instead — LiteLLM " + f"automatically uses the first element when the field value is a list." + ) raise Exception( - f"No team found in token. Checked team_id field '{team_id_field}' and team_alias field '{team_alias_field}'" + f"No team found in token. Checked team_id field '{team_id_field}' and team_alias field '{team_alias_field}'.{hint}" ) return individual_team_id, team_object diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index b56d13bb932..8418dde5e9c 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -1485,4 +1485,273 @@ async def test_get_objects_resolves_org_by_name(): ) +# --------------------------------------------------------------------------- +# Fix 1: OIDC discovery URL resolution +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_resolve_jwks_url_passthrough_for_direct_jwks_url(): + """Non-discovery URLs are returned unchanged.""" + from unittest.mock import AsyncMock, MagicMock + + from litellm.caching.dual_cache import DualCache + + handler = JWTHandler() + handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth(), + ) + url = "https://login.microsoftonline.com/common/discovery/keys" + result = await handler._resolve_jwks_url(url) + assert result == url + + +@pytest.mark.asyncio +async def test_resolve_jwks_url_resolves_oidc_discovery_document(): + """ + A .well-known/openid-configuration URL should be fetched and its + jwks_uri returned. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.caching.dual_cache import DualCache + + handler = JWTHandler() + cache = DualCache() + handler.update_environment( + prisma_client=None, + user_api_key_cache=cache, + litellm_jwtauth=LiteLLM_JWTAuth(), + ) + + discovery_url = "https://login.microsoftonline.com/tenant/.well-known/openid-configuration" + jwks_url = "https://login.microsoftonline.com/tenant/discovery/keys" + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"jwks_uri": jwks_url, "issuer": "https://..."} + + with patch.object(handler.http_handler, "get", new_callable=AsyncMock, return_value=mock_response) as mock_get: + result = await handler._resolve_jwks_url(discovery_url) + + assert result == jwks_url + mock_get.assert_called_once_with(discovery_url) + + +@pytest.mark.asyncio +async def test_resolve_jwks_url_caches_resolved_jwks_uri(): + """Resolved jwks_uri is cached — second call does not hit the network.""" + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.caching.dual_cache import DualCache + + handler = JWTHandler() + cache = DualCache() + handler.update_environment( + prisma_client=None, + user_api_key_cache=cache, + litellm_jwtauth=LiteLLM_JWTAuth(), + ) + + discovery_url = "https://login.microsoftonline.com/tenant/.well-known/openid-configuration" + jwks_url = "https://login.microsoftonline.com/tenant/discovery/keys" + + mock_response = MagicMock() + mock_response.json.return_value = {"jwks_uri": jwks_url} + + with patch.object(handler.http_handler, "get", new_callable=AsyncMock, return_value=mock_response) as mock_get: + first = await handler._resolve_jwks_url(discovery_url) + second = await handler._resolve_jwks_url(discovery_url) + + assert first == jwks_url + assert second == jwks_url + # Network should only be hit once + assert mock_get.call_count == 1 + + +@pytest.mark.asyncio +async def test_resolve_jwks_url_raises_if_no_jwks_uri_in_discovery_doc(): + """Raise a helpful error if the discovery document has no jwks_uri.""" + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.caching.dual_cache import DualCache + + handler = JWTHandler() + handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth(), + ) + + discovery_url = "https://example.com/.well-known/openid-configuration" + mock_response = MagicMock() + mock_response.json.return_value = {"issuer": "https://example.com"} # no jwks_uri + + with patch.object(handler.http_handler, "get", new_callable=AsyncMock, return_value=mock_response): + with pytest.raises(Exception, match="jwks_uri"): + await handler._resolve_jwks_url(discovery_url) + + +# --------------------------------------------------------------------------- +# Fix 2: handle array values in team_id_jwt_field (e.g. AAD "roles" claim) +# --------------------------------------------------------------------------- + + +def _make_jwt_handler(team_id_jwt_field: str) -> JWTHandler: + from litellm.caching.dual_cache import DualCache + + handler = JWTHandler() + handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth(team_id_jwt_field=team_id_jwt_field), + ) + return handler + + +def test_get_team_id_returns_first_element_when_roles_is_list(): + """ + AAD sends roles as a list. get_team_id() must return the first string + element rather than the raw list (which would later crash with + 'unhashable type: list'). + """ + handler = _make_jwt_handler("roles") + token = {"oid": "user-oid", "roles": ["team1"]} + result = handler.get_team_id(token=token, default_value=None) + assert result == "team1" + + +def test_get_team_id_returns_first_element_from_multi_value_roles_list(): + """When roles has multiple entries, the first one is used.""" + handler = _make_jwt_handler("roles") + token = {"roles": ["team2", "team1"]} + result = handler.get_team_id(token=token, default_value=None) + assert result == "team2" + + +def test_get_team_id_returns_default_when_roles_list_is_empty(): + """Empty list should fall back to default_value.""" + handler = _make_jwt_handler("roles") + token = {"roles": []} + result = handler.get_team_id(token=token, default_value="fallback") + assert result == "fallback" + + +def test_get_team_id_still_works_with_string_value(): + """String values (non-array) continue to work as before.""" + handler = _make_jwt_handler("appid") + token = {"appid": "my-team-id"} + result = handler.get_team_id(token=token, default_value=None) + assert result == "my-team-id" + + +def test_get_team_id_list_result_is_hashable(): + """ + The value returned by get_team_id() must be hashable so it can be + added to a set (the operation that previously crashed). + """ + handler = _make_jwt_handler("roles") + token = {"roles": ["team1"]} + result = handler.get_team_id(token=token, default_value=None) + # This must not raise TypeError + s: set = set() + s.add(result) + assert "team1" in s + + +# --------------------------------------------------------------------------- +# Fix 3: helpful error message for dot-notation array indexing (roles.0) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_find_and_validate_specific_team_id_hints_bracket_notation(): + """ + When team_id_jwt_field is set to 'roles.0' (unsupported dot-notation for + array indexing) and no team is found, the exception message should suggest + using 'roles' instead (and explain LiteLLM auto-unwraps list values). + """ + from unittest.mock import MagicMock + + from litellm.caching.dual_cache import DualCache + + handler = _make_jwt_handler("roles.0") + # token has roles as a list — dot-notation won't find anything + token = {"roles": ["team1"]} + + with pytest.raises(Exception) as exc_info: + await JWTAuthManager.find_and_validate_specific_team_id( + jwt_handler=handler, + jwt_valid_token=token, + prisma_client=None, + user_api_key_cache=DualCache(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + + error_msg = str(exc_info.value) + # Should mention the bad field name and suggest the fix + assert "roles.0" in error_msg, f"Expected field name in: {error_msg}" + assert "roles" in error_msg and "list" in error_msg, ( + f"Expected hint about using 'roles' instead: {error_msg}" + ) + + +@pytest.mark.asyncio +async def test_find_and_validate_specific_team_id_hints_bracket_index_notation(): + """ + When team_id_jwt_field is set to 'roles[0]' (bracket indexing, also unsupported + in get_nested_value) the error message should suggest using 'roles' instead. + """ + from unittest.mock import MagicMock + + from litellm.caching.dual_cache import DualCache + + handler = _make_jwt_handler("roles[0]") + token = {"roles": ["team1"]} + + with pytest.raises(Exception) as exc_info: + await JWTAuthManager.find_and_validate_specific_team_id( + jwt_handler=handler, + jwt_valid_token=token, + prisma_client=None, + user_api_key_cache=DualCache(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + + error_msg = str(exc_info.value) + assert "roles[0]" in error_msg, f"Expected field name in: {error_msg}" + assert "roles" in error_msg and "list" in error_msg, ( + f"Expected hint about using 'roles' instead: {error_msg}" + ) + + +@pytest.mark.asyncio +async def test_find_and_validate_specific_team_id_no_hint_for_valid_field(): + """ + When team_id_jwt_field is a normal field name (no dot-notation) the + error message should not contain a spurious bracket-notation hint. + """ + from unittest.mock import AsyncMock, MagicMock + + from litellm.caching.dual_cache import DualCache + + handler = _make_jwt_handler("appid") + token = {} # no appid — triggers the "no team found" path + + with pytest.raises(Exception) as exc_info: + await JWTAuthManager.find_and_validate_specific_team_id( + jwt_handler=handler, + jwt_valid_token=token, + prisma_client=None, + user_api_key_cache=DualCache(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + + error_msg = str(exc_info.value) + assert "Hint" not in error_msg From 9f2453712ec054b8ad8888d314bb24f6c42316c6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 27 Feb 2026 20:45:53 -0800 Subject: [PATCH 35/52] =?UTF-8?q?perf:=20streaming=20latency=20improvement?= =?UTF-8?q?s=20=E2=80=94=204=20targeted=20hot-path=20fixes=20(#22346)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf: raise aiohttp connection pool limits (300→1000, 50/host→500) * perf: skip model_copy() on every chunk — only copy usage-bearing chunks * perf: replace list+join O(n²) with str+= O(n) in async_data_generator * perf: cache model-level guardrail lookup per request, not per chunk --- litellm/constants.py | 4 +- .../litellm_core_utils/streaming_handler.py | 21 ++- litellm/proxy/proxy_server.py | 10 +- litellm/proxy/utils.py | 149 +++++++++++------- 4 files changed, 112 insertions(+), 72 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 3d2cebf2224..4c38ecd74b5 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -193,9 +193,9 @@ _DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600 # 1 hour, re-use the same httpx client fo # Aiohttp connection pooling - prevents memory leaks from unbounded connection growth # Set to 0 for unlimited (not recommended for production) -AIOHTTP_CONNECTOR_LIMIT = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 300)) +AIOHTTP_CONNECTOR_LIMIT = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 1000)) AIOHTTP_CONNECTOR_LIMIT_PER_HOST = int( - os.getenv("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 50) + os.getenv("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 500) ) AIOHTTP_KEEPALIVE_TIMEOUT = int(os.getenv("AIOHTTP_KEEPALIVE_TIMEOUT", 120)) AIOHTTP_TTL_DNS_CACHE = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300)) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index baf274f2c62..3b75a56fcc9 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1968,22 +1968,24 @@ class CustomStreamWrapper: self.rules.post_call_rules( input=self.response_uptil_now, model=self.model ) - # Store a shallow copy so usage stripping below - # does not mutate the stored chunk. - self.chunks.append(processed_chunk.model_copy()) - # Add mcp_list_tools to first chunk if present if not self.sent_first_chunk: processed_chunk = self._add_mcp_list_tools_to_first_chunk(processed_chunk) self.sent_first_chunk = True - if ( + + _has_usage = ( hasattr(processed_chunk, "usage") and getattr(processed_chunk, "usage", None) is not None - ): + ) + + if _has_usage: + # Store a copy ONLY when usage stripping below will mutate + # the chunk. For non-usage chunks (vast majority), store + # directly to avoid expensive model_copy() per chunk. + self.chunks.append(processed_chunk.model_copy()) + # Strip usage from the outgoing chunk so it's not sent twice # (once in the chunk, once in _hidden_params). - # Create a new object without usage, matching sync behavior. - # The copy in self.chunks retains usage for calculate_total_usage(). obj_dict = processed_chunk.model_dump() if "usage" in obj_dict: del obj_dict["usage"] @@ -1995,6 +1997,9 @@ class CustomStreamWrapper: ) if is_empty: continue + else: + # No usage data — safe to store directly without copying + self.chunks.append(processed_chunk) # add usage as hidden param if self.sent_last_chunk is True and self.stream_options is None: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 48025863641..bd5b5309e0f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5302,13 +5302,15 @@ async def async_data_generator( ): verbose_proxy_logger.debug("inside generator") try: - # Use a list to accumulate response segments to avoid O(n^2) string concatenation - str_so_far_parts: list[str] = [] error_message: Optional[str] = None requested_model_from_client = _get_client_requested_model_for_streaming( request_data=request_data ) model_mismatch_logged = False + # Use a running string instead of list + join to avoid O(n^2) overhead. + # Previously "".join(str_so_far_parts) was called every chunk, re-joining + # the entire accumulated response. String += is O(n) amortized total. + _str_so_far: str = "" async for chunk in proxy_logging_obj.async_post_call_streaming_iterator_hook( user_api_key_dict=user_api_key_dict, response=response, @@ -5319,12 +5321,12 @@ async def async_data_generator( user_api_key_dict=user_api_key_dict, response=chunk, data=request_data, - str_so_far="".join(str_so_far_parts), + str_so_far=_str_so_far if _str_so_far else None, ) if isinstance(chunk, (ModelResponse, ModelResponseStream)): response_str = litellm.get_response_string(response_obj=chunk) - str_so_far_parts.append(response_str) + _str_so_far += response_str chunk, model_mismatch_logged = _restamp_streaming_chunk_model( chunk=chunk, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index f6613b5548f..5e0d5336aa9 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -23,23 +23,31 @@ from typing import ( ) from litellm import _custom_logger_compatible_callbacks_literal -from litellm.constants import (DEFAULT_MODEL_CREATED_AT_TIME, - MAX_TEAM_LIST_LIMIT) -from litellm.proxy._types import (DB_CONNECTION_ERROR_TYPES, CommonProxyErrors, - ProxyErrorTypes, ProxyException, - SpendLogsMetadata, SpendLogsPayload) +from litellm.constants import DEFAULT_MODEL_CREATED_AT_TIME, MAX_TEAM_LIST_LIMIT +from litellm.proxy._types import ( + DB_CONNECTION_ERROR_TYPES, + CommonProxyErrors, + ProxyErrorTypes, + ProxyException, + SpendLogsMetadata, + SpendLogsPayload, +) from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import CallTypes, CallTypesLiteral try: - from litellm_enterprise.enterprise_callbacks.send_emails.base_email import \ - BaseEmailLogger - from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import \ - ResendEmailLogger - from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import \ - SendGridEmailLogger - from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import \ - SMTPEmailLogger + from litellm_enterprise.enterprise_callbacks.send_emails.base_email import ( + BaseEmailLogger, + ) + from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import ( + ResendEmailLogger, + ) + from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import ( + SendGridEmailLogger, + ) + from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import ( + SMTPEmailLogger, + ) except ImportError: BaseEmailLogger = None # type: ignore SendGridEmailLogger = None # type: ignore @@ -58,56 +66,70 @@ from fastapi import HTTPException, status import litellm import litellm.litellm_core_utils import litellm.litellm_core_utils.litellm_logging -from litellm import (EmbeddingResponse, ImageResponse, ModelResponse, - ModelResponseStream, Router) +from litellm import ( + EmbeddingResponse, + ImageResponse, + ModelResponse, + ModelResponseStream, + Router, +) from litellm._logging import verbose_proxy_logger from litellm._service_logger import ServiceLogging, ServiceTypes from litellm.caching.caching import DualCache, RedisCache from litellm.caching.dual_cache import LimitedSizeOrderedDict from litellm.exceptions import RejectedRequestError -from litellm.integrations.custom_guardrail import (CustomGuardrail, - ModifyResponseException) +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + ModifyResponseException, +) from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting -from litellm.integrations.SlackAlerting.utils import \ - _add_langfuse_trace_id_to_alert +from litellm.integrations.SlackAlerting.utils import _add_langfuse_trace_id_to_alert from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -from litellm.proxy._types import (AlertType, CallInfo, - LiteLLM_VerificationTokenView, Member, - UserAPIKeyAuth) +from litellm.proxy._types import ( + AlertType, + CallInfo, + LiteLLM_VerificationTokenView, + Member, + UserAPIKeyAuth, +) from litellm.proxy.auth.route_checks import RouteChecks -from litellm.proxy.db.create_views import (create_missing_views, - should_create_missing_views) +from litellm.proxy.db.create_views import ( + create_missing_views, + should_create_missing_views, +) from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.proxy.db.log_db_metrics import log_db_metrics from litellm.proxy.db.prisma_client import PrismaWrapper -from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import \ - UnifiedLLMGuardrails +from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( + UnifiedLLMGuardrails, +) from litellm.proxy.hooks import PROXY_HOOKS, get_proxy_hook from litellm.proxy.hooks.cache_control_check import _PROXY_CacheControlCheck from litellm.proxy.hooks.max_budget_limiter import _PROXY_MaxBudgetLimiter -from litellm.proxy.hooks.parallel_request_limiter import \ - _PROXY_MaxParallelRequestsHandler +from litellm.proxy.hooks.parallel_request_limiter import ( + _PROXY_MaxParallelRequestsHandler, +) from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy.policy_engine.pipeline_executor import PipelineExecutor from litellm.secret_managers.main import str_to_bool from litellm.types.integrations.slack_alerting import DEFAULT_ALERT_TYPES -from litellm.types.mcp import (MCPDuringCallResponseObject, - MCPPreCallRequestObject, - MCPPreCallResponseObject) -from litellm.types.proxy.policy_engine.pipeline_types import \ - PipelineExecutionResult +from litellm.types.mcp import ( + MCPDuringCallResponseObject, + MCPPreCallRequestObject, + MCPPreCallResponseObject, +) +from litellm.types.proxy.policy_engine.pipeline_types import PipelineExecutionResult from litellm.types.utils import LLMResponseTypes, LoggedLiteLLMParams if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - from litellm.litellm_core_utils.litellm_logging import \ - Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj Span = Union[_Span, Any] else: @@ -1050,9 +1072,10 @@ class ProxyLogging: """Process prompt template if applicable.""" from litellm.proxy.prompts.prompt_endpoints import ( - construct_versioned_prompt_id, get_latest_version_prompt_id) - from litellm.proxy.prompts.prompt_registry import \ - IN_MEMORY_PROMPT_REGISTRY + construct_versioned_prompt_id, + get_latest_version_prompt_id, + ) + from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY from litellm.utils import get_non_default_completion_params if prompt_version is None: @@ -1102,8 +1125,9 @@ class ProxyLogging: def _process_guardrail_metadata(self, data: dict) -> None: """Process guardrails from metadata and add to applied_guardrails.""" - from litellm.proxy.common_utils.callback_utils import \ - add_guardrail_to_applied_guardrails_header + from litellm.proxy.common_utils.callback_utils import ( + add_guardrail_to_applied_guardrails_header, + ) metadata_standard = data.get("metadata") or {} metadata_litellm = data.get("litellm_metadata") or {} @@ -2000,27 +2024,32 @@ class ProxyLogging: if isinstance(response, (ModelResponse, ModelResponseStream)): response_str = litellm.get_response_string(response_obj=response) elif isinstance(response, dict) and self.is_a2a_streaming_response(response): - from litellm.llms.a2a.common_utils import \ - extract_text_from_a2a_response + from litellm.llms.a2a.common_utils import extract_text_from_a2a_response response_str = extract_text_from_a2a_response(response) if response_str is not None: + # Cache model-level guardrails check per-request to avoid repeated + # dict lookups + llm_router.get_deployment() per callback per chunk. + _cached_guardrail_data: Optional[dict] = None + _guardrail_data_computed = False + for callback in litellm.callbacks: try: _callback: Optional[CustomLogger] = None if isinstance(callback, CustomGuardrail): # Main - V2 Guardrails implementation - from litellm.types.guardrails import \ - GuardrailEventHooks + from litellm.types.guardrails import GuardrailEventHooks - ## CHECK FOR MODEL-LEVEL GUARDRAILS - modified_data = _check_and_merge_model_level_guardrails( - data=data, llm_router=llm_router - ) + ## CHECK FOR MODEL-LEVEL GUARDRAILS (cached per-request) + if not _guardrail_data_computed: + _cached_guardrail_data = _check_and_merge_model_level_guardrails( + data=data, llm_router=llm_router + ) + _guardrail_data_computed = True if ( callback.should_run_guardrail( - data=modified_data, + data=_cached_guardrail_data, event_type=GuardrailEventHooks.post_call, ) is not True @@ -4626,8 +4655,9 @@ async def update_spend_logs_job( # Guardrail/policy usage tracking (same batch, outside spend-logs update) try: - from litellm.proxy.guardrails.usage_tracking import \ - process_spend_logs_guardrail_usage + from litellm.proxy.guardrails.usage_tracking import ( + process_spend_logs_guardrail_usage, + ) await process_spend_logs_guardrail_usage( prisma_client=prisma_client, logs_to_process=logs_to_process, @@ -4653,8 +4683,10 @@ async def _monitor_spend_logs_queue( db_writer_client: Optional HTTP handler for external spend logs endpoint proxy_logging_obj: Proxy logging object """ - from litellm.constants import (SPEND_LOG_QUEUE_POLL_INTERVAL, - SPEND_LOG_QUEUE_SIZE_THRESHOLD) + from litellm.constants import ( + SPEND_LOG_QUEUE_POLL_INTERVAL, + SPEND_LOG_QUEUE_SIZE_THRESHOLD, + ) threshold = SPEND_LOG_QUEUE_SIZE_THRESHOLD base_interval = SPEND_LOG_QUEUE_POLL_INTERVAL @@ -5175,11 +5207,12 @@ async def get_available_models_for_user( List of model names available to the user """ from litellm.proxy.auth.auth_checks import get_team_object - from litellm.proxy.auth.model_checks import (get_complete_model_list, - get_key_models, - get_team_models) - from litellm.proxy.management_endpoints.team_endpoints import \ - validate_membership + from litellm.proxy.auth.model_checks import ( + get_complete_model_list, + get_key_models, + get_team_models, + ) + from litellm.proxy.management_endpoints.team_endpoints import validate_membership # Get proxy model list and access groups if llm_router is None: From c4a0174e0072218b00574053f952ac2c4e33db69 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 27 Feb 2026 21:16:41 -0800 Subject: [PATCH 36/52] test: add comprehensive Vitest coverage for CostTrackingSettings Add 88 tests across 9 test files for the CostTrackingSettings component directory: - provider_display_helpers.test.ts: 9 tests for helper functions - how_it_works.test.tsx: 9 tests for discount calculator component - add_provider_form.test.tsx: 7 tests for provider form validation - add_margin_form.test.tsx: 9 tests for margin form with type toggle - provider_discount_table.test.tsx: 12 tests for table editing and interactions - provider_margin_table.test.tsx: 13 tests for margin table with sorting - use_discount_config.test.ts: 11 tests for discount hook logic - use_margin_config.test.ts: 12 tests for margin hook logic - cost_tracking_settings.test.tsx: 15 tests for main component and role-based rendering All tests passing. Coverage includes form validation, user interactions, API calls, state management, and conditional rendering. Co-Authored-By: Claude Haiku 4.5 --- .../add_margin_form.test.tsx | 148 ++++++++ .../add_provider_form.test.tsx | 98 ++++++ .../cost_tracking_settings.test.tsx | 201 +++++++++++ .../how_it_works.test.tsx | 95 ++++++ .../provider_discount_table.test.tsx | 241 +++++++++++++ .../provider_display_helpers.test.ts | 91 +++++ .../provider_margin_table.test.tsx | 246 ++++++++++++++ .../use_discount_config.test.ts | 244 ++++++++++++++ .../use_margin_config.test.ts | 316 ++++++++++++++++++ 9 files changed, 1680 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/add_margin_form.test.tsx create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/add_provider_form.test.tsx create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/cost_tracking_settings.test.tsx create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/how_it_works.test.tsx create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/provider_discount_table.test.tsx create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/provider_display_helpers.test.ts create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/provider_margin_table.test.tsx create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/use_discount_config.test.ts create mode 100644 ui/litellm-dashboard/src/components/CostTrackingSettings/use_margin_config.test.ts diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/add_margin_form.test.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/add_margin_form.test.tsx new file mode 100644 index 00000000000..d61f0987ac2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/add_margin_form.test.tsx @@ -0,0 +1,148 @@ +import React from "react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders } from "../../../tests/test-utils"; +import AddMarginForm from "./add_margin_form"; +import { MarginConfig } from "./types"; + +vi.mock("../provider_info_helpers", () => ({ + Providers: { + OpenAI: "OpenAI", + Anthropic: "Anthropic", + }, + provider_map: { + OpenAI: "openai", + Anthropic: "anthropic", + }, + providerLogoMap: { + OpenAI: "https://example.com/openai.png", + Anthropic: "https://example.com/anthropic.png", + }, +})); + +vi.mock("./provider_display_helpers", () => ({ + handleImageError: vi.fn(), +})); + +const DEFAULT_PROPS = { + marginConfig: {} as MarginConfig, + selectedProvider: undefined, + marginType: "percentage" as const, + percentageValue: "", + fixedAmountValue: "", + onProviderChange: vi.fn(), + onMarginTypeChange: vi.fn(), + onPercentageChange: vi.fn(), + onFixedAmountChange: vi.fn(), + onAddProvider: vi.fn(), +}; + +describe("AddMarginForm", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should render", () => { + renderWithProviders(); + expect(screen.getByRole("button", { name: /add provider margin/i })).toBeInTheDocument(); + }); + + it("should show the percentage input when marginType is percentage", () => { + renderWithProviders(); + expect(screen.getByPlaceholderText("10")).toBeInTheDocument(); + }); + + it("should show the fixed amount input when marginType is fixed", () => { + renderWithProviders(); + expect(screen.getByPlaceholderText("0.001")).toBeInTheDocument(); + }); + + it("should not show the fixed amount input when marginType is percentage", () => { + renderWithProviders(); + expect(screen.queryByPlaceholderText("0.001")).not.toBeInTheDocument(); + }); + + it("should not show the percentage input when marginType is fixed", () => { + renderWithProviders(); + expect(screen.queryByPlaceholderText("10")).not.toBeInTheDocument(); + }); + + it("should show the Percentage-based and Fixed Amount radio options", () => { + renderWithProviders(); + expect(screen.getByText("Percentage-based")).toBeInTheDocument(); + expect(screen.getByText("Fixed Amount")).toBeInTheDocument(); + }); + + it("should disable the submit button when no provider is selected (percentage mode)", () => { + renderWithProviders( + + ); + expect(screen.getByRole("button", { name: /add provider margin/i })).toBeDisabled(); + }); + + it("should disable the submit button when provider is selected but no percentage value (percentage mode)", () => { + renderWithProviders( + + ); + expect(screen.getByRole("button", { name: /add provider margin/i })).toBeDisabled(); + }); + + it("should enable the submit button when provider and percentage value are both provided", () => { + renderWithProviders( + + ); + expect(screen.getByRole("button", { name: /add provider margin/i })).not.toBeDisabled(); + }); + + it("should disable the submit button in fixed mode when no fixed amount is provided", () => { + renderWithProviders( + + ); + expect(screen.getByRole("button", { name: /add provider margin/i })).toBeDisabled(); + }); + + it("should enable the submit button in fixed mode when provider and fixed amount are provided", () => { + renderWithProviders( + + ); + expect(screen.getByRole("button", { name: /add provider margin/i })).not.toBeDisabled(); + }); + + it("should call onAddProvider when the enabled submit button is clicked", async () => { + const onAddProvider = vi.fn(); + const user = userEvent.setup(); + renderWithProviders( + + ); + + await user.click(screen.getByRole("button", { name: /add provider margin/i })); + expect(onAddProvider).toHaveBeenCalledTimes(1); + }); + + it("should call onMarginTypeChange when the Fixed Amount radio is clicked", async () => { + const onMarginTypeChange = vi.fn(); + const user = userEvent.setup(); + renderWithProviders( + + ); + + await user.click(screen.getByText("Fixed Amount")); + expect(onMarginTypeChange).toHaveBeenCalledWith("fixed"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/add_provider_form.test.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/add_provider_form.test.tsx new file mode 100644 index 00000000000..611c8609c36 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/add_provider_form.test.tsx @@ -0,0 +1,98 @@ +import React from "react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders } from "../../../tests/test-utils"; +import AddProviderForm from "./add_provider_form"; +import { DiscountConfig } from "./types"; + +vi.mock("../provider_info_helpers", () => ({ + Providers: { + OpenAI: "OpenAI", + Anthropic: "Anthropic", + }, + provider_map: { + OpenAI: "openai", + Anthropic: "anthropic", + }, + providerLogoMap: { + OpenAI: "https://example.com/openai.png", + Anthropic: "https://example.com/anthropic.png", + }, +})); + +vi.mock("./provider_display_helpers", () => ({ + handleImageError: vi.fn(), +})); + +const DEFAULT_PROPS = { + discountConfig: {} as DiscountConfig, + selectedProvider: undefined, + newDiscount: "", + onProviderChange: vi.fn(), + onDiscountChange: vi.fn(), + onAddProvider: vi.fn(), +}; + +describe("AddProviderForm", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should render", () => { + renderWithProviders(); + expect(screen.getByRole("button", { name: /add provider discount/i })).toBeInTheDocument(); + }); + + it("should render the discount percentage input field", () => { + renderWithProviders(); + expect(screen.getByPlaceholderText("5")).toBeInTheDocument(); + }); + + it("should disable the submit button when no provider is selected and no discount is entered", () => { + renderWithProviders(); + expect(screen.getByRole("button", { name: /add provider discount/i })).toBeDisabled(); + }); + + it("should disable the submit button when a provider is selected but no discount is entered", () => { + renderWithProviders( + + ); + expect(screen.getByRole("button", { name: /add provider discount/i })).toBeDisabled(); + }); + + it("should disable the submit button when a discount is entered but no provider is selected", () => { + renderWithProviders( + + ); + expect(screen.getByRole("button", { name: /add provider discount/i })).toBeDisabled(); + }); + + it("should enable the submit button when both a provider and a discount value are provided", () => { + renderWithProviders( + + ); + expect(screen.getByRole("button", { name: /add provider discount/i })).not.toBeDisabled(); + }); + + it("should call onAddProvider when the enabled submit button is clicked", async () => { + const onAddProvider = vi.fn(); + const user = userEvent.setup(); + renderWithProviders( + + ); + + await user.click(screen.getByRole("button", { name: /add provider discount/i })); + expect(onAddProvider).toHaveBeenCalledTimes(1); + }); + + it("should show the percent sign next to the discount input", () => { + renderWithProviders(); + expect(screen.getByText("%")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/cost_tracking_settings.test.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/cost_tracking_settings.test.tsx new file mode 100644 index 00000000000..db6899ba17f --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/cost_tracking_settings.test.tsx @@ -0,0 +1,201 @@ +import React from "react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders } from "../../../tests/test-utils"; +import CostTrackingSettings from "./cost_tracking_settings"; + +// Mock sub-hooks so we can control their state without network calls +const mockDiscountConfig = vi.fn(() => ({})); +const mockMarginConfig = vi.fn(() => ({})); + +vi.mock("./use_discount_config", () => ({ + useDiscountConfig: () => ({ + discountConfig: mockDiscountConfig(), + fetchDiscountConfig: vi.fn().mockResolvedValue(undefined), + handleAddProvider: vi.fn().mockResolvedValue(true), + handleRemoveProvider: vi.fn().mockResolvedValue(undefined), + handleDiscountChange: vi.fn().mockResolvedValue(undefined), + }), +})); + +vi.mock("./use_margin_config", () => ({ + useMarginConfig: () => ({ + marginConfig: mockMarginConfig(), + fetchMarginConfig: vi.fn().mockResolvedValue(undefined), + handleAddMargin: vi.fn().mockResolvedValue(true), + handleRemoveMargin: vi.fn().mockResolvedValue(undefined), + handleMarginChange: vi.fn().mockResolvedValue(undefined), + }), +})); + +vi.mock("./pricing_calculator/index", () => ({ + default: () =>
Pricing Calculator
, +})); + +vi.mock("../playground/llm_calls/fetch_models", () => ({ + fetchAvailableModels: vi.fn().mockResolvedValue([]), +})); + +vi.mock("../HelpLink", () => ({ + DocsMenu: () => null, +})); + +vi.mock("./how_it_works", () => ({ + default: () =>
How It Works
, +})); + +vi.mock("../provider_info_helpers", () => ({ + Providers: { OpenAI: "OpenAI" }, + provider_map: { OpenAI: "openai" }, + providerLogoMap: {}, +})); + +vi.mock("./provider_display_helpers", () => ({ + getProviderDisplayInfo: vi.fn(() => ({ displayName: "OpenAI", logo: "", enumKey: "OpenAI" })), + handleImageError: vi.fn(), +})); + +const ADMIN_PROPS = { + userID: "user-1", + userRole: "proxy_admin", + accessToken: "test-token", +}; + +describe("CostTrackingSettings", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockDiscountConfig.mockReturnValue({}); + mockMarginConfig.mockReturnValue({}); + }); + + it("should return nothing when accessToken is null", () => { + const { container } = renderWithProviders( + + ); + expect(container.firstChild).toBeNull(); + }); + + it("should render the page title", () => { + renderWithProviders(); + expect(screen.getByText("Cost Tracking Settings")).toBeInTheDocument(); + }); + + it("should show the Provider Discounts accordion header for proxy_admin", () => { + renderWithProviders(); + expect(screen.getByText("Provider Discounts")).toBeInTheDocument(); + }); + + it("should show the Fee/Price Margin accordion header for proxy_admin", () => { + renderWithProviders(); + expect(screen.getByText("Fee/Price Margin")).toBeInTheDocument(); + }); + + it("should always show the Pricing Calculator section", () => { + renderWithProviders(); + // The accordion header text appears in the DOM; getAllByText tolerates duplicates + expect(screen.getAllByText("Pricing Calculator").length).toBeGreaterThan(0); + }); + + it("should show the pricing calculator component", async () => { + renderWithProviders(); + expect(await screen.findByTestId("pricing-calculator")).toBeInTheDocument(); + }); + + it("should not show Provider Discounts section for a non-admin role", () => { + renderWithProviders( + + ); + expect(screen.queryByText("Provider Discounts")).not.toBeInTheDocument(); + }); + + it("should not show Fee/Price Margin section for a non-admin role", () => { + renderWithProviders( + + ); + expect(screen.queryByText("Fee/Price Margin")).not.toBeInTheDocument(); + }); + + it("should show Provider Discounts for the 'Admin' role as well", () => { + renderWithProviders( + + ); + expect(screen.getByText("Provider Discounts")).toBeInTheDocument(); + }); + + it("should show the subtitle describing discount/margin configuration", () => { + renderWithProviders(); + expect( + screen.getByText(/configure cost discounts and margins/i) + ).toBeInTheDocument(); + }); + + describe("Add Provider Discount modal", () => { + it("should open the Add Provider Discount modal when the button is clicked", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + // The button lives inside the Provider Discounts accordion — click the header to expand first + const accordionHeader = screen.getByText("Provider Discounts").closest("button"); + if (accordionHeader) { + await user.click(accordionHeader); + } + + const addButton = await screen.findByRole("button", { name: /add provider discount/i }); + await user.click(addButton); + + expect( + await screen.findByText("Add Provider Discount", { selector: "h2" }) + ).toBeInTheDocument(); + }); + }); + + describe("Add Provider Margin modal", () => { + it("should open the Add Provider Margin modal when the button is clicked", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + const accordionHeader = screen.getByText("Fee/Price Margin").closest("button"); + if (accordionHeader) { + await user.click(accordionHeader); + } + + const addButton = await screen.findByRole("button", { name: /add provider margin/i }); + await user.click(addButton); + + expect( + await screen.findByText("Add Provider Margin", { selector: "h2" }) + ).toBeInTheDocument(); + }); + }); + + describe("empty state messages", () => { + it("should show the empty state message when no discount config is loaded", async () => { + mockDiscountConfig.mockReturnValue({}); + renderWithProviders(); + + const accordionHeader = screen.getByText("Provider Discounts").closest("button"); + if (accordionHeader) { + await userEvent.setup().click(accordionHeader); + } + + expect( + await screen.findByText(/no provider discounts configured/i) + ).toBeInTheDocument(); + }); + + it("should show the empty state message when no margin config is loaded", async () => { + mockMarginConfig.mockReturnValue({}); + renderWithProviders(); + + const accordionHeader = screen.getByText("Fee/Price Margin").closest("button"); + if (accordionHeader) { + await userEvent.setup().click(accordionHeader); + } + + expect( + await screen.findByText(/no provider margins configured/i) + ).toBeInTheDocument(); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/how_it_works.test.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/how_it_works.test.tsx new file mode 100644 index 00000000000..fa608f555ce --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/how_it_works.test.tsx @@ -0,0 +1,95 @@ +import React from "react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders } from "../../../tests/test-utils"; +import HowItWorks from "./how_it_works"; + +vi.mock("@/app/(dashboard)/api-reference/components/CodeBlock", () => ({ + default: ({ code }: { code: string }) => 
{code}
, +})); + +describe("HowItWorks", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should render", () => { + renderWithProviders(); + expect(screen.getByText("Cost Calculation")).toBeInTheDocument(); + }); + + it("should display the cost calculation formula", () => { + renderWithProviders(); + expect(screen.getByText(/final_cost = base_cost/i)).toBeInTheDocument(); + }); + + it("should display the valid range information", () => { + renderWithProviders(); + expect(screen.getByText(/0% and 100%/i)).toBeInTheDocument(); + }); + + it("should render the code block with a curl example", () => { + renderWithProviders(); + expect(screen.getByTestId("code-block")).toBeInTheDocument(); + expect(screen.getByTestId("code-block").textContent).toContain("curl"); + }); + + it("should show the response header names for discount verification", () => { + renderWithProviders(); + expect(screen.getByText("x-litellm-response-cost")).toBeInTheDocument(); + expect(screen.getByText("x-litellm-response-cost-original")).toBeInTheDocument(); + expect(screen.getByText("x-litellm-response-cost-discount-amount")).toBeInTheDocument(); + }); + + it("should not show calculated results initially when no input is provided", () => { + renderWithProviders(); + expect(screen.queryByText("Calculated Results")).not.toBeInTheDocument(); + }); + + it("should not show calculated results when only response cost is entered", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + const responseCostInput = screen.getByPlaceholderText("0.0171938125"); + await user.type(responseCostInput, "0.01"); + + expect(screen.queryByText("Calculated Results")).not.toBeInTheDocument(); + }); + + it("should not show calculated results when only discount amount is entered", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + const discountAmountInput = screen.getByPlaceholderText("0.0009049375"); + await user.type(discountAmountInput, "0.001"); + + expect(screen.queryByText("Calculated Results")).not.toBeInTheDocument(); + }); + + it("should show calculated results when both fields are filled", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + const responseCostInput = screen.getByPlaceholderText("0.0171938125"); + const discountAmountInput = screen.getByPlaceholderText("0.0009049375"); + + await user.type(responseCostInput, "0.0171938125"); + await user.type(discountAmountInput, "0.0009049375"); + + expect(await screen.findByText("Calculated Results")).toBeInTheDocument(); + }); + + it("should show original cost, final cost, and discount amount in results", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.type(screen.getByPlaceholderText("0.0171938125"), "0.0171938125"); + await user.type(screen.getByPlaceholderText("0.0009049375"), "0.0009049375"); + + expect(await screen.findByText("Original Cost:")).toBeInTheDocument(); + expect(screen.getByText("Final Cost:")).toBeInTheDocument(); + expect(screen.getByText("Discount Amount:")).toBeInTheDocument(); + expect(screen.getByText("Discount Applied:")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/provider_discount_table.test.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/provider_discount_table.test.tsx new file mode 100644 index 00000000000..7697c6e7686 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/provider_discount_table.test.tsx @@ -0,0 +1,241 @@ +import React from "react"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders } from "../../../tests/test-utils"; +import ProviderDiscountTable from "./provider_discount_table"; + +vi.mock("@heroicons/react/outline", () => ({ + TrashIcon: function TrashIcon() { return null; }, + PencilAltIcon: function PencilAltIcon() { return null; }, + CheckIcon: function CheckIcon() { return null; }, + XIcon: function XIcon() { return null; }, +})); + +vi.mock("@tremor/react", () => ({ + Table: ({ children }: any) => {children}
, + TableHead: ({ children }: any) => {children}, + TableRow: ({ children }: any) => {children}, + TableHeaderCell: ({ children }: any) => {children}, + TableBody: ({ children }: any) => {children}, + TableCell: ({ children }: any) => {children}, + Text: ({ children }: any) => {children}, + TextInput: ({ value, onValueChange, onKeyDown, placeholder, ...rest }: any) => ( + onValueChange?.(e.target.value)} + onKeyDown={onKeyDown} + placeholder={placeholder} + {...rest} + /> + ), + Icon: ({ icon: IconComponent, onClick }: any) => { + const name = IconComponent?.displayName ?? IconComponent?.name ?? "icon"; + return + + + {/* Project Details */} + + + + {project.description || "\u2014"} + + {new Date(project.created_at).toLocaleString()} + {project.created_by && ( + +  {"by"}  + + + )} + + + {new Date(project.updated_at).toLocaleString()} + {project.updated_by && ( + +  {"by"}  + + + )} + + + + + + {/* Spend / Budget */} + + + + + Budget + + } + style={{ height: "100%" }} + > + +
+ + ${spend.toFixed(2)} + +
+ {hasLimit ? `of $${maxBudget.toFixed(2)} budget` : "No budget limit"} +
+ {hasLimit && ( +
+ + + {(Math.round(spendPercent * 10) / 10).toFixed(1)}% utilized + +
+ )} + + + + + + {modelSpendData.length > 0 ? ( + `$${value.toFixed(4)}`} + yAxisWidth={140} + showLegend={false} + style={{ height: Math.max(modelSpendData.length * 40, 120) }} + /> + ) : ( + + )} + + + + + {/* Keys & Team */} + + + + + Keys + + } + style={{ height: "100%" }} + > + + + + + + + Team + + } + style={{ height: "100%" }} + > + {teamInfo ? ( + (() => { + const teamBudget = teamInfo.max_budget ?? null; + const teamSpend = teamInfo.spend ?? 0; + const teamHasLimit = teamBudget != null && teamBudget > 0; + const teamPercent = teamHasLimit ? Math.min((teamSpend / teamBudget) * 100, 100) : 0; + const teamColor = teamPercent >= 90 ? "#f5222d" : teamPercent >= 70 ? "#faad14" : "#52c41a"; + + return ( + + {/* Team name + ID */} +
+ + {teamInfo.team_alias || teamInfo.team_id} + +
+ + ID:{" "} + + {teamInfo.team_id} + + +
+ + {/* Models */} +
+ + Models + + {(teamInfo.models?.length ?? 0) > 0 ? ( + + {teamInfo.models?.map((m: string) => ( + + {m} + + ))} + + ) : ( + All models + )} +
+ + {/* Budget + Spend compact */} +
+ + + Spend + + + ${teamSpend.toFixed(2)} + {teamHasLimit ? ( + + {" "} + / ${teamBudget.toFixed(2)} + + ) : ( + + {" "} + (Unlimited) + + )} + + + {teamHasLimit && ( + + )} +
+ + {/* Members */} + + + Members + + {teamInfo.members_with_roles?.length ?? 0} + + + ); + })() + ) : project.team_id ? ( + + } size="small" /> + + ) : ( + + )} + + + + + {/* Edit Modal */} + setIsEditModalVisible(false)} /> + + ); +} diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx index 14b4d70b743..4010699360b 100644 --- a/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx @@ -1,95 +1,39 @@ -import { useEffect, useState } from "react"; +import { Modal, Form, Button, Typography, message } from "antd"; +import { FolderAddOutlined } from "@ant-design/icons"; import { - Alert, - Modal, - Form, - Input, - Select, - Switch, - InputNumber, - Collapse, - Button, - Col, - Flex, - Row, - Space, - Divider, - Typography, - message, -} from "antd"; -import { FolderAddOutlined, PlusOutlined, MinusCircleOutlined } from "@ant-design/icons"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; -import { useCreateProject, ProjectCreateParams } from "@/app/(dashboard)/hooks/projects/useCreateProject"; -import { Team } from "../../key_team_helpers/key_list"; -import { fetchTeamModels } from "../../organisms/create_key_button"; -import { getModelDisplayName } from "../../key_team_helpers/fetch_available_models_team_key"; + useCreateProject, + ProjectCreateParams, +} from "@/app/(dashboard)/hooks/projects/useCreateProject"; +import { + ProjectBaseForm, + ProjectFormValues, +} from "./ProjectBaseForm"; +import { buildProjectApiParams } from "./projectFormUtils"; interface CreateProjectModalProps { isOpen: boolean; onClose: () => void; } -export function CreateProjectModal({ isOpen, onClose }: CreateProjectModalProps) { - const [form] = Form.useForm(); - const { accessToken, userId, userRole } = useAuthorized(); - const { data: teams } = useTeams(); +export function CreateProjectModal({ + isOpen, + onClose, +}: CreateProjectModalProps) { + const [form] = Form.useForm(); const createMutation = useCreateProject(); - const [selectedTeam, setSelectedTeam] = useState(null); - const [modelsToPick, setModelsToPick] = useState([]); - - // Fetch team-scoped models when team selection changes - useEffect(() => { - if (userId && userRole && accessToken && selectedTeam) { - fetchTeamModels(userId, userRole, accessToken, selectedTeam.team_id).then((models) => { - const allModels = Array.from(new Set([...(selectedTeam.models ?? []), ...models])); - setModelsToPick(allModels); - }); - } else { - setModelsToPick([]); - } - form.setFieldValue("models", []); - }, [selectedTeam, accessToken, userId, userRole, form]); - const handleSubmit = async () => { try { const values = await form.validateFields(); - - // Build model-specific limits from the dynamic form list - const modelRpmLimit: Record = {}; - const modelTpmLimit: Record = {}; - for (const entry of values.modelLimits ?? []) { - if (entry.model) { - if (entry.rpm != null) modelRpmLimit[entry.model] = entry.rpm; - if (entry.tpm != null) modelTpmLimit[entry.model] = entry.tpm; - } - } - - // Build metadata from the dynamic form list - const metadata: Record = {}; - for (const entry of values.metadata ?? []) { - if (entry.key) metadata[entry.key] = entry.value; - } - const params: ProjectCreateParams = { - project_alias: values.project_alias, - description: values.description, + ...buildProjectApiParams(values), team_id: values.team_id, - models: values.models ?? [], - max_budget: values.max_budget, - blocked: values.isBlocked ?? false, - ...(Object.keys(modelRpmLimit).length > 0 && { model_rpm_limit: modelRpmLimit }), - ...(Object.keys(modelTpmLimit).length > 0 && { model_tpm_limit: modelTpmLimit }), - ...(Object.keys(metadata).length > 0 && { metadata }), }; createMutation.mutate(params, { onSuccess: () => { message.success("Project created successfully"); form.resetFields(); - setSelectedTeam(null); - setModelsToPick([]); onClose(); }, onError: (error) => { @@ -103,16 +47,9 @@ export function CreateProjectModal({ isOpen, onClose }: CreateProjectModalProps) const handleCancel = () => { form.resetFields(); - setSelectedTeam(null); - setModelsToPick([]); onClose(); }; - const handleTeamChange = (teamId: string) => { - const team = teams?.find((t) => t.team_id === teamId) ?? null; - setSelectedTeam(team); - }; - return ( Cancel , - , ]} > -
- {/* Basic Info */} - - Basic Information - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {/* Advanced Settings */} - - - - - Advanced Settings - - } - key="1" - > - - Block Project - - - - - prev.isBlocked !== cur.isBlocked}> - {({ getFieldValue }) => - getFieldValue("isBlocked") ? ( - - ) : null - } - - - - - - Model-Specific Limits - - - {(fields, { add, remove }) => ( - <> - {fields.map(({ key, name, ...restField }) => ( - - - - - - - - - - - remove(name)} style={{ color: "#ef4444" }} /> - - ))} - - - - - )} - - - - - - Metadata - - - {(fields, { add, remove }) => ( - <> - {fields.map(({ key, name, ...restField }) => ( - - - - - - - - remove(name)} style={{ color: "#ef4444" }} /> - - ))} - - - - - )} - - - - - - +
); } diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx new file mode 100644 index 00000000000..75f56b1373f --- /dev/null +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx @@ -0,0 +1,126 @@ +import { useEffect } from "react"; +import { Modal, Form, Button, Typography, message } from "antd"; +import { SaveOutlined } from "@ant-design/icons"; +import { ProjectResponse } from "@/app/(dashboard)/hooks/projects/useProjects"; +import { + useUpdateProject, + ProjectUpdateParams, +} from "@/app/(dashboard)/hooks/projects/useUpdateProject"; +import { ProjectBaseForm, ProjectFormValues } from "./ProjectBaseForm"; +import { buildProjectApiParams } from "./projectFormUtils"; + +interface EditProjectModalProps { + isOpen: boolean; + project: ProjectResponse; + onClose: () => void; + onSuccess?: () => void; +} + +export function EditProjectModal({ + isOpen, + project, + onClose, + onSuccess, +}: EditProjectModalProps) { + const [form] = Form.useForm(); + const updateMutation = useUpdateProject(); + + // Populate form with existing project data when modal opens + useEffect(() => { + if (isOpen && project) { + // Model limits are stored inside metadata by the backend + const metadataObj = (project.metadata ?? {}) as Record; + const rpmLimits = (metadataObj.model_rpm_limit ?? {}) as Record; + const tpmLimits = (metadataObj.model_tpm_limit ?? {}) as Record; + + const modelLimits: ProjectFormValues["modelLimits"] = []; + const allLimitModels = new Set([ + ...Object.keys(rpmLimits), + ...Object.keys(tpmLimits), + ]); + for (const model of allLimitModels) { + modelLimits.push({ + model, + rpm: rpmLimits[model], + tpm: tpmLimits[model], + }); + } + + // Filter out internal keys from user-facing metadata + const internalKeys = new Set(["model_rpm_limit", "model_tpm_limit"]); + const metadata: ProjectFormValues["metadata"] = []; + for (const [key, value] of Object.entries(metadataObj)) { + if (!internalKeys.has(key)) { + metadata.push({ key, value: String(value) }); + } + } + + form.setFieldsValue({ + project_alias: project.project_alias ?? "", + team_id: project.team_id ?? "", + description: project.description ?? "", + models: project.models ?? [], + max_budget: project.litellm_budget_table?.max_budget ?? undefined, + isBlocked: project.blocked, + modelLimits: modelLimits.length > 0 ? modelLimits : undefined, + metadata: metadata.length > 0 ? metadata : undefined, + }); + } + }, [isOpen, project, form]); + + const handleSubmit = async () => { + try { + const values = await form.validateFields(); + const params: ProjectUpdateParams = { + ...buildProjectApiParams(values), + team_id: values.team_id, + }; + + updateMutation.mutate( + { projectId: project.project_id, params }, + { + onSuccess: () => { + message.success("Project updated successfully"); + onSuccess?.(); + onClose(); + }, + onError: (error) => { + message.error(error.message || "Failed to update project"); + }, + }, + ); + } catch (error) { + console.error("Validation failed:", error); + } + }; + + return ( + + Edit Project + + } + open={isOpen} + onCancel={onClose} + width={720} + destroyOnHidden + footer={[ + , + , + ]} + > + + + ); +} diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.tsx new file mode 100644 index 00000000000..bf1eca882c3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.tsx @@ -0,0 +1,401 @@ +import { useEffect, useState } from "react"; +import { + Alert, + Col, + Collapse, + Divider, + Flex, + Form, + Input, + InputNumber, + Row, + Select, + Space, + Switch, + Typography, + Button, +} from "antd"; +import type { FormInstance } from "antd"; +import { PlusOutlined, MinusCircleOutlined } from "@ant-design/icons"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; +import { Team } from "../../key_team_helpers/key_list"; +import { fetchTeamModels } from "../../organisms/create_key_button"; +import { getModelDisplayName } from "../../key_team_helpers/fetch_available_models_team_key"; + +export interface ProjectFormValues { + project_alias: string; + team_id: string; + description?: string; + models: string[]; + max_budget?: number; + isBlocked: boolean; + modelLimits?: { model: string; tpm?: number; rpm?: number }[]; + metadata?: { key: string; value: string }[]; +} + +interface ProjectBaseFormProps { + form: FormInstance; +} + +export function ProjectBaseForm({ + form, +}: ProjectBaseFormProps) { + const { accessToken, userId, userRole } = useAuthorized(); + const { data: teams } = useTeams(); + + const [selectedTeam, setSelectedTeam] = useState(null); + const [modelsToPick, setModelsToPick] = useState([]); + + // Sync selectedTeam from form value (needed for edit mode pre-fill) + const teamIdValue = Form.useWatch("team_id", form); + useEffect(() => { + if (teamIdValue && teams) { + const team = teams.find((t) => t.team_id === teamIdValue) ?? null; + if (team && team.team_id !== selectedTeam?.team_id) { + setSelectedTeam(team); + } + } + }, [teamIdValue, teams, selectedTeam?.team_id]); + + // Fetch team-scoped models when team selection changes + useEffect(() => { + if (userId && userRole && accessToken && selectedTeam) { + fetchTeamModels(userId, userRole, accessToken, selectedTeam.team_id).then( + (models) => { + const allModels = Array.from( + new Set([...(selectedTeam.models ?? []), ...models]), + ); + setModelsToPick(allModels); + }, + ); + } else { + setModelsToPick([]); + } + }, [selectedTeam, accessToken, userId, userRole]); + + const handleTeamChange = (teamId: string) => { + const team = teams?.find((t) => t.team_id === teamId) ?? null; + setSelectedTeam(team); + form.setFieldValue("models", []); + }; + + return ( +
+ {/* Basic Info */} + + Basic Information + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {/* Advanced Settings */} + + + + Advanced Settings + + ), + children: ( + <> + + Block Project + + + + + prev.isBlocked !== cur.isBlocked} + > + {({ getFieldValue }) => + getFieldValue("isBlocked") ? ( + + ) : null + } + + + + + + Model-Specific Limits + + + {(fields, { add, remove }) => ( + <> + {fields.map(({ key, name, ...restField }) => ( + + { + if (!value) return Promise.resolve(); + const all = form.getFieldValue("modelLimits") ?? []; + const dupes = all.filter( + (entry: { model?: string }) => entry?.model === value, + ); + if (dupes.length > 1) { + return Promise.reject(new Error("Duplicate model")); + } + return Promise.resolve(); + }, + }, + ]} + > + + + + + + + + + remove(name)} + style={{ color: "#ef4444" }} + /> + + ))} + + + + + )} + + + + + + Metadata + + + {(fields, { add, remove }) => ( + <> + {fields.map(({ key, name, ...restField }) => ( + + { + if (!value) return Promise.resolve(); + const all = form.getFieldValue("metadata") ?? []; + const dupes = all.filter( + (entry: { key?: string }) => entry?.key === value, + ); + if (dupes.length > 1) { + return Promise.reject(new Error("Duplicate key")); + } + return Promise.resolve(); + }, + }, + ]} + > + + + + + + remove(name)} + style={{ color: "#ef4444" }} + /> + + ))} + + + + + )} + + + ), + }, + ]} + /> + + + + ); +} diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/projectFormUtils.ts b/ui/litellm-dashboard/src/components/Projects/ProjectModals/projectFormUtils.ts new file mode 100644 index 00000000000..97c093b57d9 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/projectFormUtils.ts @@ -0,0 +1,36 @@ +import { ProjectFormValues } from "./ProjectBaseForm"; + +/** + * Transforms ProjectFormValues into the flat API param shape + * shared by both create and update endpoints. + */ +export function buildProjectApiParams(values: ProjectFormValues) { + const modelRpmLimit: Record = {}; + const modelTpmLimit: Record = {}; + for (const entry of values.modelLimits ?? []) { + if (entry.model) { + if (entry.rpm != null) modelRpmLimit[entry.model] = entry.rpm; + if (entry.tpm != null) modelTpmLimit[entry.model] = entry.tpm; + } + } + + const metadata: Record = {}; + for (const entry of values.metadata ?? []) { + if (entry.key) metadata[entry.key] = entry.value; + } + + return { + project_alias: values.project_alias, + description: values.description, + models: values.models ?? [], + max_budget: values.max_budget, + blocked: values.isBlocked ?? false, + ...(Object.keys(modelRpmLimit).length > 0 && { + model_rpm_limit: modelRpmLimit, + }), + ...(Object.keys(modelTpmLimit).length > 0 && { + model_tpm_limit: modelTpmLimit, + }), + ...(Object.keys(metadata).length > 0 && { metadata }), + }; +} diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx index 40ab5045703..f0b593c2e49 100644 --- a/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx +++ b/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx @@ -1,13 +1,15 @@ import { useProjects, ProjectResponse } from "@/app/(dashboard)/hooks/projects/useProjects"; import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; -import { PlusOutlined } from "@ant-design/icons"; +import { LoadingOutlined, PlusOutlined } from "@ant-design/icons"; import { Button, Card, Flex, Input, Layout, + Pagination, Space, + Spin, Table, Tag, theme, @@ -18,6 +20,7 @@ import type { ColumnsType } from "antd/es/table"; import { LayersIcon, SearchIcon } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; import { CreateProjectModal } from "./ProjectModals/CreateProjectModal"; +import { ProjectDetail } from "./ProjectDetailsPage"; const { Title, Text } = Typography; const { Content } = Layout; @@ -25,8 +28,9 @@ const { Content } = Layout; export function ProjectsPage() { const { token } = theme.useToken(); const { data: projects, isLoading } = useProjects(); - const { data: teams } = useTeams(); + const { data: teams, isLoading: isTeamsLoading } = useTeams(); + const [selectedProjectId, setSelectedProjectId] = useState(null); const [isCreateModalVisible, setIsCreateModalVisible] = useState(false); const [searchText, setSearchText] = useState(""); const [currentPage, setCurrentPage] = useState(1); @@ -74,6 +78,7 @@ export function ProjectsPage() { ellipsis className="text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs cursor-pointer" style={{ fontSize: 14, padding: "1px 8px" }} + onClick={() => setSelectedProjectId(id)} > {id} @@ -96,8 +101,11 @@ export function ProjectsPage() { return aAlias.localeCompare(bAlias); }, render: (_: unknown, record: ProjectResponse) => { - const alias = teamAliasMap.get(record.team_id ?? ""); - return alias ?? record.team_id ?? "—"; + if (!record.team_id) return "—"; + const alias = teamAliasMap.get(record.team_id); + if (alias) return alias; + if (isTeamsLoading) return } size="small" />; + return record.team_id; }, }, { @@ -144,6 +152,15 @@ export function ProjectsPage() { }, ]; + if (selectedProjectId) { + return ( + setSelectedProjectId(null)} + /> + ); + } + return ( setSearchText(e.target.value)} allowClear /> + setCurrentPage(page)} + size="small" + showTotal={(total) => `${total} projects`} + showSizeChanger={false} + /> setCurrentPage(page), - size: "small", - showTotal: (total) => `${total} projects`, - showSizeChanger: false, - }} + pagination={false} /> From 50dc7b520cc7ba0211af064598b69a435b9da661 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 27 Feb 2026 22:11:04 -0800 Subject: [PATCH 39/52] Update ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../components/Projects/ProjectModals/CreateProjectModal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx index 4010699360b..e490f89303f 100644 --- a/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/CreateProjectModal.tsx @@ -60,7 +60,7 @@ export function CreateProjectModal({ open={isOpen} onCancel={handleCancel} width={720} - destroyOnClose + destroyOnHidden footer={[ - - - Keys - - } - style={{ height: "100%" }} - > - - + (""); + + const { data, isLoading } = useKeys(page, PAGE_SIZE, { + projectID: projectId, + selectedKeyAlias: keyAlias || null, + }); + + // Reset to page 1 when filter changes + useEffect(() => { + setPage(1); + }, [keyAlias]); + + const keys = data?.keys ?? []; + const totalCount = data?.total_count ?? 0; + + return ( + + + Keys + + } + style={{ height: "100%" }} + > + + } + placeholder="Filter by key name..." + style={{ maxWidth: 220 }} + value={keyAlias} + onChange={(e) => setKeyAlias(e.target.value)} + allowClear + size="small" + /> + `${total} keys`} + /> + + } /> } : false} + /> + + ); +} diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectKeysTable.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectKeysTable.tsx new file mode 100644 index 00000000000..cb80d0e27a5 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Projects/ProjectKeysTable.tsx @@ -0,0 +1,58 @@ +import { KeyResponse } from "@/components/key_team_helpers/key_list"; +import { Empty, Table, Tooltip } from "antd"; +import type { ColumnsType } from "antd/es/table"; +import type { SpinProps } from "antd"; +import DefaultProxyAdminTag from "../common_components/DefaultProxyAdminTag"; + +interface ProjectKeysTableProps { + keys: KeyResponse[]; + loading?: boolean | SpinProps; +} + +const columns: ColumnsType = [ + { + title: "Key Name", + dataIndex: "key_alias", + key: "key_alias", + render: (alias: string | null) => alias || "—", + }, + { + title: "Owner", + key: "owner", + render: (_: unknown, record: KeyResponse) => { + const email = record.user?.user_email ?? record.user_id ?? null; + if (!email) return "—"; + return ( + + + + ); + }, + }, + { + title: "Created", + dataIndex: "created_at", + key: "created_at", + render: (date: string) => (date ? new Date(date).toLocaleDateString() : "—"), + }, + { + title: "Last Active", + dataIndex: "last_active", + key: "last_active", + render: (date: string | null) => (date ? new Date(date).toLocaleDateString() : "Never"), + }, +]; + +export function ProjectKeysTable({ keys, loading }: ProjectKeysTableProps) { + return ( +
}} + /> + ); +} diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx index f0b593c2e49..9c75e19ac4e 100644 --- a/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx +++ b/ui/litellm-dashboard/src/components/Projects/ProjectsPage.tsx @@ -2,6 +2,7 @@ import { useProjects, ProjectResponse } from "@/app/(dashboard)/hooks/projects/u import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import { LoadingOutlined, PlusOutlined } from "@ant-design/icons"; import { + Alert, Button, Card, Flex, @@ -165,6 +166,12 @@ export function ProjectsPage() { + void; + disabled?: boolean; + loading?: boolean; + /** When set, only show projects belonging to this team */ + teamId?: string | null; +} + +const ProjectDropdown: React.FC = ({ + projects, + value, + onChange, + disabled, + loading, + teamId, +}) => { + const filtered = teamId + ? projects?.filter((p) => p.team_id === teamId) + : projects; + + return ( + + ); +}; + +export default ProjectDropdown; diff --git a/ui/litellm-dashboard/src/components/common_components/team_dropdown.tsx b/ui/litellm-dashboard/src/components/common_components/team_dropdown.tsx index d54724da2a7..9e79ea2950a 100644 --- a/ui/litellm-dashboard/src/components/common_components/team_dropdown.tsx +++ b/ui/litellm-dashboard/src/components/common_components/team_dropdown.tsx @@ -7,10 +7,10 @@ interface TeamDropdownProps { value?: string; onChange?: (value: string) => void; disabled?: boolean; + loading?: boolean; } -const TeamDropdown: React.FC = ({ teams, value, onChange, disabled }) => { - console.log("disabled", disabled); +const TeamDropdown: React.FC = ({ teams, value, onChange, disabled, loading }) => { return ( onChange?.(e.target.value)} + /> + ), +})); + vi.mock("../common_components/AccessGroupSelector", () => ({ default: ({ value = [], onChange }: { value?: string[]; onChange?: (v: string[]) => void }) => ( = ({ team, teams, data, addKey }) => { const { accessToken, userId: userID, userRole, premiumUser } = useAuthorized(); + const { data: projects, isLoading: isProjectsLoading } = useProjects(); const queryClient = useQueryClient(); const [form] = Form.useForm(); const [isModalVisible, setIsModalVisible] = useState(false); @@ -157,6 +160,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey }) => { const [promptsList, setPromptsList] = useState([]); const [loggingSettings, setLoggingSettings] = useState([]); const [selectedCreateKeyTeam, setSelectedCreateKeyTeam] = useState(team); + const [selectedProjectId, setSelectedProjectId] = useState(null); const [isCreateUserModalVisible, setIsCreateUserModalVisible] = useState(false); const [newlyCreatedUserId, setNewlyCreatedUserId] = useState(null); const [possibleUIRoles, setPossibleUIRoles] = useState>>({}); @@ -184,6 +188,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey }) => { setRouterSettings(null); setRouterSettingsKey((prev) => prev + 1); setSelectedAgentId(null); + setSelectedProjectId(null); }; const handleCancel = () => { @@ -200,6 +205,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey }) => { setRouterSettings(null); setRouterSettingsKey((prev) => prev + 1); setSelectedAgentId(null); + setSelectedProjectId(null); }; useEffect(() => { @@ -468,6 +474,14 @@ const CreateKey: React.FC = ({ team, teams, data, addKey }) => { }; useEffect(() => { + if (selectedProjectId) { + // When a project is selected, use the project's models + const project = projects?.find((p) => p.project_id === selectedProjectId); + const projectModels = project?.models ?? []; + setModelsToPick(projectModels); + form.setFieldValue("models", []); + return; + } if (userID && userRole && accessToken) { fetchTeamModels(userID, userRole, accessToken, selectedCreateKeyTeam?.team_id ?? null).then((models) => { let allModels = Array.from(new Set([...(selectedCreateKeyTeam?.models ?? []), ...models])); @@ -475,7 +489,22 @@ const CreateKey: React.FC = ({ team, teams, data, addKey }) => { }); } form.setFieldValue("models", []); - }, [selectedCreateKeyTeam, accessToken, userID, userRole]); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [selectedCreateKeyTeam, selectedProjectId, accessToken, userID, userRole]); + + // Sync team when project is selected but teams loaded later (race condition) + useEffect(() => { + if (!selectedProjectId || !teams) return; + const project = projects?.find((p) => p.project_id === selectedProjectId); + if (!project?.team_id) return; + // If team is already set correctly, skip + if (selectedCreateKeyTeam?.team_id === project.team_id) return; + const projectTeam = teams.find((t) => t.team_id === project.team_id) || null; + if (projectTeam) { + setSelectedCreateKeyTeam(projectTeam); + form.setFieldValue("team_id", projectTeam.team_id); + } + }, [teams, selectedProjectId, projects]); // Add a callback function to handle user creation const handleUserCreated = (userId: string) => { @@ -653,9 +682,40 @@ const CreateKey: React.FC = ({ team, teams, data, addKey }) => { > { const selectedTeam = teams?.find((t) => t.team_id === teamId) || null; setSelectedCreateKeyTeam(selectedTeam); + setSelectedProjectId(null); + form.setFieldValue("project_id", undefined); + }} + /> + + + Project{" "} + + + + + } + name="project_id" + className="mt-4" + > + { + if (!projectId) { + setSelectedProjectId(null); + setSelectedCreateKeyTeam(null); + form.setFieldValue("team_id", undefined); + return; + } + setSelectedProjectId(projectId); }} /> @@ -735,9 +795,11 @@ const CreateKey: React.FC = ({ team, teams, data, addKey }) => { } }} > - + {!selectedProjectId && ( + + )} {modelsToPick.map((model: string) => (