From ba0aa4d9d14ab0294403ee56935bb3d8865fb85f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 06:11:44 -0700 Subject: [PATCH 1/6] fix(guardrails): retry a guardrail translation package that failed to import instead of caching the gap Discovery cached whichever handler map it got on the first lookup, so one package failing to import (a poisoned sys.modules entry in tests, a broken install in production) left every later guardrail lookup in that process without the handler, and the streaming hook passed responses through unscanned with nothing in the log Discovery now records which packages failed and why, every later lookup retries only those packages until they import, the failure is logged once with its reason and the recovery once, and the streaming hook warns with the guardrail and route it left unscanned --- litellm/llms/__init__.py | 213 +++++++++++------- .../unified_guardrail/unified_guardrail.py | 53 +++-- .../test_guardrail_translation_discovery.py | 104 +++++++++ .../test_mcp_guardrail_handler.py | 4 +- .../test_openai_moderation_streaming.py | 89 +++++++- .../test_unified_guardrail.py | 79 +++++++ 6 files changed, 432 insertions(+), 110 deletions(-) create mode 100644 tests/test_litellm/llms/test_guardrail_translation_discovery.py diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index 88a44f38c57..23aa2a94a97 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -1,5 +1,8 @@ import importlib import os +from collections.abc import Iterable, Iterator, Mapping +from dataclasses import dataclass +from types import MappingProxyType from typing import TYPE_CHECKING, Final from litellm._logging import verbose_logger @@ -80,93 +83,137 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo return None -def discover_guardrail_translation_mappings() -> dict[CallTypes, type["BaseTranslation"]]: +_GUARDRAIL_TRANSLATION_PACKAGE: Final = "guardrail_translation" +_MCP_GUARDRAIL_TRANSLATION_MODULE: Final = "litellm.proxy._experimental.mcp_server.guardrail_translation" +_NO_MAPPINGS: Final[Mapping[CallTypes, type["BaseTranslation"]]] = MappingProxyType({}) + + +@dataclass(frozen=True, slots=True) +class GuardrailTranslationDiscovery: + """ + The outcome of one scan for guardrail translation handlers. + + unavailable maps each bundled package that failed to import to the reason, which is what tells a complete + result apart from one that is missing handlers and therefore has to be retried. + """ + + mappings: Mapping[CallTypes, type["BaseTranslation"]] + unavailable: Mapping[str, str] + + +def _bundled_guardrail_translation_modules() -> Iterator[str]: + """Yield the import path of every guardrail_translation package shipped under litellm/llms.""" + llms_dir: Final = os.path.dirname(__file__) + for root, dirs, files in os.walk(llms_dir): + dirs[:] = tuple(d for d in dirs if not d.startswith("__") and d != "base_llm") + if os.path.basename(root) == _GUARDRAIL_TRANSLATION_PACKAGE and "__init__.py" in files: + yield "litellm." + os.path.relpath(root, os.path.dirname(llms_dir)).replace(os.sep, ".") + + +def _import_guardrail_translations(module_path: str) -> Mapping[CallTypes, type["BaseTranslation"]] | str: + """Import one guardrail_translation package, returning the reason as a string when that fails.""" + try: + module: Final = importlib.import_module(module_path) + except Exception as e: # noqa: BLE001 # a package failing at import time for any reason is unavailable, not fatal + return f"{type(e).__name__}: {e}" + mappings: Final = getattr(module, "guardrail_translation_mappings", None) + if not isinstance(mappings, dict): + return _NO_MAPPINGS + declared: Final[Mapping[CallTypes, type[BaseTranslation]]] = mappings + return declared + + +def _optional_mcp_guardrail_translation_mappings() -> Mapping[CallTypes, type["BaseTranslation"]]: + """MCP call types live outside litellm/llms and are absent from installs without the MCP server.""" + try: + from litellm.proxy._experimental.mcp_server.guardrail_translation import ( + guardrail_translation_mappings as mcp_guardrail_translation_mappings, + ) + except ImportError: + verbose_logger.debug("%s not available; skipping", _MCP_GUARDRAIL_TRANSLATION_MODULE) + return _NO_MAPPINGS + return mcp_guardrail_translation_mappings + + +def _discover( + module_paths: Iterable[str], already_found: Mapping[CallTypes, type["BaseTranslation"]] +) -> GuardrailTranslationDiscovery: + imported: Final = tuple((module_path, _import_guardrail_translations(module_path)) for module_path in module_paths) + found: Final = (already_found, *(result for _, result in imported if not isinstance(result, str))) + return GuardrailTranslationDiscovery( + mappings=MappingProxyType( + {call_type: handler for mappings in found for call_type, handler in mappings.items()} + ), + unavailable=MappingProxyType( + {module_path: result for module_path, result in imported if isinstance(result, str)} + ), + ) + + +def discover_guardrail_translations() -> GuardrailTranslationDiscovery: + """ + Scan the llms tree, plus the optional MCP package, for guardrail translation handlers. + + Returns: + GuardrailTranslationDiscovery: the handlers found, and the bundled packages that failed to import + """ + return _discover( + _bundled_guardrail_translation_modules(), already_found=_optional_mcp_guardrail_translation_mappings() + ) + + +def discover_guardrail_translation_mappings() -> Mapping[CallTypes, type["BaseTranslation"]]: """ Discover guardrail translation mappings by scanning the llms directory structure. - Scans for modules with guardrail_translation_mappings dictionaries and aggregates them. - Returns: - Dict[CallTypes, Type[BaseTranslation]]: A dictionary mapping call types to their translation handler classes + Mapping[CallTypes, Type[BaseTranslation]]: the call types that have a translation handler class """ - discovered_mappings: Final[dict[CallTypes, type[BaseTranslation]]] = {} + return discover_guardrail_translations().mappings - try: - # Get the path to the llms directory - current_dir: Final = os.path.dirname(__file__) - llms_dir: Final = current_dir - if not os.path.exists(llms_dir): - verbose_logger.debug("llms directory not found") - return discovered_mappings - - # Recursively scan for guardrail_translation directories - for root, dirs, files in os.walk(llms_dir): - # Skip __pycache__ and base_llm directories - dirs[:] = [d for d in dirs if not d.startswith("__") and d != "base_llm"] - - # Check if this is a guardrail_translation directory with __init__.py - if os.path.basename(root) == "guardrail_translation" and "__init__.py" in files: - # Build the module path relative to litellm - rel_path = os.path.relpath(root, os.path.dirname(llms_dir)) - module_path = "litellm." + rel_path.replace(os.sep, ".") - - try: - # Import the module - verbose_logger.debug("Discovering guardrail translations in: %s", module_path) - - module = importlib.import_module(module_path) - - # Check for guardrail_translation_mappings dictionary - if hasattr(module, "guardrail_translation_mappings"): - mappings = getattr(module, "guardrail_translation_mappings") - if isinstance(mappings, dict): - discovered_mappings.update(mappings) - verbose_logger.debug( - "Found guardrail_translation_mappings in %s: %s", module_path, list(mappings.keys()) - ) - - except ImportError as e: - verbose_logger.error("Could not import %s: %s", module_path, e) - continue - except Exception as e: - verbose_logger.error("Error processing %s: %s", module_path, e) - continue - - try: - from litellm.proxy._experimental.mcp_server.guardrail_translation import ( - guardrail_translation_mappings as mcp_guardrail_translation_mappings, - ) - - discovered_mappings.update(mcp_guardrail_translation_mappings) - verbose_logger.debug( - "Loaded MCP guardrail translation mappings: %s", - list(mcp_guardrail_translation_mappings.keys()), - ) - except ImportError: - verbose_logger.debug("MCP guardrail translation mappings not available; skipping") - - verbose_logger.debug( - "Discovered %s guardrail translation mappings: %s", - len(discovered_mappings), - list(discovered_mappings.keys()), +def _announce_discovery(previous: GuardrailTranslationDiscovery | None, current: GuardrailTranslationDiscovery) -> None: + if previous is None and not current.unavailable: + return + if previous is None: + verbose_logger.error( + "Could not import guardrail translation handlers from %s; guardrails cannot run for their call types " + "until the import succeeds, which every lookup retries. %s", + ", ".join(current.unavailable), + "; ".join(f"{module_path}: {reason}" for module_path, reason in current.unavailable.items()), ) - - except Exception as e: - verbose_logger.error("Error discovering guardrail translation mappings: %s", e) - - return discovered_mappings + return + recovered: Final = tuple( + module_path for module_path in previous.unavailable if module_path not in current.unavailable + ) + if not recovered: + return + verbose_logger.info("Guardrail translation handlers from %s are available again.", ", ".join(recovered)) -# Cache the discovered mappings -endpoint_guardrail_translation_mappings: dict[CallTypes, type["BaseTranslation"]] | None = None +guardrail_translation_discovery: GuardrailTranslationDiscovery | None = None -def load_guardrail_translation_mappings(): - global endpoint_guardrail_translation_mappings - if endpoint_guardrail_translation_mappings is None: - endpoint_guardrail_translation_mappings = discover_guardrail_translation_mappings() - return endpoint_guardrail_translation_mappings +def load_guardrail_translation_mappings() -> Mapping[CallTypes, type["BaseTranslation"]]: + """ + Return the guardrail translation handlers, retrying any bundled package that could not be imported last time. + + Serving an incomplete scan as if it were complete would silently strip the missing call types off every + guardrail for the rest of the process, so the packages that failed are imported again on each lookup, and + only the part that succeeded is kept. + """ + global guardrail_translation_discovery + cached: Final = guardrail_translation_discovery + if cached is not None and not cached.unavailable: + return cached.mappings + discovery: Final = ( + discover_guardrail_translations() + if cached is None + else _discover(cached.unavailable, already_found=cached.mappings) + ) + _announce_discovery(previous=cached, current=discovery) + guardrail_translation_discovery = discovery + return discovery.mappings def get_guardrail_translation_mapping(call_type: CallTypes) -> type["BaseTranslation"]: @@ -182,18 +229,10 @@ def get_guardrail_translation_mapping(call_type: CallTypes) -> type["BaseTransla Raises: ValueError: If no translation mapping exists for the given call type """ - global endpoint_guardrail_translation_mappings - - # Lazy load the mappings on first access - if endpoint_guardrail_translation_mappings is None: - endpoint_guardrail_translation_mappings = discover_guardrail_translation_mappings() - - # Get the translation handler class for the call type - if call_type not in endpoint_guardrail_translation_mappings: + mappings: Final = load_guardrail_translation_mappings() + if call_type not in mappings: raise ValueError( f"No guardrail translation mapping found for call_type: {call_type}. " - f"Available mappings: {list(endpoint_guardrail_translation_mappings.keys())}" + f"Available mappings: {list(mappings.keys())}" ) - - # Return the handler class directly - return endpoint_guardrail_translation_mappings[call_type] + return mappings[call_type] diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index ffa322da288..24a29eeb378 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -133,6 +133,30 @@ def _ensure_litellm_metadata(data: dict, user_api_key_dict: UserAPIKeyAuth) -> N data["litellm_metadata"] = user_metadata +def _warn_left_unscanned( + guardrail_to_apply: "CustomGuardrail", + user_api_key_dict: UserAPIKeyAuth, + call_type: str | None, + consequence: str, +) -> None: + if call_type is None: + verbose_proxy_logger.warning( + "Guardrail '%s' selected for route '%s' but its call type could not be resolved; %s. " + "Add the route to API_ROUTE_TO_CALL_TYPES.", + guardrail_to_apply.guardrail_name, + user_api_key_dict.request_route, + consequence, + ) + return + verbose_proxy_logger.warning( + "Guardrail '%s' selected for route '%s' but call type '%s' has no guardrail translation handler; %s.", + guardrail_to_apply.guardrail_name, + user_api_key_dict.request_route, + call_type, + consequence, + ) + + class UnifiedLLMGuardrails(CustomLogger): def __init__( self, @@ -297,24 +321,13 @@ class UnifiedLLMGuardrails(CustomLogger): ): call_type = logging_call_type - if call_type is None: - verbose_proxy_logger.warning( - "Guardrail '%s' selected for route '%s' but its call type could not be resolved; " - "skipping post-call scanning. Add the route to API_ROUTE_TO_CALL_TYPES.", - guardrail_to_apply.guardrail_name, - user_api_key_dict.request_route, - ) - return response - mappings: Final = load_guardrail_translation_mappings() - - if CallTypes(call_type) not in mappings: - verbose_proxy_logger.warning( - "Guardrail '%s' selected for route '%s' but call type '%s' has no guardrail translation handler; " - "skipping post-call scanning.", - guardrail_to_apply.guardrail_name, - user_api_key_dict.request_route, - call_type, + if call_type is None or CallTypes(call_type) not in mappings: + _warn_left_unscanned( + guardrail_to_apply=guardrail_to_apply, + user_api_key_dict=user_api_key_dict, + call_type=call_type, + consequence="skipping post-call scanning", ) return response @@ -1018,6 +1031,12 @@ class UnifiedLLMGuardrails(CustomLogger): # If call type not supported, just pass through all chunks if call_type is None or CallTypes(call_type) not in mappings: + _warn_left_unscanned( + guardrail_to_apply=guardrail_to_apply, + user_api_key_dict=user_api_key_dict, + call_type=call_type, + consequence="streaming this response to the client unscanned", + ) yield item async for remaining_item in response: yield remaining_item diff --git a/tests/test_litellm/llms/test_guardrail_translation_discovery.py b/tests/test_litellm/llms/test_guardrail_translation_discovery.py new file mode 100644 index 00000000000..4d10ff5f4aa --- /dev/null +++ b/tests/test_litellm/llms/test_guardrail_translation_discovery.py @@ -0,0 +1,104 @@ +import logging +import sys +from collections.abc import Iterator +from contextlib import contextmanager + +import pytest + +import litellm.llms as llms_package +from litellm._logging import verbose_logger +from litellm.types.utils import CallTypes + +OPENAI_CHAT_TRANSLATION_MODULE = "litellm.llms.openai.chat.guardrail_translation" + + +@contextmanager +def unimportable(module_path: str) -> Iterator[None]: + with pytest.MonkeyPatch.context() as mp: + mp.setitem(sys.modules, module_path, None) + yield + + +@contextmanager +def capturing(caplog: pytest.LogCaptureFixture, level: int) -> Iterator[None]: + with pytest.MonkeyPatch.context() as mp: + mp.setattr(verbose_logger, "propagate", True) + caplog.set_level(level, logger=verbose_logger.name) + yield + + +@pytest.fixture(autouse=True) +def reset_guardrail_translation_discovery(): + saved = llms_package.guardrail_translation_discovery + llms_package.guardrail_translation_discovery = None + yield + llms_package.guardrail_translation_discovery = saved + + +def test_discovery_reports_the_handler_package_it_could_not_import(): + with unimportable(OPENAI_CHAT_TRANSLATION_MODULE): + discovery = llms_package.discover_guardrail_translations() + + assert tuple(discovery.unavailable) == (OPENAI_CHAT_TRANSLATION_MODULE,) + assert "None in sys.modules" in discovery.unavailable[OPENAI_CHAT_TRANSLATION_MODULE] + assert CallTypes.acompletion not in discovery.mappings + assert CallTypes.completion not in discovery.mappings + assert CallTypes.aembedding in discovery.mappings + + +def test_complete_discovery_reports_nothing_unavailable(): + discovery = llms_package.discover_guardrail_translations() + + assert not discovery.unavailable + assert CallTypes.acompletion in discovery.mappings + + +def test_the_next_lookup_retries_a_package_that_failed_to_import(): + with unimportable(OPENAI_CHAT_TRANSLATION_MODULE): + partial = llms_package.load_guardrail_translation_mappings() + + assert CallTypes.acompletion not in partial + assert CallTypes.aembedding in partial + + recovered = llms_package.load_guardrail_translation_mappings() + + assert CallTypes.acompletion in recovered + assert CallTypes.completion in recovered + assert CallTypes.aembedding in recovered + assert not llms_package.guardrail_translation_discovery.unavailable + + +def test_a_complete_discovery_is_cached(): + first = llms_package.load_guardrail_translation_mappings() + second = llms_package.load_guardrail_translation_mappings() + + assert first is second + + +def test_a_package_that_keeps_failing_is_reported_once_and_its_recovery_announced(caplog): + with capturing(caplog, logging.INFO), unimportable(OPENAI_CHAT_TRANSLATION_MODULE): + for _ in range(3): + llms_package.load_guardrail_translation_mappings() + + errors = [record for record in caplog.records if record.levelno == logging.ERROR] + assert len(errors) == 1, [record.getMessage() for record in errors] + assert OPENAI_CHAT_TRANSLATION_MODULE in errors[0].getMessage() + assert "None in sys.modules" in errors[0].getMessage() + + caplog.clear() + with capturing(caplog, logging.INFO): + llms_package.load_guardrail_translation_mappings() + llms_package.load_guardrail_translation_mappings() + + recoveries = [record for record in caplog.records if "available again" in record.getMessage()] + assert len(recoveries) == 1 + assert OPENAI_CHAT_TRANSLATION_MODULE in recoveries[0].getMessage() + assert not [record for record in caplog.records if record.levelno == logging.ERROR] + + +def test_lookup_recovers_after_a_failed_discovery(): + with unimportable(OPENAI_CHAT_TRANSLATION_MODULE): + with pytest.raises(ValueError, match="acompletion"): + llms_package.get_guardrail_translation_mapping(CallTypes.acompletion) + + assert llms_package.get_guardrail_translation_mapping(CallTypes.acompletion) is not None diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/guardrail_translation/test_mcp_guardrail_handler.py b/tests/test_litellm/proxy/_experimental/mcp_server/guardrail_translation/test_mcp_guardrail_handler.py index 28959054195..853bc1adfa0 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/guardrail_translation/test_mcp_guardrail_handler.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/guardrail_translation/test_mcp_guardrail_handler.py @@ -133,8 +133,8 @@ def restore_callbacks(monkeypatch): monkeypatch.setattr(litellm, "callbacks", litellm.callbacks) monkeypatch.setattr( litellm_llms, - "endpoint_guardrail_translation_mappings", - litellm_llms.endpoint_guardrail_translation_mappings, + "guardrail_translation_discovery", + litellm_llms.guardrail_translation_discovery, ) yield ProxyLogging._callback_capabilities_cache.clear() diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_openai_moderation_streaming.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_openai_moderation_streaming.py index cb6772977ec..f9ae5b27ddb 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_openai_moderation_streaming.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_openai_moderation_streaming.py @@ -1,14 +1,24 @@ -import pytest -from unittest.mock import MagicMock, patch import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +import litellm.llms as llms_package +from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.openai.moderations import ( OpenAIModerationGuardrail, ) from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( UnifiedLLMGuardrails, ) -from litellm.types.utils import ModelResponseStream, ModelResponse -from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.utils import ( + CallTypes, + Delta, + ModelResponse, + ModelResponseStream, + StreamingChoices, +) @pytest.mark.asyncio @@ -516,3 +526,74 @@ async def test_openai_moderation_streaming_sampled_when_end_of_stream_only_disab f"because chunk 6 already scanned the full text), " f"got {patched_make_request.await_count}" ) + + +@pytest.fixture +def reset_guardrail_translation_discovery(): + saved = llms_package.guardrail_translation_discovery + llms_package.guardrail_translation_discovery = None + yield llms_package + llms_package.guardrail_translation_discovery = saved + + +@pytest.mark.asyncio +async def test_moderation_still_runs_after_a_failed_translation_discovery( + reset_guardrail_translation_discovery, +): + """ + A guardrail translation discovery that could not import the chat handler must not silently + disable moderation for the rest of the process. + """ + with pytest.MonkeyPatch.context() as poison: + poison.setitem(sys.modules, "litellm.llms.openai.chat.guardrail_translation", None) + poisoned = llms_package.load_guardrail_translation_mappings() + assert CallTypes.acompletion not in poisoned + + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): + openai_guardrail = OpenAIModerationGuardrail( + guardrail_name="test-openai-moderation", + event_hook="post_call", + ) + unified_guardrail = UnifiedLLMGuardrails() + + mock_mod_response = MagicMock() + mock_mod_response.results = [] + + async def mock_stream(): + chunks_data = ["Hello", " ", "world", "!", " Goodbye"] + for i, content in enumerate(chunks_data): + yield ModelResponseStream( + model="gpt-4", + choices=[ + StreamingChoices( + index=0, + delta=Delta(content=content, role="assistant"), + finish_reason=( + "stop" if i == len(chunks_data) - 1 else None + ), + ) + ], + ) + + with patch.object( + openai_guardrail, "async_make_request", return_value=mock_mod_response + ) as patched_make_request: + chunks_received = 0 + async for _ in unified_guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth( + api_key="test", request_route="/chat/completions" + ), + response=mock_stream(), + request_data={ + "messages": [{"role": "user", "content": "hi"}], + "guardrail_to_apply": openai_guardrail, + "metadata": {"guardrails": ["test-openai-moderation"]}, + }, + ): + chunks_received += 1 + + assert chunks_received == 5 + assert patched_make_request.await_count > 0, ( + "Moderation never ran: the failed discovery was cached and the streaming hook " + "passed every chunk through unscanned" + ) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index a579370ad3c..91dff7b8346 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -2289,3 +2289,82 @@ class TestTranslationMappingsAreReadLive: for name, value in vars(unified_module).items() if isinstance(value, dict) and CallTypes.aocr in value ] + + +class TestUnscannedStreamIsAnnounced: + """The streaming hook must never forward a whole response unscanned without saying so.""" + + @staticmethod + async def _drive(caplog, monkeypatch, request_route, mappings, response_chunks): + _patch_translation_mappings(monkeypatch, mappings) + + async def stream(): + for chunk in response_chunks: + yield chunk + + caplog.set_level(logging.WARNING, logger="LiteLLM Proxy") + unified_module.verbose_proxy_logger.addHandler(caplog.handler) + try: + chunks = [ + chunk + async for chunk in UnifiedLLMGuardrails().async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth( + api_key="test", request_route=request_route + ), + response=stream(), + request_data={ + "guardrail_to_apply": RecordingGuardrail(), + "metadata": {"guardrails": ["recording-guardrail"]}, + }, + ) + ] + finally: + unified_module.verbose_proxy_logger.removeHandler(caplog.handler) + + return chunks, [ + record.getMessage() + for record in caplog.records + if record.levelno >= logging.WARNING + ] + + @pytest.mark.asyncio + async def test_warns_when_the_call_type_has_no_translation_handler( + self, caplog, monkeypatch + ): + chunks, warnings = await self._drive( + caplog, + monkeypatch, + request_route="/chat/completions", + mappings={CallTypes.aembedding: _NoopTranslation}, + response_chunks=[ + ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(content=content))] + ) + for content in ("a", "b", "c") + ], + ) + + assert len(chunks) == 3 + assert any( + "no guardrail translation handler" in message + and "recording-guardrail" in message + and "/chat/completions" in message + for message in warnings + ), warnings + + @pytest.mark.asyncio + async def test_warns_when_the_call_type_cannot_be_resolved(self, caplog, monkeypatch): + chunks, warnings = await self._drive( + caplog, + monkeypatch, + request_route="/v1/not-a-mapped-route", + mappings=load_guardrail_translation_mappings(), + response_chunks=[{"event": "delta", "text": content} for content in ("a", "b", "c")], + ) + + assert len(chunks) == 3 + assert any( + "call type could not be resolved" in message + and "recording-guardrail" in message + for message in warnings + ), warnings From 1a8d1272630cc7f24975930482f04b8bc480d270 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 20:56:04 -0700 Subject: [PATCH 2/6] fix(guardrails): keep a failing translation package from breaking every guarded request Discovery caught only ImportError on the optional MCP package, so any other failure at import time propagated out of load_guardrail_translation_mappings and took down every request with a guardrail on it. Import failures are classified instead: a package that fails for any reason is reported as unavailable and retried on the next lookup, and only an mcp this install does not ship stays cached and quiet. The pre-call and during-call hooks warned nothing when they skipped a scan, and the during-call hook raised ValueError on a call type outside the enum. Both now warn with the guardrail and the route they left unscanned, and the recovery line moved to WARNING so an install running at WARNING or above sees the outage end --- litellm/llms/__init__.py | 76 ++++++++++----- .../unified_guardrail/unified_guardrail.py | 40 +++++--- .../test_guardrail_translation_discovery.py | 62 +++++++++++++ .../test_unified_guardrail.py | 93 +++++++++++++++++++ 4 files changed, 239 insertions(+), 32 deletions(-) diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index 23aa2a94a97..90a07222d35 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -93,7 +93,7 @@ class GuardrailTranslationDiscovery: """ The outcome of one scan for guardrail translation handlers. - unavailable maps each bundled package that failed to import to the reason, which is what tells a complete + unavailable maps each package that failed to import to the reason, which is what tells a complete result apart from one that is missing handlers and therefore has to be retried. """ @@ -110,12 +110,30 @@ def _bundled_guardrail_translation_modules() -> Iterator[str]: yield "litellm." + os.path.relpath(root, os.path.dirname(llms_dir)).replace(os.sep, ".") -def _import_guardrail_translations(module_path: str) -> Mapping[CallTypes, type["BaseTranslation"]] | str: - """Import one guardrail_translation package, returning the reason as a string when that fails.""" +@dataclass(frozen=True, slots=True) +class _UnavailablePackage: + """ + Why one guardrail_translation package could not be imported. + + missing_dependency is set when the package asked for a module outside litellm that this install does not + have, which is the one failure that says the package is absent rather than momentarily unimportable. + """ + + reason: str + missing_dependency: bool + + +def _import_guardrail_translations( + module_path: str, +) -> Mapping[CallTypes, type["BaseTranslation"]] | _UnavailablePackage: + """Import one guardrail_translation package, reporting why that failed instead of raising.""" try: module: Final = importlib.import_module(module_path) except Exception as e: # noqa: BLE001 # a package failing at import time for any reason is unavailable, not fatal - return f"{type(e).__name__}: {e}" + return _UnavailablePackage( + reason=f"{type(e).__name__}: {e}", + missing_dependency=isinstance(e, ModuleNotFoundError) and not (e.name or "").startswith("litellm"), + ) mappings: Final = getattr(module, "guardrail_translation_mappings", None) if not isinstance(mappings, dict): return _NO_MAPPINGS @@ -123,29 +141,47 @@ def _import_guardrail_translations(module_path: str) -> Mapping[CallTypes, type[ return declared -def _optional_mcp_guardrail_translation_mappings() -> Mapping[CallTypes, type["BaseTranslation"]]: - """MCP call types live outside litellm/llms and are absent from installs without the MCP server.""" - try: - from litellm.proxy._experimental.mcp_server.guardrail_translation import ( - guardrail_translation_mappings as mcp_guardrail_translation_mappings, - ) - except ImportError: - verbose_logger.debug("%s not available; skipping", _MCP_GUARDRAIL_TRANSLATION_MODULE) +def _guardrail_translations_from( + module_path: str, +) -> Mapping[CallTypes, type["BaseTranslation"]] | _UnavailablePackage: + """ + Import one package's handlers, tolerating an install that does not ship the optional MCP server. + + litellm ships every package under llms, so a failure there is a gap to retry rather than a fact about the + install. The MCP package instead arrives with the proxy extra, and a dependency it cannot import means this + install serves no MCP endpoints for a guardrail to scan, so there is nothing to retry or report. + """ + result: Final = _import_guardrail_translations(module_path) + if ( + module_path == _MCP_GUARDRAIL_TRANSLATION_MODULE + and isinstance(result, _UnavailablePackage) + and result.missing_dependency + ): + verbose_logger.debug("%s is not installed: %s", module_path, result.reason) return _NO_MAPPINGS - return mcp_guardrail_translation_mappings + return result + + +def _guardrail_translation_modules() -> Iterator[str]: + """Yield every module that can declare guardrail translation handlers, the optional MCP one first.""" + yield _MCP_GUARDRAIL_TRANSLATION_MODULE + yield from _bundled_guardrail_translation_modules() def _discover( module_paths: Iterable[str], already_found: Mapping[CallTypes, type["BaseTranslation"]] ) -> GuardrailTranslationDiscovery: - imported: Final = tuple((module_path, _import_guardrail_translations(module_path)) for module_path in module_paths) - found: Final = (already_found, *(result for _, result in imported if not isinstance(result, str))) + imported: Final = tuple((module_path, _guardrail_translations_from(module_path)) for module_path in module_paths) + found: Final = ( + already_found, + *(result for _, result in imported if not isinstance(result, _UnavailablePackage)), + ) return GuardrailTranslationDiscovery( mappings=MappingProxyType( {call_type: handler for mappings in found for call_type, handler in mappings.items()} ), unavailable=MappingProxyType( - {module_path: result for module_path, result in imported if isinstance(result, str)} + {module_path: result.reason for module_path, result in imported if isinstance(result, _UnavailablePackage)} ), ) @@ -155,11 +191,9 @@ def discover_guardrail_translations() -> GuardrailTranslationDiscovery: Scan the llms tree, plus the optional MCP package, for guardrail translation handlers. Returns: - GuardrailTranslationDiscovery: the handlers found, and the bundled packages that failed to import + GuardrailTranslationDiscovery: the handlers found, and the packages that failed to import """ - return _discover( - _bundled_guardrail_translation_modules(), already_found=_optional_mcp_guardrail_translation_mappings() - ) + return _discover(_guardrail_translation_modules(), already_found=_NO_MAPPINGS) def discover_guardrail_translation_mappings() -> Mapping[CallTypes, type["BaseTranslation"]]: @@ -188,7 +222,7 @@ def _announce_discovery(previous: GuardrailTranslationDiscovery | None, current: ) if not recovered: return - verbose_logger.info("Guardrail translation handlers from %s are available again.", ", ".join(recovered)) + verbose_logger.warning("Guardrail translation handlers from %s are available again.", ", ".join(recovered)) guardrail_translation_discovery: GuardrailTranslationDiscovery | None = None diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index 24a29eeb378..f17674f3e3a 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -133,6 +133,14 @@ def _ensure_litellm_metadata(data: dict, user_api_key_dict: UserAPIKeyAuth) -> N data["litellm_metadata"] = user_metadata +def _resolved_call_type(call_type: str | None) -> CallTypes | None: + """Return the CallTypes member a route's call type names, or None when the enum has no member for it.""" + try: + return CallTypes(call_type) + except ValueError: + return None + + def _warn_left_unscanned( guardrail_to_apply: "CustomGuardrail", user_api_key_dict: UserAPIKeyAuth, @@ -141,8 +149,8 @@ def _warn_left_unscanned( ) -> None: if call_type is None: verbose_proxy_logger.warning( - "Guardrail '%s' selected for route '%s' but its call type could not be resolved; %s. " - "Add the route to API_ROUTE_TO_CALL_TYPES.", + "Guardrail '%s' selected for route '%s' but its call type could not be resolved, so no guardrail " + "can run on that route; %s.", guardrail_to_apply.guardrail_name, user_api_key_dict.request_route, consequence, @@ -207,14 +215,17 @@ class UnifiedLLMGuardrails(CustomLogger): return data mappings: Final = load_guardrail_translation_mappings() + resolved_call_type: Final = _resolved_call_type(call_type) + if resolved_call_type is None or resolved_call_type not in mappings: + _warn_left_unscanned( + guardrail_to_apply=guardrail_to_apply, + user_api_key_dict=user_api_key_dict, + call_type=call_type, + consequence="skipping pre-call scanning", + ) + return data - try: - if CallTypes(call_type) not in mappings: - return data - except ValueError: - return data # handle unmapped call types - - endpoint_translation: Final = _as_endpoint_translation(mappings[CallTypes(call_type)]()) + endpoint_translation: Final = _as_endpoint_translation(mappings[resolved_call_type]()) _ensure_litellm_metadata(data, user_api_key_dict) @@ -257,10 +268,17 @@ class UnifiedLLMGuardrails(CustomLogger): return data mappings: Final = load_guardrail_translation_mappings() - if call_type is not None and CallTypes(call_type) not in mappings: + resolved_call_type: Final = _resolved_call_type(call_type) + if resolved_call_type is None or resolved_call_type not in mappings: + _warn_left_unscanned( + guardrail_to_apply=guardrail_to_apply, + user_api_key_dict=user_api_key_dict, + call_type=call_type, + consequence="skipping during-call scanning", + ) return data - endpoint_translation: Final = _as_endpoint_translation(mappings[CallTypes(call_type)]()) + endpoint_translation: Final = _as_endpoint_translation(mappings[resolved_call_type]()) _ensure_litellm_metadata(data, user_api_key_dict) diff --git a/tests/test_litellm/llms/test_guardrail_translation_discovery.py b/tests/test_litellm/llms/test_guardrail_translation_discovery.py index 4d10ff5f4aa..0507a645ab3 100644 --- a/tests/test_litellm/llms/test_guardrail_translation_discovery.py +++ b/tests/test_litellm/llms/test_guardrail_translation_discovery.py @@ -1,7 +1,10 @@ +import importlib.abc +import importlib.util import logging import sys from collections.abc import Iterator from contextlib import contextmanager +from types import ModuleType import pytest @@ -10,6 +13,34 @@ from litellm._logging import verbose_logger from litellm.types.utils import CallTypes OPENAI_CHAT_TRANSLATION_MODULE = "litellm.llms.openai.chat.guardrail_translation" +MCP_TRANSLATION_MODULE = "litellm.proxy._experimental.mcp_server.guardrail_translation" + + +class RaisingLoader(importlib.abc.MetaPathFinder, importlib.abc.Loader): + """Serves one module path, and raises the given error when Python executes it.""" + + def __init__(self, module_path: str, error: BaseException) -> None: + self.module_path = module_path + self.error = error + + def find_spec(self, fullname: str, path=None, target=None): + if fullname != self.module_path: + return None + return importlib.util.spec_from_loader(fullname, self) + + def create_module(self, spec) -> ModuleType | None: + return None + + def exec_module(self, module: ModuleType) -> None: + raise self.error + + +@contextmanager +def raising_on_import(module_path: str, error: BaseException) -> Iterator[None]: + with pytest.MonkeyPatch.context() as mp: + mp.delitem(sys.modules, module_path, raising=False) + mp.setattr(sys, "meta_path", [RaisingLoader(module_path, error), *sys.meta_path]) + yield @contextmanager @@ -93,6 +124,7 @@ def test_a_package_that_keeps_failing_is_reported_once_and_its_recovery_announce recoveries = [record for record in caplog.records if "available again" in record.getMessage()] assert len(recoveries) == 1 assert OPENAI_CHAT_TRANSLATION_MODULE in recoveries[0].getMessage() + assert recoveries[0].levelno >= logging.WARNING assert not [record for record in caplog.records if record.levelno == logging.ERROR] @@ -102,3 +134,33 @@ def test_lookup_recovers_after_a_failed_discovery(): llms_package.get_guardrail_translation_mapping(CallTypes.acompletion) assert llms_package.get_guardrail_translation_mapping(CallTypes.acompletion) is not None + + +def test_an_mcp_package_that_fails_to_import_is_reported_and_retried(): + with raising_on_import(MCP_TRANSLATION_MODULE, AttributeError("module 'mcp.types' has no attribute 'ToolCall'")): + partial = llms_package.load_guardrail_translation_mappings() + + assert CallTypes.call_mcp_tool not in partial + assert CallTypes.acompletion in partial + assert tuple(llms_package.guardrail_translation_discovery.unavailable) == (MCP_TRANSLATION_MODULE,) + assert "AttributeError" in llms_package.guardrail_translation_discovery.unavailable[MCP_TRANSLATION_MODULE] + + recovered = llms_package.load_guardrail_translation_mappings() + + assert CallTypes.call_mcp_tool in recovered + assert CallTypes.acompletion in recovered + assert not llms_package.guardrail_translation_discovery.unavailable + + +def test_an_install_without_the_mcp_server_is_discovered_once_and_quietly(caplog): + absent = ModuleNotFoundError("No module named 'mcp'", name="mcp") + + with capturing(caplog, logging.DEBUG), raising_on_import(MCP_TRANSLATION_MODULE, absent): + first = llms_package.load_guardrail_translation_mappings() + second = llms_package.load_guardrail_translation_mappings() + + assert first is second + assert CallTypes.call_mcp_tool not in first + assert CallTypes.acompletion in first + assert not llms_package.guardrail_translation_discovery.unavailable + assert not [record for record in caplog.records if record.levelno >= logging.WARNING] diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 91dff7b8346..f09ecfa392d 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -1,6 +1,7 @@ """Tests for unified guardrail.""" import logging +from contextlib import contextmanager import pytest @@ -2368,3 +2369,95 @@ class TestUnscannedStreamIsAnnounced: and "recording-guardrail" in message for message in warnings ), warnings + + +class TestUnscannedRequestIsAnnounced: + """A request hook that cannot scan must say so instead of passing the request through in silence.""" + + @staticmethod + def _request(guardrail): + return { + "guardrail_to_apply": guardrail, + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hello world"}], + } + + @staticmethod + @contextmanager + def _capturing(caplog): + caplog.set_level(logging.WARNING, logger="LiteLLM Proxy") + unified_module.verbose_proxy_logger.addHandler(caplog.handler) + try: + yield + finally: + unified_module.verbose_proxy_logger.removeHandler(caplog.handler) + + @staticmethod + def _warnings(caplog): + return [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING] + + @pytest.mark.asyncio + async def test_pre_call_warns_when_the_call_type_has_no_translation_handler(self, caplog, monkeypatch): + _patch_translation_mappings(monkeypatch, {CallTypes.aembedding: _NoopTranslation}) + guardrail = RecordingGuardrail() + data = self._request(guardrail) + + with self._capturing(caplog): + returned = await UnifiedLLMGuardrails().async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", request_route="/v1/moderations"), + cache=DualCache(), + data=data, + call_type=CallTypes.acompletion.value, + ) + + assert guardrail.apply_calls == [] + assert returned["messages"] == [{"role": "user", "content": "hello world"}] + assert any( + "no guardrail translation handler" in message + and "skipping pre-call scanning" in message + and "recording-guardrail" in message + and "/v1/moderations" in message + and "acompletion" in message + for message in self._warnings(caplog) + ), self._warnings(caplog) + + @pytest.mark.asyncio + async def test_during_call_warns_when_the_call_type_has_no_translation_handler(self, caplog, monkeypatch): + _patch_translation_mappings(monkeypatch, {CallTypes.aembedding: _NoopTranslation}) + guardrail = RecordingGuardrail() + data = self._request(guardrail) + + with self._capturing(caplog): + returned = await UnifiedLLMGuardrails().async_moderation_hook( + data=data, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", request_route="/v1/moderations"), + call_type=CallTypes.acompletion.value, + ) + + assert guardrail.apply_calls == [] + assert returned["messages"] == [{"role": "user", "content": "hello world"}] + assert any( + "skipping during-call scanning" in message and "recording-guardrail" in message + for message in self._warnings(caplog) + ), self._warnings(caplog) + + @pytest.mark.asyncio + async def test_pre_call_warns_instead_of_raising_on_a_call_type_outside_the_enum(self, caplog, monkeypatch): + _patch_translation_mappings(monkeypatch, load_guardrail_translation_mappings()) + guardrail = RecordingGuardrail() + data = self._request(guardrail) + + with self._capturing(caplog): + returned = await UnifiedLLMGuardrails().async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", request_route="/v1/moderations"), + cache=DualCache(), + data=data, + call_type="moderation", + ) + + assert guardrail.apply_calls == [] + assert returned["messages"] == [{"role": "user", "content": "hello world"}] + assert any( + "moderation" in message and "skipping pre-call scanning" in message + for message in self._warnings(caplog) + ), self._warnings(caplog) From 74afe43b920c2b0bfd046f8e5e02392cdeec489c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 21:12:03 -0700 Subject: [PATCH 3/6] fix(guardrails): name the call type by value in the post-call skip warning The post-call hook passed the CallTypes member into the warning, so it printed call type 'CallTypes.aresponses' while the pre-call, during-call, and streaming paths printed 'aresponses'. Pass the value like the other three do. Also fixes the pre-call unresolvable-call-type test, which passed a call type that is in the enum, and adds the during-call twin. --- .../unified_guardrail/unified_guardrail.py | 2 +- .../test_unified_guardrail.py | 44 ++++++++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index f17674f3e3a..07573158950 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -323,7 +323,7 @@ class UnifiedLLMGuardrails(CustomLogger): if user_api_key_dict.request_route is not None: call_types: Final = get_call_types_for_route(user_api_key_dict.request_route) if call_types is not None and len(call_types) > 0: - call_type = call_types[0] + call_type = call_types[0].value if call_type is None: call_type = _infer_call_type(call_type=None, completion_response=response) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index f09ecfa392d..9f4a0d8683f 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -2452,12 +2452,52 @@ class TestUnscannedRequestIsAnnounced: user_api_key_dict=UserAPIKeyAuth(api_key="test-key", request_route="/v1/moderations"), cache=DualCache(), data=data, - call_type="moderation", + call_type="not_a_call_type", ) assert guardrail.apply_calls == [] assert returned["messages"] == [{"role": "user", "content": "hello world"}] assert any( - "moderation" in message and "skipping pre-call scanning" in message + "call type 'not_a_call_type' has no guardrail translation handler" in message + and "skipping pre-call scanning" in message for message in self._warnings(caplog) ), self._warnings(caplog) + + @pytest.mark.asyncio + async def test_during_call_warns_instead_of_raising_on_a_call_type_outside_the_enum(self, caplog, monkeypatch): + _patch_translation_mappings(monkeypatch, load_guardrail_translation_mappings()) + guardrail = RecordingGuardrail() + data = self._request(guardrail) + + with self._capturing(caplog): + returned = await UnifiedLLMGuardrails().async_moderation_hook( + data=data, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", request_route="/v1/moderations"), + call_type="not_a_call_type", + ) + + assert guardrail.apply_calls == [] + assert returned["messages"] == [{"role": "user", "content": "hello world"}] + assert any( + "call type 'not_a_call_type' has no guardrail translation handler" in message + and "skipping during-call scanning" in message + for message in self._warnings(caplog) + ), self._warnings(caplog) + + @pytest.mark.asyncio + async def test_post_call_names_the_call_type_the_route_maps_to(self, caplog, monkeypatch): + _patch_translation_mappings(monkeypatch, {CallTypes.aembedding: _NoopTranslation}) + guardrail = RecordingGuardrail() + + with self._capturing(caplog): + await UnifiedLLMGuardrails().async_post_call_success_hook( + data={"guardrail_to_apply": guardrail, "model": "gpt-4o"}, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", request_route="/v1/responses"), + response=litellm.ModelResponse(), + ) + + assert guardrail.apply_calls == [] + unscanned = [message for message in self._warnings(caplog) if "skipping post-call scanning" in message] + assert unscanned, self._warnings(caplog) + assert "call type 'aresponses'" in unscanned[0], unscanned[0] + assert "CallTypes." not in unscanned[0], unscanned[0] From 109de6f0e315987e43e572f800cebdc583a0d5b3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 22:41:20 -0700 Subject: [PATCH 4/6] fix(guardrails): retry a broken dependency and warn once per unscanned route A ModuleNotFoundError from a dependency the install actually has is a broken install, not a lean one, so it now lands in the unavailable set and is retried instead of dropping its handlers for the life of the process. The unscanned warning is cached on the guardrail, route, call type and reason, since most proxy routes have no translation handler and never will, and both remaining skip sites now resolve the call type instead of raising on a route string the enum never had. --- litellm/llms/__init__.py | 21 +++++-- .../unified_guardrail/unified_guardrail.py | 62 ++++++++++++++----- .../test_guardrail_translation_discovery.py | 22 ++++++- .../test_unified_guardrail.py | 44 ++++++++++++- 4 files changed, 124 insertions(+), 25 deletions(-) diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index 90a07222d35..a4a43630ba0 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -1,4 +1,5 @@ import importlib +import importlib.util import os from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass @@ -115,14 +116,23 @@ class _UnavailablePackage: """ Why one guardrail_translation package could not be imported. - missing_dependency is set when the package asked for a module outside litellm that this install does not - have, which is the one failure that says the package is absent rather than momentarily unimportable. + missing_dependency is set when the package asked for a module this install does not have at all, which is + the one failure that says the package is absent rather than momentarily unimportable. """ reason: str missing_dependency: bool +def _is_absent(module_name: str) -> bool: + """Whether this install has no module of that name, as opposed to one that is present but failed to import.""" + root: Final = module_name.partition(".")[0] + try: + return importlib.util.find_spec(root) is None + except (ImportError, ValueError): + return False + + def _import_guardrail_translations( module_path: str, ) -> Mapping[CallTypes, type["BaseTranslation"]] | _UnavailablePackage: @@ -132,7 +142,7 @@ def _import_guardrail_translations( except Exception as e: # noqa: BLE001 # a package failing at import time for any reason is unavailable, not fatal return _UnavailablePackage( reason=f"{type(e).__name__}: {e}", - missing_dependency=isinstance(e, ModuleNotFoundError) and not (e.name or "").startswith("litellm"), + missing_dependency=isinstance(e, ModuleNotFoundError) and _is_absent(e.name or ""), ) mappings: Final = getattr(module, "guardrail_translation_mappings", None) if not isinstance(mappings, dict): @@ -148,8 +158,9 @@ def _guardrail_translations_from( Import one package's handlers, tolerating an install that does not ship the optional MCP server. litellm ships every package under llms, so a failure there is a gap to retry rather than a fact about the - install. The MCP package instead arrives with the proxy extra, and a dependency it cannot import means this - install serves no MCP endpoints for a guardrail to scan, so there is nothing to retry or report. + install. The MCP package instead arrives with the proxy extra, and a dependency this install does not have + at all means it serves no MCP endpoints for a guardrail to scan, so there is nothing to retry or report. A + dependency that is installed and still fails to import is a broken install, which is reported and retried. """ result: Final = _import_guardrail_translations(module_path) if ( diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index 07573158950..64d5af0f772 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -9,6 +9,7 @@ Unified Guardrail, leveraging LiteLLM's /applyGuardrail endpoint import copy import json from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Callable, Mapping, Sequence +from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, Protocol from fastapi import HTTPException @@ -133,6 +134,9 @@ def _ensure_litellm_metadata(data: dict, user_api_key_dict: UserAPIKeyAuth) -> N data["litellm_metadata"] = user_metadata +_UNSCANNED_WARNING_KEYS: Final = 256 + + def _resolved_call_type(call_type: str | None) -> CallTypes | None: """Return the CallTypes member a route's call type names, or None when the enum has no member for it.""" try: @@ -141,27 +145,51 @@ def _resolved_call_type(call_type: str | None) -> CallTypes | None: return None +@lru_cache(maxsize=_UNSCANNED_WARNING_KEYS) +def _warn_left_unscanned_once( + guardrail_name: str | None, + request_route: str | None, + call_type: str | None, + consequence: str, +) -> None: + if _resolved_call_type(call_type) is not None: + verbose_proxy_logger.warning( + "Guardrail '%s' selected for route '%s' but call type '%s' has no guardrail translation handler; %s.", + guardrail_name, + request_route, + call_type, + consequence, + ) + return + unscannable: Final = ( + f"call type '{call_type}' is not one litellm can scan" if call_type else "its call type could not be resolved" + ) + verbose_proxy_logger.warning( + "Guardrail '%s' selected for route '%s' but %s, so no guardrail can run on that route; %s.", + guardrail_name, + request_route, + unscannable, + consequence, + ) + + def _warn_left_unscanned( guardrail_to_apply: "CustomGuardrail", user_api_key_dict: UserAPIKeyAuth, call_type: str | None, consequence: str, ) -> None: - if call_type is None: - verbose_proxy_logger.warning( - "Guardrail '%s' selected for route '%s' but its call type could not be resolved, so no guardrail " - "can run on that route; %s.", - guardrail_to_apply.guardrail_name, - user_api_key_dict.request_route, - consequence, - ) - return - verbose_proxy_logger.warning( - "Guardrail '%s' selected for route '%s' but call type '%s' has no guardrail translation handler; %s.", - guardrail_to_apply.guardrail_name, - user_api_key_dict.request_route, - call_type, - consequence, + """ + Say that a selected guardrail could not scan this request, once per route and reason. + + Most proxy routes have no translation handler and never will, so a line per request would bury the + outage it is meant to surface, and repeating it adds nothing an operator can act on twice. + """ + _warn_left_unscanned_once( + guardrail_name=guardrail_to_apply.guardrail_name, + request_route=user_api_key_dict.request_route, + call_type=call_type, + consequence=consequence, ) @@ -340,7 +368,7 @@ class UnifiedLLMGuardrails(CustomLogger): call_type = logging_call_type mappings: Final = load_guardrail_translation_mappings() - if call_type is None or CallTypes(call_type) not in mappings: + if _resolved_call_type(call_type) not in mappings: _warn_left_unscanned( guardrail_to_apply=guardrail_to_apply, user_api_key_dict=user_api_key_dict, @@ -1048,7 +1076,7 @@ class UnifiedLLMGuardrails(CustomLogger): call_type = _infer_call_type(call_type=None, completion_response=item) # If call type not supported, just pass through all chunks - if call_type is None or CallTypes(call_type) not in mappings: + if _resolved_call_type(call_type) not in mappings: _warn_left_unscanned( guardrail_to_apply=guardrail_to_apply, user_api_key_dict=user_api_key_dict, diff --git a/tests/test_litellm/llms/test_guardrail_translation_discovery.py b/tests/test_litellm/llms/test_guardrail_translation_discovery.py index 0507a645ab3..3b4446e4da5 100644 --- a/tests/test_litellm/llms/test_guardrail_translation_discovery.py +++ b/tests/test_litellm/llms/test_guardrail_translation_discovery.py @@ -155,7 +155,7 @@ def test_an_mcp_package_that_fails_to_import_is_reported_and_retried(): def test_an_install_without_the_mcp_server_is_discovered_once_and_quietly(caplog): absent = ModuleNotFoundError("No module named 'mcp'", name="mcp") - with capturing(caplog, logging.DEBUG), raising_on_import(MCP_TRANSLATION_MODULE, absent): + with capturing(caplog, logging.DEBUG), unimportable("mcp"), raising_on_import(MCP_TRANSLATION_MODULE, absent): first = llms_package.load_guardrail_translation_mappings() second = llms_package.load_guardrail_translation_mappings() @@ -164,3 +164,23 @@ def test_an_install_without_the_mcp_server_is_discovered_once_and_quietly(caplog assert CallTypes.acompletion in first assert not llms_package.guardrail_translation_discovery.unavailable assert not [record for record in caplog.records if record.levelno >= logging.WARNING] + + +def test_a_broken_mcp_dependency_is_reported_and_retried(caplog): + """An mcp the install has but cannot import is a broken install, not a lean one, so it must be loud.""" + broken = ModuleNotFoundError("No module named 'mcp.types'", name="mcp.types") + + with capturing(caplog, logging.DEBUG), raising_on_import(MCP_TRANSLATION_MODULE, broken): + partial = llms_package.load_guardrail_translation_mappings() + + assert CallTypes.call_mcp_tool not in partial + assert CallTypes.acompletion in partial + assert tuple(llms_package.guardrail_translation_discovery.unavailable) == (MCP_TRANSLATION_MODULE,) + errors = [record for record in caplog.records if record.levelno >= logging.ERROR] + assert len(errors) == 1, [record.getMessage() for record in caplog.records] + assert "mcp.types" in errors[0].getMessage() + + recovered = llms_package.load_guardrail_translation_mappings() + + assert CallTypes.call_mcp_tool in recovered + assert not llms_package.guardrail_translation_discovery.unavailable diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 9f4a0d8683f..65fab6e1270 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -85,6 +85,14 @@ def _patch_translation_mappings(monkeypatch, mappings): monkeypatch.setattr(unified_module, "load_guardrail_translation_mappings", lambda: mappings) +@pytest.fixture(autouse=True) +def _forget_unscanned_warnings(): + """The unscanned warning fires once per route and reason, so each test starts with nothing remembered.""" + unified_module._warn_left_unscanned_once.cache_clear() + yield + unified_module._warn_left_unscanned_once.cache_clear() + + @pytest.fixture(autouse=True) def _inject_mcp_handler_mapping(monkeypatch): """Inject MCP handler mapping so the unified guardrail can run inside tests.""" @@ -2396,6 +2404,12 @@ class TestUnscannedRequestIsAnnounced: def _warnings(caplog): return [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING] + @staticmethod + def _distinct_warnings(caplog): + """caplog holds every record twice here, once through the handler above and once through propagation.""" + by_record = {id(record): record for record in caplog.records if record.levelno >= logging.WARNING} + return [record.getMessage() for record in by_record.values()] + @pytest.mark.asyncio async def test_pre_call_warns_when_the_call_type_has_no_translation_handler(self, caplog, monkeypatch): _patch_translation_mappings(monkeypatch, {CallTypes.aembedding: _NoopTranslation}) @@ -2458,7 +2472,7 @@ class TestUnscannedRequestIsAnnounced: assert guardrail.apply_calls == [] assert returned["messages"] == [{"role": "user", "content": "hello world"}] assert any( - "call type 'not_a_call_type' has no guardrail translation handler" in message + "call type 'not_a_call_type' is not one litellm can scan" in message and "skipping pre-call scanning" in message for message in self._warnings(caplog) ), self._warnings(caplog) @@ -2479,11 +2493,37 @@ class TestUnscannedRequestIsAnnounced: assert guardrail.apply_calls == [] assert returned["messages"] == [{"role": "user", "content": "hello world"}] assert any( - "call type 'not_a_call_type' has no guardrail translation handler" in message + "call type 'not_a_call_type' is not one litellm can scan" in message and "skipping during-call scanning" in message for message in self._warnings(caplog) ), self._warnings(caplog) + @pytest.mark.asyncio + async def test_a_route_with_no_handler_warns_once_instead_of_once_per_request(self, caplog, monkeypatch): + _patch_translation_mappings(monkeypatch, {CallTypes.aembedding: _NoopTranslation}) + guardrail = RecordingGuardrail() + + with self._capturing(caplog): + for _ in range(5): + await UnifiedLLMGuardrails().async_moderation_hook( + data=self._request(guardrail), + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", request_route="/v1/responses/resp_123"), + call_type="aget_responses", + ) + await UnifiedLLMGuardrails().async_moderation_hook( + data=self._request(guardrail), + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", request_route="/v1/moderations"), + call_type="aget_responses", + ) + + unscanned = [ + message for message in self._distinct_warnings(caplog) if "skipping during-call scanning" in message + ] + assert len(unscanned) == 2, unscanned + assert "/v1/responses/resp_123" in unscanned[0] + assert "aget_responses" in unscanned[0] + assert "/v1/moderations" in unscanned[1] + @pytest.mark.asyncio async def test_post_call_names_the_call_type_the_route_maps_to(self, caplog, monkeypatch): _patch_translation_mappings(monkeypatch, {CallTypes.aembedding: _NoopTranslation}) From c196aa6f4d26f38aa345111be0fdf8dc2536d0ba Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 23:31:58 -0700 Subject: [PATCH 5/6] fix(guardrails): name the remedy in the unscanned warning and resolve call types once The warning says how to close the gap again: add a handler for the call type, map the route to a CallTypes member, or add the route to API_ROUTE_TO_CALL_TYPES. Its cache now holds every guardrail, route, call type and consequence a normal install can produce rather than evicting after 256 keys. The A2A check and the end-of-stream check resolve the call type instead of handing the enum a route string it may never have had, discovery yields the MCP package last again so it keeps winning handler collisions, and the fixture that forgets warnings between tests moved to a conftest so the file that ran first cannot decide what another file sees. --- litellm/llms/__init__.py | 4 +-- .../unified_guardrail/unified_guardrail.py | 29 ++++++++++++------- .../test_guardrail_translation_discovery.py | 19 ++++++++++++ .../test_litellm/proxy/guardrails/conftest.py | 13 +++++++++ .../test_unified_guardrail.py | 12 +++----- 5 files changed, 57 insertions(+), 20 deletions(-) create mode 100644 tests/test_litellm/proxy/guardrails/conftest.py diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index a4a43630ba0..bc12ccd42d3 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -174,9 +174,9 @@ def _guardrail_translations_from( def _guardrail_translation_modules() -> Iterator[str]: - """Yield every module that can declare guardrail translation handlers, the optional MCP one first.""" - yield _MCP_GUARDRAIL_TRANSLATION_MODULE + """Yield every module that can declare guardrail translation handlers, the optional MCP one last.""" yield from _bundled_guardrail_translation_modules() + yield _MCP_GUARDRAIL_TRANSLATION_MODULE def _discover( diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index 64d5af0f772..ffb788be497 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -134,7 +134,7 @@ def _ensure_litellm_metadata(data: dict, user_api_key_dict: UserAPIKeyAuth) -> N data["litellm_metadata"] = user_metadata -_UNSCANNED_WARNING_KEYS: Final = 256 +_UNSCANNED_WARNING_KEYS: Final = 4096 def _resolved_call_type(call_type: str | None) -> CallTypes | None: @@ -154,22 +154,29 @@ def _warn_left_unscanned_once( ) -> None: if _resolved_call_type(call_type) is not None: verbose_proxy_logger.warning( - "Guardrail '%s' selected for route '%s' but call type '%s' has no guardrail translation handler; %s.", + "Guardrail '%s' selected for route '%s' but call type '%s' has no guardrail translation handler; %s. " + "Add a guardrail translation handler for that call type.", guardrail_name, request_route, call_type, consequence, ) return - unscannable: Final = ( - f"call type '{call_type}' is not one litellm can scan" if call_type else "its call type could not be resolved" + unscannable, remedy = ( + ( + f"call type '{call_type}' is not one litellm can scan", + "Map the route to a CallTypes member in API_ROUTE_TO_CALL_TYPES", + ) + if call_type + else ("its call type could not be resolved", "Add the route to API_ROUTE_TO_CALL_TYPES") ) verbose_proxy_logger.warning( - "Guardrail '%s' selected for route '%s' but %s, so no guardrail can run on that route; %s.", + "Guardrail '%s' selected for route '%s' but %s, so no guardrail can run on that route; %s. %s.", guardrail_name, request_route, unscannable, consequence, + remedy, ) @@ -368,7 +375,8 @@ class UnifiedLLMGuardrails(CustomLogger): call_type = logging_call_type mappings: Final = load_guardrail_translation_mappings() - if _resolved_call_type(call_type) not in mappings: + resolved_call_type: Final = _resolved_call_type(call_type) + if resolved_call_type is None or resolved_call_type not in mappings: _warn_left_unscanned( guardrail_to_apply=guardrail_to_apply, user_api_key_dict=user_api_key_dict, @@ -377,7 +385,7 @@ class UnifiedLLMGuardrails(CustomLogger): ) return response - endpoint_translation: Final = _as_endpoint_translation(mappings[CallTypes(call_type)]()) + endpoint_translation: Final = _as_endpoint_translation(mappings[resolved_call_type]()) try: response = await endpoint_translation.process_output_response( @@ -479,7 +487,7 @@ class UnifiedLLMGuardrails(CustomLogger): has no in-stream error frame) the exception is re-raised so the proxy can report it with a real HTTP status. """ - if call_type is not None and CallTypes(call_type) in A2A_CALL_TYPES: + if _resolved_call_type(call_type) in A2A_CALL_TYPES: yield _a2a_jsonrpc_error_chunk(exc, _get_a2a_request_id(responses_so_far, request_data)) return if stream_started and endpoint_translation is not None: @@ -1188,14 +1196,15 @@ class UnifiedLLMGuardrails(CustomLogger): yield item # Stream has ended - do final processing with all collected chunks - if call_type is not None and CallTypes(call_type) in mappings: + final_call_type: Final = _resolved_call_type(call_type) + if final_call_type is not None and final_call_type in mappings: verbose_proxy_logger.debug( "Processing final streaming response with all %s chunks for guardrail %s", len(responses_so_far), guardrail_to_apply.guardrail_name, ) - endpoint_translation = mappings[CallTypes(call_type)]() + endpoint_translation = mappings[final_call_type]() # When buffering, snapshot the original chunks before moderation. # A shallow copy suffices: end-of-stream diff --git a/tests/test_litellm/llms/test_guardrail_translation_discovery.py b/tests/test_litellm/llms/test_guardrail_translation_discovery.py index 3b4446e4da5..1879b15f5fd 100644 --- a/tests/test_litellm/llms/test_guardrail_translation_discovery.py +++ b/tests/test_litellm/llms/test_guardrail_translation_discovery.py @@ -10,6 +10,9 @@ import pytest import litellm.llms as llms_package from litellm._logging import verbose_logger +from litellm.proxy._experimental.mcp_server.guardrail_translation.handler import ( + MCPGuardrailTranslationHandler, +) from litellm.types.utils import CallTypes OPENAI_CHAT_TRANSLATION_MODULE = "litellm.llms.openai.chat.guardrail_translation" @@ -184,3 +187,19 @@ def test_a_broken_mcp_dependency_is_reported_and_retried(caplog): assert CallTypes.call_mcp_tool in recovered assert not llms_package.guardrail_translation_discovery.unavailable + + +class _StandInMCPHandler: + """A bundled package's handler that claims the call type the MCP package owns.""" + + +def test_the_mcp_package_wins_a_handler_collision_with_a_bundled_package(monkeypatch): + """MCP handlers are the specialised ones, so a bundled package declaring the same call type must not shadow them.""" + colliding = ModuleType("litellm.llms.colliding_stub.guardrail_translation") + colliding.guardrail_translation_mappings = {CallTypes.call_mcp_tool: _StandInMCPHandler} + monkeypatch.setitem(sys.modules, colliding.__name__, colliding) + monkeypatch.setattr(llms_package, "_bundled_guardrail_translation_modules", lambda: iter((colliding.__name__,))) + + discovery = llms_package.discover_guardrail_translations() + + assert discovery.mappings[CallTypes.call_mcp_tool] is MCPGuardrailTranslationHandler diff --git a/tests/test_litellm/proxy/guardrails/conftest.py b/tests/test_litellm/proxy/guardrails/conftest.py new file mode 100644 index 00000000000..8a6341440f4 --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/conftest.py @@ -0,0 +1,13 @@ +import pytest + +from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail import ( + unified_guardrail as unified_module, +) + + +@pytest.fixture(autouse=True) +def _forget_unscanned_warnings(): + """The unscanned warning fires once per route and reason, so every guardrail test starts with nothing remembered.""" + unified_module._warn_left_unscanned_once.cache_clear() + yield + unified_module._warn_left_unscanned_once.cache_clear() diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 65fab6e1270..cc3437c46f3 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -85,14 +85,6 @@ def _patch_translation_mappings(monkeypatch, mappings): monkeypatch.setattr(unified_module, "load_guardrail_translation_mappings", lambda: mappings) -@pytest.fixture(autouse=True) -def _forget_unscanned_warnings(): - """The unscanned warning fires once per route and reason, so each test starts with nothing remembered.""" - unified_module._warn_left_unscanned_once.cache_clear() - yield - unified_module._warn_left_unscanned_once.cache_clear() - - @pytest.fixture(autouse=True) def _inject_mcp_handler_mapping(monkeypatch): """Inject MCP handler mapping so the unified guardrail can run inside tests.""" @@ -2356,6 +2348,7 @@ class TestUnscannedStreamIsAnnounced: assert len(chunks) == 3 assert any( "no guardrail translation handler" in message + and "Add a guardrail translation handler for that call type." in message and "recording-guardrail" in message and "/chat/completions" in message for message in warnings @@ -2374,6 +2367,7 @@ class TestUnscannedStreamIsAnnounced: assert len(chunks) == 3 assert any( "call type could not be resolved" in message + and "Add the route to API_ROUTE_TO_CALL_TYPES." in message and "recording-guardrail" in message for message in warnings ), warnings @@ -2473,6 +2467,7 @@ class TestUnscannedRequestIsAnnounced: assert returned["messages"] == [{"role": "user", "content": "hello world"}] assert any( "call type 'not_a_call_type' is not one litellm can scan" in message + and "Map the route to a CallTypes member in API_ROUTE_TO_CALL_TYPES." in message and "skipping pre-call scanning" in message for message in self._warnings(caplog) ), self._warnings(caplog) @@ -2494,6 +2489,7 @@ class TestUnscannedRequestIsAnnounced: assert returned["messages"] == [{"role": "user", "content": "hello world"}] assert any( "call type 'not_a_call_type' is not one litellm can scan" in message + and "Map the route to a CallTypes member in API_ROUTE_TO_CALL_TYPES." in message and "skipping during-call scanning" in message for message in self._warnings(caplog) ), self._warnings(caplog) From 409b402553d2d7e55eecd5b88d8bdeef4a1f33c7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 6 Sep 2026 00:10:48 -0700 Subject: [PATCH 6/6] test(guardrails): pin the discovery cache by behavior instead of object identity --- .../llms/test_guardrail_translation_discovery.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/llms/test_guardrail_translation_discovery.py b/tests/test_litellm/llms/test_guardrail_translation_discovery.py index 1879b15f5fd..cf2fd7e293d 100644 --- a/tests/test_litellm/llms/test_guardrail_translation_discovery.py +++ b/tests/test_litellm/llms/test_guardrail_translation_discovery.py @@ -102,11 +102,16 @@ def test_the_next_lookup_retries_a_package_that_failed_to_import(): assert not llms_package.guardrail_translation_discovery.unavailable -def test_a_complete_discovery_is_cached(): - first = llms_package.load_guardrail_translation_mappings() - second = llms_package.load_guardrail_translation_mappings() +def test_a_complete_discovery_is_not_scanned_again(): + healthy = llms_package.load_guardrail_translation_mappings() - assert first is second + assert CallTypes.acompletion in healthy + + with unimportable(OPENAI_CHAT_TRANSLATION_MODULE): + after_the_package_breaks = llms_package.load_guardrail_translation_mappings() + + assert CallTypes.acompletion in after_the_package_breaks + assert not llms_package.guardrail_translation_discovery.unavailable def test_a_package_that_keeps_failing_is_reported_once_and_its_recovery_announced(caplog):