From b4f90b5fabe7ea14ce6e9a6186752cccfa5d8597 Mon Sep 17 00:00:00 2001 From: Dev Shah <49101333+devshahofficial@users.noreply.github.com> Date: Tue, 14 Apr 2026 16:08:22 -0700 Subject: [PATCH] feat(guardrails): add Resemble AI Detect guardrail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a custom guardrail that scans audio, video, and image URLs referenced in LLM proxy requests for deepfake / synthetic content via the Resemble AI Detect API (https://app.resemble.ai/api/v2/detect). Hooks: async_pre_call_hook and async_moderation_hook. Extracts every media URL from request messages (multimodal content parts, regex over text) and from data["metadata"][metadata_key] (string or list), submits each to POST /detect, polls GET /detect/{uuid} asynchronously, and raises HTTPException(400) when Resemble labels the media as `fake` or when the aggregated score exceeds the configured threshold. Adds: - litellm/proxy/guardrails/guardrail_hooks/resemble/ __init__.py — initialize_guardrail + registry mappings resemble.py — ResembleGuardrail class - litellm/types/proxy/guardrails/guardrail_hooks/resemble.py ResembleGuardrailConfigModel pydantic schema - tests/test_litellm/proxy/guardrails/guardrail_hooks/test_resemble.py 32 unit tests covering init, extraction, evaluation, HTTP flow, multi-URL scanning, polling races, and hook integration - docs/my-website/docs/proxy/guardrails/resemble_detect.md quick start + config schema + examples - SupportedGuardrailIntegrations.RESEMBLE enum entry - ResembleGuardrailParamsConfigModel in litellm/types/guardrails.py - sidebar link in docs/my-website/sidebars.js --- .../docs/proxy/guardrails/resemble_detect.md | 187 ++++ docs/my-website/sidebars.js | 1 + .../guardrail_hooks/resemble/__init__.py | 50 ++ .../guardrail_hooks/resemble/resemble.py | 512 +++++++++++ litellm/types/guardrails.py | 70 ++ .../guardrails/guardrail_hooks/resemble.py | 90 ++ .../guardrail_hooks/test_resemble.py | 850 ++++++++++++++++++ uv.lock | 4 +- 8 files changed, 1762 insertions(+), 2 deletions(-) create mode 100644 docs/my-website/docs/proxy/guardrails/resemble_detect.md create mode 100644 litellm/proxy/guardrails/guardrail_hooks/resemble/__init__.py create mode 100644 litellm/proxy/guardrails/guardrail_hooks/resemble/resemble.py create mode 100644 litellm/types/proxy/guardrails/guardrail_hooks/resemble.py create mode 100644 tests/test_litellm/proxy/guardrails/guardrail_hooks/test_resemble.py diff --git a/docs/my-website/docs/proxy/guardrails/resemble_detect.md b/docs/my-website/docs/proxy/guardrails/resemble_detect.md new file mode 100644 index 00000000000..ebafbaa8e4a --- /dev/null +++ b/docs/my-website/docs/proxy/guardrails/resemble_detect.md @@ -0,0 +1,187 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Resemble AI Detect + +Use [Resemble AI Detect](https://www.resemble.ai/detect) to scan audio, video, and image URLs referenced in LLM requests for deepfake / synthetic media. The guardrail blocks the request when Resemble labels the media as `fake` or when the aggregated score exceeds the configured threshold. + +Resemble Detect works across three modalities: + +- **Audio** — detects cloned voices and TTS-generated speech (ElevenLabs, Resemble AI, OpenAI, PlayHT, etc.) +- **Image** — detects facial deepfakes and generative image manipulation +- **Video** — frame-level detection with a single aggregated verdict + +It runs asynchronously: LiteLLM submits the media URL, polls for the verdict, and either passes or blocks the LLM call. + +## Quick Start + +### 1. Get an API key + +Create an API token at [app.resemble.ai/account/api](https://app.resemble.ai/account/api). + +### 2. Define the guardrail in your `config.yaml` + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY + +guardrails: + - guardrail_name: "resemble-deepfake-detect" + litellm_params: + guardrail: resemble + mode: "pre_call" + api_key: os.environ/RESEMBLE_API_KEY + # Optional: override the API base + # api_base: https://app.resemble.ai/api/v2 + # Block media with aggregated_score >= threshold (default 0.5) + resemble_threshold: 0.5 + # Optional: force a modality (audio | video | image) + # resemble_media_type: audio + # Identify the TTS vendor that produced flagged audio + resemble_audio_source_tracing: true + # Do not persist media on Resemble after the scan + resemble_zero_retention_mode: true + # Block the request if Resemble is unreachable (default: fail open) + resemble_fail_closed: false +``` + +#### Supported values for `mode` + +- `pre_call` — runs **before** the LLM call. Blocks the request if the media is flagged. +- `during_call` — runs **in parallel** with the LLM call for lower latency. Still blocks on flagged media. + +### 3. Start LiteLLM + +```shell +litellm --config config.yaml --detailed_debug +``` + +### 4. Send a multimodal request + +The guardrail looks for media URLs in (in order): + +1. OpenAI-style multimodal content parts (`image_url`, `input_audio`) +2. Anthropic-style `source.url` parts (image, document) +3. Any `https://…` URL in message text that ends in a known audio/video/image extension +4. `metadata.mediaUrl` (key configurable via `resemble_metadata_key`) + + + + +```shell +curl -i http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Is this photo real?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/face.jpg"}} + ] + } + ], + "guardrails": ["resemble-deepfake-detect"] + }' +``` + + + + + +```shell +curl -i http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o", + "messages": [ + {"role": "user", "content": "Transcribe https://example.com/clip.mp3 please"} + ], + "guardrails": ["resemble-deepfake-detect"] + }' +``` + + + + + +```shell +curl -i http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o", + "messages": [ + {"role": "user", "content": "Analyze the uploaded clip"} + ], + "metadata": {"mediaUrl": "https://example.com/clip.wav"}, + "guardrails": ["resemble-deepfake-detect"] + }' +``` + + + + +### 5. Example blocked response + +```json +{ + "error": { + "message": { + "error": "Resemble Detect flagged media as synthetic", + "resemble": { + "uuid": "a1b2c3d4-5e6f-7890-abcd-ef0123456789", + "media_url": "https://example.com/clip.mp3", + "media_type": "audio", + "label": "fake", + "score": 0.95, + "threshold": 0.5, + "reason": "Resemble Detect flagged media as fake (score=0.95, threshold=0.5)", + "audio_source_tracing": { + "label": "elevenlabs", + "error_message": null + } + } + } + } +} +``` + +## Configuration reference + +| Parameter | Type | Default | Description | +| -------------------------------- | ---------- | ----------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| `api_key` | string | `RESEMBLE_API_KEY` env var | Resemble AI API token. | +| `api_base` | string | `https://app.resemble.ai/api/v2` | Override the Resemble API base URL (useful for sovereign deployments). | +| `resemble_threshold` | number | `0.5` | Aggregated score above which media is treated as fake (0.0–1.0). | +| `resemble_media_type` | enum | auto | Force `audio`, `video`, or `image`. Omit for auto-detect from extension / content type. | +| `resemble_audio_source_tracing` | bool | `false` | Return which TTS vendor generated flagged audio (ElevenLabs, Resemble AI, OpenAI, etc.). | +| `resemble_use_reverse_search` | bool | `false` | (Image only) search the web for matching images to improve accuracy. | +| `resemble_zero_retention_mode` | bool | `false` | Automatically delete submitted media after detection. URLs are redacted and filenames are tokenized. | +| `resemble_metadata_key` | string | `"mediaUrl"` | Key under request `metadata` to read the media URL from when it is not present in the message content. | +| `resemble_poll_interval_seconds` | number | `2.0` | How often to poll Resemble for the detection result. | +| `resemble_poll_timeout_seconds` | number | `60.0` | Maximum total time to wait for a detection result before failing. | +| `resemble_fail_closed` | bool | `false` | If `true`, Resemble API errors **block** the request. If `false` (default), errors are logged and ignored. | + +## Zero Retention Mode + +For workflows where you cannot retain media on Resemble's infrastructure (e.g. HIPAA/financial compliance), set `resemble_zero_retention_mode: true`. Resemble will tokenize filenames, redact submitted URLs from logs, and delete the media artifact after the scan completes. The verdict is still returned synchronously. + +## Audio source tracing + +When `resemble_audio_source_tracing: true`, the blocked-response `resemble.audio_source_tracing` object contains the source model that produced the cloned audio: + +```json +{ + "audio_source_tracing": { + "label": "elevenlabs", + "error_message": null + } +} +``` + +Possible labels include `elevenlabs`, `resemble_ai`, `openai`, `playht`, `azure_neural`, `google_tts`, and others. This is useful for incident triage and attributing cloned-voice abuse back to the generating vendor. diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index b2ac8433911..712a59356c6 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -86,6 +86,7 @@ const sidebars = { "proxy/guardrails/promptguard", "proxy/guardrails/pii_masking_v2", "proxy/guardrails/panw_prisma_airs", + "proxy/guardrails/resemble_detect", "proxy/guardrails/secret_detection", "proxy/guardrails/custom_guardrail", "proxy/guardrails/custom_code_guardrail", diff --git a/litellm/proxy/guardrails/guardrail_hooks/resemble/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/resemble/__init__.py new file mode 100644 index 00000000000..166e381d4f9 --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/resemble/__init__.py @@ -0,0 +1,50 @@ +from typing import TYPE_CHECKING + +from litellm.types.guardrails import SupportedGuardrailIntegrations + +from .resemble import ResembleGuardrail + +if TYPE_CHECKING: + from litellm.types.guardrails import Guardrail, LitellmParams + + +def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"): + import litellm + + _resemble_callback = ResembleGuardrail( + guardrail_name=guardrail.get("guardrail_name", ""), + api_key=litellm_params.api_key, + api_base=litellm_params.api_base, + event_hook=litellm_params.mode, + default_on=litellm_params.default_on, + threshold=getattr(litellm_params, "resemble_threshold", None), + media_type=getattr(litellm_params, "resemble_media_type", None), + audio_source_tracing=getattr( + litellm_params, "resemble_audio_source_tracing", None + ), + use_reverse_search=getattr(litellm_params, "resemble_use_reverse_search", None), + zero_retention_mode=getattr( + litellm_params, "resemble_zero_retention_mode", None + ), + metadata_key=getattr(litellm_params, "resemble_metadata_key", None), + poll_interval_seconds=getattr( + litellm_params, "resemble_poll_interval_seconds", None + ), + poll_timeout_seconds=getattr( + litellm_params, "resemble_poll_timeout_seconds", None + ), + fail_closed=getattr(litellm_params, "resemble_fail_closed", None), + ) + litellm.logging_callback_manager.add_litellm_callback(_resemble_callback) + + return _resemble_callback + + +guardrail_initializer_registry = { + SupportedGuardrailIntegrations.RESEMBLE.value: initialize_guardrail, +} + + +guardrail_class_registry = { + SupportedGuardrailIntegrations.RESEMBLE.value: ResembleGuardrail, +} diff --git a/litellm/proxy/guardrails/guardrail_hooks/resemble/resemble.py b/litellm/proxy/guardrails/guardrail_hooks/resemble/resemble.py new file mode 100644 index 00000000000..57ec3c10dcc --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/resemble/resemble.py @@ -0,0 +1,512 @@ +# +-------------------------------------------------------------+ +# +# Use Resemble AI Detect for your LLM calls +# https://www.resemble.ai/ +# +# Scans audio, video, and image URLs referenced in requests +# for deepfake / synthetic content. Blocks requests whose +# media inputs Resemble labels as fake or whose aggregated +# score exceeds the configured threshold. +# +# +-------------------------------------------------------------+ + +import asyncio +import os +import re +import time +from typing import ( + TYPE_CHECKING, + Any, + Dict, + List, + Literal, + Optional, + Tuple, + Type, + Union, +) + +from fastapi import HTTPException + +from litellm import DualCache +from litellm._logging import verbose_proxy_logger +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, +) +from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.guardrails import GuardrailEventHooks + +if TYPE_CHECKING: + from litellm.types.proxy.guardrails.guardrail_hooks.base import ( + GuardrailConfigModel, + ) + + +# Matches HTTPS URLs ending in common audio/video/image extensions. Query +# strings and fragments are allowed. Kept intentionally simple — multimodal +# content parts and metadata lookups handle the non-URL-in-text cases. +MEDIA_URL_REGEX = re.compile( + r"https?://[^\s<>\"')\]}]+?\.(?:mp3|wav|m4a|flac|ogg|opus|aac|webm|mp4|mov|avi|mkv|jpg|jpeg|png|webp|gif)(?:\?[^\s<>\"')\]}]*)?", + re.IGNORECASE, +) + +RESEMBLE_DEFAULT_API_BASE = "https://app.resemble.ai/api/v2" + + +class ResembleGuardrailMissingSecrets(Exception): + """Raised when the Resemble API key is not configured.""" + + pass + + +class ResembleGuardrailAPIError(Exception): + """Raised when the Resemble API returns an unexpected error.""" + + pass + + +class ResembleGuardrail(CustomGuardrail): + """ + Resemble AI Detect guardrail for LiteLLM. + + Extracts media URLs from LLM request inputs (multimodal content parts, + regex from text, or metadata) and submits them to Resemble Detect. Blocks + the request when the returned label is ``fake`` or when the aggregated + score exceeds the configured threshold. + """ + + def __init__( + self, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + threshold: Optional[float] = None, + media_type: Optional[Literal["audio", "video", "image"]] = None, + audio_source_tracing: Optional[bool] = None, + use_reverse_search: Optional[bool] = None, + zero_retention_mode: Optional[bool] = None, + metadata_key: Optional[str] = None, + poll_interval_seconds: Optional[float] = None, + poll_timeout_seconds: Optional[float] = None, + fail_closed: Optional[bool] = None, + **kwargs, + ): + self.async_handler = get_async_httpx_client( + llm_provider=httpxSpecialProvider.GuardrailCallback + ) + + resolved_api_key = api_key or os.environ.get("RESEMBLE_API_KEY") + if not resolved_api_key: + raise ResembleGuardrailMissingSecrets( + "Couldn't get Resemble API key. Set the `RESEMBLE_API_KEY` " + "environment variable or pass `api_key` in the guardrail config." + ) + self.api_key: str = resolved_api_key + + self.api_base: str = ( + api_base or os.environ.get("RESEMBLE_API_BASE") or RESEMBLE_DEFAULT_API_BASE + ).rstrip("/") + + self.threshold: float = threshold if threshold is not None else 0.5 + self.media_type: Optional[str] = media_type + self.audio_source_tracing: bool = bool(audio_source_tracing) + self.use_reverse_search: bool = bool(use_reverse_search) + self.zero_retention_mode: bool = bool(zero_retention_mode) + self.metadata_key: str = metadata_key or "mediaUrl" + self.poll_interval_seconds: float = ( + poll_interval_seconds if poll_interval_seconds is not None else 2.0 + ) + self.poll_timeout_seconds: float = ( + poll_timeout_seconds if poll_timeout_seconds is not None else 60.0 + ) + self.fail_closed: bool = bool(fail_closed) + + verbose_proxy_logger.debug( + "Resemble guardrail initialized: name=%s threshold=%s media_type=%s " + "audio_source_tracing=%s use_reverse_search=%s zero_retention_mode=%s " + "fail_closed=%s", + kwargs.get("guardrail_name", "unknown"), + self.threshold, + self.media_type, + self.audio_source_tracing, + self.use_reverse_search, + self.zero_retention_mode, + self.fail_closed, + ) + + super().__init__(**kwargs) + + # ------------------------------------------------------------------ + # Hook entrypoints + # ------------------------------------------------------------------ + + @log_guardrail_information + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: dict, + call_type: Literal[ + "completion", + "text_completion", + "embeddings", + "image_generation", + "moderation", + "audio_transcription", + "pass_through_endpoint", + "rerank", + "mcp_call", + "anthropic_messages", + ], + ) -> Union[Exception, str, dict, None]: + event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call + if self.should_run_guardrail(data=data, event_type=event_type) is not True: + return data + + await self._scan_request(data) + return data + + @log_guardrail_information + async def async_moderation_hook( + self, + data: dict, + user_api_key_dict: UserAPIKeyAuth, + call_type: Literal[ + "completion", + "embeddings", + "image_generation", + "moderation", + "audio_transcription", + "responses", + "mcp_call", + "anthropic_messages", + ], + ): + event_type: GuardrailEventHooks = GuardrailEventHooks.during_call + if self.should_run_guardrail(data=data, event_type=event_type) is not True: + return data + + await self._scan_request(data) + return data + + # ------------------------------------------------------------------ + # Core scan logic + # ------------------------------------------------------------------ + + async def _scan_request(self, data: dict) -> None: + media_urls = self._extract_media_urls(data) + if not media_urls: + verbose_proxy_logger.debug( + "Resemble guardrail: no media URL found in request — passing through" + ) + return + + for media_url in media_urls: + await self._scan_single_url(media_url) + + async def _scan_single_url(self, media_url: str) -> None: + try: + item = await self._create_and_poll_detection(media_url) + except HTTPException: + raise + except Exception as exc: + self._handle_api_error(exc, media_url) + return + + if item.get("status") == "failed": + self._handle_api_error( + ResembleGuardrailAPIError( + f"Resemble detection failed: {item.get('error_message') or 'unknown reason'}" + ), + media_url, + ) + return + + evaluation = self._evaluate_detection(item) + if not evaluation["verdict"]: + detail: Dict[str, Any] = { + "error": "Resemble Detect flagged media as synthetic", + "resemble": { + "uuid": item.get("uuid"), + "media_url": media_url, + "media_type": item.get("media_type"), + "label": evaluation["label"], + "score": evaluation["score"], + "threshold": self.threshold, + "reason": evaluation["reason"], + "audio_source_tracing": item.get("audio_source_tracing"), + }, + } + raise HTTPException(status_code=400, detail=detail) + + verbose_proxy_logger.debug( + "Resemble guardrail: passed (label=%s score=%s threshold=%s uuid=%s)", + evaluation["label"], + evaluation["score"], + self.threshold, + item.get("uuid"), + ) + + def _handle_api_error(self, exc: Exception, media_url: str) -> None: + verbose_proxy_logger.warning( + "Resemble guardrail API error for %s: %s", media_url, exc + ) + if self.fail_closed: + raise HTTPException( + status_code=500, + detail={ + "error": "Resemble Detect API call failed", + "resemble": { + "media_url": media_url, + "reason": str(exc), + }, + }, + ) + # Fail open — swallow the error so the LLM request proceeds. + + # ------------------------------------------------------------------ + # URL extraction + # ------------------------------------------------------------------ + + def _extract_media_urls(self, data: dict) -> List[str]: + """ + Collect every media URL referenced in the request. + + Scans all multimodal content parts across every message, then every + URL in the joined request text, then the metadata fallback. Returns + a de-duplicated list preserving discovery order. + + Returning only the first match (the previous behaviour) let callers + sneak a synthetic URL past the guardrail by placing a benign one + earlier in the content array. + """ + seen: Dict[str, None] = {} + + def _add(url: Optional[str]) -> None: + if isinstance(url, str) and url and url not in seen: + seen[url] = None + + messages = data.get("messages") or [] + for url in self._urls_from_content_parts(messages): + _add(url) + for url in self._urls_from_text(messages, data): + _add(url) + for url in self._urls_from_metadata(data): + _add(url) + + return list(seen.keys()) + + @staticmethod + def _urls_from_content_parts(messages: Any) -> List[str]: + """Pull every URL out of multimodal content parts across all messages.""" + urls: List[str] = [] + if not isinstance(messages, list): + return urls + for message in messages: + if not isinstance(message, dict): + continue + content = message.get("content") + if not isinstance(content, list): + continue + for part in content: + if not isinstance(part, dict): + continue + part_type = part.get("type") + if part_type == "input_audio": + audio = part.get("input_audio") + if isinstance(audio, dict) and isinstance(audio.get("url"), str): + urls.append(audio["url"]) + elif part_type == "image_url": + image_url = part.get("image_url") + if isinstance(image_url, dict) and isinstance( + image_url.get("url"), str + ): + urls.append(image_url["url"]) + elif part_type in ("image", "document"): + source = part.get("source") + if ( + isinstance(source, dict) + and source.get("type") == "url" + and isinstance(source.get("url"), str) + ): + urls.append(source["url"]) + return urls + + @staticmethod + def _urls_from_text(messages: Any, data: dict) -> List[str]: + """Find every media URL embedded in plain text fields (messages, prompt, input).""" + text_chunks: List[str] = [] + if isinstance(messages, list): + for message in messages: + if not isinstance(message, dict): + continue + content = message.get("content") + if isinstance(content, str): + text_chunks.append(content) + elif isinstance(content, list): + for part in content: + if ( + isinstance(part, dict) + and part.get("type") == "text" + and isinstance(part.get("text"), str) + ): + text_chunks.append(part["text"]) + prompt = data.get("prompt") + if isinstance(prompt, str): + text_chunks.append(prompt) + input_value = data.get("input") + if isinstance(input_value, str): + text_chunks.append(input_value) + + joined = "\n".join(text_chunks) + return [match.group(0) for match in MEDIA_URL_REGEX.finditer(joined)] + + def _urls_from_metadata(self, data: dict) -> List[str]: + """Metadata fallback — accepts a single URL string or a list of URLs.""" + metadata = data.get("metadata") or {} + candidate = metadata.get(self.metadata_key) + if isinstance(candidate, str): + return [candidate] + if isinstance(candidate, list): + return [entry for entry in candidate if isinstance(entry, str)] + return [] + + # ------------------------------------------------------------------ + # Resemble API + # ------------------------------------------------------------------ + + async def _create_and_poll_detection(self, media_url: str) -> Dict[str, Any]: + create_payload: Dict[str, Any] = {"url": media_url} + if self.media_type: + create_payload["media_type"] = self.media_type + if self.audio_source_tracing: + create_payload["audio_source_tracing"] = True + if self.use_reverse_search: + create_payload["use_reverse_search"] = True + if self.zero_retention_mode: + create_payload["zero_retention_mode"] = True + + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + } + + create_response = await self.async_handler.post( + url=f"{self.api_base}/detect", + headers=headers, + json=create_payload, + timeout=10.0, + ) + create_response.raise_for_status() + create_body = create_response.json() + + item = create_body.get("item") or {} + uuid = item.get("uuid") + if not uuid: + raise ResembleGuardrailAPIError( + "Resemble /detect response is missing item.uuid — cannot poll" + ) + + # If the server returned a completed item synchronously, skip polling. + if item.get("status") == "completed" and self._item_has_metrics(item): + return item + + return await self._poll_detection(uuid, headers) + + @staticmethod + def _item_has_metrics(item: Dict[str, Any]) -> bool: + """True when the detect item carries the scoring fields we evaluate.""" + return bool( + item.get("metrics") + or item.get("image_metrics") + or item.get("video_metrics") + ) + + async def _poll_detection( + self, uuid: str, headers: Dict[str, str] + ) -> Dict[str, Any]: + deadline = time.monotonic() + self.poll_timeout_seconds + poll_url = f"{self.api_base}/detect/{uuid}" + + while time.monotonic() < deadline: + response = await self.async_handler.get( + url=poll_url, + headers=headers, + ) + response.raise_for_status() + body = response.json() + item = body.get("item") or {} + if not item: + raise ResembleGuardrailAPIError( + f"Resemble GET /detect/{uuid} returned no item" + ) + status = item.get("status") + # "failed" is terminal; "completed" is only terminal once metrics + # have landed — otherwise a racy API response could slip a + # metric-less item past _evaluate_detection (which would treat + # missing metrics as "unknown/0.0" and silently pass the request). + if status == "failed": + return item + if status == "completed" and self._item_has_metrics(item): + return item + await asyncio.sleep(self.poll_interval_seconds) + + raise ResembleGuardrailAPIError( + f"Resemble detection timed out after {self.poll_timeout_seconds}s (uuid={uuid})" + ) + + # ------------------------------------------------------------------ + # Evaluation + # ------------------------------------------------------------------ + + def _evaluate_detection(self, item: Dict[str, Any]) -> Dict[str, Any]: + label, score = self._extract_label_and_score(item) + is_fake = label == "fake" or score >= self.threshold + reason = ( + f"Resemble Detect flagged media as {label} (score={score}, " + f"threshold={self.threshold})" + if is_fake + else f"Resemble Detect passed: label={label}, score={score}" + ) + return { + "verdict": not is_fake, + "label": label, + "score": score, + "reason": reason, + } + + def _extract_label_and_score(self, item: Dict[str, Any]) -> Tuple[str, float]: + metrics = item.get("metrics") + if isinstance(metrics, dict): + return ( + str(metrics.get("label") or "unknown").lower(), + float(metrics.get("aggregated_score") or 0), + ) + + image_metrics = item.get("image_metrics") + if isinstance(image_metrics, dict): + return ( + str(image_metrics.get("label") or "unknown").lower(), + float(image_metrics.get("score") or 0), + ) + + video_metrics = item.get("video_metrics") + if isinstance(video_metrics, dict): + return ( + str(video_metrics.get("label") or "unknown").lower(), + float(video_metrics.get("score") or 0), + ) + + return ("unknown", 0.0) + + @staticmethod + def get_config_model() -> Optional[Type["GuardrailConfigModel"]]: + from litellm.types.proxy.guardrails.guardrail_hooks.resemble import ( + ResembleGuardrailConfigModel, + ) + + return ResembleGuardrailConfigModel diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index f2319942c14..e99cd1cc448 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -88,6 +88,7 @@ class SupportedGuardrailIntegrations(Enum): BLOCK_CODE_EXECUTION = "block_code_execution" AKTO = "akto" MCP_JWT_SIGNER = "mcp_jwt_signer" + RESEMBLE = "resemble" class Role(Enum): @@ -434,6 +435,74 @@ class LassoGuardrailConfigModel(BaseModel): ) +class ResembleGuardrailParamsConfigModel(BaseModel): + """Configuration parameters for the Resemble AI Detect guardrail""" + + resemble_threshold: Optional[float] = Field( + default=0.5, + description=( + "Aggregated score above which media is treated as fake (0.0–1.0). " + "Default 0.5." + ), + ) + resemble_media_type: Optional[Literal["audio", "video", "image"]] = Field( + default=None, + description=( + "Optionally force audio / video / image. If omitted, Resemble " + "auto-detects from the file extension or content type." + ), + ) + resemble_audio_source_tracing: Optional[bool] = Field( + default=False, + description=( + "Identify which TTS vendor (elevenlabs, resemble_ai, etc.) generated " + "the audio when it is flagged as fake." + ), + ) + resemble_use_reverse_search: Optional[bool] = Field( + default=False, + description=( + "For image detections, search the web for matching images to " + "improve accuracy." + ), + ) + resemble_zero_retention_mode: Optional[bool] = Field( + default=False, + description=( + "Automatically delete submitted media after detection completes. " + "URLs are redacted and filenames are tokenized." + ), + ) + resemble_metadata_key: Optional[str] = Field( + default="mediaUrl", + description=( + "Key in request `metadata` to read the media URL from when it is " + "not present in the message content. Default `mediaUrl`." + ), + ) + resemble_poll_interval_seconds: Optional[float] = Field( + default=2.0, + description=( + "How often to poll Resemble for the detection result. Default 2s." + ), + ) + resemble_poll_timeout_seconds: Optional[float] = Field( + default=60.0, + description=( + "Maximum total time (in seconds) to wait for a detection result " + "before failing open. Default 60s." + ), + ) + resemble_fail_closed: Optional[bool] = Field( + default=False, + description=( + "If true, Resemble API errors (network, auth, timeout) will BLOCK " + "the request. If false (default), errors are logged but the " + "request passes through." + ), + ) + + class PillarGuardrailConfigModel(BaseModel): """Configuration parameters for the Pillar Security guardrail""" @@ -763,6 +832,7 @@ class LitellmParams( IBMGuardrailsBaseConfigModel, QualifireGuardrailConfigModel, BlockCodeExecutionGuardrailConfigModel, + ResembleGuardrailParamsConfigModel, ): guardrail: str = Field(description="The type of guardrail integration to use") mode: Union[str, List[str], Mode] = Field( diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/resemble.py b/litellm/types/proxy/guardrails/guardrail_hooks/resemble.py new file mode 100644 index 00000000000..15387240232 --- /dev/null +++ b/litellm/types/proxy/guardrails/guardrail_hooks/resemble.py @@ -0,0 +1,90 @@ +from typing import Literal, Optional + +from pydantic import Field + +from .base import GuardrailConfigModel + + +class ResembleGuardrailConfigModel(GuardrailConfigModel): + api_key: Optional[str] = Field( + default=None, + description=( + "The Resemble AI API token. If not provided, the `RESEMBLE_API_KEY` " + "environment variable is checked." + ), + ) + api_base: Optional[str] = Field( + default=None, + description=( + "Override the Resemble API base URL. If not provided, the " + "`RESEMBLE_API_BASE` environment variable is checked and falls " + "back to `https://app.resemble.ai/api/v2`." + ), + ) + resemble_threshold: Optional[float] = Field( + default=0.5, + description=( + "Aggregated score above which media is treated as fake (0.0–1.0). " + "Default 0.5." + ), + ) + resemble_media_type: Optional[Literal["audio", "video", "image"]] = Field( + default=None, + description=( + "Optionally force audio / video / image. If omitted, Resemble " + "auto-detects from the file extension or content type." + ), + ) + resemble_audio_source_tracing: Optional[bool] = Field( + default=False, + description=( + "Identify which TTS vendor (elevenlabs, resemble_ai, etc.) generated " + "the audio when it is flagged as fake." + ), + ) + resemble_use_reverse_search: Optional[bool] = Field( + default=False, + description=( + "For image detections, search the web for matching images to " + "improve accuracy." + ), + ) + resemble_zero_retention_mode: Optional[bool] = Field( + default=False, + description=( + "Automatically delete submitted media after detection completes. " + "URLs are redacted and filenames are tokenized." + ), + ) + resemble_metadata_key: Optional[str] = Field( + default="mediaUrl", + description=( + "Key in request `metadata` to read the media URL from when it is " + "not present in the message content. Default `mediaUrl`." + ), + ) + resemble_poll_interval_seconds: Optional[float] = Field( + default=2.0, + description=( + "How often to poll Resemble for the detection result. Default 2s." + ), + ) + resemble_poll_timeout_seconds: Optional[float] = Field( + default=60.0, + description=( + "Maximum total time (in seconds) to wait for a detection result " + "before failing open. Default 60s." + ), + ) + resemble_fail_closed: Optional[bool] = Field( + default=False, + description=( + "If true, Resemble API errors (network, auth, timeout) will BLOCK " + "the request. If false (default), errors are logged but the " + "request passes through." + ), + ) + + @staticmethod + def ui_friendly_name() -> str: + return "Resemble AI Detect" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_resemble.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_resemble.py new file mode 100644 index 00000000000..e8542ed788f --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_resemble.py @@ -0,0 +1,850 @@ +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi import HTTPException +from httpx import Request, Response + +sys.path.insert(0, os.path.abspath("../..")) + +import litellm # noqa: E402 +from litellm import DualCache # noqa: E402 +from litellm.proxy._types import UserAPIKeyAuth # noqa: E402 +from litellm.proxy.guardrails.guardrail_hooks.resemble.resemble import ( # noqa: E402 + RESEMBLE_DEFAULT_API_BASE, + ResembleGuardrail, + ResembleGuardrailMissingSecrets, +) +from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 # noqa: E402 + + +def _make_guardrail(**overrides): + defaults = dict( + api_key="test-key", + guardrail_name="resemble-test", + event_hook="pre_call", + default_on=True, + poll_interval_seconds=0.001, + poll_timeout_seconds=1.0, + ) + defaults.update(overrides) + return ResembleGuardrail(**defaults) + + +def _fake_post_response(body, status_code=200, request=None): + return Response( + status_code=status_code, + json=body, + request=request + or Request(method="POST", url="https://app.resemble.ai/api/v2/detect"), + ) + + +def _fake_get_response(body, status_code=200, uuid="abc-123"): + return Response( + status_code=status_code, + json=body, + request=Request( + method="GET", url=f"https://app.resemble.ai/api/v2/detect/{uuid}" + ), + ) + + +# --------------------------------------------------------------------------- +# Init / config tests +# --------------------------------------------------------------------------- + + +def test_resemble_guard_registered_via_init_guardrails_v2(monkeypatch): + """`resemble` is accepted by init_guardrails_v2 and loads the class.""" + litellm.set_verbose = True + litellm.guardrail_name_config_map = {} + monkeypatch.setenv("RESEMBLE_API_KEY", "test-key") + + init_guardrails_v2( + all_guardrails=[ + { + "guardrail_name": "deepfake-detect", + "litellm_params": { + "guardrail": "resemble", + "mode": "pre_call", + "default_on": True, + }, + } + ], + config_file_path="", + ) + + +def test_missing_api_key_raises(): + with pytest.raises(ResembleGuardrailMissingSecrets): + ResembleGuardrail(guardrail_name="r") + + +def test_api_base_strips_trailing_slash(monkeypatch): + guard = _make_guardrail(api_base="https://custom.example/api/v2/") + assert guard.api_base == "https://custom.example/api/v2" + + +def test_default_api_base_fallback(monkeypatch): + monkeypatch.delenv("RESEMBLE_API_BASE", raising=False) + guard = _make_guardrail() + assert guard.api_base == RESEMBLE_DEFAULT_API_BASE + + +# --------------------------------------------------------------------------- +# URL extraction tests +# --------------------------------------------------------------------------- + + +class TestExtractMediaUrls: + def setup_method(self): + self.guard = _make_guardrail() + + def test_plain_text_audio_url(self): + data = { + "messages": [ + {"role": "user", "content": "Check https://cdn.example.com/c.mp3 pls"} + ] + } + assert self.guard._extract_media_urls(data) == ["https://cdn.example.com/c.mp3"] + + def test_openai_image_url_part(self): + data = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Is this real?"}, + { + "type": "image_url", + "image_url": {"url": "https://cdn.example.com/face.png"}, + }, + ], + } + ] + } + assert self.guard._extract_media_urls(data) == [ + "https://cdn.example.com/face.png" + ] + + def test_openai_input_audio_part(self): + data = { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "input_audio", + "input_audio": {"url": "https://cdn.example.com/a.wav"}, + } + ], + } + ] + } + assert self.guard._extract_media_urls(data) == ["https://cdn.example.com/a.wav"] + + def test_anthropic_source_url(self): + data = { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "url", + "url": "https://cdn.example.com/x.jpg", + }, + } + ], + } + ] + } + assert self.guard._extract_media_urls(data) == ["https://cdn.example.com/x.jpg"] + + def test_metadata_fallback_when_no_content_url(self): + data = { + "messages": [{"role": "user", "content": "no url here"}], + "metadata": {"mediaUrl": "https://cdn.example.com/clip.mp4"}, + } + assert self.guard._extract_media_urls(data) == [ + "https://cdn.example.com/clip.mp4" + ] + + def test_custom_metadata_key(self): + guard = _make_guardrail(metadata_key="audio_src") + data = { + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"audio_src": "https://cdn.example.com/x.m4a"}, + } + assert guard._extract_media_urls(data) == ["https://cdn.example.com/x.m4a"] + + def test_metadata_list_accepted(self): + guard = _make_guardrail() + data = { + "messages": [{"role": "user", "content": "hi"}], + "metadata": { + "mediaUrl": [ + "https://cdn.example.com/a.mp3", + "https://cdn.example.com/b.mp3", + ] + }, + } + assert guard._extract_media_urls(data) == [ + "https://cdn.example.com/a.mp3", + "https://cdn.example.com/b.mp3", + ] + + def test_returns_empty_list_when_no_url(self): + data = {"messages": [{"role": "user", "content": "nothing to see"}]} + assert self.guard._extract_media_urls(data) == [] + + def test_multiple_content_part_urls_preserved_in_order(self): + """ + P1 regression test: the extractor must return every URL referenced in + a multimodal content array, in order. Returning only the first URL + let callers smuggle a synthetic URL past the guardrail by placing a + benign one earlier in the array. + """ + data = { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://cdn.example.com/real.jpg"}, + }, + { + "type": "image_url", + "image_url": {"url": "https://cdn.example.com/fake.jpg"}, + }, + ], + } + ] + } + assert self.guard._extract_media_urls(data) == [ + "https://cdn.example.com/real.jpg", + "https://cdn.example.com/fake.jpg", + ] + + def test_duplicate_urls_are_deduped_in_order(self): + data = { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://cdn.example.com/a.jpg"}, + }, + { + "type": "image_url", + "image_url": {"url": "https://cdn.example.com/a.jpg"}, + }, + ], + } + ] + } + assert self.guard._extract_media_urls(data) == ["https://cdn.example.com/a.jpg"] + + def test_urls_across_multiple_messages(self): + data = { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://cdn.example.com/one.jpg"}, + }, + ], + }, + {"role": "assistant", "content": "ok"}, + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://cdn.example.com/two.jpg"}, + }, + ], + }, + ] + } + assert self.guard._extract_media_urls(data) == [ + "https://cdn.example.com/one.jpg", + "https://cdn.example.com/two.jpg", + ] + + +# --------------------------------------------------------------------------- +# Evaluation tests +# --------------------------------------------------------------------------- + + +class TestEvaluateDetection: + def setup_method(self): + self.guard = _make_guardrail(threshold=0.5) + + def test_fake_label_always_fails(self): + item = { + "metrics": { + "label": "fake", + "aggregated_score": "0.2", + "score": ["0.1"], + } + } + result = self.guard._evaluate_detection(item) + assert result["verdict"] is False + assert result["label"] == "fake" + + def test_real_low_score_passes(self): + item = { + "metrics": { + "label": "real", + "aggregated_score": "0.1", + "score": ["0.1"], + } + } + result = self.guard._evaluate_detection(item) + assert result["verdict"] is True + assert result["score"] == 0.1 + + def test_real_high_score_fails_on_threshold(self): + item = { + "metrics": { + "label": "real", + "aggregated_score": "0.7", + "score": ["0.7"], + } + } + result = self.guard._evaluate_detection(item) + assert result["verdict"] is False + + def test_image_metrics_shape(self): + item = {"image_metrics": {"label": "fake", "score": 0.9}} + result = self.guard._evaluate_detection(item) + assert result["verdict"] is False + assert result["score"] == 0.9 + + def test_video_metrics_shape(self): + item = {"video_metrics": {"label": "real", "score": 0.2}} + result = self.guard._evaluate_detection(item) + assert result["verdict"] is True + + +# --------------------------------------------------------------------------- +# Hook behaviour tests (mocked HTTP) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_pre_call_passes_without_media_url(): + guard = _make_guardrail() + data = {"messages": [{"role": "user", "content": "plain text, no media"}]} + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post" + ) as post_mock: + result = await guard.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + assert result == data + post_mock.assert_not_called() + + +@pytest.mark.asyncio +async def test_pre_call_passes_when_audio_is_real(): + guard = _make_guardrail() + data = { + "messages": [ + {"role": "user", "content": "check https://cdn.example.com/clip.mp3"} + ] + } + + create_response = _fake_post_response( + { + "success": True, + "item": {"uuid": "u-1", "status": "processing"}, + } + ) + poll_response = _fake_get_response( + { + "success": True, + "item": { + "uuid": "u-1", + "media_type": "audio", + "status": "completed", + "metrics": { + "label": "real", + "score": ["0.1", "0.2"], + "aggregated_score": "0.15", + "consistency": "0.9", + }, + }, + }, + uuid="u-1", + ) + + with ( + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=create_response, + ), + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get", + return_value=poll_response, + ), + ): + result = await guard.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + assert result == data + + +@pytest.mark.asyncio +async def test_pre_call_blocks_fake_audio(): + guard = _make_guardrail() + data = { + "messages": [{"role": "user", "content": "https://cdn.example.com/cloned.wav"}] + } + + create_response = _fake_post_response( + {"success": True, "item": {"uuid": "u-2", "status": "processing"}} + ) + poll_response = _fake_get_response( + { + "success": True, + "item": { + "uuid": "u-2", + "media_type": "audio", + "status": "completed", + "metrics": { + "label": "fake", + "score": ["0.9"], + "aggregated_score": "0.95", + }, + "audio_source_tracing": { + "label": "elevenlabs", + "error_message": None, + }, + }, + }, + uuid="u-2", + ) + + with ( + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=create_response, + ), + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get", + return_value=poll_response, + ), + ): + with pytest.raises(HTTPException) as exc_info: + await guard.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + + assert exc_info.value.status_code == 400 + detail = exc_info.value.detail + assert isinstance(detail, dict) + assert detail["resemble"]["label"] == "fake" + assert detail["resemble"]["media_url"] == "https://cdn.example.com/cloned.wav" + assert detail["resemble"]["audio_source_tracing"]["label"] == "elevenlabs" + + +@pytest.mark.asyncio +async def test_pre_call_blocks_image_threshold_exceeded(): + guard = _make_guardrail(threshold=0.5) + data = { + "messages": [{"role": "user", "content": "https://cdn.example.com/photo.jpg"}] + } + + create_response = _fake_post_response( + { + "success": True, + "item": { + "uuid": "img-1", + "status": "completed", + "media_type": "image", + "image_metrics": {"label": "real", "score": 0.85, "type": "facial"}, + }, + } + ) + + # Synchronous completion — no GET is expected. + with ( + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=create_response, + ) as post_mock, + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get" + ) as get_mock, + ): + with pytest.raises(HTTPException): + await guard.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + + post_mock.assert_called_once() + get_mock.assert_not_called() + + +@pytest.mark.asyncio +async def test_pre_call_fails_open_on_api_error(): + guard = _make_guardrail() + data = {"messages": [{"role": "user", "content": "https://cdn.example.com/x.mp3"}]} + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + side_effect=Exception("network down"), + ): + result = await guard.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + assert result == data # fail-open: untouched + + +@pytest.mark.asyncio +async def test_pre_call_fails_closed_on_api_error_when_configured(): + guard = _make_guardrail(fail_closed=True) + data = {"messages": [{"role": "user", "content": "https://cdn.example.com/x.mp3"}]} + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + side_effect=Exception("network down"), + ): + with pytest.raises(HTTPException) as exc_info: + await guard.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + assert exc_info.value.status_code == 500 + + +@pytest.mark.asyncio +async def test_pre_call_times_out_and_fails_open(): + guard = _make_guardrail(poll_interval_seconds=0.001, poll_timeout_seconds=0.02) + data = { + "messages": [{"role": "user", "content": "https://cdn.example.com/slow.mp3"}] + } + + create_response = _fake_post_response( + {"success": True, "item": {"uuid": "slow-1", "status": "processing"}} + ) + polling_response = _fake_get_response( + {"success": True, "item": {"uuid": "slow-1", "status": "processing"}}, + uuid="slow-1", + ) + + with ( + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=create_response, + ), + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get", + return_value=polling_response, + ), + ): + result = await guard.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + assert result == data + + +@pytest.mark.asyncio +async def test_create_payload_includes_flags(): + guard = _make_guardrail( + media_type="audio", + audio_source_tracing=True, + use_reverse_search=True, + zero_retention_mode=True, + ) + data = { + "messages": [{"role": "user", "content": "https://cdn.example.com/clip.mp3"}] + } + + create_response = _fake_post_response( + { + "success": True, + "item": { + "uuid": "c-1", + "status": "completed", + "media_type": "audio", + "metrics": { + "label": "real", + "score": ["0.1"], + "aggregated_score": "0.1", + }, + }, + } + ) + + post_mock = MagicMock(return_value=create_response) + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new=post_mock, + ): + await guard.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + + call_kwargs = post_mock.call_args.kwargs + assert call_kwargs["url"].endswith("/detect") + assert call_kwargs["headers"]["Authorization"] == "Bearer test-key" + body = call_kwargs["json"] + assert body["url"] == "https://cdn.example.com/clip.mp3" + assert body["media_type"] == "audio" + assert body["audio_source_tracing"] is True + assert body["use_reverse_search"] is True + assert body["zero_retention_mode"] is True + + +@pytest.mark.asyncio +async def test_moderation_hook_also_scans(): + guard = _make_guardrail(event_hook="during_call") + data = {"messages": [{"role": "user", "content": "https://cdn.example.com/x.mp3"}]} + + create_response = _fake_post_response( + { + "success": True, + "item": { + "uuid": "m-1", + "status": "completed", + "media_type": "audio", + "metrics": { + "label": "fake", + "score": ["0.9"], + "aggregated_score": "0.95", + }, + }, + } + ) + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=create_response, + ): + with pytest.raises(HTTPException): + await guard.async_moderation_hook( + data=data, + user_api_key_dict=UserAPIKeyAuth(), + call_type="completion", + ) + + +@pytest.mark.asyncio +async def test_pre_call_blocks_when_second_of_two_urls_is_fake(): + """ + P1 regression (end-to-end): a request with [real, fake] image URLs must be + blocked. Previously we returned the first URL from the extractor, so the + fake one was never sent to Resemble. + """ + guard = _make_guardrail() + data = { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://cdn.example.com/real.jpg"}, + }, + { + "type": "image_url", + "image_url": {"url": "https://cdn.example.com/fake.jpg"}, + }, + ], + } + ] + } + + async def _post_side_effect(*args, **kwargs): + body = kwargs.get("json") or {} + url = body.get("url") + if url == "https://cdn.example.com/real.jpg": + return _fake_post_response( + { + "success": True, + "item": { + "uuid": "real-1", + "status": "completed", + "media_type": "image", + "image_metrics": {"label": "real", "score": 0.05}, + }, + } + ) + if url == "https://cdn.example.com/fake.jpg": + return _fake_post_response( + { + "success": True, + "item": { + "uuid": "fake-1", + "status": "completed", + "media_type": "image", + "image_metrics": {"label": "fake", "score": 0.95}, + }, + } + ) + raise AssertionError(f"Unexpected POST url={url}") + + post_mock = AsyncMock(side_effect=_post_side_effect) + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new=post_mock, + ): + with pytest.raises(HTTPException) as exc_info: + await guard.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + + # Both URLs must have been submitted for scanning. + assert post_mock.call_count == 2 + submitted = [call.kwargs["json"]["url"] for call in post_mock.call_args_list] + assert submitted == [ + "https://cdn.example.com/real.jpg", + "https://cdn.example.com/fake.jpg", + ] + # And the fake URL must be what surfaces in the error. + detail = exc_info.value.detail + assert isinstance(detail, dict) + assert detail["resemble"]["media_url"] == "https://cdn.example.com/fake.jpg" + assert detail["resemble"]["label"] == "fake" + + +@pytest.mark.asyncio +async def test_poll_treats_metric_less_completed_as_still_processing(): + """ + P2 regression: if the API reports ``completed`` but has no metrics yet, + the poll loop must not return that item — otherwise _evaluate_detection + falls through to ``unknown / 0.0`` and silently passes the request. + """ + guard = _make_guardrail(poll_interval_seconds=0.001, poll_timeout_seconds=0.05) + data = {"messages": [{"role": "user", "content": "https://cdn.example.com/x.mp3"}]} + + create_response = _fake_post_response( + {"success": True, "item": {"uuid": "mless", "status": "processing"}} + ) + # "completed" without any of metrics/image_metrics/video_metrics — must + # NOT be treated as terminal. The poll should keep looping until the + # deadline fires. + metric_less_response = _fake_get_response( + {"success": True, "item": {"uuid": "mless", "status": "completed"}}, + uuid="mless", + ) + + with ( + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=create_response, + ), + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get", + return_value=metric_less_response, + ) as get_mock, + ): + # With fail_closed=False (the default), a timeout fails open — the + # request passes through untouched. The important thing is that the + # poll loop kept polling instead of short-circuiting on the empty + # completed item. + result = await guard.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + + assert result == data + assert get_mock.call_count >= 2 # polled more than once, did not return early + + +@pytest.mark.asyncio +async def test_poll_returns_as_soon_as_metrics_arrive(): + """Companion to the P2 test: once metrics land, polling terminates.""" + guard = _make_guardrail(poll_interval_seconds=0.001, poll_timeout_seconds=1.0) + data = {"messages": [{"role": "user", "content": "https://cdn.example.com/x.mp3"}]} + + create_response = _fake_post_response( + {"success": True, "item": {"uuid": "late-metrics", "status": "processing"}} + ) + responses = [ + # First poll: completed but no metrics — keep polling. + _fake_get_response( + { + "success": True, + "item": {"uuid": "late-metrics", "status": "completed"}, + }, + uuid="late-metrics", + ), + # Second poll: metrics land, label = real → pass. + _fake_get_response( + { + "success": True, + "item": { + "uuid": "late-metrics", + "status": "completed", + "media_type": "audio", + "metrics": { + "label": "real", + "score": ["0.1"], + "aggregated_score": "0.1", + }, + }, + }, + uuid="late-metrics", + ), + ] + + with ( + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=create_response, + ), + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get", + side_effect=responses, + ) as get_mock, + ): + result = await guard.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + + assert result == data + assert get_mock.call_count == 2 diff --git a/uv.lock b/uv.lock index 04224dc5374..6f0c3814ed4 100644 --- a/uv.lock +++ b/uv.lock @@ -11,7 +11,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-04-08T16:01:27.663665Z" +exclude-newer = "2026-04-12T17:24:11.806442Z" exclude-newer-span = "P3D" [manifest] @@ -3602,7 +3602,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.83.6" +version = "1.83.7" source = { editable = "." } dependencies = [ { name = "aiohttp" },