scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-03 02:22:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 80

fix(guardrails): harden Resemble Detect integration

Browse source
This commit is contained in:
devshahofficial 2026-05-26 16:20:32 -07:00
parent dad97087e2
commit cf30a36447
6 changed files with 59 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
docs/my-website/docs/proxy/guardrails/resemble_detect.md
View file

@ -39,8 +39,6 @@ guardrails:
# api_base: https://app.resemble.ai/api/v2
# Block media with aggregated_score >= threshold (default 0.5)
resemble_threshold: 0.5
# Optional: force a modality (audio | video | image)
# resemble_media_type: audio
# Identify the TTS vendor that produced flagged audio
resemble_audio_source_tracing: true
# Do not persist media on Resemble after the scan
@ -66,7 +64,7 @@ The guardrail looks for media URLs in (in order):
1. OpenAI-style multimodal content parts (`image_url`, `input_audio`)
2. Anthropic-style `source.url` parts (image, document)
3. Any `https://…` URL in message text that ends in a known audio/video/image extension
3. Any `https://...` URL in message text that ends in a Resemble-supported audio/video/image extension
4. `metadata.mediaUrl` (key configurable via `resemble_metadata_key`)
<Tabs>
@ -158,7 +156,6 @@ curl -i http://0.0.0.0:4000/v1/chat/completions \
| `api_key` | string | `RESEMBLE_API_KEY` env var | Resemble AI API token. |
| `api_base` | string | `https://app.resemble.ai/api/v2` | Override the Resemble API base URL (useful for sovereign deployments). |
| `resemble_threshold` | number | `0.5` | Aggregated score above which media is treated as fake (0.0–1.0). |
| `resemble_media_type` | enum | auto | Force `audio`, `video`, or `image`. Omit for auto-detect from extension / content type. |
| `resemble_audio_source_tracing` | bool | `false` | Return which TTS vendor generated flagged audio (ElevenLabs, Resemble AI, OpenAI, etc.). |
| `resemble_use_reverse_search` | bool | `false` | (Image only) search the web for matching images to improve accuracy. |
| `resemble_zero_retention_mode` | bool | `false` | Automatically delete submitted media after detection. URLs are redacted and filenames are tokenized. |

1
litellm/proxy/guardrails/guardrail_hooks/resemble/__init__.py
View file

@ -18,7 +18,6 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
event_hook=litellm_params.mode,
default_on=litellm_params.default_on,
threshold=getattr(litellm_params, "resemble_threshold", None),
media_type=getattr(litellm_params, "resemble_media_type", None),
audio_source_tracing=getattr(
litellm_params, "resemble_audio_source_tracing", None
),

29
litellm/proxy/guardrails/guardrail_hooks/resemble/resemble.py
View file

@ -53,7 +53,7 @@ if TYPE_CHECKING:
# strings and fragments are allowed. Kept intentionally simple — multimodal
# content parts and metadata lookups handle the non-URL-in-text cases.
MEDIA_URL_REGEX = re.compile(
r"https?://[^\s<>\"')\]}]+?\.(?:mp3|wav|m4a|flac|ogg|opus|aac|webm|mp4|mov|avi|mkv|jpg|jpeg|png|webp|gif)(?:\?[^\s<>\"')\]}]*)?",
r"https?://[^\s<>\"'),\]}]+?\.(?:mp3|wav|m4a|flac|ogg|opus|aac|amr|3gpp|3gp|webm|mp4|mov|avi|mkv|jpg|jpeg|png|webp|gif)(?:[?#][^\s<>\"'),\]}]*)?",
re.IGNORECASE,
)
@ -87,7 +87,6 @@ class ResembleGuardrail(CustomGuardrail):
api_key: Optional[str] = None,
api_base: Optional[str] = None,
threshold: Optional[float] = None,
media_type: Optional[Literal["audio", "video", "image"]] = None,
audio_source_tracing: Optional[bool] = None,
use_reverse_search: Optional[bool] = None,
zero_retention_mode: Optional[bool] = None,
@ -114,7 +113,6 @@ class ResembleGuardrail(CustomGuardrail):
).rstrip("/")
self.threshold: float = threshold if threshold is not None else 0.5
self.media_type: Optional[str] = media_type
self.audio_source_tracing: bool = bool(audio_source_tracing)
self.use_reverse_search: bool = bool(use_reverse_search)
self.zero_retention_mode: bool = bool(zero_retention_mode)
@ -128,12 +126,11 @@ class ResembleGuardrail(CustomGuardrail):
self.fail_closed: bool = bool(fail_closed)
verbose_proxy_logger.debug(
"Resemble guardrail initialized: name=%s threshold=%s media_type=%s "
"audio_source_tracing=%s use_reverse_search=%s zero_retention_mode=%s "
"fail_closed=%s",
"Resemble guardrail initialized: name=%s threshold=%s "
"audio_source_tracing=%s use_reverse_search=%s "
"zero_retention_mode=%s fail_closed=%s",
kwargs.get("guardrail_name", "unknown"),
self.threshold,
self.media_type,
self.audio_source_tracing,
self.use_reverse_search,
self.zero_retention_mode,
@ -444,8 +441,6 @@ class ResembleGuardrail(CustomGuardrail):
async def _create_and_poll_detection(self, media_url: str) -> Dict[str, Any]:
create_payload: Dict[str, Any] = {"url": media_url}
if self.media_type:
create_payload["media_type"] = self.media_type
if self.audio_source_tracing:
create_payload["audio_source_tracing"] = True
if self.use_reverse_search:
@ -499,7 +494,6 @@ class ResembleGuardrail(CustomGuardrail):
response = await self.async_handler.get(
url=poll_url,
headers=headers,
timeout=10.0,
)
response.raise_for_status()
body = response.json()
@ -545,21 +539,23 @@ class ResembleGuardrail(CustomGuardrail):
def _extract_label_and_score(self, item: Dict[str, Any]) -> Tuple[str, float]:
metrics = item.get("metrics")
if isinstance(metrics, dict):
if isinstance(metrics, dict) and metrics:
return (
str(metrics.get("label") or "unknown").lower(),
self._coerce_score(metrics.get("aggregated_score")),
self._coerce_score(
metrics.get("aggregated_score", metrics.get("score"))
),
)
image_metrics = item.get("image_metrics")
if isinstance(image_metrics, dict):
if isinstance(image_metrics, dict) and image_metrics:
return (
str(image_metrics.get("label") or "unknown").lower(),
self._coerce_score(image_metrics.get("score")),
)
video_metrics = item.get("video_metrics")
if isinstance(video_metrics, dict):
if isinstance(video_metrics, dict) and video_metrics:
return (
str(video_metrics.get("label") or "unknown").lower(),
self._coerce_score(video_metrics.get("score")),
@ -569,7 +565,10 @@ class ResembleGuardrail(CustomGuardrail):
@staticmethod
def _coerce_score(score: Any) -> float:
return float(score if score is not None else 0)
try:
return float(score if score is not None else 0)
except (TypeError, ValueError):
return 0.0
@staticmethod
def get_config_model() -> Optional[Type["GuardrailConfigModel"]]:

7
litellm/types/guardrails.py
View file

@ -453,13 +453,6 @@ class ResembleGuardrailParamsConfigModel(BaseModel):
"Default 0.5."
),
)
resemble_media_type: Optional[Literal["audio", "video", "image"]] = Field(
default=None,
description=(
"Optionally force audio / video / image. If omitted, Resemble "
"auto-detects from the file extension or content type."
),
)
resemble_audio_source_tracing: Optional[bool] = Field(
default=False,
description=(

9
litellm/types/proxy/guardrails/guardrail_hooks/resemble.py
View file

@ -1,4 +1,4 @@
from typing import Literal, Optional
from typing import Optional
from pydantic import Field
@ -28,13 +28,6 @@ class ResembleGuardrailConfigModel(GuardrailConfigModel):
"Default 0.5."
),
)
resemble_media_type: Optional[Literal["audio", "video", "image"]] = Field(
default=None,
description=(
"Optionally force audio / video / image. If omitted, Resemble "
"auto-detects from the file extension or content type."
),
)
resemble_audio_source_tracing: Optional[bool] = Field(
default=False,
description=(

47
tests/test_litellm/proxy/guardrails/guardrail_hooks/test_resemble.py
View file

@ -110,6 +110,25 @@ class TestExtractMediaUrls:
}
assert self.guard._extract_media_urls(data) == ["https://cdn.example.com/c.mp3"]
def test_plain_text_supported_resemble_extensions(self):
data = {
"messages": [
{
"role": "user",
"content": (
"Scan https://cdn.example.com/a.amr, "
"https://cdn.example.com/b.3gp#clip, and "
"https://cdn.example.com/c.3gpp?download=1"
),
}
]
}
assert self.guard._extract_media_urls(data) == [
"https://cdn.example.com/a.amr",
"https://cdn.example.com/b.3gp#clip",
"https://cdn.example.com/c.3gpp?download=1",
]
def test_openai_image_url_part(self):
data = {
"messages": [
@ -330,6 +349,16 @@ class TestEvaluateDetection:
assert result["verdict"] is False
assert result["score"] == 0.9
def test_empty_metrics_falls_through_to_image_metrics(self):
item = {
"metrics": {},
"image_metrics": {"label": "Fake", "score": 1.0},
}
result = self.guard._evaluate_detection(item)
assert result["verdict"] is False
assert result["label"] == "fake"
assert result["score"] == 1.0
def test_video_metrics_shape(self):
item = {"video_metrics": {"label": "real", "score": 0.2}}
result = self.guard._evaluate_detection(item)
@ -340,6 +369,18 @@ class TestEvaluateDetection:
result = self.guard._evaluate_detection(item)
assert result["score"] == 0.0
def test_non_numeric_score_falls_back_to_zero(self):
item = {"metrics": {"label": "real", "aggregated_score": "N/A"}}
result = self.guard._evaluate_detection(item)
assert result["score"] == 0.0
assert result["verdict"] is True
def test_metrics_score_is_used_when_aggregated_score_is_absent(self):
item = {"metrics": {"label": "fake", "score": 0.9}}
result = self.guard._evaluate_detection(item)
assert result["score"] == 0.9
assert result["verdict"] is False
# ---------------------------------------------------------------------------
# Hook behaviour tests (mocked HTTP)
@ -393,8 +434,7 @@ async def test_apply_guardrail_scans_generic_image_inputs():
assert result == inputs
assert (
post_mock.call_args.kwargs["json"]["url"]
== "https://cdn.example.com/image.jpg"
post_mock.call_args.kwargs["json"]["url"] == "https://cdn.example.com/image.jpg"
)
@ -651,7 +691,6 @@ async def test_pre_call_times_out_and_fails_open():
@pytest.mark.asyncio
async def test_create_payload_includes_flags():
guard = _make_guardrail(
media_type="audio",
audio_source_tracing=True,
use_reverse_search=True,
zero_retention_mode=True,
@ -693,7 +732,7 @@ async def test_create_payload_includes_flags():
assert call_kwargs["headers"]["Authorization"] == "Bearer test-key"
body = call_kwargs["json"]
assert body["url"] == "https://cdn.example.com/clip.mp3"
assert body["media_type"] == "audio"
assert "media_type" not in body
assert body["audio_source_tracing"] is True
assert body["use_reverse_search"] is True
assert body["zero_retention_mode"] is True