From fd4476b1308feba1da8e86cd162401ce1bb2da4c Mon Sep 17 00:00:00 2001 From: yassin Date: Fri, 18 Sep 2026 04:49:50 +0000 Subject: [PATCH] fix(guardrails): bound typesafe tuning params, preserve result tail, log fail-open status Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../guardrail_hooks/typesafe/typesafe.py | 26 +++++++++++++++++- .../guardrails/guardrail_hooks/typesafe.py | 7 ++++- .../guardrail_hooks/test_typesafe.py | 27 ++++++++++++------- 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/typesafe/typesafe.py b/litellm/proxy/guardrails/guardrail_hooks/typesafe/typesafe.py index 9fca6eaed20..b3cf5bcf800 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/typesafe/typesafe.py +++ b/litellm/proxy/guardrails/guardrail_hooks/typesafe/typesafe.py @@ -53,6 +53,7 @@ _JEV_TIMEOUT_SECONDS: Final = 30.0 DROPPED_RESULT_TEXT: Final = ( "[Tool result removed by TypeSafe compaction: judged no longer relevant to the current task]" ) +_ELISION_MARKER: Final = "\n... [middle truncated] ...\n" _STR_OBJECT_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) @@ -99,6 +100,17 @@ class _JevSystemOneResponse(BaseModel): _JEV_RESPONSE_ADAPTER: Final = TypeAdapter(_JevSystemOneResponse) +def _truncate_for_state(text: str, max_chars: int) -> str: + """Keeps the head and tail within ``max_chars`` so Jev sees both ends of a long result.""" + if len(text) <= max_chars: + return text + if max_chars <= len(_ELISION_MARKER): + return text[:max_chars] + budget: Final = max_chars - len(_ELISION_MARKER) + head: Final = budget // 2 + return text[:head] + _ELISION_MARKER + text[len(text) - (budget - head) :] + + def _question_instructions(question_id: str) -> str: return ( f"Is tool exchange `{question_id}` in `tool_exchanges` still needed by the assistant to " @@ -231,7 +243,7 @@ class TypeSafeGuardrail(CustomGuardrail): ) tool_exchanges[f"e{ordinal}"] = { "tool_calls": _tool_call_entries(messages[group[0]]), - "result": result_text[: self.max_result_chars_in_state], + "result": _truncate_for_state(result_text, self.max_result_chars_in_state), } return {"task": task, "system": system, "tool_exchanges": tool_exchanges} @@ -324,6 +336,18 @@ class TypeSafeGuardrail(CustomGuardrail): response: Final = await self._call_systemone(state, question_ids) end_time: Final = time.monotonic() if response is None: + self.add_standard_logging_guardrail_information_to_request_data( # pyright: ignore[reportUnknownMemberType] # untyped base helper + guardrail_json_response={ + "error": "TypeSafe evaluation unavailable; request forwarded uncompacted", + "model": self.jev_model, + }, + request_data=request_data, + guardrail_status="guardrail_failed_to_respond", + guardrail_provider="typesafe", + start_time=start_time, + end_time=end_time, + duration=end_time - start_time, + ) return inputs dropped_ordinals: Final = frozenset( diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/typesafe.py b/litellm/types/proxy/guardrails/guardrail_hooks/typesafe.py index 4e742bfc7be..59482d2e190 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/typesafe.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/typesafe.py @@ -10,6 +10,8 @@ class TypeSafeGuardrailOptionalParams(BaseModel): relevance_threshold: float | None = Field( default=None, + ge=0.0, + le=1.0, description=( "Relevance cutoff in [0, 1]. A completed tool exchange is dropped when Jev " "scores the probability that it is still needed below this value. Defaults to 0.2." @@ -17,14 +19,17 @@ class TypeSafeGuardrailOptionalParams(BaseModel): ) min_chars_to_evaluate: int | None = Field( default=None, + ge=0, description=( "Skip tool exchanges whose combined tool-result text is shorter than this many characters. Defaults to 200." ), ) max_result_chars_in_state: int | None = Field( default=None, + ge=1, description=( - "Tool result text is truncated to this many characters when sent to the Jev evaluator. Defaults to 4000." + "Tool result text is truncated to this many characters when sent to the Jev evaluator, " + "keeping the head and tail. Defaults to 4000." ), ) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_typesafe.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_typesafe.py index 8b732091d7c..e2e5fc2fefc 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_typesafe.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_typesafe.py @@ -40,7 +40,7 @@ TOOL_OUTPUT_LONG = "Result: EV range comparison. " * 40 # > 200 chars TOOL_OUTPUT_SHORT = "short" -def _exchange(call_id: str, tool_text: str, name: str = "web_search") -> list[dict]: +def _exchange(call_id: str, tool_text: str, name: str = "web_search") -> list[dict[str, object]]: return [ { "role": "assistant", @@ -57,7 +57,7 @@ def _exchange(call_id: str, tool_text: str, name: str = "web_search") -> list[di ] -def _messages(*, tail: list | None = None) -> list[dict]: +def _messages(*, tail: list[dict[str, object]] | None = None) -> list[dict[str, object]]: base = [ {"role": "system", "content": SYSTEM_TEXT}, {"role": "user", "content": USER_TEXT}, @@ -65,16 +65,21 @@ def _messages(*, tail: list | None = None) -> list[dict]: return base + (tail or []) -def _make_guardrail(handler: MagicMock | None = None, **kwargs) -> TypeSafeGuardrail: - defaults = dict( +def _make_guardrail( + handler: MagicMock | None = None, + *, + max_result_chars_in_state: int | None = None, + unreachable_fallback: str | None = None, +) -> TypeSafeGuardrail: + return TypeSafeGuardrail( api_base=FAKE_API_BASE, api_key=FAKE_API_KEY, guardrail_name="typesafe", default_on=True, async_handler=handler or _make_handler({"e0": 0.9}), + max_result_chars_in_state=max_result_chars_in_state, + unreachable_fallback=unreachable_fallback, ) - defaults.update(kwargs) - return TypeSafeGuardrail(**defaults) def _make_handler(answers: dict[str, float], status: int = 200) -> MagicMock: @@ -91,11 +96,13 @@ def _make_handler(answers: dict[str, float], status: int = 200) -> MagicMock: return handler -def _inputs(messages: list) -> GenericGuardrailAPIInputs: +def _inputs(messages: list[dict[str, object]]) -> GenericGuardrailAPIInputs: return GenericGuardrailAPIInputs(structured_messages=messages) -async def _apply(guardrail: TypeSafeGuardrail, messages: list, input_type: str = "request"): +async def _apply( + guardrail: TypeSafeGuardrail, messages: list[dict[str, object]], input_type: str = "request" +) -> GenericGuardrailAPIInputs: return await guardrail.apply_guardrail( inputs=_inputs(messages), request_data={}, @@ -188,7 +195,9 @@ async def test_request_body_shape_and_truncation(): assert "e0" in payload["questions"]["e0"]["instructions"] assert payload["state"]["task"] == USER_TEXT exchange = payload["state"]["tool_exchanges"]["e0"] - assert exchange["result"] == TOOL_OUTPUT_LONG[:50] + assert len(exchange["result"]) == 50 + assert exchange["result"].startswith(TOOL_OUTPUT_LONG[:10]) + assert exchange["result"].endswith(TOOL_OUTPUT_LONG[-11:]) assert exchange["tool_calls"] == [{"name": "web_search", "arguments": '{"query": "ev"}'}]