mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-03 07:23:33 +00:00
fix(logging): replace a pydantic input_value whole instead of walking each cut
The scrub walked every "..." in the error and tried each quote around it to find the two ends pydantic shows of a long input. A request of dots made that walk cost about 48 Python steps per character on the event loop, with no cap tripping. Replace everything from a line's first input_value= to its last , input_type= (or to the line's end without one) instead. Finding both labels is linear, the scrub no longer needs the prompt's cut sides, and the cut-side work cap goes with them. A value that is not request text, such as None or a bad role, is now blanked too. A quote of the prompt that holds input_value= is replaced first, so the label replace can't split it. Other quotes go after, and a text's trailing label is left out of its quote, so a message such as "value" or ", input_value=" can't break the label of a real error.
This commit is contained in:
yucheng
parent
6909f27886
commit
91aa6a16b2
2 changed files with 166 additions and 83 deletions
|
|
@ -376,12 +376,8 @@ _REQUEST_TEXT_KEYS: Final = ("messages", "input", "prompt", "system", "instructi
|
|||
_STRUCTURAL_KEYS: Final = frozenset({"role", "type", "id", "tool_call_id", "name"})
|
||||
_MIN_UNQUOTED_TEXT_LENGTH: Final = 4
|
||||
_ESCAPED_CHARACTERS: Final = ('"', "\\")
|
||||
# pydantic shows a long input value as its repr's first 25 and last 24 characters around "...", quotes and
|
||||
# container keys included, so each side of the cut shows at most 24 characters of a quoted text
|
||||
_CUT_SIDE_LENGTH: Final = 24
|
||||
_CUT: Final = "..."
|
||||
_CUT_STARTS: Final = re.compile(f"(?={re.escape(_CUT)})")
|
||||
_QUOTES: Final = "'\""
|
||||
_INPUT_VALUE_LABEL: Final = "input_value="
|
||||
_INPUT_TYPE_LABEL: Final = ", input_type="
|
||||
_MAX_SCRUB_COMPARISONS: Final = 10_000_000
|
||||
|
||||
|
||||
|
|
@ -429,63 +425,11 @@ def _echo_forms(text: str) -> frozenset[str]:
|
|||
return frozenset({text, json.dumps(text)[1:-1], json.dumps(text, ensure_ascii=False)[1:-1], repr(text)[1:-1]})
|
||||
|
||||
|
||||
def _cut_sides(text: str) -> str:
|
||||
"""
|
||||
The start and end of ``text`` that a pydantic cut can show, for a text too long to be quoted whole, with
|
||||
``text``'s quote characters between them so repr escapes the sides as it escapes the whole ``text``.
|
||||
"""
|
||||
quotes: Final = "".join(quote for quote in "'\"" if quote in text)
|
||||
return f"{text[:_CUT_SIDE_LENGTH]}{quotes}{text[-_CUT_SIDE_LENGTH:]}"
|
||||
|
||||
|
||||
def _cut_echoes(text: str, forms: Collection[str]) -> tuple[tuple[str, str], ...] | None:
|
||||
"""
|
||||
Each side of a quoted value ``text`` cuts short with ``...`` that starts or ends one of ``forms``, as
|
||||
pydantic shows a long input value, with its pattern anchored to the quote and the cut. A side can hold
|
||||
the request's own quotes, so every quote within a side's length of the cut is tried as its opening or
|
||||
closing one. ``None`` when checking every side against ``forms`` is too large a search for a failure path.
|
||||
"""
|
||||
cuts: Final = tuple(match.start() for match in _CUT_STARTS.finditer(text))
|
||||
heads: Final = frozenset(itertools.chain.from_iterable(_cut_heads(text, cut) for cut in cuts))
|
||||
tails: Final = frozenset(itertools.chain.from_iterable(_cut_tails(text, cut) for cut in cuts))
|
||||
if (len(heads) + len(tails)) * len(forms) > _MAX_SCRUB_COMPARISONS:
|
||||
return None
|
||||
return (
|
||||
*(
|
||||
(head, f"(?<=['\"]){re.escape(head)}(?=\\.\\.\\.)")
|
||||
for head in heads
|
||||
if any(form.startswith(head) for form in forms)
|
||||
),
|
||||
*(
|
||||
(tail, f"(?<=\\.\\.\\.){re.escape(tail)}(?=['\"])")
|
||||
for tail in tails
|
||||
if any(form.endswith(tail) for form in forms)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _cut_heads(text: str, cut: int) -> tuple[str, ...]:
|
||||
"""Each run of ``text`` from just after a quote up to the ``...`` at ``cut``, no longer than a cut side."""
|
||||
return tuple(
|
||||
text[quote + 1 : cut] for quote in range(max(0, cut - _CUT_SIDE_LENGTH - 1), cut - 1) if text[quote] in _QUOTES
|
||||
)
|
||||
|
||||
|
||||
def _cut_tails(text: str, cut: int) -> tuple[str, ...]:
|
||||
"""Each run of ``text`` from the end of the ``...`` at ``cut`` up to just before a quote, no longer than a cut side."""
|
||||
start: Final = cut + len(_CUT)
|
||||
return tuple(
|
||||
text[start:quote]
|
||||
for quote in range(start + 1, min(len(text), start + _CUT_SIDE_LENGTH + 1))
|
||||
if text[quote] in _QUOTES
|
||||
)
|
||||
|
||||
|
||||
def scrub_request_text(text: str, texts: Collection[str]) -> str:
|
||||
"""
|
||||
``text`` with every quote of the request's own ``texts`` replaced by ``REDACTED_BY_LITELLM``, so an
|
||||
error that does not echo the prompt keeps its wording. A search too large for a failure path
|
||||
replaces the whole text.
|
||||
``text`` with every quote of the request's own ``texts`` and every pydantic ``input_value`` replaced by
|
||||
``REDACTED_BY_LITELLM``, so an error that holds neither keeps its wording. A search too large for a
|
||||
failure path replaces the whole text.
|
||||
"""
|
||||
if not text or not texts:
|
||||
return text
|
||||
|
|
@ -494,20 +438,38 @@ def scrub_request_text(text: str, texts: Collection[str]) -> str:
|
|||
forms: Final = frozenset(
|
||||
itertools.chain.from_iterable(_echo_forms(each) for each in texts if len(each) <= len(text))
|
||||
)
|
||||
cut_forms: Final = frozenset(
|
||||
itertools.chain.from_iterable(_echo_forms(_cut_sides(each)) for each in texts if len(each) > len(text))
|
||||
)
|
||||
cut_echoes: Final = _cut_echoes(text, forms | cut_forms)
|
||||
if cut_echoes is None:
|
||||
return REDACTED_BY_LITELLM
|
||||
echoes: Final = sorted(
|
||||
(*((form, _echo_pattern(form)) for form in forms if form in text), *cut_echoes),
|
||||
key=lambda echo: len(echo[0]),
|
||||
(form for form in (each.removesuffix(_INPUT_VALUE_LABEL) for each in forms) if form and form in text),
|
||||
key=len,
|
||||
reverse=True,
|
||||
)
|
||||
label_quotes: Final = tuple(form for form in echoes if _INPUT_VALUE_LABEL in form)
|
||||
other_quotes: Final = tuple(form for form in echoes if _INPUT_VALUE_LABEL not in form)
|
||||
return _with_echoes_replaced(_with_input_values_replaced(_with_echoes_replaced(text, label_quotes)), other_quotes)
|
||||
|
||||
|
||||
def _with_echoes_replaced(text: str, echoes: Collection[str]) -> str:
|
||||
"""``text`` with one pass that replaces each of ``echoes``, in order, by ``REDACTED_BY_LITELLM``."""
|
||||
if not echoes:
|
||||
return text
|
||||
return re.sub("|".join(pattern for _, pattern in echoes), REDACTED_BY_LITELLM, text)
|
||||
return re.sub("|".join(_echo_pattern(form) for form in echoes), REDACTED_BY_LITELLM, text)
|
||||
|
||||
|
||||
def _with_input_values_replaced(text: str) -> str:
|
||||
"""``text`` with the pydantic ``input_value`` on each of its lines replaced by ``REDACTED_BY_LITELLM``."""
|
||||
if _INPUT_VALUE_LABEL not in text:
|
||||
return text
|
||||
return "\n".join(_line_with_input_value_replaced(line) for line in text.split("\n"))
|
||||
|
||||
|
||||
def _line_with_input_value_replaced(line: str) -> str:
|
||||
"""``line`` from its first ``input_value=`` up to its last ``, input_type=``, or to its end without one, replaced."""
|
||||
start: Final = line.find(_INPUT_VALUE_LABEL)
|
||||
if start < 0:
|
||||
return line
|
||||
value_start: Final = start + len(_INPUT_VALUE_LABEL)
|
||||
type_start: Final = line.rfind(_INPUT_TYPE_LABEL, value_start)
|
||||
return f"{line[:value_start]}{REDACTED_BY_LITELLM}{line[type_start:] if type_start >= 0 else ''}"
|
||||
|
||||
|
||||
def _echo_pattern(form: str) -> str:
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ import copy
|
|||
import importlib
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
import tracemalloc
|
||||
from collections.abc import AsyncIterator, Callable, Mapping
|
||||
from datetime import datetime
|
||||
|
|
@ -2128,14 +2128,14 @@ def test_scrub_request_text_replaces_an_escaped_echo_of_a_prompt_that_escaping_c
|
|||
"text_longer_than_the_error_cut_inside_an_escaped_quote",
|
||||
],
|
||||
)
|
||||
def test_scrub_request_text_replaces_both_sides_of_a_pydantic_cut_input_value(value: object) -> None:
|
||||
def test_scrub_request_text_replaces_a_pydantic_cut_input_value(value: object) -> None:
|
||||
with pytest.raises(ValidationError, match="input_value=") as raised:
|
||||
TypeAdapter(int).validate_python(value)
|
||||
|
||||
scrubbed: Final = scrub_request_text(str(raised.value), request_texts(value))
|
||||
|
||||
assert "my ssn" not in scrubbed and "4321" not in scrubbed
|
||||
assert "redacted-by-litellm...redacted-by-litellm" in scrubbed
|
||||
assert "input_value=redacted-by-litellm, input_type=" in scrubbed
|
||||
|
||||
|
||||
SINGLE_QUOTED_PROMPT: Final = "account 'secret' " + "x" * 1_000 + " end 'private'"
|
||||
|
|
@ -2164,14 +2164,143 @@ DOTTED_PROMPT: Final = "account secret is there." + "x" * 1_000 + ".private secr
|
|||
"dots_beside_both_cuts",
|
||||
],
|
||||
)
|
||||
def test_scrub_request_text_replaces_pydantic_cut_sides_that_hold_the_prompts_own_quotes(value: object) -> None:
|
||||
def test_scrub_request_text_replaces_a_pydantic_input_value_that_holds_the_prompts_own_quotes(value: object) -> None:
|
||||
with pytest.raises(ValidationError, match="input_value=") as raised:
|
||||
TypeAdapter(int).validate_python(value)
|
||||
|
||||
scrubbed: Final = scrub_request_text(str(raised.value), request_texts(value))
|
||||
|
||||
assert not any(word in scrubbed for word in ("account", "said", "secret", "private"))
|
||||
assert "redacted-by-litellm...redacted-by-litellm" in scrubbed
|
||||
assert "input_value=redacted-by-litellm, input_type=" in scrubbed
|
||||
|
||||
|
||||
def test_scrub_request_text_replaces_every_line_of_a_pydantic_error_and_keeps_its_wording() -> None:
|
||||
prompt: Final = "account secret " + "x" * 200 + " private"
|
||||
with pytest.raises(ValidationError) as raised:
|
||||
TypeAdapter(tuple[int, int]).validate_python([prompt, prompt])
|
||||
|
||||
scrubbed: Final = scrub_request_text(str(raised.value), request_texts(prompt))
|
||||
|
||||
assert not any(word in scrubbed for word in ("account", "secret", "private"))
|
||||
assert scrubbed.count("input_value=redacted-by-litellm, input_type=str]") == 2
|
||||
assert "Input should be a valid integer" in scrubbed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"wrapped",
|
||||
[repr, lambda error: json.dumps({"detail": error}), lambda error: str({"error": {"message": error}})],
|
||||
ids=["repr", "json", "dict_str"],
|
||||
)
|
||||
def test_scrub_request_text_replaces_the_input_values_of_a_pydantic_error_wrapped_onto_one_line(
|
||||
wrapped: Callable[[str], str],
|
||||
) -> None:
|
||||
prompt: Final = "account 'secret' " + "x" * 200 + ' end "private"'
|
||||
with pytest.raises(ValidationError) as raised:
|
||||
TypeAdapter(tuple[int, int]).validate_python([prompt, prompt])
|
||||
|
||||
scrubbed: Final = scrub_request_text(wrapped(str(raised.value)), request_texts(prompt))
|
||||
|
||||
assert not any(word in scrubbed for word in ("account", "secret", "private"))
|
||||
assert "input_value=redacted-by-litellm" in scrubbed
|
||||
|
||||
|
||||
def test_scrub_request_text_replaces_an_input_value_that_holds_the_input_type_label() -> None:
|
||||
prompt: Final = "account, input_type=str] secret input_value='a' " + "x" * 200 + " private"
|
||||
with pytest.raises(ValidationError) as raised:
|
||||
TypeAdapter(int).validate_python(prompt)
|
||||
|
||||
scrubbed: Final = scrub_request_text(str(raised.value), request_texts(prompt))
|
||||
|
||||
assert not any(word in scrubbed for word in ("account", "secret", "private"))
|
||||
assert "input_value=redacted-by-litellm, input_type=str]" in scrubbed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"other_message",
|
||||
[
|
||||
"value",
|
||||
"input",
|
||||
"input_value",
|
||||
"input_value=",
|
||||
", input_value=",
|
||||
"int_parsing, input_value=",
|
||||
"_value=",
|
||||
"type",
|
||||
"input_type",
|
||||
", input_type=",
|
||||
],
|
||||
)
|
||||
def test_scrub_request_text_replaces_an_input_value_when_another_message_matches_its_label(other_message: str) -> None:
|
||||
prompt: Final = "account secret " + "x" * 200 + " private"
|
||||
with pytest.raises(ValidationError) as raised:
|
||||
TypeAdapter(int).validate_python(prompt)
|
||||
|
||||
scrubbed: Final = scrub_request_text(str(raised.value), request_texts([other_message, prompt]))
|
||||
|
||||
assert not any(word in scrubbed for word in ("account", "secret", "private"))
|
||||
|
||||
|
||||
def test_scrub_request_text_leaves_empty_quotes_when_a_message_is_only_the_input_value_label() -> None:
|
||||
error: Final = "empty value '' rejected"
|
||||
|
||||
assert scrub_request_text(error, request_texts("input_value=")) == error
|
||||
|
||||
|
||||
def test_scrub_request_text_replaces_to_the_end_of_the_line_without_an_input_type() -> None:
|
||||
error: Final = "bad value [type=int_parsing, input_value='account sec\nnext line"
|
||||
|
||||
assert scrub_request_text(error, request_texts("hi there")) == (
|
||||
"bad value [type=int_parsing, input_value=redacted-by-litellm\nnext line"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prompt",
|
||||
["account secret input_value=1 private", "input_value=account secret private"],
|
||||
ids=["middle", "start"],
|
||||
)
|
||||
def test_scrub_request_text_replaces_the_whole_of_a_quoted_prompt_that_holds_the_input_value_label(prompt: str) -> None:
|
||||
assert scrub_request_text(f"Invalid content: <{prompt}> rejected", request_texts(prompt)) == (
|
||||
"Invalid content: <redacted-by-litellm> rejected"
|
||||
)
|
||||
|
||||
|
||||
def test_scrub_request_text_replaces_a_quoted_prompt_that_ends_in_the_input_value_label() -> None:
|
||||
prompt: Final = "account secret private input_value="
|
||||
|
||||
scrubbed: Final = scrub_request_text(f"Invalid content: <{prompt}> rejected", request_texts(prompt))
|
||||
|
||||
assert scrubbed.startswith("Invalid content: <redacted-by-litellm")
|
||||
assert not any(word in scrubbed for word in ("account", "secret", "private"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [None, "critic"], ids=["none", "str_that_is_not_request_text"])
|
||||
def test_scrub_request_text_replaces_an_input_value_that_is_not_request_text(value: object) -> None:
|
||||
with pytest.raises(ValidationError) as raised:
|
||||
TypeAdapter(int).validate_python(value)
|
||||
|
||||
assert "input_value=redacted-by-litellm, input_type=" in scrub_request_text(
|
||||
str(raised.value), request_texts("hi there")
|
||||
)
|
||||
|
||||
|
||||
def test_scrub_request_text_keeps_an_input_value_when_the_request_has_no_text() -> None:
|
||||
error: Final = "bad value [type=int_type, input_value=None, input_type=NoneType]"
|
||||
|
||||
assert scrub_request_text(error, request_texts(None)) == error
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
["." * 1_000_000, "input_value=" * 100_000, "input_value=x, input_type=y\n" * 40_000, " ...'x" * 200_000],
|
||||
ids=["dots", "input_value_labels_on_one_line", "input_values_on_many_lines", "quoted_cuts"],
|
||||
)
|
||||
def test_scrub_request_text_stays_fast_on_a_large_error(error: str) -> None:
|
||||
started: Final = time.perf_counter()
|
||||
|
||||
scrub_request_text(error, request_texts("hi there"))
|
||||
|
||||
assert time.perf_counter() - started < 0.5
|
||||
|
||||
|
||||
def test_scrub_request_text_leaves_an_error_that_does_not_quote_the_prompt() -> None:
|
||||
|
|
@ -2190,14 +2319,6 @@ def test_scrub_request_text_replaces_the_whole_text_when_the_search_is_too_large
|
|||
assert scrub_request_text("x" * 5_000, texts) == "redacted-by-litellm"
|
||||
|
||||
|
||||
def test_scrub_request_text_replaces_the_whole_text_when_the_cut_side_search_is_too_large() -> None:
|
||||
rng: Final = random.Random(0)
|
||||
error: Final = "".join(rng.choice("'ab") if index % 10 else "..." for index in range(10_000))
|
||||
texts: Final = request_texts([f'say "hi" {index}' for index in range(800)])
|
||||
|
||||
assert scrub_request_text(error, texts) == "redacted-by-litellm"
|
||||
|
||||
|
||||
def test_scrub_request_text_never_copies_a_request_text_longer_than_the_error() -> None:
|
||||
error: Final = "litellm.RateLimitError: rate limit reached for gpt-4o"
|
||||
image_urls: Final = request_texts([f"data:image/png;base64,{index}{'A' * 1_000_000}" for index in range(4)])
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue