mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
fix(complexity_router): propagate turn_off_message_logging to internal sub-calls
The classifier and semantic-embedding sub-calls now capture proxy_server_request, but neither forwarded the caller's turn_off_message_logging opt-out. A caller who disabled message logging still had their prompt stored in the clear in these internal sub-calls' spend-log rows, since should_redact_message_logging reads the flag per-call and this internal call never inherited it.
This commit is contained in:
parent
78207064d8
commit
3d5b8e5960
2 changed files with 90 additions and 0 deletions
|
|
@ -113,6 +113,14 @@ def _classifier_call_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]
|
|||
}
|
||||
|
||||
|
||||
def _effective_turn_off_message_logging(request_kwargs: dict[str, Any] | None) -> bool | None:
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
|
||||
initialize_standard_callback_dynamic_params,
|
||||
)
|
||||
|
||||
return initialize_standard_callback_dynamic_params(request_kwargs or {}).get("turn_off_message_logging")
|
||||
|
||||
|
||||
class DimensionScore:
|
||||
"""Represents a score for a single dimension with optional signal."""
|
||||
|
||||
|
|
@ -430,6 +438,7 @@ class ComplexityRouter(CustomLogger):
|
|||
# internal classifier call) is responsible for reconciling.
|
||||
request_metadata = (request_kwargs or {}).get("litellm_metadata") or (request_kwargs or {}).get("metadata")
|
||||
metadata = _classifier_call_metadata(request_metadata)
|
||||
turn_off_message_logging = _effective_turn_off_message_logging(request_kwargs)
|
||||
|
||||
proxy_server_request = {
|
||||
"body": {
|
||||
|
|
@ -446,6 +455,7 @@ class ComplexityRouter(CustomLogger):
|
|||
timeout=llm_config.timeout_ms / 1000,
|
||||
metadata=metadata,
|
||||
proxy_server_request=proxy_server_request,
|
||||
turn_off_message_logging=turn_off_message_logging,
|
||||
)
|
||||
content = response.choices[0].message.content
|
||||
if not content:
|
||||
|
|
@ -832,6 +842,7 @@ class ComplexityRouter(CustomLogger):
|
|||
# key/team budget. Key/team attribution fields are preserved for spend logging.
|
||||
metadata = _classifier_call_metadata(request_kwargs.get("metadata"))
|
||||
litellm_metadata = _classifier_call_metadata(request_kwargs.get("litellm_metadata"))
|
||||
turn_off_message_logging = _effective_turn_off_message_logging(request_kwargs)
|
||||
proxy_server_request = {"body": {"model": self.config.embedding_model, "input": [user_message]}}
|
||||
query_vector = (
|
||||
await encoder.aencode_queries(
|
||||
|
|
@ -839,6 +850,7 @@ class ComplexityRouter(CustomLogger):
|
|||
metadata=metadata,
|
||||
litellm_metadata=litellm_metadata,
|
||||
proxy_server_request=proxy_server_request,
|
||||
turn_off_message_logging=turn_off_message_logging,
|
||||
)
|
||||
)[0]
|
||||
route_choice = await routelayer.acall(vector=query_vector)
|
||||
|
|
|
|||
|
|
@ -1465,6 +1465,54 @@ class TestLLMClassifier:
|
|||
"REASONING",
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aclassify_propagates_top_level_turn_off_message_logging(
|
||||
self, llm_complexity_router, mock_router_instance
|
||||
):
|
||||
"""A caller's top-level turn_off_message_logging must reach the classifier call.
|
||||
|
||||
Without this, a caller who opts a request out of message logging still has their
|
||||
prompt captured in full by the classifier's proxy_server_request: the spend-log
|
||||
redaction gate (should_redact_message_logging) reads turn_off_message_logging off
|
||||
the classifier call's own kwargs, and this internal call is not the caller's
|
||||
request, so it never inherits the opt-out unless it's forwarded explicitly.
|
||||
"""
|
||||
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}'))
|
||||
await llm_complexity_router.aclassify("secret prompt", request_kwargs={"turn_off_message_logging": True})
|
||||
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
|
||||
assert call_kwargs["turn_off_message_logging"] is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aclassify_propagates_metadata_slot_turn_off_message_logging(
|
||||
self, llm_complexity_router, mock_router_instance
|
||||
):
|
||||
"""turn_off_message_logging set inside metadata/litellm_metadata must also propagate.
|
||||
|
||||
initialize_standard_callback_dynamic_params reads this flag from either the
|
||||
top-level request kwargs or the metadata/litellm_metadata dicts (the same slots a
|
||||
real HTTP request populates), so the classifier call must resolve it from there too.
|
||||
"""
|
||||
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}'))
|
||||
await llm_complexity_router.aclassify(
|
||||
"secret prompt", request_kwargs={"litellm_metadata": {"turn_off_message_logging": True}}
|
||||
)
|
||||
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
|
||||
assert call_kwargs["turn_off_message_logging"] is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aclassify_defaults_turn_off_message_logging_to_none(
|
||||
self, llm_complexity_router, mock_router_instance
|
||||
):
|
||||
"""With no caller opt-out, the classifier call must not force redaction on or off.
|
||||
|
||||
Passing None (rather than omitting the kwarg or defaulting to False) preserves the
|
||||
existing header- and global-setting fallbacks in should_redact_message_logging.
|
||||
"""
|
||||
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}'))
|
||||
await llm_complexity_router.aclassify("hi")
|
||||
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
|
||||
assert call_kwargs["turn_off_message_logging"] is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aclassify_strips_budget_reservation_from_classifier_metadata(
|
||||
self, llm_complexity_router, mock_router_instance
|
||||
|
|
@ -2250,6 +2298,36 @@ class TestSemanticKeywordTierRules:
|
|||
assert body["model"] == "fake-embed"
|
||||
assert body["input"] == ["roll out my k8s cluster"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_embedding_call_propagates_turn_off_message_logging(self, basic_config):
|
||||
"""A caller's turn_off_message_logging must reach the query embedding call.
|
||||
|
||||
The embedding now captures the user's prompt in proxy_server_request, so a caller
|
||||
who opts out of message logging must have that opt-out forwarded; otherwise the
|
||||
embedding's spend-log row stores the prompt in the clear despite the parent request
|
||||
being redacted, exposing it to anyone authorized to read the team's spend logs.
|
||||
"""
|
||||
fake_router = FakeEmbeddingRouter()
|
||||
config = {
|
||||
**basic_config,
|
||||
"keyword_tier_rules": [{"keywords": ["kubernetes deployment"], "tier": "REASONING"}],
|
||||
"semantic_keyword_matching": True,
|
||||
"embedding_model": "fake-embed",
|
||||
"match_threshold": 0.5,
|
||||
}
|
||||
router = ComplexityRouter(
|
||||
model_name="test-router",
|
||||
litellm_router_instance=fake_router,
|
||||
complexity_router_config=config,
|
||||
)
|
||||
await router.async_pre_routing_hook(
|
||||
model="test-model",
|
||||
request_kwargs={"turn_off_message_logging": True},
|
||||
messages=[{"role": "user", "content": "roll out my k8s cluster"}],
|
||||
)
|
||||
assert fake_router.async_embedding_kwargs, "expected an embedding call for the prompt"
|
||||
assert fake_router.async_embedding_kwargs[0]["turn_off_message_logging"] is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_embedding_call_strips_budget_reservation(self, basic_config):
|
||||
"""The embedding call must not carry the parent request's budget reservation.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue