diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index 949076aabf3..7e74bf5a579 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -1,7 +1,23 @@ -from typing import Dict, Optional +from typing import Any, Dict, Iterator, Optional from litellm.types.utils import StandardCallbackDynamicParams +_CLIENT_CALLBACK_METADATA_SLOTS: tuple[str, ...] = ("litellm_metadata", "metadata") + + +def iter_client_callback_metadata_dicts( + kwargs: dict[str, Any], +) -> Iterator[tuple[str, dict[str, Any]]]: + litellm_params = kwargs.get("litellm_params") + if isinstance(litellm_params, dict): + nested = litellm_params.get("metadata") + if isinstance(nested, dict): + yield "litellm_params.metadata", nested + for key in _CLIENT_CALLBACK_METADATA_SLOTS: + candidate = kwargs.get(key) + if isinstance(candidate, dict): + yield key, candidate + def _is_env_reference(value: object) -> bool: return isinstance(value, str) and "os.environ/" in value @@ -57,6 +73,7 @@ _supported_callback_params = [ "dd_site", "dd_agent_host", "dd_agent_port", + "turn_off_message_logging", ] _request_blocked_callback_params = { @@ -91,20 +108,14 @@ def initialize_standard_callback_dynamic_params( ) standard_callback_dynamic_params[param] = _param_value # type: ignore - # 2. Fallback: check "metadata" or "litellm_params" -> "metadata" - metadata = (kwargs.get("metadata") or {}).copy() - litellm_params = kwargs.get("litellm_params") or {} - if isinstance(litellm_params, dict): - metadata.update(litellm_params.get("metadata") or {}) - - if isinstance(metadata, dict): + for slot_label, metadata in iter_client_callback_metadata_dicts(kwargs): for param in _supported_callback_params: if param in _request_blocked_callback_params: continue if param not in standard_callback_dynamic_params and param in metadata: _param_value = metadata.get(param) validate_no_callback_env_reference( - param, _param_value, source="metadata" + param, _param_value, source=slot_label ) standard_callback_dynamic_params[param] = _param_value # type: ignore diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index e90b8a31a6e..93385204e4a 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -374,6 +374,18 @@ def is_request_body_safe( metadata = _coerce_metadata_to_dict(request_body.get(metadata_key)) if metadata is not None: _check_banned_params(metadata, general_settings, llm_router, model) + litellm_params = _coerce_metadata_to_dict(request_body.get("litellm_params")) + if litellm_params is not None: + litellm_params_metadata = _coerce_metadata_to_dict( + litellm_params.get("metadata") + ) + if litellm_params_metadata is not None: + _check_banned_params( + litellm_params_metadata, + general_settings, + llm_router, + model, + ) return True diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 22657f6250c..f4ddf96ddd0 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -15,6 +15,9 @@ from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging from litellm.constants import PRE_CALL_EXECUTED_GUARDRAILS_KEY from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + iter_client_callback_metadata_dicts, +) from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.url_utils import is_url_destination_allowed_by_host from litellm.proxy._types import ( @@ -296,6 +299,28 @@ def _key_or_team_allows_client_pricing_override( ) +def _strip_client_message_redaction_opt_out(data: dict[str, Any]) -> None: + stripped: list[str] = [] + if "turn_off_message_logging" in data and _is_false_like( + data["turn_off_message_logging"] + ): + stripped.append("turn_off_message_logging") + data.pop("turn_off_message_logging", None) + for slot_label, metadata in iter_client_callback_metadata_dicts(data): + if "turn_off_message_logging" in metadata and _is_false_like( + metadata["turn_off_message_logging"] + ): + stripped.append(f"{slot_label}.turn_off_message_logging") + metadata.pop("turn_off_message_logging", None) + if stripped: + verbose_proxy_logger.debug( + "Stripped client-supplied message-redaction opt-out fields from request body: %s. " + "Set `allow_client_message_redaction_opt_out: true` on the key or team metadata " + "to keep these values.", + ", ".join(stripped), + ) + + def _strip_client_pricing_overrides(data: Dict[str, Any]) -> None: """Drop pricing overrides from the request body and any metadata variant. @@ -1374,13 +1399,6 @@ async def add_litellm_data_to_request( # noqa: PLR0915 _headers, allow_client_message_redaction_opt_out=_allow_client_message_redaction_opt_out, ) - if ( - not _allow_client_message_redaction_opt_out - and litellm.turn_off_message_logging is True - and "turn_off_message_logging" in data - and _is_false_like(data["turn_off_message_logging"]) - ): - data.pop("turn_off_message_logging", None) verbose_proxy_logger.debug(f"Request Headers: {_headers}") verbose_proxy_logger.debug(f"Raw Headers: {_raw_headers}") @@ -1540,6 +1558,12 @@ async def add_litellm_data_to_request( # noqa: PLR0915 if not _key_or_team_allows_client_pricing_override(user_api_key_dict): _strip_client_pricing_overrides(data) + if ( + not _allow_client_message_redaction_opt_out + and litellm.turn_off_message_logging is True + ): + _strip_client_message_redaction_opt_out(data) + # Fill in the proxy_server_request body snapshot now that metadata has # been parsed. Consumers (standard_logging_payload, lago, # spend_tracking_utils, streaming_iterator) read `body` to audit the diff --git a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py index 3f4b446bea5..891e5020f37 100644 --- a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py +++ b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py @@ -56,69 +56,56 @@ async def test_global_redaction_on(): ) -@pytest.mark.parametrize("turn_off_message_logging", [True, False]) +@pytest.mark.parametrize( + "dynamic_turn_off, expect_redacted", + [(True, True), (False, False)], +) @pytest.mark.asyncio -async def test_global_redaction_ignores_dynamic_param(turn_off_message_logging): - """ - Request-body `turn_off_message_logging` is no longer honored as a dynamic - callback param — global setting (or admin-configured key/team config) wins. - With global redaction ON, the caller cannot disable redaction via the - request body. - """ +async def test_dynamic_turn_off_message_logging_overrides_global_on(dynamic_turn_off, expect_redacted): litellm.turn_off_message_logging = True test_custom_logger = TestCustomLogger() litellm.callbacks = [test_custom_logger] - response = await litellm.acompletion( + await litellm.acompletion( model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], - turn_off_message_logging=turn_off_message_logging, + turn_off_message_logging=dynamic_turn_off, mock_response="hello", ) await asyncio.sleep(1) standard_logging_payload = test_custom_logger.logged_standard_logging_payload assert standard_logging_payload is not None - print( - "logged standard logging payload", - json.dumps(standard_logging_payload, indent=2), - ) - response = standard_logging_payload["response"] - assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" - assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" + expected_response_content = "redacted-by-litellm" if expect_redacted else "hello" + expected_message_content = "redacted-by-litellm" if expect_redacted else "hi" + assert standard_logging_payload["response"]["choices"][0]["message"]["content"] == expected_response_content + assert standard_logging_payload["messages"][0]["content"] == expected_message_content -@pytest.mark.parametrize("turn_off_message_logging", [True, False]) +@pytest.mark.parametrize( + "dynamic_turn_off, expect_redacted", + [(True, True), (False, False)], +) @pytest.mark.asyncio -async def test_global_redaction_off_ignores_dynamic_param(turn_off_message_logging): - """ - Request-body `turn_off_message_logging` is no longer honored as a dynamic - callback param — global setting (or admin-configured key/team config) wins. - With global redaction OFF, the caller cannot enable redaction via the - request body. - """ +async def test_dynamic_turn_off_message_logging_overrides_global_off(dynamic_turn_off, expect_redacted): litellm.turn_off_message_logging = False test_custom_logger = TestCustomLogger() litellm.callbacks = [test_custom_logger] - response = await litellm.acompletion( + await litellm.acompletion( model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], - turn_off_message_logging=turn_off_message_logging, + turn_off_message_logging=dynamic_turn_off, mock_response="hello", ) await asyncio.sleep(1) standard_logging_payload = test_custom_logger.logged_standard_logging_payload assert standard_logging_payload is not None - print( - "logged standard logging payload", - json.dumps(standard_logging_payload, indent=2), - ) - assert ( - standard_logging_payload["response"]["choices"][0]["message"]["content"] - == "hello" - ) - assert standard_logging_payload["messages"][0]["content"] == "hi" + + expected_response_content = "redacted-by-litellm" if expect_redacted else "hello" + expected_message_content = "redacted-by-litellm" if expect_redacted else "hi" + assert standard_logging_payload["response"]["choices"][0]["message"]["content"] == expected_response_content + assert standard_logging_payload["messages"][0]["content"] == expected_message_content @pytest.mark.asyncio diff --git a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py index f63216b96d4..0dca4f3a1b1 100644 --- a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py +++ b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py @@ -7,9 +7,53 @@ sys.path.insert(0, os.path.abspath("../../..")) from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( initialize_standard_callback_dynamic_params, + iter_client_callback_metadata_dicts, ) +def test_iter_client_callback_metadata_dicts_covers_all_read_paths(): + md = {"m": 1} + lm = {"lm": 1} + lp_md = {"lp": 1} + slots = dict( + iter_client_callback_metadata_dicts( + { + "metadata": md, + "litellm_metadata": lm, + "litellm_params": {"metadata": lp_md}, + } + ) + ) + assert slots == { + "metadata": md, + "litellm_metadata": lm, + "litellm_params.metadata": lp_md, + } + + +def test_iter_client_callback_metadata_dicts_skips_non_dict_slots(): + slots = list( + iter_client_callback_metadata_dicts( + { + "metadata": "not-a-dict", + "litellm_metadata": None, + "litellm_params": {"metadata": []}, + } + ) + ) + assert slots == [] + + +def test_extractor_reads_turn_off_message_logging_from_every_slot(): + for kwargs in ( + {"metadata": {"turn_off_message_logging": True}}, + {"litellm_metadata": {"turn_off_message_logging": True}}, + {"litellm_params": {"metadata": {"turn_off_message_logging": True}}}, + ): + params = initialize_standard_callback_dynamic_params(kwargs) + assert params.get("turn_off_message_logging") is True, kwargs + + def test_resolves_plain_values_at_top_level(): kwargs = { "langfuse_public_key": "pk-test", @@ -36,6 +80,33 @@ def test_resolves_plain_values_from_metadata(): assert params.get("langfuse_host") == "https://test.langfuse.com" +def test_litellm_params_metadata_overrides_metadata(): + kwargs = { + "metadata": { + "langfuse_public_key": "pk-meta", + }, + "litellm_params": { + "metadata": { + "langfuse_public_key": "pk-litellm-params", + } + }, + } + + params = initialize_standard_callback_dynamic_params(kwargs) + + assert params.get("langfuse_public_key") == "pk-litellm-params" + + +def test_top_level_kwargs_overrides_metadata_slots(): + kwargs = { + "langfuse_public_key": "from-top-level", + "metadata": {"langfuse_public_key": "from-metadata"}, + "litellm_params": {"metadata": {"langfuse_public_key": "from-litellm-params"}}, + } + params = initialize_standard_callback_dynamic_params(kwargs) + assert params.get("langfuse_public_key") == "from-top-level" + + def test_env_reference_at_top_level_raises_with_guidance(): kwargs = {"langfuse_public_key": "os.environ/LANGFUSE_PUBLIC_KEY"} @@ -100,11 +171,17 @@ def test_non_string_values_are_not_flagged(): assert params.get("langsmith_sampling_rate") == 0.5 -def test_turn_off_message_logging_not_extracted_from_request(): - """turn_off_message_logging is admin-only — must not be settable via request.""" - kwargs = {"turn_off_message_logging": True} +@pytest.mark.parametrize( + "kwargs,expected", + [ + ({"turn_off_message_logging": False}, False), + ({"turn_off_message_logging": "False"}, "False"), + ({"metadata": {"turn_off_message_logging": True}}, True), + ], +) +def test_turn_off_message_logging_extracted_from_kwargs(kwargs, expected): params = initialize_standard_callback_dynamic_params(kwargs) - assert params.get("turn_off_message_logging") is None + assert params.get("turn_off_message_logging") == expected def test_empty_kwargs_returns_empty_params(): diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index 089c69c6229..a187d2ad9f2 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -1770,6 +1770,21 @@ class TestObservabilityCallbackBans: ) assert field in str(exc.value) + def test_observability_field_in_litellm_params_metadata_is_rejected(self): + with pytest.raises(ValueError) as exc: + is_request_body_safe( + request_body={ + "model": "gpt-4", + "litellm_params": { + "metadata": {"turn_off_message_logging": False} + }, + }, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + assert "turn_off_message_logging" in str(exc.value) + @pytest.mark.parametrize( "metadata_key", ["metadata", "litellm_metadata"], diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index f336c632546..91ea8e42ee4 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -824,10 +824,19 @@ async def test_add_litellm_data_to_request_strips_client_redaction_bypass_contro "model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "hello"}], "turn_off_message_logging": False, - "metadata": {"headers": {"litellm-disable-message-redaction": "true"}}, + "metadata": { + "headers": {"litellm-disable-message-redaction": "true"}, + "turn_off_message_logging": False, + }, "litellm_metadata": json.dumps( - {"headers": {"LiteLLM-Disable-Message-Redaction": "true"}} + { + "headers": {"LiteLLM-Disable-Message-Redaction": "true"}, + "turn_off_message_logging": "false", + } ), + "litellm_params": { + "metadata": {"turn_off_message_logging": False}, + }, }, request=request_mock, user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), @@ -839,6 +848,9 @@ async def test_add_litellm_data_to_request_strips_client_redaction_bypass_contro litellm.turn_off_message_logging = original_turn_off_message_logging assert "turn_off_message_logging" not in updated + assert "turn_off_message_logging" not in (updated.get("litellm_params") or {}).get("metadata", {}) + assert "turn_off_message_logging" not in updated["metadata"] + assert "turn_off_message_logging" not in (updated.get("litellm_metadata") or {}) assert "litellm-disable-message-redaction" not in { header.lower() for header in updated["metadata"]["headers"] } @@ -859,6 +871,158 @@ async def test_add_litellm_data_to_request_strips_client_redaction_bypass_contro } +@pytest.mark.parametrize( + "admin_metadata_kwargs", + [ + { + "metadata": { + "logging": [ + { + "callback_name": "langfuse", + "callback_type": "success_and_failure", + "callback_vars": {"turn_off_message_logging": False}, + } + ] + } + }, + { + "team_metadata": { + "logging": [ + { + "callback_name": "langfuse", + "callback_type": "success_and_failure", + "callback_vars": {"turn_off_message_logging": False}, + } + ] + } + }, + ], +) +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_admin_callback_vars_turn_off_message_logging_overrides_global( + admin_metadata_kwargs, +): + from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + initialize_standard_callback_dynamic_params, + ) + from litellm.litellm_core_utils.redact_messages import should_redact_message_logging + + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/v1/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + original_turn_off_message_logging = litellm.turn_off_message_logging + litellm.turn_off_message_logging = True + try: + updated = await add_litellm_data_to_request( + data={ + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hello"}], + }, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", **admin_metadata_kwargs), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated.get("turn_off_message_logging") == "False" + + dynamic_params = initialize_standard_callback_dynamic_params(updated) + assert dynamic_params.get("turn_off_message_logging") == "False" + + assert ( + should_redact_message_logging( + {"standard_callback_dynamic_params": dynamic_params} + ) + is False + ) + finally: + litellm.turn_off_message_logging = original_turn_off_message_logging + + +@pytest.mark.parametrize( + "admin_metadata_kwargs", + [ + { + "metadata": { + "logging": [ + { + "callback_name": "langfuse", + "callback_type": "success_and_failure", + "callback_vars": {"turn_off_message_logging": True}, + } + ] + } + }, + { + "team_metadata": { + "logging": [ + { + "callback_name": "langfuse", + "callback_type": "success_and_failure", + "callback_vars": {"turn_off_message_logging": True}, + } + ] + } + }, + ], +) +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_admin_callback_vars_turn_off_message_logging_enables_redaction_when_global_off( + admin_metadata_kwargs, +): + from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + initialize_standard_callback_dynamic_params, + ) + from litellm.litellm_core_utils.redact_messages import should_redact_message_logging + + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/v1/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + original_turn_off_message_logging = litellm.turn_off_message_logging + litellm.turn_off_message_logging = False + try: + updated = await add_litellm_data_to_request( + data={ + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hello"}], + }, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", **admin_metadata_kwargs), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated.get("turn_off_message_logging") == "True" + + dynamic_params = initialize_standard_callback_dynamic_params(updated) + assert dynamic_params.get("turn_off_message_logging") == "True" + + assert ( + should_redact_message_logging( + {"standard_callback_dynamic_params": dynamic_params} + ) + is True + ) + finally: + litellm.turn_off_message_logging = original_turn_off_message_logging + + @pytest.mark.parametrize( "auth_kwargs", [ @@ -891,7 +1055,10 @@ async def test_add_litellm_data_to_request_allows_redaction_opt_out_with_admin_o "model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "hello"}], "turn_off_message_logging": False, - "metadata": {"headers": {"litellm-disable-message-redaction": "true"}}, + "metadata": { + "headers": {"litellm-disable-message-redaction": "true"}, + "turn_off_message_logging": False, + }, "litellm_metadata": json.dumps( {"headers": {"LiteLLM-Disable-Message-Redaction": "true"}} ), @@ -906,6 +1073,7 @@ async def test_add_litellm_data_to_request_allows_redaction_opt_out_with_admin_o litellm.turn_off_message_logging = original_turn_off_message_logging assert updated["turn_off_message_logging"] is False + assert updated["metadata"]["turn_off_message_logging"] is False assert "litellm-disable-message-redaction" in { header.lower() for header in updated["metadata"]["headers"] }