diff --git a/litellm/main.py b/litellm/main.py index 8a2d554da2f..703af1339e3 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -954,7 +954,8 @@ async def _async_streaming(response, model, custom_llm_provider, args): if asyncio.iscoroutine(response): response = await response async for line in response: - print_verbose(f"line in async streaming: {line}") + if not litellm.redact_messages_in_debug_logs: + print_verbose(f"line in async streaming: {line}") yield line except Exception as e: custom_llm_provider = custom_llm_provider or "openai" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 980dc818ae3..d91af4fa144 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4997,27 +4997,45 @@ def sentry_is_configured(general_settings: Mapping[str, object]) -> bool: ) -def _scrub_guardrail_inner(inner: dict[str, JsonValue]) -> None: +_DbConfigWarning: TypeAlias = tuple[str, tuple[object, ...]] + + +def _warn_db_config( + message: str, *args: object, warning_sink: Callable[[_DbConfigWarning], None] | None = None +) -> None: + if warning_sink is None: + verbose_proxy_logger.warning(message, *args) + else: + warning_sink((message, args)) + + +def _scrub_guardrail_inner( + inner: dict[str, JsonValue], warning_sink: Callable[[_DbConfigWarning], None] | None = None +) -> None: """Strip remote-URL entries from a guardrail's ``callbacks`` list and ``guardrail`` (v2 module-path) field. Mutates in place.""" cbs: Final = inner.get("callbacks") if isinstance(cbs, list): cleaned: Final = [c for c in cbs if not _is_remote_module_url(c)] if len(cleaned) != len(cbs): - verbose_proxy_logger.warning( + _warn_db_config( "Refused %d remote-URL entries from DB-overlay litellm_settings.guardrails[...].callbacks", len(cbs) - len(cleaned), + warning_sink=warning_sink, ) inner["callbacks"] = cleaned if _is_remote_module_url(inner.get("guardrail")): - verbose_proxy_logger.warning( + _warn_db_config( "Refused remote-URL guardrail module from DB-overlay litellm_settings.guardrails[...].guardrail: %r", inner.get("guardrail"), + warning_sink=warning_sink, ) inner["guardrail"] = None -def _scrub_db_overlay_remote_module_loads(section: str, db_value: object) -> object: +def _scrub_db_overlay_remote_module_loads( + section: str, db_value: object, warning_sink: Callable[[_DbConfigWarning], None] | None = None +) -> object: """Strip ``s3://`` / ``gcs://`` entries from the DB-overlay value for fields whose contents reach ``get_instance_fn``. The same scheme is allowed from a YAML config (the documented operator flow) but a @@ -5033,12 +5051,13 @@ def _scrub_db_overlay_remote_module_loads(section: str, db_value: object) -> obj for field in str_fields: v = sanitized.get(field) if _is_remote_module_url(v): - verbose_proxy_logger.warning( + _warn_db_config( "Refused remote-URL value for DB-overlay %s.%s=%r; only " "config.yaml entries may reference s3:// / gcs:// modules.", section, field, v, + warning_sink=warning_sink, ) sanitized[field] = None for field in list_fields: @@ -5046,13 +5065,14 @@ def _scrub_db_overlay_remote_module_loads(section: str, db_value: object) -> obj if isinstance(v, list): cleaned = [item for item in v if not _is_remote_module_url(item)] if len(cleaned) != len(v): - verbose_proxy_logger.warning( + _warn_db_config( "Refused %d remote-URL entries from DB-overlay %s.%s; " "only config.yaml entries may reference s3:// / gcs:// " "modules.", len(v) - len(cleaned), section, field, + warning_sink=warning_sink, ) sanitized[field] = cleaned # ``custom_provider_map`` is a list of dicts with ``custom_handler`` — @@ -5062,9 +5082,10 @@ def _scrub_db_overlay_remote_module_loads(section: str, db_value: object) -> obj if isinstance(cpm, list): for item in cpm: if isinstance(item, dict) and _is_remote_module_url(item.get("custom_handler")): - verbose_proxy_logger.warning( + _warn_db_config( "Refused remote-URL custom_handler from DB-overlay litellm_settings.custom_provider_map: %r", item.get("custom_handler"), + warning_sink=warning_sink, ) item["custom_handler"] = None # ``litellm_settings.guardrails`` is a list of single-key dicts in @@ -5082,19 +5103,20 @@ def _scrub_db_overlay_remote_module_loads(section: str, db_value: object) -> obj for inner in entry.values(): if not isinstance(inner, dict): continue - _scrub_guardrail_inner(inner) + _scrub_guardrail_inner(inner, warning_sink=warning_sink) lp = entry.get("litellm_params") if isinstance(lp, dict): - _scrub_guardrail_inner(lp) + _scrub_guardrail_inner(lp, warning_sink=warning_sink) # ``general_settings.litellm_jwtauth.custom_validate`` is a nested # string field. if section == "general_settings": jwt: Final = sanitized.get("litellm_jwtauth") if isinstance(jwt, dict) and _is_remote_module_url(jwt.get("custom_validate")): - verbose_proxy_logger.warning( + _warn_db_config( "Refused remote-URL custom_validate from DB-overlay general_settings.litellm_jwtauth: %r", jwt.get("custom_validate"), + warning_sink=warning_sink, ) jwt["custom_validate"] = None # ``pass_through_endpoints`` is a list of dicts whose ``target`` @@ -5106,12 +5128,13 @@ def _scrub_db_overlay_remote_module_loads(section: str, db_value: object) -> obj if isinstance(pte, list): for entry in pte: if isinstance(entry, dict) and _is_remote_module_url(entry.get("target")): - verbose_proxy_logger.warning( + _warn_db_config( "Refused remote-URL target from DB-overlay " "general_settings.pass_through_endpoints " "(path=%r): %r", entry.get("path"), entry.get("target"), + warning_sink=warning_sink, ) entry["target"] = None return sanitized @@ -6415,12 +6438,16 @@ class ProxyConfig: self._load_yaml_settings_stores(config) + pending_db_diagnostics: Final[list[tuple[str, JsonValue]]] = [] # mutable-ok: defer rows until privacy resolves + pending_db_warnings: Final[list[_DbConfigWarning]] = [] # mutable-ok: defer warnings until privacy resolves ## UPDATE CONFIG WITH DB if prisma_client is not None and store_model_in_db is True: config = await self._update_config_from_db( config=config, prisma_client=prisma_client, store_model_in_db=store_model_in_db, + diagnostic_sink=pending_db_diagnostics.append, + warning_sink=pending_db_warnings.append, ) config = self._config_with_resolved_settings(config) @@ -6433,6 +6460,10 @@ class ProxyConfig: config = self._check_for_os_environ_vars(config=config) self._apply_resolved_runtime_settings(config) + for section, param_value in pending_db_diagnostics: + _log_db_config_param(section, param_value) + for message, args in pending_db_warnings: + verbose_proxy_logger.warning(message, *args) self.update_config_state(config=config) @@ -8476,6 +8507,8 @@ class ProxyConfig: prisma_client: PrismaClient, config: dict, store_model_in_db: bool | None, + diagnostic_sink: Callable[[tuple[str, JsonValue]], None] | None = None, + warning_sink: Callable[[_DbConfigWarning], None] | None = None, ): if store_model_in_db is not True: verbose_proxy_logger.info("'store_model_in_db' is not True, skipping db updates") @@ -8493,13 +8526,14 @@ class ProxyConfig: if param_value is None: continue - verbose_proxy_logger.debug( - "param_name=%s, param_value=%s", - section, - _redact_config_param_value_for_logging(section, param_value), - ) + if diagnostic_sink is None: + _log_db_config_param(section, param_value) + else: + diagnostic_sink((section, param_value)) if section == "litellm_settings": - self._apply_litellm_settings_db_values(self._prepared_db_settings_values(section, param_value)) + self._apply_litellm_settings_db_values( + self._prepared_db_settings_values(section, param_value, warning_sink=warning_sink) + ) elif section == "environment_variables": await self._apply_db_environment_variables_row( prisma_client, response, environment_before_db, environment_read @@ -8507,10 +8541,10 @@ class ProxyConfig: else: self._settings_stores[section].apply_db_row( section, - self._prepared_db_settings_values(section, param_value), + self._prepared_db_settings_values(section, param_value, warning_sink=warning_sink), ) - self._warn_about_shadowed_db_settings() + self._warn_about_shadowed_db_settings(warning_sink=warning_sink) return self._config_with_resolved_settings(config) async def _apply_db_environment_variables_row( @@ -8559,18 +8593,24 @@ class ProxyConfig: revision: Final = _config_row_revision(row) return revision is None or revision < applied_revision - def _warn_about_shadowed_db_settings(self) -> None: + def _warn_about_shadowed_db_settings(self, warning_sink: Callable[[_DbConfigWarning], None] | None = None) -> None: shadowed: Final[frozenset[tuple[Section, str]]] = frozenset( (section, key) for section, store in self._settings_stores.items() for key in store.shadowed_db_keys() ) for section, key in sorted(shadowed - self._warned_shadowed_keys): - verbose_proxy_logger.warning( - "%s", config_ownership_message(section=section, key=key, shadows_db_value=True) + _warn_db_config( + "%s", + config_ownership_message(section=section, key=key, shadows_db_value=True), + warning_sink=warning_sink, ) self._warned_shadowed_keys = shadowed - def _prepared_db_settings_values(self, section: Section, value: object) -> Mapping[str, SettingsJsonValue]: - scrubbed: Final = _scrub_db_overlay_remote_module_loads(section=section, db_value=value) + def _prepared_db_settings_values( + self, section: Section, value: object, warning_sink: Callable[[_DbConfigWarning], None] | None = None + ) -> Mapping[str, SettingsJsonValue]: + scrubbed: Final = _scrub_db_overlay_remote_module_loads( + section=section, db_value=value, warning_sink=warning_sink + ) return _as_settings_mapping(scrubbed) def _apply_db_environment_variables( @@ -20175,6 +20215,14 @@ def _redact_config_param_value_for_logging(param_name: str | None, param_value: return param_value +def _log_db_config_param(section: str, param_value: JsonValue) -> None: + verbose_proxy_logger.debug( + "param_name=%s, param_value=%s", + section, + _redact_config_param_value_for_logging(section, param_value), + ) + + def _redact_general_setting_value(field_name: str, value: JsonValue, is_full_admin: bool) -> JsonValue: if is_full_admin: return value diff --git a/tests/unit/llms/codestral/completion/test_codestral_completion_handler.py b/tests/unit/llms/codestral/completion/test_codestral_completion_handler.py index 9ce50d8ceca..5d6b1cf5d27 100644 --- a/tests/unit/llms/codestral/completion/test_codestral_completion_handler.py +++ b/tests/unit/llms/codestral/completion/test_codestral_completion_handler.py @@ -48,3 +48,32 @@ async def test_fim_stream_keeps_unicode_line_separators_inside_a_delta( assert texts == [_LINE_SEPARATED_TEXT, None] assert json.loads(route.calls.last.request.content)["stream"] is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize("privacy", [False, True]) +async def test_fim_response_content_is_preserved_when_diagnostics_are_private( + privacy: bool, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "redact_messages_in_debug_logs", privacy) + monkeypatch.setattr(litellm, "set_verbose", True) + route: Final = respx_mock.post("https://codestral.mistral.ai/v1/fim/completions").mock( + return_value=httpx.Response(200, json={ + "id": "fim-private", "object": "text_completion", "created": 1, "model": "codestral-latest", + "choices": [{"index": 0, "message": {"content": "PRIVATE_FIM_REPLY"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 2, "completion_tokens": 3, "total_tokens": 5}, + }) + ) + + response: Final = await litellm.atext_completion( + model="text-completion-codestral/codestral-latest", + prompt="PRIVATE_FIM_PROMPT", api_key="test-codestral-key", max_retries=0, + ) + + assert response.choices[0].text == "PRIVATE_FIM_REPLY" + assert json.loads(route.calls.last.request.content)["prompt"] == "PRIVATE_FIM_PROMPT" + assert ("PRIVATE_FIM_REPLY" in capsys.readouterr().out) is not privacy diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index 6bd1468cfc1..93d0b778e26 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -6271,3 +6271,109 @@ async def test_proxy_config_validates_advertised_mcp_versions_at_load(tmp_path, return with pytest.raises(ValidationError): await ProxyConfig().load_config(router=None, config_file_path=str(config)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("privacy", [False, True]) +async def test_proxy_config_load_applies_diagnostic_privacy( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, privacy: bool +) -> None: + config_file: Final = tmp_path / "privacy.yaml" + config_file.write_text( + "model_list: []\nlitellm_settings:\n redact_messages_in_debug_logs: " + str(privacy).lower() + "\n" + ) + monkeypatch.setattr(litellm, "redact_messages_in_debug_logs", not privacy) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + + await ProxyConfig().load_config(router=None, config_file_path=str(config_file)) + + assert litellm.redact_messages_in_debug_logs is privacy + + +@pytest.mark.asyncio +@pytest.mark.parametrize("file_privacy", [None, False, True]) +@pytest.mark.parametrize("db_privacy", [False, True, "invalid-boolean"]) +@pytest.mark.parametrize("env_reference", [False, True]) +async def test_proxy_config_privacy_precedes_database_settings_diagnostics( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + file_privacy: bool | None, + db_privacy: bool | str, + env_reference: bool, +) -> None: + from litellm.caching.dual_cache import DualCache + from litellm.proxy import proxy_server + from litellm.proxy import utils as proxy_utils + + monkeypatch.setenv("LITELLM_QA_DB_PRIVACY", str(db_privacy).lower()) + monkeypatch.setenv("LITELLM_QA_FILE_PRIVACY", str(file_privacy).lower()) + file_value: Final = "os.environ/LITELLM_QA_FILE_PRIVACY" if env_reference else file_privacy + remote_module: Final = "s3://example-bucket/PRIVATE_DB_CONTENT.py" + stored: Final = { + "general_settings": { + "alerting": ["PRIVATE_DB_CONTENT"], + "custom_auth": remote_module, + "litellm_jwtauth": {"custom_validate": remote_module}, + "pass_through_endpoints": [{"path": "/PRIVATE_DB_CONTENT", "target": remote_module}], + "PRIVATE_DB_SETTING": "database-value", + }, + "router_settings": {"model_group_alias": {"PRIVATE_DB_CONTENT": "qa-model"}}, + "litellm_settings": { + "redact_messages_in_debug_logs": "os.environ/LITELLM_QA_DB_PRIVACY" if env_reference else db_privacy, + "model_alias_map": {"PRIVATE_DB_CONTENT": "qa-model"}, + "callbacks": [remote_module, "prometheus"], + "custom_provider_map": [{"provider": "qa", "custom_handler": remote_module}], + "guardrails": [{"qa": {"callbacks": [remote_module], "guardrail": remote_module}}], + }, + } + + async def get_generic_data(key: str, value: str, table_name: str) -> SimpleNamespace | None: + assert (key, table_name) == ("param_name", "config") + row: Final = stored.get(value) + return SimpleNamespace(param_name=value, param_value=row) if row is not None else None + + monkeypatch.setattr(litellm, "redact_messages_in_debug_logs", False) + monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(get_generic_data=get_generic_data)) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_utils, "litellm_config_cache", DualCache()) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + config_file: Final = tmp_path / "db-privacy.yaml" + config_file.write_text( + json.dumps( + { + "litellm_settings": {} if file_privacy is None else {"redact_messages_in_debug_logs": file_value}, + "general_settings": {"PRIVATE_DB_SETTING": "file-value"}, + } + ) + ) + + enabled: Final = db_privacy if file_privacy is None else file_privacy + if isinstance(enabled, str): + with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"), pytest.raises(ValidationError): + await ProxyConfig().get_config(str(config_file)) + assert "PRIVATE_DB_CONTENT" not in caplog.text + assert litellm.redact_messages_in_debug_logs is False + return + + with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"): + loaded: Final = await ProxyConfig().get_config(str(config_file)) + + assert ("PRIVATE_DB_CONTENT" in caplog.text) is not enabled + assert litellm.redact_messages_in_debug_logs is enabled + assert loaded["litellm_settings"]["redact_messages_in_debug_logs"] is enabled + assert loaded["litellm_settings"]["model_alias_map"] == {"PRIVATE_DB_CONTENT": "qa-model"} + assert loaded["general_settings"] == { + "alerting": ["PRIVATE_DB_CONTENT"], + "litellm_jwtauth": {"custom_validate": None}, + "pass_through_endpoints": [{"path": "/PRIVATE_DB_CONTENT", "target": None}], + "PRIVATE_DB_SETTING": "file-value", + } + assert loaded["litellm_settings"]["callbacks"] == ["prometheus"] + assert loaded["litellm_settings"]["custom_provider_map"] == [{"provider": "qa", "custom_handler": None}] + assert loaded["litellm_settings"]["guardrails"] == [{"qa": {"callbacks": [], "guardrail": None}}] + assert stored["general_settings"]["custom_auth"] == remote_module + assert any(record.levelno == logging.WARNING for record in caplog.records) + assert loaded["router_settings"] == stored["router_settings"] diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index a1d083ee3a9..a4d5d073352 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -12619,3 +12619,104 @@ async def test_stream_failure_retains_error_status_without_debug_logging( else: assert any("PRIVATE_STREAM_FAILURE_42896" in message for message in messages) assert all("event=request_failed" not in message for message in messages) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("privacy", [False, True]) +async def test_nonstream_response_keeps_content_with_private_ready_diagnostic( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, privacy: bool +) -> None: + import logging + + import respx + + from litellm._logging import verbose_proxy_logger + from litellm.proxy.proxy_server import ProxyConfig as ServerProxyConfig + + monkeypatch.setattr(litellm, "redact_messages_in_debug_logs", privacy) + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(verbose_proxy_logger, "level", logging.DEBUG) + ProxyLogging._callback_capabilities_cache.clear() + content: Final = "PRIVATE_NONSTREAM_RESPONSE_42896" + processor: Final = ProxyBaseLLMRequestProcessing( + data={ + "model": "openai/gpt-4o-mini", + "api_base": "https://example.com/v1", + "api_key": "test-key", + "messages": [{"role": "user", "content": "PRIVATE_NONSTREAM_INPUT_42896"}], + } + ) + with respx.mock() as upstream: + upstream.post("https://example.com/v1/chat/completions").respond( + 200, + json={ + "id": "chatcmpl-example", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }, + ) + response: Final = await processor.base_process_llm_request( + request=Request( + {"type": "http", "method": "POST", "path": "/v1/chat/completions", "headers": [], "query_string": b""} + ), + fastapi_response=Response(), + user_api_key_dict=ProxyUserAPIKeyAuth(), + route_type="acompletion", + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + general_settings={}, + proxy_config=ServerProxyConfig(), + ) + assert isinstance(response, litellm.ModelResponse) + assert isinstance(response.choices[0], litellm.Choices) + assert response.choices[0].message.content == content + assert response.choices[0].finish_reason == "stop" + messages: Final = tuple( + record.getMessage() for record in caplog.records if record.name == verbose_proxy_logger.name + ) + if privacy: + assert any("event=response_ready status=200 elapsed_seconds=" in message for message in messages) + assert all("PRIVATE_NONSTREAM_" not in message for message in messages) + else: + assert all("event=response_ready" not in message for message in messages) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("privacy", [False, True]) +async def test_nonstream_failure_preserves_client_error_and_safe_status_without_debug( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, privacy: bool +) -> None: + import logging + + from litellm._logging import verbose_proxy_logger + + monkeypatch.setattr(litellm, "redact_messages_in_debug_logs", privacy) + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(verbose_proxy_logger, "level", logging.ERROR) + ProxyLogging._callback_capabilities_cache.clear() + processor: Final = ProxyBaseLLMRequestProcessing(data={"model": "example-model"}) + failure: Final = HTTPException(status_code=429, detail="PRIVATE_NONSTREAM_FAILURE_42896") + with pytest.raises(ProxyException) as raised: + await processor.handle_llm_api_exception( + e=failure, + user_api_key_dict=ProxyUserAPIKeyAuth(), + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + assert raised.value.code == "429" + assert raised.value.message == "PRIVATE_NONSTREAM_FAILURE_42896" + messages: Final = tuple( + record.getMessage() for record in caplog.records if record.name == verbose_proxy_logger.name + ) + if privacy: + assert any("event=request_failed status=429" in message for message in messages) + assert all("PRIVATE_NONSTREAM_FAILURE_42896" not in message for message in messages) + else: + assert any("PRIVATE_NONSTREAM_FAILURE_42896" in message for message in messages) + assert all("event=request_failed" not in message for message in messages) diff --git a/tests/unit/responses/mcp/test_mcp_streaming_iterator.py b/tests/unit/responses/mcp/test_mcp_streaming_iterator.py index a9d4e7067a1..74026c0d0db 100644 --- a/tests/unit/responses/mcp/test_mcp_streaming_iterator.py +++ b/tests/unit/responses/mcp/test_mcp_streaming_iterator.py @@ -591,3 +591,40 @@ async def test_stream_without_auto_execute_is_forwarded_unchanged(monkeypatch): assert [c.output_index for c in chunks if hasattr(c, "output_index")] == [0, 0] assert [c.sequence_number for c in chunks if hasattr(c, "sequence_number")] == [0, 1, 2] aresponses_mock.assert_not_called() + + +@pytest.mark.parametrize("redact", (False, True)) +@pytest.mark.asyncio +async def test_malformed_provider_tool_arguments_keep_error_diagnostics_private( + monkeypatch: pytest.MonkeyPatch, redact: bool, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.setattr(litellm, "redact_messages_in_debug_logs", redact) + response: Final = ResponsesAPIResponse( + id="resp-malformed-arguments", + created_at=0, + output=[ + { + "type": "function_call", + "call_id": "call-1", + "name": "read_wiki_contents", + "arguments": {"text": "PRIVATE_TOOL_ARGUMENT"}, + } + ], + ) + iterator: Final = _make_iterator([]) + iterator.collected_response = response + + await iterator._generate_tool_execution_events() + + assert iterator.tool_results == [] + assert iterator._tool_results_for_response is response + assert iterator.collected_response.output == [ + { + "type": "function_call", + "call_id": "call-1", + "name": "read_wiki_contents", + "arguments": {"text": "PRIVATE_TOOL_ARGUMENT"}, + } + ] + assert [event.type for event in iterator.tool_execution_events] == [ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED] + assert ("PRIVATE_TOOL_ARGUMENT" in capsys.readouterr().err) is not redact diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index e10352e0b3d..f4e6acb93b4 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -10713,3 +10713,39 @@ async def test_bedrock_strict_tools_limit_rejection_is_not_sent_again_when_no_to assert error is not None and "Too many strict tools" in str(error) assert len(sent) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("privacy", [False, True]) +async def test_async_streaming_keeps_chunks_without_rendering_private_content( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], privacy: bool +) -> None: + from collections.abc import AsyncIterator + + class Chunk: + content: Final = "PRIVATE_ASYNC_STREAM_42896" + + def __str__(self) -> str: + if privacy: + raise AssertionError("private chunks must not be rendered for diagnostics") + return self.content + + monkeypatch.setattr(litellm, "redact_messages_in_debug_logs", privacy) + monkeypatch.setattr(litellm, "set_verbose", True) + chunk: Final = Chunk() + + async def upstream() -> AsyncIterator[Chunk]: + yield chunk + + stream: Final = litellm_main._async_streaming(upstream(), "example-model", "openai", {}) + observed: Final = tuple([item async for item in stream]) + + assert len(observed) == 1 + assert observed[0] is chunk + assert chunk.content == "PRIVATE_ASYNC_STREAM_42896" + output: Final = capsys.readouterr().out + assert ("PRIVATE_ASYNC_STREAM_42896" in output) is not privacy + if privacy: + assert output.strip() == "redacted-by-litellm" + else: + assert "line in async streaming: PRIVATE_ASYNC_STREAM_42896" in output