mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-06 07:48:06 +00:00
fix(logging): protect database config and streaming diagnostics
This commit is contained in:
moe-berri
parent
56372ea04f
commit
9cc438c228
7 changed files with 383 additions and 25 deletions
|
|
@ -954,7 +954,8 @@ async def _async_streaming(response, model, custom_llm_provider, args):
|
|||
if asyncio.iscoroutine(response):
|
||||
response = await response
|
||||
async for line in response:
|
||||
print_verbose(f"line in async streaming: {line}")
|
||||
if not litellm.redact_messages_in_debug_logs:
|
||||
print_verbose(f"line in async streaming: {line}")
|
||||
yield line
|
||||
except Exception as e:
|
||||
custom_llm_provider = custom_llm_provider or "openai"
|
||||
|
|
|
|||
|
|
@ -4997,27 +4997,45 @@ def sentry_is_configured(general_settings: Mapping[str, object]) -> bool:
|
|||
)
|
||||
|
||||
|
||||
def _scrub_guardrail_inner(inner: dict[str, JsonValue]) -> None:
|
||||
_DbConfigWarning: TypeAlias = tuple[str, tuple[object, ...]]
|
||||
|
||||
|
||||
def _warn_db_config(
|
||||
message: str, *args: object, warning_sink: Callable[[_DbConfigWarning], None] | None = None
|
||||
) -> None:
|
||||
if warning_sink is None:
|
||||
verbose_proxy_logger.warning(message, *args)
|
||||
else:
|
||||
warning_sink((message, args))
|
||||
|
||||
|
||||
def _scrub_guardrail_inner(
|
||||
inner: dict[str, JsonValue], warning_sink: Callable[[_DbConfigWarning], None] | None = None
|
||||
) -> None:
|
||||
"""Strip remote-URL entries from a guardrail's ``callbacks`` list
|
||||
and ``guardrail`` (v2 module-path) field. Mutates in place."""
|
||||
cbs: Final = inner.get("callbacks")
|
||||
if isinstance(cbs, list):
|
||||
cleaned: Final = [c for c in cbs if not _is_remote_module_url(c)]
|
||||
if len(cleaned) != len(cbs):
|
||||
verbose_proxy_logger.warning(
|
||||
_warn_db_config(
|
||||
"Refused %d remote-URL entries from DB-overlay litellm_settings.guardrails[...].callbacks",
|
||||
len(cbs) - len(cleaned),
|
||||
warning_sink=warning_sink,
|
||||
)
|
||||
inner["callbacks"] = cleaned
|
||||
if _is_remote_module_url(inner.get("guardrail")):
|
||||
verbose_proxy_logger.warning(
|
||||
_warn_db_config(
|
||||
"Refused remote-URL guardrail module from DB-overlay litellm_settings.guardrails[...].guardrail: %r",
|
||||
inner.get("guardrail"),
|
||||
warning_sink=warning_sink,
|
||||
)
|
||||
inner["guardrail"] = None
|
||||
|
||||
|
||||
def _scrub_db_overlay_remote_module_loads(section: str, db_value: object) -> object:
|
||||
def _scrub_db_overlay_remote_module_loads(
|
||||
section: str, db_value: object, warning_sink: Callable[[_DbConfigWarning], None] | None = None
|
||||
) -> object:
|
||||
"""Strip ``s3://`` / ``gcs://`` entries from the DB-overlay value for
|
||||
fields whose contents reach ``get_instance_fn``. The same scheme is
|
||||
allowed from a YAML config (the documented operator flow) but a
|
||||
|
|
@ -5033,12 +5051,13 @@ def _scrub_db_overlay_remote_module_loads(section: str, db_value: object) -> obj
|
|||
for field in str_fields:
|
||||
v = sanitized.get(field)
|
||||
if _is_remote_module_url(v):
|
||||
verbose_proxy_logger.warning(
|
||||
_warn_db_config(
|
||||
"Refused remote-URL value for DB-overlay %s.%s=%r; only "
|
||||
"config.yaml entries may reference s3:// / gcs:// modules.",
|
||||
section,
|
||||
field,
|
||||
v,
|
||||
warning_sink=warning_sink,
|
||||
)
|
||||
sanitized[field] = None
|
||||
for field in list_fields:
|
||||
|
|
@ -5046,13 +5065,14 @@ def _scrub_db_overlay_remote_module_loads(section: str, db_value: object) -> obj
|
|||
if isinstance(v, list):
|
||||
cleaned = [item for item in v if not _is_remote_module_url(item)]
|
||||
if len(cleaned) != len(v):
|
||||
verbose_proxy_logger.warning(
|
||||
_warn_db_config(
|
||||
"Refused %d remote-URL entries from DB-overlay %s.%s; "
|
||||
"only config.yaml entries may reference s3:// / gcs:// "
|
||||
"modules.",
|
||||
len(v) - len(cleaned),
|
||||
section,
|
||||
field,
|
||||
warning_sink=warning_sink,
|
||||
)
|
||||
sanitized[field] = cleaned
|
||||
# ``custom_provider_map`` is a list of dicts with ``custom_handler`` —
|
||||
|
|
@ -5062,9 +5082,10 @@ def _scrub_db_overlay_remote_module_loads(section: str, db_value: object) -> obj
|
|||
if isinstance(cpm, list):
|
||||
for item in cpm:
|
||||
if isinstance(item, dict) and _is_remote_module_url(item.get("custom_handler")):
|
||||
verbose_proxy_logger.warning(
|
||||
_warn_db_config(
|
||||
"Refused remote-URL custom_handler from DB-overlay litellm_settings.custom_provider_map: %r",
|
||||
item.get("custom_handler"),
|
||||
warning_sink=warning_sink,
|
||||
)
|
||||
item["custom_handler"] = None
|
||||
# ``litellm_settings.guardrails`` is a list of single-key dicts in
|
||||
|
|
@ -5082,19 +5103,20 @@ def _scrub_db_overlay_remote_module_loads(section: str, db_value: object) -> obj
|
|||
for inner in entry.values():
|
||||
if not isinstance(inner, dict):
|
||||
continue
|
||||
_scrub_guardrail_inner(inner)
|
||||
_scrub_guardrail_inner(inner, warning_sink=warning_sink)
|
||||
lp = entry.get("litellm_params")
|
||||
if isinstance(lp, dict):
|
||||
_scrub_guardrail_inner(lp)
|
||||
_scrub_guardrail_inner(lp, warning_sink=warning_sink)
|
||||
|
||||
# ``general_settings.litellm_jwtauth.custom_validate`` is a nested
|
||||
# string field.
|
||||
if section == "general_settings":
|
||||
jwt: Final = sanitized.get("litellm_jwtauth")
|
||||
if isinstance(jwt, dict) and _is_remote_module_url(jwt.get("custom_validate")):
|
||||
verbose_proxy_logger.warning(
|
||||
_warn_db_config(
|
||||
"Refused remote-URL custom_validate from DB-overlay general_settings.litellm_jwtauth: %r",
|
||||
jwt.get("custom_validate"),
|
||||
warning_sink=warning_sink,
|
||||
)
|
||||
jwt["custom_validate"] = None
|
||||
# ``pass_through_endpoints`` is a list of dicts whose ``target``
|
||||
|
|
@ -5106,12 +5128,13 @@ def _scrub_db_overlay_remote_module_loads(section: str, db_value: object) -> obj
|
|||
if isinstance(pte, list):
|
||||
for entry in pte:
|
||||
if isinstance(entry, dict) and _is_remote_module_url(entry.get("target")):
|
||||
verbose_proxy_logger.warning(
|
||||
_warn_db_config(
|
||||
"Refused remote-URL target from DB-overlay "
|
||||
"general_settings.pass_through_endpoints "
|
||||
"(path=%r): %r",
|
||||
entry.get("path"),
|
||||
entry.get("target"),
|
||||
warning_sink=warning_sink,
|
||||
)
|
||||
entry["target"] = None
|
||||
return sanitized
|
||||
|
|
@ -6415,12 +6438,16 @@ class ProxyConfig:
|
|||
|
||||
self._load_yaml_settings_stores(config)
|
||||
|
||||
pending_db_diagnostics: Final[list[tuple[str, JsonValue]]] = [] # mutable-ok: defer rows until privacy resolves
|
||||
pending_db_warnings: Final[list[_DbConfigWarning]] = [] # mutable-ok: defer warnings until privacy resolves
|
||||
## UPDATE CONFIG WITH DB
|
||||
if prisma_client is not None and store_model_in_db is True:
|
||||
config = await self._update_config_from_db(
|
||||
config=config,
|
||||
prisma_client=prisma_client,
|
||||
store_model_in_db=store_model_in_db,
|
||||
diagnostic_sink=pending_db_diagnostics.append,
|
||||
warning_sink=pending_db_warnings.append,
|
||||
)
|
||||
|
||||
config = self._config_with_resolved_settings(config)
|
||||
|
|
@ -6433,6 +6460,10 @@ class ProxyConfig:
|
|||
|
||||
config = self._check_for_os_environ_vars(config=config)
|
||||
self._apply_resolved_runtime_settings(config)
|
||||
for section, param_value in pending_db_diagnostics:
|
||||
_log_db_config_param(section, param_value)
|
||||
for message, args in pending_db_warnings:
|
||||
verbose_proxy_logger.warning(message, *args)
|
||||
|
||||
self.update_config_state(config=config)
|
||||
|
||||
|
|
@ -8476,6 +8507,8 @@ class ProxyConfig:
|
|||
prisma_client: PrismaClient,
|
||||
config: dict,
|
||||
store_model_in_db: bool | None,
|
||||
diagnostic_sink: Callable[[tuple[str, JsonValue]], None] | None = None,
|
||||
warning_sink: Callable[[_DbConfigWarning], None] | None = None,
|
||||
):
|
||||
if store_model_in_db is not True:
|
||||
verbose_proxy_logger.info("'store_model_in_db' is not True, skipping db updates")
|
||||
|
|
@ -8493,13 +8526,14 @@ class ProxyConfig:
|
|||
if param_value is None:
|
||||
continue
|
||||
|
||||
verbose_proxy_logger.debug(
|
||||
"param_name=%s, param_value=%s",
|
||||
section,
|
||||
_redact_config_param_value_for_logging(section, param_value),
|
||||
)
|
||||
if diagnostic_sink is None:
|
||||
_log_db_config_param(section, param_value)
|
||||
else:
|
||||
diagnostic_sink((section, param_value))
|
||||
if section == "litellm_settings":
|
||||
self._apply_litellm_settings_db_values(self._prepared_db_settings_values(section, param_value))
|
||||
self._apply_litellm_settings_db_values(
|
||||
self._prepared_db_settings_values(section, param_value, warning_sink=warning_sink)
|
||||
)
|
||||
elif section == "environment_variables":
|
||||
await self._apply_db_environment_variables_row(
|
||||
prisma_client, response, environment_before_db, environment_read
|
||||
|
|
@ -8507,10 +8541,10 @@ class ProxyConfig:
|
|||
else:
|
||||
self._settings_stores[section].apply_db_row(
|
||||
section,
|
||||
self._prepared_db_settings_values(section, param_value),
|
||||
self._prepared_db_settings_values(section, param_value, warning_sink=warning_sink),
|
||||
)
|
||||
|
||||
self._warn_about_shadowed_db_settings()
|
||||
self._warn_about_shadowed_db_settings(warning_sink=warning_sink)
|
||||
return self._config_with_resolved_settings(config)
|
||||
|
||||
async def _apply_db_environment_variables_row(
|
||||
|
|
@ -8559,18 +8593,24 @@ class ProxyConfig:
|
|||
revision: Final = _config_row_revision(row)
|
||||
return revision is None or revision < applied_revision
|
||||
|
||||
def _warn_about_shadowed_db_settings(self) -> None:
|
||||
def _warn_about_shadowed_db_settings(self, warning_sink: Callable[[_DbConfigWarning], None] | None = None) -> None:
|
||||
shadowed: Final[frozenset[tuple[Section, str]]] = frozenset(
|
||||
(section, key) for section, store in self._settings_stores.items() for key in store.shadowed_db_keys()
|
||||
)
|
||||
for section, key in sorted(shadowed - self._warned_shadowed_keys):
|
||||
verbose_proxy_logger.warning(
|
||||
"%s", config_ownership_message(section=section, key=key, shadows_db_value=True)
|
||||
_warn_db_config(
|
||||
"%s",
|
||||
config_ownership_message(section=section, key=key, shadows_db_value=True),
|
||||
warning_sink=warning_sink,
|
||||
)
|
||||
self._warned_shadowed_keys = shadowed
|
||||
|
||||
def _prepared_db_settings_values(self, section: Section, value: object) -> Mapping[str, SettingsJsonValue]:
|
||||
scrubbed: Final = _scrub_db_overlay_remote_module_loads(section=section, db_value=value)
|
||||
def _prepared_db_settings_values(
|
||||
self, section: Section, value: object, warning_sink: Callable[[_DbConfigWarning], None] | None = None
|
||||
) -> Mapping[str, SettingsJsonValue]:
|
||||
scrubbed: Final = _scrub_db_overlay_remote_module_loads(
|
||||
section=section, db_value=value, warning_sink=warning_sink
|
||||
)
|
||||
return _as_settings_mapping(scrubbed)
|
||||
|
||||
def _apply_db_environment_variables(
|
||||
|
|
@ -20175,6 +20215,14 @@ def _redact_config_param_value_for_logging(param_name: str | None, param_value:
|
|||
return param_value
|
||||
|
||||
|
||||
def _log_db_config_param(section: str, param_value: JsonValue) -> None:
|
||||
verbose_proxy_logger.debug(
|
||||
"param_name=%s, param_value=%s",
|
||||
section,
|
||||
_redact_config_param_value_for_logging(section, param_value),
|
||||
)
|
||||
|
||||
|
||||
def _redact_general_setting_value(field_name: str, value: JsonValue, is_full_admin: bool) -> JsonValue:
|
||||
if is_full_admin:
|
||||
return value
|
||||
|
|
|
|||
|
|
@ -48,3 +48,32 @@ async def test_fim_stream_keeps_unicode_line_separators_inside_a_delta(
|
|||
|
||||
assert texts == [_LINE_SEPARATED_TEXT, None]
|
||||
assert json.loads(route.calls.last.request.content)["stream"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("privacy", [False, True])
|
||||
async def test_fim_response_content_is_preserved_when_diagnostics_are_private(
|
||||
privacy: bool,
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "redact_messages_in_debug_logs", privacy)
|
||||
monkeypatch.setattr(litellm, "set_verbose", True)
|
||||
route: Final = respx_mock.post("https://codestral.mistral.ai/v1/fim/completions").mock(
|
||||
return_value=httpx.Response(200, json={
|
||||
"id": "fim-private", "object": "text_completion", "created": 1, "model": "codestral-latest",
|
||||
"choices": [{"index": 0, "message": {"content": "PRIVATE_FIM_REPLY"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 2, "completion_tokens": 3, "total_tokens": 5},
|
||||
})
|
||||
)
|
||||
|
||||
response: Final = await litellm.atext_completion(
|
||||
model="text-completion-codestral/codestral-latest",
|
||||
prompt="PRIVATE_FIM_PROMPT", api_key="test-codestral-key", max_retries=0,
|
||||
)
|
||||
|
||||
assert response.choices[0].text == "PRIVATE_FIM_REPLY"
|
||||
assert json.loads(route.calls.last.request.content)["prompt"] == "PRIVATE_FIM_PROMPT"
|
||||
assert ("PRIVATE_FIM_REPLY" in capsys.readouterr().out) is not privacy
|
||||
|
|
|
|||
|
|
@ -6271,3 +6271,109 @@ async def test_proxy_config_validates_advertised_mcp_versions_at_load(tmp_path,
|
|||
return
|
||||
with pytest.raises(ValidationError):
|
||||
await ProxyConfig().load_config(router=None, config_file_path=str(config))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("privacy", [False, True])
|
||||
async def test_proxy_config_load_applies_diagnostic_privacy(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, privacy: bool
|
||||
) -> None:
|
||||
config_file: Final = tmp_path / "privacy.yaml"
|
||||
config_file.write_text(
|
||||
"model_list: []\nlitellm_settings:\n redact_messages_in_debug_logs: " + str(privacy).lower() + "\n"
|
||||
)
|
||||
monkeypatch.setattr(litellm, "redact_messages_in_debug_logs", not privacy)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False)
|
||||
monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False)
|
||||
|
||||
await ProxyConfig().load_config(router=None, config_file_path=str(config_file))
|
||||
|
||||
assert litellm.redact_messages_in_debug_logs is privacy
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("file_privacy", [None, False, True])
|
||||
@pytest.mark.parametrize("db_privacy", [False, True, "invalid-boolean"])
|
||||
@pytest.mark.parametrize("env_reference", [False, True])
|
||||
async def test_proxy_config_privacy_precedes_database_settings_diagnostics(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
file_privacy: bool | None,
|
||||
db_privacy: bool | str,
|
||||
env_reference: bool,
|
||||
) -> None:
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy import utils as proxy_utils
|
||||
|
||||
monkeypatch.setenv("LITELLM_QA_DB_PRIVACY", str(db_privacy).lower())
|
||||
monkeypatch.setenv("LITELLM_QA_FILE_PRIVACY", str(file_privacy).lower())
|
||||
file_value: Final = "os.environ/LITELLM_QA_FILE_PRIVACY" if env_reference else file_privacy
|
||||
remote_module: Final = "s3://example-bucket/PRIVATE_DB_CONTENT.py"
|
||||
stored: Final = {
|
||||
"general_settings": {
|
||||
"alerting": ["PRIVATE_DB_CONTENT"],
|
||||
"custom_auth": remote_module,
|
||||
"litellm_jwtauth": {"custom_validate": remote_module},
|
||||
"pass_through_endpoints": [{"path": "/PRIVATE_DB_CONTENT", "target": remote_module}],
|
||||
"PRIVATE_DB_SETTING": "database-value",
|
||||
},
|
||||
"router_settings": {"model_group_alias": {"PRIVATE_DB_CONTENT": "qa-model"}},
|
||||
"litellm_settings": {
|
||||
"redact_messages_in_debug_logs": "os.environ/LITELLM_QA_DB_PRIVACY" if env_reference else db_privacy,
|
||||
"model_alias_map": {"PRIVATE_DB_CONTENT": "qa-model"},
|
||||
"callbacks": [remote_module, "prometheus"],
|
||||
"custom_provider_map": [{"provider": "qa", "custom_handler": remote_module}],
|
||||
"guardrails": [{"qa": {"callbacks": [remote_module], "guardrail": remote_module}}],
|
||||
},
|
||||
}
|
||||
|
||||
async def get_generic_data(key: str, value: str, table_name: str) -> SimpleNamespace | None:
|
||||
assert (key, table_name) == ("param_name", "config")
|
||||
row: Final = stored.get(value)
|
||||
return SimpleNamespace(param_name=value, param_value=row) if row is not None else None
|
||||
|
||||
monkeypatch.setattr(litellm, "redact_messages_in_debug_logs", False)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(get_generic_data=get_generic_data))
|
||||
monkeypatch.setattr(proxy_server, "store_model_in_db", True)
|
||||
monkeypatch.setattr(proxy_utils, "litellm_config_cache", DualCache())
|
||||
monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False)
|
||||
config_file: Final = tmp_path / "db-privacy.yaml"
|
||||
config_file.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"litellm_settings": {} if file_privacy is None else {"redact_messages_in_debug_logs": file_value},
|
||||
"general_settings": {"PRIVATE_DB_SETTING": "file-value"},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
enabled: Final = db_privacy if file_privacy is None else file_privacy
|
||||
if isinstance(enabled, str):
|
||||
with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"), pytest.raises(ValidationError):
|
||||
await ProxyConfig().get_config(str(config_file))
|
||||
assert "PRIVATE_DB_CONTENT" not in caplog.text
|
||||
assert litellm.redact_messages_in_debug_logs is False
|
||||
return
|
||||
|
||||
with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"):
|
||||
loaded: Final = await ProxyConfig().get_config(str(config_file))
|
||||
|
||||
assert ("PRIVATE_DB_CONTENT" in caplog.text) is not enabled
|
||||
assert litellm.redact_messages_in_debug_logs is enabled
|
||||
assert loaded["litellm_settings"]["redact_messages_in_debug_logs"] is enabled
|
||||
assert loaded["litellm_settings"]["model_alias_map"] == {"PRIVATE_DB_CONTENT": "qa-model"}
|
||||
assert loaded["general_settings"] == {
|
||||
"alerting": ["PRIVATE_DB_CONTENT"],
|
||||
"litellm_jwtauth": {"custom_validate": None},
|
||||
"pass_through_endpoints": [{"path": "/PRIVATE_DB_CONTENT", "target": None}],
|
||||
"PRIVATE_DB_SETTING": "file-value",
|
||||
}
|
||||
assert loaded["litellm_settings"]["callbacks"] == ["prometheus"]
|
||||
assert loaded["litellm_settings"]["custom_provider_map"] == [{"provider": "qa", "custom_handler": None}]
|
||||
assert loaded["litellm_settings"]["guardrails"] == [{"qa": {"callbacks": [], "guardrail": None}}]
|
||||
assert stored["general_settings"]["custom_auth"] == remote_module
|
||||
assert any(record.levelno == logging.WARNING for record in caplog.records)
|
||||
assert loaded["router_settings"] == stored["router_settings"]
|
||||
|
|
|
|||
|
|
@ -12619,3 +12619,104 @@ async def test_stream_failure_retains_error_status_without_debug_logging(
|
|||
else:
|
||||
assert any("PRIVATE_STREAM_FAILURE_42896" in message for message in messages)
|
||||
assert all("event=request_failed" not in message for message in messages)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("privacy", [False, True])
|
||||
async def test_nonstream_response_keeps_content_with_private_ready_diagnostic(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, privacy: bool
|
||||
) -> None:
|
||||
import logging
|
||||
|
||||
import respx
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy.proxy_server import ProxyConfig as ServerProxyConfig
|
||||
|
||||
monkeypatch.setattr(litellm, "redact_messages_in_debug_logs", privacy)
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "success_callback", [])
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", [])
|
||||
monkeypatch.setattr(verbose_proxy_logger, "level", logging.DEBUG)
|
||||
ProxyLogging._callback_capabilities_cache.clear()
|
||||
content: Final = "PRIVATE_NONSTREAM_RESPONSE_42896"
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(
|
||||
data={
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_base": "https://example.com/v1",
|
||||
"api_key": "test-key",
|
||||
"messages": [{"role": "user", "content": "PRIVATE_NONSTREAM_INPUT_42896"}],
|
||||
}
|
||||
)
|
||||
with respx.mock() as upstream:
|
||||
upstream.post("https://example.com/v1/chat/completions").respond(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-example",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [
|
||||
{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}
|
||||
],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
},
|
||||
)
|
||||
response: Final = await processor.base_process_llm_request(
|
||||
request=Request(
|
||||
{"type": "http", "method": "POST", "path": "/v1/chat/completions", "headers": [], "query_string": b""}
|
||||
),
|
||||
fastapi_response=Response(),
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(),
|
||||
route_type="acompletion",
|
||||
proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()),
|
||||
general_settings={},
|
||||
proxy_config=ServerProxyConfig(),
|
||||
)
|
||||
assert isinstance(response, litellm.ModelResponse)
|
||||
assert isinstance(response.choices[0], litellm.Choices)
|
||||
assert response.choices[0].message.content == content
|
||||
assert response.choices[0].finish_reason == "stop"
|
||||
messages: Final = tuple(
|
||||
record.getMessage() for record in caplog.records if record.name == verbose_proxy_logger.name
|
||||
)
|
||||
if privacy:
|
||||
assert any("event=response_ready status=200 elapsed_seconds=" in message for message in messages)
|
||||
assert all("PRIVATE_NONSTREAM_" not in message for message in messages)
|
||||
else:
|
||||
assert all("event=response_ready" not in message for message in messages)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("privacy", [False, True])
|
||||
async def test_nonstream_failure_preserves_client_error_and_safe_status_without_debug(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, privacy: bool
|
||||
) -> None:
|
||||
import logging
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
monkeypatch.setattr(litellm, "redact_messages_in_debug_logs", privacy)
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
monkeypatch.setattr(verbose_proxy_logger, "level", logging.ERROR)
|
||||
ProxyLogging._callback_capabilities_cache.clear()
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(data={"model": "example-model"})
|
||||
failure: Final = HTTPException(status_code=429, detail="PRIVATE_NONSTREAM_FAILURE_42896")
|
||||
with pytest.raises(ProxyException) as raised:
|
||||
await processor.handle_llm_api_exception(
|
||||
e=failure,
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(),
|
||||
proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()),
|
||||
)
|
||||
assert raised.value.code == "429"
|
||||
assert raised.value.message == "PRIVATE_NONSTREAM_FAILURE_42896"
|
||||
messages: Final = tuple(
|
||||
record.getMessage() for record in caplog.records if record.name == verbose_proxy_logger.name
|
||||
)
|
||||
if privacy:
|
||||
assert any("event=request_failed status=429" in message for message in messages)
|
||||
assert all("PRIVATE_NONSTREAM_FAILURE_42896" not in message for message in messages)
|
||||
else:
|
||||
assert any("PRIVATE_NONSTREAM_FAILURE_42896" in message for message in messages)
|
||||
assert all("event=request_failed" not in message for message in messages)
|
||||
|
|
|
|||
|
|
@ -591,3 +591,40 @@ async def test_stream_without_auto_execute_is_forwarded_unchanged(monkeypatch):
|
|||
assert [c.output_index for c in chunks if hasattr(c, "output_index")] == [0, 0]
|
||||
assert [c.sequence_number for c in chunks if hasattr(c, "sequence_number")] == [0, 1, 2]
|
||||
aresponses_mock.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("redact", (False, True))
|
||||
@pytest.mark.asyncio
|
||||
async def test_malformed_provider_tool_arguments_keep_error_diagnostics_private(
|
||||
monkeypatch: pytest.MonkeyPatch, redact: bool, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "redact_messages_in_debug_logs", redact)
|
||||
response: Final = ResponsesAPIResponse(
|
||||
id="resp-malformed-arguments",
|
||||
created_at=0,
|
||||
output=[
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call-1",
|
||||
"name": "read_wiki_contents",
|
||||
"arguments": {"text": "PRIVATE_TOOL_ARGUMENT"},
|
||||
}
|
||||
],
|
||||
)
|
||||
iterator: Final = _make_iterator([])
|
||||
iterator.collected_response = response
|
||||
|
||||
await iterator._generate_tool_execution_events()
|
||||
|
||||
assert iterator.tool_results == []
|
||||
assert iterator._tool_results_for_response is response
|
||||
assert iterator.collected_response.output == [
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call-1",
|
||||
"name": "read_wiki_contents",
|
||||
"arguments": {"text": "PRIVATE_TOOL_ARGUMENT"},
|
||||
}
|
||||
]
|
||||
assert [event.type for event in iterator.tool_execution_events] == [ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED]
|
||||
assert ("PRIVATE_TOOL_ARGUMENT" in capsys.readouterr().err) is not redact
|
||||
|
|
|
|||
|
|
@ -10713,3 +10713,39 @@ async def test_bedrock_strict_tools_limit_rejection_is_not_sent_again_when_no_to
|
|||
|
||||
assert error is not None and "Too many strict tools" in str(error)
|
||||
assert len(sent) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("privacy", [False, True])
|
||||
async def test_async_streaming_keeps_chunks_without_rendering_private_content(
|
||||
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], privacy: bool
|
||||
) -> None:
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
class Chunk:
|
||||
content: Final = "PRIVATE_ASYNC_STREAM_42896"
|
||||
|
||||
def __str__(self) -> str:
|
||||
if privacy:
|
||||
raise AssertionError("private chunks must not be rendered for diagnostics")
|
||||
return self.content
|
||||
|
||||
monkeypatch.setattr(litellm, "redact_messages_in_debug_logs", privacy)
|
||||
monkeypatch.setattr(litellm, "set_verbose", True)
|
||||
chunk: Final = Chunk()
|
||||
|
||||
async def upstream() -> AsyncIterator[Chunk]:
|
||||
yield chunk
|
||||
|
||||
stream: Final = litellm_main._async_streaming(upstream(), "example-model", "openai", {})
|
||||
observed: Final = tuple([item async for item in stream])
|
||||
|
||||
assert len(observed) == 1
|
||||
assert observed[0] is chunk
|
||||
assert chunk.content == "PRIVATE_ASYNC_STREAM_42896"
|
||||
output: Final = capsys.readouterr().out
|
||||
assert ("PRIVATE_ASYNC_STREAM_42896" in output) is not privacy
|
||||
if privacy:
|
||||
assert output.strip() == "redacted-by-litellm"
|
||||
else:
|
||||
assert "line in async streaming: PRIVATE_ASYNC_STREAM_42896" in output
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue