feat(errors): add stream and safe config flags to the bug report link (#42428)

Proxy bug reports now carry the request's stream flag and a config block
built from dotted paths like router_settings.routing_strategy. A line is
emitted only when its key is defined by a LiteLLM schema and its value is
a bool or a LiteLLM-defined value (providers, callbacks, routing
strategies, cache types, guardrail integrations and modes, key management
systems). Secrets, URLs, numbers and custom values leave no line
This commit is contained in:
ryan-crabbe-berri 2026-09-21 22:51:24 -07:00 • committed by GitHub
parent 77d656a8ba
commit a9cea9d644
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 417 additions and 17 deletions

View file

@ -31,6 +31,8 @@ class BugReport:
python_version: str
call_type: str | None
custom_llm_provider: str | None
stream: bool | None
config_lines: tuple[str, ...]
def bug_report_enabled() -> bool:
@ -73,6 +75,8 @@ def build_bug_report(
surface: Surface,
call_type: str | None = None,
custom_llm_provider: object = None,
stream: object = None,
config_lines: tuple[str, ...] = (),
) -> BugReport:
return BugReport(
surface=surface,
@ -82,6 +86,8 @@ def build_bug_report(
python_version=platform.python_version(),
call_type=call_type,
custom_llm_provider=allowlisted(custom_llm_provider, KNOWN_PROVIDERS),
stream=stream if isinstance(stream, bool) else None,
config_lines=config_lines,
)
@ -98,8 +104,14 @@ def _title(report: BugReport, frames: tuple[str, ...]) -> str:
return f"[Bug]: {report.exception_type} in {location}"
def _description(report: BugReport, frames: tuple[str, ...]) -> str:
def _description(report: BugReport, frames: tuple[str, ...], config_lines: tuple[str, ...]) -> str:
frame_block: Final = "LiteLLM frames:\n```\n" + "\n".join(frames) + "\n```\n\n" if frames else ""
stream_line: Final = "" if report.stream is None else f"Stream: {str(report.stream).lower()}\n"
config_block: Final = (
"\nConfig (true/false flags and LiteLLM-defined values only):\n```\n" + "\n".join(config_lines) + "\n```\n"
if config_lines
else ""
)
return (
"Auto-generated by LiteLLM's bug report link. It carries no request data or error text. "
"Please describe what you were doing, and paste the error message from your log below "
@ -112,10 +124,12 @@ def _description(report: BugReport, frames: tuple[str, ...]) -> str:
f"Provider: {report.custom_llm_provider or 'unknown'}\n"
f"LiteLLM: {report.litellm_version}\n"
f"Python: {report.python_version}\n"
f"{stream_line}"
f"{config_block}"
)
def _issue_url(report: BugReport, frames: tuple[str, ...]) -> str:
def _issue_url(report: BugReport, frames: tuple[str, ...], config_lines: tuple[str, ...]) -> str:
deployment: Final[tuple[tuple[str, str], ...]] = (
(("deployment", "pip / Python SDK"),)
if report.surface == "sdk"
@ -129,15 +143,26 @@ def _issue_url(report: BugReport, frames: tuple[str, ...]) -> str:
("title", _title(report, frames)),
("version", report.litellm_version),
("domain", _domain(report)),
("description", _description(report, frames)),
("description", _description(report, frames, config_lines)),
) + deployment
return f"{ISSUE_URL_BASE}?{urlencode(fields)}"
def bug_report_issue_url(report: BugReport) -> str:
frames: Final = report.litellm_frames
candidates: Final = tuple(_issue_url(report, frames[index:]) for index in range(len(frames) + 1))
return next((candidate for candidate in candidates if len(candidate) <= MAX_URL_LENGTH), candidates[-1])
config_lines: Final = report.config_lines
candidates: Final = (
*((frames, config_lines[:count]) for count in range(len(config_lines), -1, -1)),
*((frames[index:], ()) for index in range(1, len(frames) + 1)),
)
return next(
(
url
for candidate_frames, candidate_config in candidates
if len(url := _issue_url(report, candidate_frames, candidate_config)) <= MAX_URL_LENGTH
),
_issue_url(report, (), ()),
)
def strip_bug_report_notice(message: str) -> str:

View file

@ -0,0 +1,185 @@
from __future__ import annotations
import ast
import functools
import inspect
from collections.abc import Mapping, Sequence
from pathlib import Path
from types import MappingProxyType
from typing import Final
from pydantic import JsonValue, TypeAdapter, ValidationError
import litellm
from litellm.litellm_core_utils.bug_report import KNOWN_PROVIDERS, BugReport, allowlisted, build_bug_report
from litellm.proxy._types import ConfigGeneralSettings
from litellm.router_utils.routing_groups import VALID_ROUTING_STRATEGIES
from litellm.types.caching import LiteLLMCacheType
from litellm.types.guardrails import GuardrailEventHooks, LitellmParams, SupportedGuardrailIntegrations
from litellm.types.secret_managers.main import KeyManagementSystem
_OBJECT_MAP: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
_OBJECT_LIST: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...])
_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
def _object_map(value: object) -> Mapping[str, object]:
try:
return _OBJECT_MAP.validate_python(value)
except ValidationError:
return MappingProxyType({})
def _object_list(value: object) -> Sequence[object]:
try:
return _OBJECT_LIST.validate_python(value)
except ValidationError:
return ()
@functools.cache
def _known_values() -> frozenset[str]:
from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry
return frozenset(
(
*VALID_ROUTING_STRATEGIES,
*KNOWN_PROVIDERS,
*litellm._known_custom_logger_compatible_callbacks, # pyright: ignore[reportPrivateUsage, reportUnknownMemberType, reportUnknownArgumentType] # untyped List of the callback Literal's args, no public alias
*CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE,
*(member.value for member in LiteLLMCacheType),
*(member.value for member in KeyManagementSystem),
*(member.value for member in SupportedGuardrailIntegrations),
*(member.value for member in GuardrailEventHooks),
)
)
def _module_level_names(node: ast.stmt) -> tuple[str, ...]:
match node:
case ast.Assign(targets=targets):
return tuple(target.id for target in targets if isinstance(target, ast.Name))
case ast.AnnAssign(target=ast.Name(id=name)):
return (name,)
case ast.ImportFrom(names=aliases):
return tuple(alias.asname or alias.name for alias in aliases)
case _:
return ()
@functools.cache
def _litellm_settings_keys() -> frozenset[str]:
tree: Final = ast.parse(Path(litellm.__file__).read_text())
return frozenset(name for node in tree.body for name in _module_level_names(node))
@functools.cache
def _router_settings_keys() -> frozenset[str]:
from litellm.router import Router
return frozenset(name for name in inspect.signature(Router.__init__).parameters if name != "self") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped params, only names are read
@functools.cache
def _cache_params_keys() -> frozenset[str]:
from litellm.caching.caching import Cache
return frozenset(name for name in inspect.signature(Cache.__init__).parameters if name != "self") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped params, only names are read
def _render_json(value: JsonValue) -> str | None:
match value:
case bool():
return str(value).lower()
case str():
return value if value in _known_values() else None
case list():
known_items: Final = tuple(rendered for item in value if (rendered := _render_json(item)) is not None)
return f"[{', '.join(known_items)}]" if known_items else None
case _:
return None
def _render(value: object) -> str | None:
try:
return _render_json(_JSON.validate_python(value))
except ValidationError:
return None
def _section_lines(section: str, values: Mapping[str, object], known_keys: frozenset[str]) -> tuple[str, ...]:
return tuple(
f"{section}.{key} = {rendered}"
for key, value in values.items()
if key in known_keys and (rendered := _render(value)) is not None
)
def _guardrail_lines(guardrails: object) -> tuple[str, ...]:
known_keys: Final = frozenset(LitellmParams.model_fields)
return tuple(
line
for index, guardrail in enumerate(_object_list(guardrails))
for line in _section_lines(
f"guardrails[{index}].litellm_params", _object_map(_object_map(guardrail).get("litellm_params")), known_keys
)
)
def _deployment_provider(model: object) -> str | None:
prefix: Final = model.split("/", 1)[0] if isinstance(model, str) and "/" in model else None
return allowlisted(prefix, KNOWN_PROVIDERS)
def _model_list_lines(model_list: object) -> tuple[str, ...]:
providers: Final = tuple(
sorted(
frozenset(
provider
for deployment in _object_list(model_list)
if (
provider := _deployment_provider(
_object_map(_object_map(deployment).get("litellm_params")).get("model")
)
)
is not None
)
)
)
return (f"model_list[*].provider = [{', '.join(providers)}]",) if providers else ()
def safe_config_lines(config: Mapping[str, object], general_settings: Mapping[str, object]) -> tuple[str, ...]:
litellm_settings: Final = _object_map(config.get("litellm_settings"))
return (
*_section_lines("general_settings", general_settings, frozenset(ConfigGeneralSettings.model_fields)),
*_section_lines("litellm_settings", litellm_settings, _litellm_settings_keys()),
*_section_lines(
"litellm_settings.cache_params", _object_map(litellm_settings.get("cache_params")), _cache_params_keys()
),
*_section_lines("router_settings", _object_map(config.get("router_settings")), _router_settings_keys()),
*_guardrail_lines(config.get("guardrails")),
*_model_list_lines(config.get("model_list")),
)
def build_proxy_bug_report(
exc: BaseException,
*,
call_type: str | None = None,
custom_llm_provider: object = None,
stream: object = None,
) -> BugReport:
from litellm.proxy import proxy_server
return build_bug_report(
exc,
surface="proxy",
call_type=call_type,
custom_llm_provider=custom_llm_provider,
stream=stream,
config_lines=safe_config_lines(
proxy_server.proxy_config.config,
_object_map(proxy_server.general_settings), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # bare dict global, validated by _object_map
),
)

View file

@ -49,7 +49,6 @@ from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.litellm_core_utils.bug_report import (
allowlisted,
bug_report_notice,
build_bug_report,
should_report_bug,
strip_bug_report_notice,
)
@ -79,6 +78,7 @@ from litellm.proxy.auth.auth_checks import (
tag_max_budget_check_for_tags,
)
from litellm.proxy.auth.auth_utils import check_response_size_is_safe, get_request_route
from litellm.proxy.bug_report_config import build_proxy_bug_report
from litellm.proxy.common_utils.callback_utils import (
get_logging_caching_headers,
get_remaining_tokens_and_requests_from_request_data,
@ -3689,11 +3689,11 @@ class ProxyBaseLLMRequestProcessing:
request_path: Final = urlparse(str(request_url)).path if request_url is not None else None
verbose_proxy_logger.error(
bug_report_notice(
build_bug_report(
build_proxy_bug_report(
e,
surface="proxy",
call_type=allowlisted(request_path, KNOWN_PROXY_ROUTES),
custom_llm_provider=self.data.get("custom_llm_provider"),
stream=self.data.get("stream"),
)
)
)

View file

@ -78,7 +78,6 @@ from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.bug_report import (
allowlisted,
bug_report_notice,
build_bug_report,
should_report_bug,
)
from litellm.litellm_core_utils.litellm_logging import (
@ -373,6 +372,7 @@ from litellm.proxy.auth.user_api_key_auth import (
user_api_key_auth_websocket,
)
from litellm.proxy.batches_endpoints.endpoints import router as batches_router
from litellm.proxy.bug_report_config import build_proxy_bug_report
## Import All Misc routes here ##
from litellm.proxy.caching_routes import router as caching_router
@ -1986,9 +1986,8 @@ async def otel_unhandled_exception_handler(request: Request, exc: Exception):
if should_report_bug(exc):
verbose_proxy_logger.error(
bug_report_notice(
build_bug_report(
build_proxy_bug_report(
exc,
surface="proxy",
call_type=allowlisted(request.url.path, KNOWN_PROXY_ROUTES),
)
)

View file

@ -59,7 +59,6 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.bug_report import (
bug_report_notice,
build_bug_report,
should_report_bug,
strip_bug_report_notice,
)
@ -70,6 +69,7 @@ from litellm.proxy._types import (
SpendLogsMetadata,
SpendLogsPayload,
)
from litellm.proxy.bug_report_config import build_proxy_bug_report
from litellm.proxy.common_utils.openai_error_payload import (
litellm_call_id_headers,
openai_error_param,
@ -7996,7 +7996,7 @@ def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None)
return with_litellm_call_id(e, litellm_call_id)
_status_code: Final = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
if should_report_bug(e):
verbose_proxy_logger.error(bug_report_notice(build_bug_report(e, surface="proxy")))
verbose_proxy_logger.error(bug_report_notice(build_proxy_bug_report(e)))
return ProxyException(
message=strip_bug_report_notice(str(e)),
type=ProxyErrorTypes.internal_server_error,

View file

@ -4,18 +4,19 @@ from typing import Final
from litellm._logging import verbose_router_logger
from litellm.types.router import RoutingGroup, RoutingStrategy
VALID_ROUTING_STRATEGIES: Final = ("simple-shuffle", "lar1", *(s.value for s in RoutingStrategy))
def validate_routing_strategy(routing_strategy: RoutingStrategy | str | None) -> None:
if routing_strategy is None:
return
valid_strategy_strings: Final = ("simple-shuffle", "lar1", *(s.value for s in RoutingStrategy))
is_valid_string: Final = isinstance(routing_strategy, str) and routing_strategy in valid_strategy_strings
is_valid_string: Final = isinstance(routing_strategy, str) and routing_strategy in VALID_ROUTING_STRATEGIES
is_valid_enum: Final = isinstance(routing_strategy, RoutingStrategy)
if not is_valid_string and not is_valid_enum:
raise ValueError(
f"Invalid routing_strategy: '{routing_strategy}'. "
f"Valid options: {list(valid_strategy_strings)}. "
f"Valid options: {list(VALID_ROUTING_STRATEGIES)}. "
f"Check 'router_settings.routing_strategy' in your config.yaml "
f"or the 'routing_strategy' parameter if using the Router SDK directly."
)

View file

@ -3,7 +3,7 @@ from __future__ import annotations
from collections.abc import Callable
from pathlib import Path
from typing import cast
from urllib.parse import parse_qs, urlparse
from urllib.parse import parse_qs, unquote_plus, urlparse
import httpx
import pytest
@ -168,3 +168,37 @@ def test_strip_bug_report_notice():
assert strip_bug_report_notice(f"boom\n\n{notice}") == "boom\n"
assert strip_bug_report_notice("boom") == "boom"
def test_issue_description_renders_stream_and_config_block():
report = build_bug_report(
RuntimeError("boom"),
surface="proxy",
stream=True,
config_lines=("router_settings.routing_strategy = least-busy", "litellm_settings.drop_params = true"),
)
description = parse_qs(urlparse(bug_report_issue_url(report)).query)["description"][0]
assert "Stream: true\n" in description
assert "```\nrouter_settings.routing_strategy = least-busy\nlitellm_settings.drop_params = true\n```" in description
@pytest.mark.parametrize("stream", [None, "true", 1])
def test_issue_description_omits_stream_unless_it_is_a_bool(stream: object):
report = build_bug_report(RuntimeError("boom"), surface="proxy", stream=stream)
assert report.stream is None
assert "Stream:" not in unquote_plus(bug_report_issue_url(report))
def test_oversized_config_is_trimmed_from_the_end_before_any_frame():
with pytest.raises(BadRequestError) as raised:
get_llm_provider(cast(str, None))
config_lines = tuple(f"general_settings.flag_{index:04d} = true" for index in range(400))
report = build_bug_report(raised.value, surface="proxy", config_lines=config_lines)
description = parse_qs(urlparse(url := bug_report_issue_url(report)).query)["description"][0]
assert len(url) <= MAX_URL_LENGTH
assert all(frame in description for frame in report.litellm_frames)
assert "general_settings.flag_0000 = true" in description
assert "general_settings.flag_0399 = true" not in description

View file

@ -0,0 +1,154 @@
from __future__ import annotations
from collections.abc import Iterator, Mapping
import pytest
from litellm.proxy import proxy_server
from litellm.proxy.bug_report_config import build_proxy_bug_report, safe_config_lines
CUSTOMER_STRINGS = (
"acme",
"sk-live-secret",
"hunter2",
"postgres://",
"10.0.0.7",
)
CUSTOMER_CONFIG: Mapping[str, object] = {
"model_list": [
{
"model_name": "acme-prod-gpt4",
"litellm_params": {
"model": "azure/acme-gpt4o-deployment",
"api_base": "https://acme-eastus.openai.azure.com",
"api_key": "sk-live-secret-1",
"rpm": 600,
"acme_extra_param": "acme",
},
},
{
"model_name": "acme-mini",
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-live-secret-2"},
},
{
"model_name": "acme-backup",
"litellm_params": {"model": "azure/acme-backup-deployment", "api_key": "sk-live-secret-3"},
},
{"model_name": "acme-bare", "litellm_params": {"model": "acme-custom-model"}},
],
"litellm_settings": {
"callbacks": ["langfuse", "acme_hooks.audit_logger"],
"drop_params": True,
"num_retries": 3,
"acme_internal_flag": True,
"cache": True,
"cache_params": {
"type": "redis",
"host": "10.0.0.7",
"port": 6379,
"password": "hunter2",
"acme_cache_option": "acme",
},
},
"router_settings": {
"routing_strategy": "latency-based-routing",
"redis_host": "10.0.0.7",
"acme_router_option": True,
},
"guardrails": [
{"guardrail_name": "acme-pii-mask", "litellm_params": {"guardrail": "presidio", "mode": "pre_call"}},
{
"guardrail_name": "acme-policy",
"litellm_params": {"guardrail": "acme_guardrails.PolicyCheck", "api_key": "sk-live-secret-4"},
},
],
"environment_variables": {"ACME_PROD_OPENAI_KEY": "sk-live-secret-5", "ACME_TENANT": "acme"},
}
CUSTOMER_GENERAL_SETTINGS: Mapping[str, object] = {
"master_key": "sk-live-secret-master",
"database_url": "postgres://user:hunter2@10.0.0.7/litellm",
"key_management_system": "aws_secret_manager",
"store_model_in_db": True,
"health_check_interval": 300,
"acme_sso_tenant": "acme-prod",
}
def test_safe_config_lines_keep_only_flags_and_litellm_defined_values():
lines = safe_config_lines(CUSTOMER_CONFIG, CUSTOMER_GENERAL_SETTINGS)
assert lines == (
"general_settings.key_management_system = aws_secret_manager",
"general_settings.store_model_in_db = true",
"litellm_settings.callbacks = [langfuse]",
"litellm_settings.drop_params = true",
"litellm_settings.cache = true",
"litellm_settings.cache_params.type = redis",
"router_settings.routing_strategy = latency-based-routing",
"guardrails[0].litellm_params.guardrail = presidio",
"guardrails[0].litellm_params.mode = pre_call",
"model_list[*].provider = [azure, openai]",
)
assert not any(customer_string in "\n".join(lines) for customer_string in CUSTOMER_STRINGS)
@pytest.mark.parametrize(
("config", "expected_lines"),
[
({"router_settings": {"routing_strategy": "acme-strategy"}}, ()),
({"litellm_settings": {"cache_params": {"type": "acme-cache"}}}, ()),
(
{"litellm_settings": {"success_callback": ["acme_logger", "langsmith"]}},
("litellm_settings.success_callback = [langsmith]",),
),
({"litellm_settings": {"callbacks": ["acme_hooks.audit_logger"]}}, ()),
],
)
def test_string_values_show_only_when_litellm_defines_them(
config: Mapping[str, object], expected_lines: tuple[str, ...]
):
assert safe_config_lines(config, {}) == expected_lines
def test_secrets_numbers_and_unknown_values_leave_no_line():
general_settings: Mapping[str, object] = {
"master_key": "sk-live-secret-master",
"health_check_interval": 300,
"store_model_in_db": object(),
"alerting": {"acme": "webhook"},
"background_health_checks": False,
}
assert safe_config_lines({}, general_settings) == ("general_settings.background_health_checks = false",)
def test_malformed_sections_produce_no_lines():
config: Mapping[str, object] = {
"litellm_settings": "acme",
"router_settings": ["acme"],
"guardrails": {"acme": {"litellm_params": {"guardrail": "presidio"}}},
"model_list": "acme",
"environment_variables": None,
}
assert safe_config_lines(config, {}) == ()
@pytest.fixture
def loaded_proxy_config(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
previous_config = proxy_server.proxy_config.get_config_state()
proxy_server.proxy_config.update_config_state(config=CUSTOMER_CONFIG)
monkeypatch.setattr(proxy_server, "general_settings", dict(CUSTOMER_GENERAL_SETTINGS))
yield
proxy_server.proxy_config.update_config_state(config=previous_config)
@pytest.mark.usefixtures("loaded_proxy_config")
def test_build_proxy_bug_report_reads_the_loaded_proxy_config():
report = build_proxy_bug_report(RuntimeError("boom"), stream=False)
assert report.surface == "proxy"
assert report.stream is False
assert report.config_lines == safe_config_lines(CUSTOMER_CONFIG, CUSTOMER_GENERAL_SETTINGS)

View file

@ -9407,6 +9407,7 @@ async def test_handle_llm_api_exception_logs_bug_report_for_unmapped_error(
"proxy_server_request": {"url": "https://example.test/v1/chat/completions?debug=true"},
"model": "acme-prod-gpt4",
"custom_llm_provider": "openai",
"stream": True,
}
)
proxy_logging_obj = MagicMock()
@ -9424,6 +9425,7 @@ async def test_handle_llm_api_exception_logs_bug_report_for_unmapped_error(
issue_url = next(word for word in caplog.text.split() if word.startswith(ISSUE_URL_BASE))
assert "Endpoint / call: /v1/chat/completions" in unquote_plus(issue_url)
assert "Provider: openai" in unquote_plus(issue_url)
assert "Stream: true" in unquote_plus(issue_url)
assert "acme-prod-gpt4" not in unquote_plus(issue_url)
assert "user@example.com" not in unquote_plus(issue_url)