diff --git a/litellm/litellm_core_utils/bug_report.py b/litellm/litellm_core_utils/bug_report.py new file mode 100644 index 00000000000..a44df268339 --- /dev/null +++ b/litellm/litellm_core_utils/bug_report.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +import os +import platform +import traceback +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal +from urllib.parse import urlencode + +import litellm +from litellm._logging import redact_secrets +from litellm._version import version as litellm_version + +ISSUE_URL_BASE: Final = "https://github.com/BerriAI/litellm/issues/new" +MAX_URL_LENGTH: Final = 6000 +MAX_MESSAGE_CHARS: Final = 600 +MAX_FRAMES: Final = 12 +DISABLE_ENV_VAR: Final = "LITELLM_DISABLE_BUG_REPORT_LINK" + +Surface = Literal["sdk", "proxy"] + + +@dataclass(frozen=True, slots=True) +class BugReport: + surface: Surface + exception_type: str + exception_message: str + litellm_frames: tuple[str, ...] + litellm_version: str + python_version: str + os_platform: str + call_type: str | None + model: str | None + custom_llm_provider: str | None + + +def bug_report_enabled() -> bool: + return os.getenv(DISABLE_ENV_VAR, "").lower() != "true" + + +def _format_frame(frame: traceback.FrameSummary, package_dir: Path, package_parent: Path) -> str | None: + frame_path: Final = Path(frame.filename).resolve() + try: + frame_path.relative_to(package_dir) + relative_path: Final = frame_path.relative_to(package_parent) + except ValueError: + return None + return f"{relative_path.as_posix()}:{frame.lineno} in {frame.name}" + + +def _get_litellm_frames(exc: BaseException) -> tuple[str, ...]: + package_file: Final = getattr(litellm, "__file__", None) + if package_file is None or exc.__traceback__ is None: + return () + package_dir: Final = Path(package_file).resolve().parent + package_parent: Final = package_dir.parent + return tuple( + frame_text + for frame in traceback.extract_tb(exc.__traceback__) + if (frame_text := _format_frame(frame, package_dir, package_parent)) is not None + )[-MAX_FRAMES:] + + +def _redact_context(value: str | None) -> str | None: + if value is None: + return None + return redact_secrets(value)[:MAX_MESSAGE_CHARS] + + +def build_bug_report( + exc: BaseException, + *, + surface: Surface, + call_type: str | None = None, + model: str | None = None, + custom_llm_provider: str | None = None, +) -> BugReport: + exception_message: Final = redact_secrets(str(exc))[:MAX_MESSAGE_CHARS] + return BugReport( + surface=surface, + exception_type=type(exc).__name__, + exception_message=exception_message, + litellm_frames=_get_litellm_frames(exc), + litellm_version=litellm_version, + python_version=platform.python_version(), + os_platform=platform.platform(terse=True), + call_type=_redact_context(call_type), + model=_redact_context(model), + custom_llm_provider=_redact_context(custom_llm_provider), + ) + + +def _domain(report: BugReport) -> str: + if report.surface == "sdk": + return "Python SDK: the litellm package itself" + if report.custom_llm_provider is not None: + return "LLM translation: a specific provider's request or response" + return "Proxy core: startup, config, health checks, endpoints" + + +def _description(report: BugReport, message: str, frames: tuple[str, ...]) -> str: + frame_text: Final = "\n".join(frames) + return ( + "Auto-generated by LiteLLM's bug report link. Please describe what you were doing above this line.\n\n" + f"```\n{report.exception_type}: {message}\n```\n\n" + f"LiteLLM frames:\n```\n{frame_text}\n```\n\n" + f"Surface: {report.surface}\n" + f"Endpoint / call: {report.call_type or 'unknown'}\n" + f"Model: {report.model or 'unknown'}\n" + f"Provider: {report.custom_llm_provider or 'unknown'}\n" + f"LiteLLM: {report.litellm_version}\n" + f"Python: {report.python_version}\n" + f"OS: {report.os_platform}\n" + ) + + +def _issue_url(report: BugReport, message: str, frames: tuple[str, ...]) -> str: + deployment: tuple[tuple[str, str], ...] = ( + (("deployment", "pip / Python SDK"),) + if report.surface == "sdk" + else (("deployment", "Docker"),) + if os.path.exists("/.dockerenv") + else () + ) + fields: Final = ( + ("template", "bug_report.yml"), + ("labels", "bug"), + ("title", f"[Bug]: {report.exception_type}: {message[:80]}"), + ("version", report.litellm_version), + ("domain", _domain(report)), + ("description", _description(report, message, frames)), + ) + deployment + return f"{ISSUE_URL_BASE}?{urlencode(fields)}" + + +def bug_report_issue_url(report: BugReport) -> str: + frame_candidates: Final = tuple(report.litellm_frames[index:] for index in range(len(report.litellm_frames) + 1)) + message_lengths: Final = ( + len(report.exception_message), + 480, + 360, + 240, + 120, + 0, + ) + frame_candidates_with_full_message: Final = ( + _issue_url(report, report.exception_message[:message_length], frames) + for frames in frame_candidates + for message_length in (len(report.exception_message),) + ) + shortest_candidate: Final = _issue_url(report, "", ()) + full_message_candidate: Final = next( + (candidate for candidate in frame_candidates_with_full_message if len(candidate) <= MAX_URL_LENGTH), + None, + ) + if full_message_candidate is not None: + return full_message_candidate + shortened_candidates: Final = ( + _issue_url(report, report.exception_message[:message_length], ()) + for message_length in message_lengths + if message_length <= len(report.exception_message) + ) + return next((candidate for candidate in shortened_candidates if len(candidate) <= MAX_URL_LENGTH), shortest_candidate) + + +def bug_report_notice(report: BugReport) -> str: + return ( + "This looks like a bug in LiteLLM rather than in your request. File it with one click " + f"(prefilled and redacted, review before submitting): {bug_report_issue_url(report)}" + ) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 61e2698dd6f..6ce64f856e9 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -10,6 +10,11 @@ import httpx import litellm from litellm._logging import _ENABLE_SECRET_REDACTION, _redact_string, verbose_logger +from litellm.litellm_core_utils.bug_report import ( + bug_report_enabled, + bug_report_notice, + build_bug_report, +) from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.types.utils import LlmProviders @@ -510,8 +515,24 @@ def _map_openai_exception( else: # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors # exception_mapping_worked = True + bug_report_message: Final = ( + f"{exception_provider} - {message}" + + ( + "\n" + + bug_report_notice( + build_bug_report( + original_exception, + surface="sdk", + model=model, + custom_llm_provider=custom_llm_provider, + ) + ) + if not hasattr(original_exception, "request") and bug_report_enabled() + else "" + ) + ) raise APIConnectionError( - message=f"APIConnectionError: {exception_provider} - {message}", + message=f"APIConnectionError: {bug_report_message}", llm_provider=custom_llm_provider, model=model, litellm_debug_info=extra_information, @@ -2673,7 +2694,22 @@ def exception_type( ) else: raise APIConnectionError( - message=f"{original_exception}\n{_redact_string(traceback.format_exc())}", + message=( + f"{original_exception}\n{_redact_string(traceback.format_exc())}" + + ( + "\n" + + bug_report_notice( + build_bug_report( + original_exception, + surface="sdk", + model=model, + custom_llm_provider=custom_llm_provider, + ) + ) + if bug_report_enabled() + else "" + ) + ), llm_provider=custom_llm_provider, model=model, request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), # stub the request diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 9484fd7c723..dd0734a3e4b 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -19,6 +19,7 @@ from typing import ( overload, runtime_checkable, ) +from urllib.parse import urlparse import anyio import httpx @@ -45,6 +46,11 @@ from litellm.constants import ( UNSAFE_PROXY_RESPONSE_HEADERS, ) from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.bug_report import ( + bug_report_enabled, + bug_report_notice, + build_bug_report, +) from litellm.litellm_core_utils.core_helpers import ( get_or_create_metadata_bucket, independent_snapshot, @@ -3658,6 +3664,25 @@ class ProxyBaseLLMRequestProcessing: _code = _exc_status_code else: _code = status.HTTP_500_INTERNAL_SERVER_ERROR + if bug_report_enabled(): + proxy_server_request: Final = self.data.get("proxy_server_request") + request_url: Final = ( + proxy_server_request.get("url") + if isinstance(proxy_server_request, Mapping) + else None + ) + request_path: Final = urlparse(str(request_url)).path if request_url is not None else None + verbose_proxy_logger.error( + bug_report_notice( + build_bug_report( + e, + surface="proxy", + call_type=request_path or None, + model=self.data.get("model"), + custom_llm_provider=self.data.get("custom_llm_provider"), + ) + ) + ) raise ProxyException( message=redact_internal_details_from_client_message(getattr(e, "message", error_msg)), type=openai_error_type(e, _code), diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index af25d418a63..b9becdf746c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -74,6 +74,11 @@ from litellm.constants import ( RUNTIME_UPDATABLE_ROUTER_SETTINGS, ) from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.bug_report import ( + bug_report_enabled, + bug_report_notice, + build_bug_report, +) from litellm.litellm_core_utils.litellm_logging import ( _init_custom_logger_compatible_class, ) @@ -1870,6 +1875,10 @@ async def otel_unhandled_exception_handler(request: Request, exc: Exception): if isinstance(exc, (ProxyException, HTTPException, RequestValidationError)): raise exc verbose_proxy_logger.exception("Unhandled exception in request: %s", type(exc).__name__) + if bug_report_enabled(): + verbose_proxy_logger.error( + bug_report_notice(build_bug_report(exc, surface="proxy", call_type=request.url.path)) + ) _close_dangling_otel_server_span(request, 500, exc=exc) return JSONResponse( status_code=500, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index f6f437bea75..9f6317316ad 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -55,6 +55,11 @@ from litellm.constants import ( SPEND_LOG_WRITE_BATCH_MAX_BYTES, SPEND_LOG_WRITE_BATCH_MAX_ROWS, ) +from litellm.litellm_core_utils.bug_report import ( + bug_report_enabled, + bug_report_notice, + build_bug_report, +) from litellm.proxy._types import ( CommonProxyErrors, ProxyErrorTypes, @@ -7836,6 +7841,8 @@ def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None) elif isinstance(e, ProxyException): return with_litellm_call_id(e, litellm_call_id) _status_code: Final = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR) + if _status_code == status.HTTP_500_INTERNAL_SERVER_ERROR and bug_report_enabled(): + verbose_proxy_logger.error(bug_report_notice(build_bug_report(e, surface="proxy"))) return ProxyException( message=str(e), type=ProxyErrorTypes.internal_server_error, diff --git a/tests/test_litellm/litellm_core_utils/test_bug_report.py b/tests/test_litellm/litellm_core_utils/test_bug_report.py new file mode 100644 index 00000000000..1c693b9a4cb --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_bug_report.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from typing import cast +from urllib.parse import parse_qs, urlparse + +import pytest + +from litellm._version import version +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.bug_report import ( + DISABLE_ENV_VAR, + ISSUE_URL_BASE, + MAX_URL_LENGTH, + BugReport, + build_bug_report, + bug_report_enabled, + bug_report_issue_url, +) +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + +def test_build_bug_report_keeps_only_redacted_litellm_frames(): + with pytest.raises(BadRequestError) as raised: + get_llm_provider(cast(str, None)) + report = build_bug_report(raised.value, surface="sdk") + + assert report.litellm_frames + assert all(frame.startswith("litellm/") for frame in report.litellm_frames) + assert all("test_bug_report.py" not in frame for frame in report.litellm_frames) + + +def test_issue_url_redacts_message_and_prefills_sdk_fields(): + report = build_bug_report( + RuntimeError("failed with key sk-abcdefghijklmnopqrstuvwxyz1234567890"), + surface="sdk", + ) + query = parse_qs(urlparse(bug_report_issue_url(report)).query) + + assert ISSUE_URL_BASE in bug_report_issue_url(report) + assert "sk-abcdef" not in str(query) + assert query["title"][0].startswith("[Bug]: RuntimeError:") + assert query["version"] == [version] + assert query["template"] == ["bug_report.yml"] + assert query["domain"] == ["Python SDK: the litellm package itself"] + assert query["deployment"] == ["pip / Python SDK"] + assert "RuntimeError" in query["description"][0] + assert "Python:" in query["description"][0] + + +def test_issue_url_is_bounded_for_long_messages(): + report = build_bug_report(RuntimeError("x" * 20_000), surface="proxy") + query = parse_qs(urlparse(bug_report_issue_url(report)).query) + + assert len(bug_report_issue_url(report)) <= MAX_URL_LENGTH + assert "RuntimeError" in query["description"][0] + + +def test_issue_url_builds_without_a_traceback(): + exc = RuntimeError("no traceback") + assert exc.__traceback__ is None + report = build_bug_report(exc, surface="proxy") + + assert report.litellm_frames == () + assert bug_report_issue_url(report).startswith(ISSUE_URL_BASE) + + +def test_bug_report_can_be_disabled(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv(DISABLE_ENV_VAR, "true") + + assert bug_report_enabled() is False + + +def test_proxy_provider_uses_translation_domain(): + report = build_bug_report( + RuntimeError("proxy failure"), + surface="proxy", + call_type="/v1/chat/completions", + model="gpt-4", + custom_llm_provider="openai", + ) + query = parse_qs(urlparse(bug_report_issue_url(report)).query) + + assert query["domain"] == ["LLM translation: a specific provider's request or response"] diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index cfe7470fa76..6045e584737 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -4,7 +4,7 @@ import pytest import litellm - +from litellm.litellm_core_utils.bug_report import DISABLE_ENV_VAR, ISSUE_URL_BASE from litellm.litellm_core_utils.exception_mapping_utils import ( ExceptionCheckers, _get_body_error_code, @@ -974,6 +974,30 @@ def test_an_unmapped_exception_with_no_model_or_provider_is_a_connection_error(q assert "boom" in raised.value.message +def test_unmapped_sdk_exception_includes_bug_report_link(quiet_exception_mapping): + with pytest.raises(litellm.APIConnectionError) as raised: + exception_type( + model="gpt-4", + original_exception=ValueError("boom"), + custom_llm_provider="openai", + ) + + assert ISSUE_URL_BASE in str(raised.value) + + +def test_unmapped_sdk_exception_bug_report_link_can_be_disabled(quiet_exception_mapping, monkeypatch): + monkeypatch.setenv(DISABLE_ENV_VAR, "true") + + with pytest.raises(litellm.APIConnectionError) as raised: + exception_type( + model="gpt-4", + original_exception=ValueError("boom"), + custom_llm_provider="openai", + ) + + assert ISSUE_URL_BASE not in str(raised.value) + + def _raise_and_map(model: str | None, original_exception: Exception, custom_llm_provider: str | None) -> None: """Calls exception_type() from inside the except block, as litellm/main.py does, so traceback.format_exc() has a real stack.""" diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index e4ca0b03d59..1f698f179aa 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -4,6 +4,7 @@ import datetime import json from types import MappingProxyType, SimpleNamespace from typing import AsyncGenerator, Callable, Final, Iterator, Optional, Sequence +from urllib.parse import unquote from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -13,6 +14,7 @@ from fastapi.responses import JSONResponse, StreamingResponse import litellm from litellm._uuid import uuid +from litellm.litellm_core_utils.bug_report import DISABLE_ENV_VAR, ISSUE_URL_BASE from litellm.constants import ( CLIENT_REQUESTED_MODEL_SCOPE_KEY, MAX_LITELLM_CALL_ID_LENGTH, @@ -9285,6 +9287,63 @@ async def test_handle_llm_api_exception_forwards_litellm_response_headers_when_r assert exc_info.value.headers["llm_provider-x-request-id"] == "req_openai_400" +@pytest.mark.asyncio +async def test_handle_llm_api_exception_logs_bug_report_for_unmapped_error( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +): + monkeypatch.delenv(DISABLE_ENV_VAR, raising=False) + processor = ProxyBaseLLMRequestProcessing( + data={ + "proxy_server_request": {"url": "https://example.test/v1/chat/completions?debug=true"}, + "model": "gpt-4", + "custom_llm_provider": "openai", + } + ) + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + with caplog.at_level("ERROR", logger="LiteLLM Proxy"): + with pytest.raises(ProxyException): + await processor._handle_llm_api_exception( + e=RuntimeError("unmapped"), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + ) + + assert ISSUE_URL_BASE in caplog.text + assert "/v1/chat/completions" in unquote(caplog.text) + + +@pytest.mark.asyncio +async def test_handle_llm_api_exception_skips_bug_report_for_provider_status( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +): + monkeypatch.delenv(DISABLE_ENV_VAR, raising=False) + + class ProviderRateLimitError(Exception): + def __init__(self, message: str): + super().__init__(message) + self.status_code = 429 + + processor = ProxyBaseLLMRequestProcessing(data={}) + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + with caplog.at_level("ERROR", logger="LiteLLM Proxy"): + with pytest.raises(ProxyException): + await processor._handle_llm_api_exception( + e=ProviderRateLimitError("rate limited"), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + ) + + assert ISSUE_URL_BASE not in caplog.text + + class TestBackgroundResponseRetrievalGovernance: """LIT-7175: retrieving a background Response attaches the model's post_call policy pipelines."""