scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-26 01:12:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 99

feat(errors): prefilled GitHub issue link on unmapped internal errors

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
ryan 2026-09-20 01:23:50 +00:00
parent d1773d96e9
commit 38475d7414
8 changed files with 417 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

171
litellm/litellm_core_utils/bug_report.py Normal file
View file

@ -0,0 +1,171 @@
from __future__ import annotations
import os
import platform
import traceback
from dataclasses import dataclass
from pathlib import Path
from typing import Final, Literal
from urllib.parse import urlencode
import litellm
from litellm._logging import redact_secrets
from litellm._version import version as litellm_version
ISSUE_URL_BASE: Final = "https://github.com/BerriAI/litellm/issues/new"
MAX_URL_LENGTH: Final = 6000
MAX_MESSAGE_CHARS: Final = 600
MAX_FRAMES: Final = 12
DISABLE_ENV_VAR: Final = "LITELLM_DISABLE_BUG_REPORT_LINK"
Surface = Literal["sdk", "proxy"]
@dataclass(frozen=True, slots=True)
class BugReport:
surface: Surface
exception_type: str
exception_message: str
litellm_frames: tuple[str, ...]
litellm_version: str
python_version: str
os_platform: str
call_type: str | None
model: str | None
custom_llm_provider: str | None
def bug_report_enabled() -> bool:
return os.getenv(DISABLE_ENV_VAR, "").lower() != "true"
def _format_frame(frame: traceback.FrameSummary, package_dir: Path, package_parent: Path) -> str | None:
frame_path: Final = Path(frame.filename).resolve()
try:
frame_path.relative_to(package_dir)
relative_path: Final = frame_path.relative_to(package_parent)
except ValueError:
return None
return f"{relative_path.as_posix()}:{frame.lineno} in {frame.name}"
def _get_litellm_frames(exc: BaseException) -> tuple[str, ...]:
package_file: Final = getattr(litellm, "__file__", None)
if package_file is None or exc.__traceback__ is None:
return ()
package_dir: Final = Path(package_file).resolve().parent
package_parent: Final = package_dir.parent
return tuple(
frame_text
for frame in traceback.extract_tb(exc.__traceback__)
if (frame_text := _format_frame(frame, package_dir, package_parent)) is not None
)[-MAX_FRAMES:]
def _redact_context(value: str | None) -> str | None:
if value is None:
return None
return redact_secrets(value)[:MAX_MESSAGE_CHARS]
def build_bug_report(
exc: BaseException,
*,
surface: Surface,
call_type: str | None = None,
model: str | None = None,
custom_llm_provider: str | None = None,
) -> BugReport:
exception_message: Final = redact_secrets(str(exc))[:MAX_MESSAGE_CHARS]
return BugReport(
surface=surface,
exception_type=type(exc).__name__,
exception_message=exception_message,
litellm_frames=_get_litellm_frames(exc),
litellm_version=litellm_version,
python_version=platform.python_version(),
os_platform=platform.platform(terse=True),
call_type=_redact_context(call_type),
model=_redact_context(model),
custom_llm_provider=_redact_context(custom_llm_provider),
)
def _domain(report: BugReport) -> str:
if report.surface == "sdk":
return "Python SDK: the litellm package itself"
if report.custom_llm_provider is not None:
return "LLM translation: a specific provider's request or response"
return "Proxy core: startup, config, health checks, endpoints"
def _description(report: BugReport, message: str, frames: tuple[str, ...]) -> str:
frame_text: Final = "\n".join(frames)
return (
"Auto-generated by LiteLLM's bug report link. Please describe what you were doing above this line.\n\n"
f"```\n{report.exception_type}: {message}\n```\n\n"
f"LiteLLM frames:\n```\n{frame_text}\n```\n\n"
f"Surface: {report.surface}\n"
f"Endpoint / call: {report.call_type or 'unknown'}\n"
f"Model: {report.model or 'unknown'}\n"
f"Provider: {report.custom_llm_provider or 'unknown'}\n"
f"LiteLLM: {report.litellm_version}\n"
f"Python: {report.python_version}\n"
f"OS: {report.os_platform}\n"
)
def _issue_url(report: BugReport, message: str, frames: tuple[str, ...]) -> str:
deployment: tuple[tuple[str, str], ...] = (
(("deployment", "pip / Python SDK"),)
if report.surface == "sdk"
else (("deployment", "Docker"),)
if os.path.exists("/.dockerenv")
else ()
)
fields: Final = (
("template", "bug_report.yml"),
("labels", "bug"),
("title", f"[Bug]: {report.exception_type}: {message[:80]}"),
("version", report.litellm_version),
("domain", _domain(report)),
("description", _description(report, message, frames)),
) + deployment
return f"{ISSUE_URL_BASE}?{urlencode(fields)}"
def bug_report_issue_url(report: BugReport) -> str:
frame_candidates: Final = tuple(report.litellm_frames[index:] for index in range(len(report.litellm_frames) + 1))
message_lengths: Final = (
len(report.exception_message),
480,
360,
240,
120,
0,
)
frame_candidates_with_full_message: Final = (
_issue_url(report, report.exception_message[:message_length], frames)
for frames in frame_candidates
for message_length in (len(report.exception_message),)
)
shortest_candidate: Final = _issue_url(report, "", ())
full_message_candidate: Final = next(
(candidate for candidate in frame_candidates_with_full_message if len(candidate) <= MAX_URL_LENGTH),
None,
)
if full_message_candidate is not None:
return full_message_candidate
shortened_candidates: Final = (
_issue_url(report, report.exception_message[:message_length], ())
for message_length in message_lengths
if message_length <= len(report.exception_message)
)
return next((candidate for candidate in shortened_candidates if len(candidate) <= MAX_URL_LENGTH), shortest_candidate)
def bug_report_notice(report: BugReport) -> str:
return (
"This looks like a bug in LiteLLM rather than in your request. File it with one click "
f"(prefilled and redacted, review before submitting): {bug_report_issue_url(report)}"
)

40
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -10,6 +10,11 @@ import httpx
import litellm
from litellm._logging import _ENABLE_SECRET_REDACTION, _redact_string, verbose_logger
from litellm.litellm_core_utils.bug_report import (
bug_report_enabled,
bug_report_notice,
build_bug_report,
)
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.types.utils import LlmProviders
@ -510,8 +515,24 @@ def _map_openai_exception(
else:
# if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
# exception_mapping_worked = True
bug_report_message: Final = (
f"{exception_provider} - {message}"
+ (
"\n"
+ bug_report_notice(
build_bug_report(
original_exception,
surface="sdk",
model=model,
custom_llm_provider=custom_llm_provider,
)
)
if not hasattr(original_exception, "request") and bug_report_enabled()
else ""
)
)
raise APIConnectionError(
message=f"APIConnectionError: {exception_provider} - {message}",
message=f"APIConnectionError: {bug_report_message}",
llm_provider=custom_llm_provider,
model=model,
litellm_debug_info=extra_information,
@ -2673,7 +2694,22 @@ def exception_type(
)
else:
raise APIConnectionError(
message=f"{original_exception}\n{_redact_string(traceback.format_exc())}",
message=(
f"{original_exception}\n{_redact_string(traceback.format_exc())}"
+ (
"\n"
+ bug_report_notice(
build_bug_report(
original_exception,
surface="sdk",
model=model,
custom_llm_provider=custom_llm_provider,
)
)
if bug_report_enabled()
else ""
)
),
llm_provider=custom_llm_provider,
model=model,
request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), # stub the request

25
litellm/proxy/common_request_processing.py
View file

@ -19,6 +19,7 @@ from typing import (
overload,
runtime_checkable,
)
from urllib.parse import urlparse
import anyio
import httpx
@ -45,6 +46,11 @@ from litellm.constants import (
UNSAFE_PROXY_RESPONSE_HEADERS,
)
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.litellm_core_utils.bug_report import (
bug_report_enabled,
bug_report_notice,
build_bug_report,
)
from litellm.litellm_core_utils.core_helpers import (
get_or_create_metadata_bucket,
independent_snapshot,
@ -3658,6 +3664,25 @@ class ProxyBaseLLMRequestProcessing:
_code = _exc_status_code
else:
_code = status.HTTP_500_INTERNAL_SERVER_ERROR
if bug_report_enabled():
proxy_server_request: Final = self.data.get("proxy_server_request")
request_url: Final = (
proxy_server_request.get("url")
if isinstance(proxy_server_request, Mapping)
else None
)
request_path: Final = urlparse(str(request_url)).path if request_url is not None else None
verbose_proxy_logger.error(
bug_report_notice(
build_bug_report(
e,
surface="proxy",
call_type=request_path or None,
model=self.data.get("model"),
custom_llm_provider=self.data.get("custom_llm_provider"),
)
)
)
raise ProxyException(
message=redact_internal_details_from_client_message(getattr(e, "message", error_msg)),
type=openai_error_type(e, _code),

9
litellm/proxy/proxy_server.py
View file

@ -74,6 +74,11 @@ from litellm.constants import (
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
)
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.bug_report import (
bug_report_enabled,
bug_report_notice,
build_bug_report,
)
from litellm.litellm_core_utils.litellm_logging import (
_init_custom_logger_compatible_class,
)
@ -1870,6 +1875,10 @@ async def otel_unhandled_exception_handler(request: Request, exc: Exception):
if isinstance(exc, (ProxyException, HTTPException, RequestValidationError)):
raise exc
verbose_proxy_logger.exception("Unhandled exception in request: %s", type(exc).__name__)
if bug_report_enabled():
verbose_proxy_logger.error(
bug_report_notice(build_bug_report(exc, surface="proxy", call_type=request.url.path))
)
_close_dangling_otel_server_span(request, 500, exc=exc)
return JSONResponse(
status_code=500,

7
litellm/proxy/utils.py
View file

@ -55,6 +55,11 @@ from litellm.constants import (
SPEND_LOG_WRITE_BATCH_MAX_BYTES,
SPEND_LOG_WRITE_BATCH_MAX_ROWS,
)
from litellm.litellm_core_utils.bug_report import (
bug_report_enabled,
bug_report_notice,
build_bug_report,
)
from litellm.proxy._types import (
CommonProxyErrors,
ProxyErrorTypes,
@ -7836,6 +7841,8 @@ def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None)
elif isinstance(e, ProxyException):
return with_litellm_call_id(e, litellm_call_id)
_status_code: Final = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
if _status_code == status.HTTP_500_INTERNAL_SERVER_ERROR and bug_report_enabled():
verbose_proxy_logger.error(bug_report_notice(build_bug_report(e, surface="proxy")))
return ProxyException(
message=str(e),
type=ProxyErrorTypes.internal_server_error,

83
tests/test_litellm/litellm_core_utils/test_bug_report.py Normal file
View file

@ -0,0 +1,83 @@
from __future__ import annotations
from typing import cast
from urllib.parse import parse_qs, urlparse
import pytest
from litellm._version import version
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.bug_report import (
DISABLE_ENV_VAR,
ISSUE_URL_BASE,
MAX_URL_LENGTH,
BugReport,
build_bug_report,
bug_report_enabled,
bug_report_issue_url,
)
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
def test_build_bug_report_keeps_only_redacted_litellm_frames():
with pytest.raises(BadRequestError) as raised:
get_llm_provider(cast(str, None))
report = build_bug_report(raised.value, surface="sdk")
assert report.litellm_frames
assert all(frame.startswith("litellm/") for frame in report.litellm_frames)
assert all("test_bug_report.py" not in frame for frame in report.litellm_frames)
def test_issue_url_redacts_message_and_prefills_sdk_fields():
report = build_bug_report(
RuntimeError("failed with key sk-abcdefghijklmnopqrstuvwxyz1234567890"),
surface="sdk",
)
query = parse_qs(urlparse(bug_report_issue_url(report)).query)
assert ISSUE_URL_BASE in bug_report_issue_url(report)
assert "sk-abcdef" not in str(query)
assert query["title"][0].startswith("[Bug]: RuntimeError:")
assert query["version"] == [version]
assert query["template"] == ["bug_report.yml"]
assert query["domain"] == ["Python SDK: the litellm package itself"]
assert query["deployment"] == ["pip / Python SDK"]
assert "RuntimeError" in query["description"][0]
assert "Python:" in query["description"][0]
def test_issue_url_is_bounded_for_long_messages():
report = build_bug_report(RuntimeError("x" * 20_000), surface="proxy")
query = parse_qs(urlparse(bug_report_issue_url(report)).query)
assert len(bug_report_issue_url(report)) <= MAX_URL_LENGTH
assert "RuntimeError" in query["description"][0]
def test_issue_url_builds_without_a_traceback():
exc = RuntimeError("no traceback")
assert exc.__traceback__ is None
report = build_bug_report(exc, surface="proxy")
assert report.litellm_frames == ()
assert bug_report_issue_url(report).startswith(ISSUE_URL_BASE)
def test_bug_report_can_be_disabled(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv(DISABLE_ENV_VAR, "true")
assert bug_report_enabled() is False
def test_proxy_provider_uses_translation_domain():
report = build_bug_report(
RuntimeError("proxy failure"),
surface="proxy",
call_type="/v1/chat/completions",
model="gpt-4",
custom_llm_provider="openai",
)
query = parse_qs(urlparse(bug_report_issue_url(report)).query)
assert query["domain"] == ["LLM translation: a specific provider's request or response"]

26
tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
View file

@ -4,7 +4,7 @@ import pytest
import litellm
from litellm.litellm_core_utils.bug_report import DISABLE_ENV_VAR, ISSUE_URL_BASE
from litellm.litellm_core_utils.exception_mapping_utils import (
ExceptionCheckers,
_get_body_error_code,
@ -974,6 +974,30 @@ def test_an_unmapped_exception_with_no_model_or_provider_is_a_connection_error(q
assert "boom" in raised.value.message
def test_unmapped_sdk_exception_includes_bug_report_link(quiet_exception_mapping):
with pytest.raises(litellm.APIConnectionError) as raised:
exception_type(
model="gpt-4",
original_exception=ValueError("boom"),
custom_llm_provider="openai",
)
assert ISSUE_URL_BASE in str(raised.value)
def test_unmapped_sdk_exception_bug_report_link_can_be_disabled(quiet_exception_mapping, monkeypatch):
monkeypatch.setenv(DISABLE_ENV_VAR, "true")
with pytest.raises(litellm.APIConnectionError) as raised:
exception_type(
model="gpt-4",
original_exception=ValueError("boom"),
custom_llm_provider="openai",
)
assert ISSUE_URL_BASE not in str(raised.value)
def _raise_and_map(model: str | None, original_exception: Exception, custom_llm_provider: str | None) -> None:
"""Calls exception_type() from inside the except block, as litellm/main.py does,
so traceback.format_exc() has a real stack."""

59
tests/test_litellm/proxy/test_common_request_processing.py
View file

@ -4,6 +4,7 @@ import datetime
import json
from types import MappingProxyType, SimpleNamespace
from typing import AsyncGenerator, Callable, Final, Iterator, Optional, Sequence
from urllib.parse import unquote
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
@ -13,6 +14,7 @@ from fastapi.responses import JSONResponse, StreamingResponse
import litellm
from litellm._uuid import uuid
from litellm.litellm_core_utils.bug_report import DISABLE_ENV_VAR, ISSUE_URL_BASE
from litellm.constants import (
CLIENT_REQUESTED_MODEL_SCOPE_KEY,
MAX_LITELLM_CALL_ID_LENGTH,
@ -9285,6 +9287,63 @@ async def test_handle_llm_api_exception_forwards_litellm_response_headers_when_r
assert exc_info.value.headers["llm_provider-x-request-id"] == "req_openai_400"
@pytest.mark.asyncio
async def test_handle_llm_api_exception_logs_bug_report_for_unmapped_error(
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
):
monkeypatch.delenv(DISABLE_ENV_VAR, raising=False)
processor = ProxyBaseLLMRequestProcessing(
data={
"proxy_server_request": {"url": "https://example.test/v1/chat/completions?debug=true"},
"model": "gpt-4",
"custom_llm_provider": "openai",
}
)
proxy_logging_obj = MagicMock()
proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={})
with caplog.at_level("ERROR", logger="LiteLLM Proxy"):
with pytest.raises(ProxyException):
await processor._handle_llm_api_exception(
e=RuntimeError("unmapped"),
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"),
proxy_logging_obj=proxy_logging_obj,
)
assert ISSUE_URL_BASE in caplog.text
assert "/v1/chat/completions" in unquote(caplog.text)
@pytest.mark.asyncio
async def test_handle_llm_api_exception_skips_bug_report_for_provider_status(
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
):
monkeypatch.delenv(DISABLE_ENV_VAR, raising=False)
class ProviderRateLimitError(Exception):
def __init__(self, message: str):
super().__init__(message)
self.status_code = 429
processor = ProxyBaseLLMRequestProcessing(data={})
proxy_logging_obj = MagicMock()
proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={})
with caplog.at_level("ERROR", logger="LiteLLM Proxy"):
with pytest.raises(ProxyException):
await processor._handle_llm_api_exception(
e=ProviderRateLimitError("rate limited"),
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"),
proxy_logging_obj=proxy_logging_obj,
)
assert ISSUE_URL_BASE not in caplog.text
class TestBackgroundResponseRetrievalGovernance:
"""LIT-7175: retrieving a background Response attaches the model's post_call policy pipelines."""