mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-05 02:41:56 +00:00
feat(errors): prefilled GitHub issue link on unmapped internal errors (#42065)
* feat(errors): prefilled GitHub issue link on unmapped internal errors Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(errors): scope bug report link to unmapped exceptions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(errors): cover bug report link on unmapped SDK exceptions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(errors): exercise unmapped branch without changing openai mapping Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style: format bug report changes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): guard bug report generation for invalid exceptions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep bug report notice out of client error bodies Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(errors): strip only the notice separator from client messages Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(errors): only link bug reports for exceptions without a provider status Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(errors): allowlist-only bug report link, drop exception message and model from URL Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): read request.url.path for the unhandled-exception bug report route Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bug_report): drop non-string provider values before the allowlist lookup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(bug_report): drop the OS line from the prefilled issue Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: ryan <ryan@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
798d45970b
commit
4936427094
9 changed files with 553 additions and 7 deletions
155
litellm/litellm_core_utils/bug_report.py
Normal file
155
litellm/litellm_core_utils/bug_report.py
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import platform
|
||||
import traceback
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final, Literal
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import litellm
|
||||
from litellm._version import version as litellm_version
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
ISSUE_URL_BASE: Final = "https://github.com/BerriAI/litellm/issues/new"
|
||||
MAX_URL_LENGTH: Final = 6000
|
||||
MAX_FRAMES: Final = 12
|
||||
DISABLE_ENV_VAR: Final = "LITELLM_DISABLE_BUG_REPORT_LINK"
|
||||
NOTICE_PREFIX: Final = "This looks like a bug in LiteLLM rather than in your request."
|
||||
KNOWN_PROVIDERS: Final = frozenset(provider.value for provider in LlmProviders)
|
||||
|
||||
Surface = Literal["sdk", "proxy"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BugReport:
|
||||
surface: Surface
|
||||
exception_type: str
|
||||
litellm_frames: tuple[str, ...]
|
||||
litellm_version: str
|
||||
python_version: str
|
||||
call_type: str | None
|
||||
custom_llm_provider: str | None
|
||||
|
||||
|
||||
def bug_report_enabled() -> bool:
|
||||
return os.getenv(DISABLE_ENV_VAR, "").lower() != "true"
|
||||
|
||||
|
||||
def should_report_bug(exc: object) -> bool:
|
||||
return bug_report_enabled() and isinstance(exc, BaseException) and getattr(exc, "status_code", None) is None
|
||||
|
||||
|
||||
def _format_frame(frame: traceback.FrameSummary, package_dir: Path, package_parent: Path) -> str | None:
|
||||
frame_path: Final = Path(frame.filename).resolve()
|
||||
try:
|
||||
frame_path.relative_to(package_dir)
|
||||
relative_path: Final = frame_path.relative_to(package_parent)
|
||||
except ValueError:
|
||||
return None
|
||||
return f"{relative_path.as_posix()}:{frame.lineno} in {frame.name}"
|
||||
|
||||
|
||||
def _get_litellm_frames(exc: BaseException) -> tuple[str, ...]:
|
||||
if exc.__traceback__ is None:
|
||||
return ()
|
||||
package_dir: Final = Path(litellm.__file__).resolve().parent
|
||||
package_parent: Final = package_dir.parent
|
||||
return tuple(
|
||||
frame_text
|
||||
for frame in traceback.extract_tb(exc.__traceback__)
|
||||
if (frame_text := _format_frame(frame, package_dir, package_parent)) is not None
|
||||
)[-MAX_FRAMES:]
|
||||
|
||||
|
||||
def allowlisted(value: object, allowed: frozenset[str]) -> str | None:
|
||||
return value if isinstance(value, str) and value in allowed else None
|
||||
|
||||
|
||||
def build_bug_report(
|
||||
exc: BaseException,
|
||||
*,
|
||||
surface: Surface,
|
||||
call_type: str | None = None,
|
||||
custom_llm_provider: object = None,
|
||||
) -> BugReport:
|
||||
return BugReport(
|
||||
surface=surface,
|
||||
exception_type=type(exc).__name__,
|
||||
litellm_frames=_get_litellm_frames(exc),
|
||||
litellm_version=litellm_version,
|
||||
python_version=platform.python_version(),
|
||||
call_type=call_type,
|
||||
custom_llm_provider=allowlisted(custom_llm_provider, KNOWN_PROVIDERS),
|
||||
)
|
||||
|
||||
|
||||
def _domain(report: BugReport) -> str:
|
||||
if report.surface == "sdk":
|
||||
return "Python SDK: the litellm package itself"
|
||||
if report.custom_llm_provider is not None:
|
||||
return "LLM translation: a specific provider's request or response"
|
||||
return "Proxy core: startup, config, health checks, endpoints"
|
||||
|
||||
|
||||
def _title(report: BugReport, frames: tuple[str, ...]) -> str:
|
||||
location: Final = frames[-1].split(":", 1)[0] if frames else "litellm"
|
||||
return f"[Bug]: {report.exception_type} in {location}"
|
||||
|
||||
|
||||
def _description(report: BugReport, frames: tuple[str, ...]) -> str:
|
||||
frame_block: Final = "LiteLLM frames:\n```\n" + "\n".join(frames) + "\n```\n\n" if frames else ""
|
||||
return (
|
||||
"Auto-generated by LiteLLM's bug report link. It carries no request data or error text. "
|
||||
"Please describe what you were doing, and paste the error message from your log below "
|
||||
"if it contains nothing sensitive.\n\n"
|
||||
"```\n\n```\n\n"
|
||||
f"Exception: `{report.exception_type}`\n\n"
|
||||
f"{frame_block}"
|
||||
f"Surface: {report.surface}\n"
|
||||
f"Endpoint / call: {report.call_type or 'unknown'}\n"
|
||||
f"Provider: {report.custom_llm_provider or 'unknown'}\n"
|
||||
f"LiteLLM: {report.litellm_version}\n"
|
||||
f"Python: {report.python_version}\n"
|
||||
)
|
||||
|
||||
|
||||
def _issue_url(report: BugReport, frames: tuple[str, ...]) -> str:
|
||||
deployment: Final[tuple[tuple[str, str], ...]] = (
|
||||
(("deployment", "pip / Python SDK"),)
|
||||
if report.surface == "sdk"
|
||||
else (("deployment", "Docker"),)
|
||||
if os.path.exists("/.dockerenv")
|
||||
else ()
|
||||
)
|
||||
fields: Final = (
|
||||
("template", "bug_report.yml"),
|
||||
("labels", "bug"),
|
||||
("title", _title(report, frames)),
|
||||
("version", report.litellm_version),
|
||||
("domain", _domain(report)),
|
||||
("description", _description(report, frames)),
|
||||
) + deployment
|
||||
return f"{ISSUE_URL_BASE}?{urlencode(fields)}"
|
||||
|
||||
|
||||
def bug_report_issue_url(report: BugReport) -> str:
|
||||
frames: Final = report.litellm_frames
|
||||
candidates: Final = tuple(_issue_url(report, frames[index:]) for index in range(len(frames) + 1))
|
||||
return next((candidate for candidate in candidates if len(candidate) <= MAX_URL_LENGTH), candidates[-1])
|
||||
|
||||
|
||||
def strip_bug_report_notice(message: str) -> str:
|
||||
index: Final = message.find(NOTICE_PREFIX)
|
||||
if index == -1:
|
||||
return message
|
||||
head: Final = message[:index]
|
||||
return head.removesuffix("\n")
|
||||
|
||||
|
||||
def bug_report_notice(report: BugReport) -> str:
|
||||
return (
|
||||
f"{NOTICE_PREFIX} File it with one click "
|
||||
f"(prefilled, no request data or error text, review before submitting): {bug_report_issue_url(report)}"
|
||||
)
|
||||
|
|
@ -10,6 +10,11 @@ import httpx
|
|||
|
||||
import litellm
|
||||
from litellm._logging import _ENABLE_SECRET_REDACTION, _redact_string, verbose_logger
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
bug_report_notice,
|
||||
build_bug_report,
|
||||
should_report_bug,
|
||||
)
|
||||
from litellm.litellm_core_utils.secret_redaction import redact_string
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
|
@ -2673,7 +2678,21 @@ def exception_type(
|
|||
)
|
||||
else:
|
||||
raise APIConnectionError(
|
||||
message=f"{original_exception}\n{_redact_string(traceback.format_exc())}",
|
||||
message=(
|
||||
f"{original_exception}\n{_redact_string(traceback.format_exc())}"
|
||||
+ (
|
||||
"\n"
|
||||
+ bug_report_notice(
|
||||
build_bug_report(
|
||||
original_exception,
|
||||
surface="sdk",
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
)
|
||||
if should_report_bug(original_exception)
|
||||
else ""
|
||||
)
|
||||
),
|
||||
llm_provider=custom_llm_provider,
|
||||
model=model,
|
||||
request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), # stub the request
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from typing import (
|
|||
overload,
|
||||
runtime_checkable,
|
||||
)
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import anyio
|
||||
import httpx
|
||||
|
|
@ -45,6 +46,13 @@ from litellm.constants import (
|
|||
UNSAFE_PROXY_RESPONSE_HEADERS,
|
||||
)
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
allowlisted,
|
||||
bug_report_notice,
|
||||
build_bug_report,
|
||||
should_report_bug,
|
||||
strip_bug_report_notice,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
get_or_create_metadata_bucket,
|
||||
independent_snapshot,
|
||||
|
|
@ -64,7 +72,7 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
|||
from litellm.litellm_core_utils.streaming_handler import (
|
||||
backfill_missing_cache_usage_fields,
|
||||
)
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
can_key_call_resolved_model,
|
||||
request_skips_budget_checks,
|
||||
|
|
@ -107,6 +115,10 @@ from litellm.types.router_weights import validate_router_weights
|
|||
_LateResponseT = TypeVar("_LateResponseT", bound=Response)
|
||||
_LlmCallT = TypeVar("_LlmCallT")
|
||||
|
||||
KNOWN_PROXY_ROUTES: Final = frozenset(
|
||||
route for member in LiteLLMRoutes for route in member.value if route.startswith("/")
|
||||
)
|
||||
|
||||
ProxyRouteType: TypeAlias = Literal[
|
||||
"acompletion",
|
||||
"aembedding",
|
||||
|
|
@ -3669,8 +3681,27 @@ class ProxyBaseLLMRequestProcessing:
|
|||
_code = _exc_status_code
|
||||
else:
|
||||
_code = status.HTTP_500_INTERNAL_SERVER_ERROR
|
||||
if should_report_bug(e):
|
||||
proxy_server_request: Final = self.data.get("proxy_server_request")
|
||||
request_url: Final = (
|
||||
proxy_server_request.get("url") if isinstance(proxy_server_request, Mapping) else None
|
||||
)
|
||||
request_path: Final = urlparse(str(request_url)).path if request_url is not None else None
|
||||
verbose_proxy_logger.error(
|
||||
bug_report_notice(
|
||||
build_bug_report(
|
||||
e,
|
||||
surface="proxy",
|
||||
call_type=allowlisted(request_path, KNOWN_PROXY_ROUTES),
|
||||
custom_llm_provider=self.data.get("custom_llm_provider"),
|
||||
)
|
||||
)
|
||||
)
|
||||
client_message: Final = getattr(e, "message", error_msg)
|
||||
raise ProxyException(
|
||||
message=redact_internal_details_from_client_message(getattr(e, "message", error_msg)),
|
||||
message=redact_internal_details_from_client_message(
|
||||
strip_bug_report_notice(client_message) if isinstance(client_message, str) else error_msg
|
||||
),
|
||||
type=openai_error_type(e, _code),
|
||||
param=openai_error_param(e),
|
||||
openai_code=getattr(e, "code", None),
|
||||
|
|
|
|||
|
|
@ -75,6 +75,12 @@ from litellm.constants import (
|
|||
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
allowlisted,
|
||||
bug_report_notice,
|
||||
build_bug_report,
|
||||
should_report_bug,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
_init_custom_logger_compatible_class,
|
||||
)
|
||||
|
|
@ -371,6 +377,7 @@ from litellm.proxy.batches_endpoints.endpoints import router as batches_router
|
|||
## Import All Misc routes here ##
|
||||
from litellm.proxy.caching_routes import router as caching_router
|
||||
from litellm.proxy.common_request_processing import (
|
||||
KNOWN_PROXY_ROUTES,
|
||||
ProxyBaseLLMRequestProcessing,
|
||||
_is_azure_model_router_request,
|
||||
_should_return_raw_model_name,
|
||||
|
|
@ -1965,6 +1972,16 @@ async def otel_unhandled_exception_handler(request: Request, exc: Exception):
|
|||
if isinstance(exc, (ProxyException, HTTPException, RequestValidationError)):
|
||||
raise exc
|
||||
verbose_proxy_logger.exception("Unhandled exception in request: %s", type(exc).__name__)
|
||||
if should_report_bug(exc):
|
||||
verbose_proxy_logger.error(
|
||||
bug_report_notice(
|
||||
build_bug_report(
|
||||
exc,
|
||||
surface="proxy",
|
||||
call_type=allowlisted(request.url.path, KNOWN_PROXY_ROUTES),
|
||||
)
|
||||
)
|
||||
)
|
||||
_close_dangling_otel_server_span(request, 500, exc=exc)
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
|
|
|
|||
|
|
@ -57,6 +57,12 @@ from litellm.constants import (
|
|||
SPEND_LOG_WRITE_BATCH_MAX_BYTES,
|
||||
SPEND_LOG_WRITE_BATCH_MAX_ROWS,
|
||||
)
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
bug_report_notice,
|
||||
build_bug_report,
|
||||
should_report_bug,
|
||||
strip_bug_report_notice,
|
||||
)
|
||||
from litellm.proxy._types import (
|
||||
CommonProxyErrors,
|
||||
ProxyErrorTypes,
|
||||
|
|
@ -7989,8 +7995,10 @@ def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None)
|
|||
elif isinstance(e, ProxyException):
|
||||
return with_litellm_call_id(e, litellm_call_id)
|
||||
_status_code: Final = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
|
||||
if should_report_bug(e):
|
||||
verbose_proxy_logger.error(bug_report_notice(build_bug_report(e, surface="proxy")))
|
||||
return ProxyException(
|
||||
message=str(e),
|
||||
message=strip_bug_report_notice(str(e)),
|
||||
type=ProxyErrorTypes.internal_server_error,
|
||||
param=openai_error_param(e),
|
||||
headers=headers,
|
||||
|
|
|
|||
170
tests/test_litellm/litellm_core_utils/test_bug_report.py
Normal file
170
tests/test_litellm/litellm_core_utils/test_bug_report.py
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm._version import version
|
||||
from litellm.exceptions import APIConnectionError, BadRequestError, InternalServerError
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
DISABLE_ENV_VAR,
|
||||
ISSUE_URL_BASE,
|
||||
MAX_FRAMES,
|
||||
MAX_URL_LENGTH,
|
||||
allowlisted,
|
||||
bug_report_enabled,
|
||||
bug_report_issue_url,
|
||||
bug_report_notice,
|
||||
build_bug_report,
|
||||
should_report_bug,
|
||||
strip_bug_report_notice,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
|
||||
def test_build_bug_report_keeps_only_litellm_frames():
|
||||
with pytest.raises(BadRequestError) as raised:
|
||||
get_llm_provider(cast(str, None))
|
||||
report = build_bug_report(raised.value, surface="sdk")
|
||||
|
||||
assert report.litellm_frames
|
||||
assert all(frame.startswith("litellm/") for frame in report.litellm_frames)
|
||||
assert all("test_bug_report.py" not in frame for frame in report.litellm_frames)
|
||||
|
||||
|
||||
def test_issue_url_never_contains_the_exception_message():
|
||||
secret = "sk-abcdefghijklmnopqrstuvwxyz1234567890"
|
||||
prompt = "my social security number is 123-45-6789"
|
||||
report = build_bug_report(RuntimeError(f"{secret} {prompt}"), surface="sdk")
|
||||
url = bug_report_issue_url(report)
|
||||
query = parse_qs(urlparse(url).query)
|
||||
|
||||
assert url.startswith(ISSUE_URL_BASE)
|
||||
assert secret not in url and "123-45-6789" not in url and "social" not in url
|
||||
assert query["title"] == ["[Bug]: RuntimeError in litellm"]
|
||||
assert query["version"] == [version]
|
||||
assert query["template"] == ["bug_report.yml"]
|
||||
assert query["domain"] == ["Python SDK: the litellm package itself"]
|
||||
assert query["deployment"] == ["pip / Python SDK"]
|
||||
assert "Exception: `RuntimeError`" in query["description"][0]
|
||||
assert "Python:" in query["description"][0]
|
||||
|
||||
|
||||
def test_issue_url_drops_unknown_provider_and_call_type():
|
||||
report = build_bug_report(
|
||||
ValueError("boom"),
|
||||
surface="proxy",
|
||||
custom_llm_provider="acme-internal-gateway",
|
||||
)
|
||||
query = parse_qs(urlparse(bug_report_issue_url(report)).query)
|
||||
|
||||
assert report.custom_llm_provider is None
|
||||
assert "acme" not in bug_report_issue_url(report)
|
||||
assert "Provider: unknown" in query["description"][0]
|
||||
assert "Endpoint / call: unknown" in query["description"][0]
|
||||
|
||||
|
||||
def test_allowlisted_only_passes_exact_members():
|
||||
allowed = frozenset({"/v1/chat/completions"})
|
||||
|
||||
assert allowlisted("/v1/chat/completions", allowed) == "/v1/chat/completions"
|
||||
assert allowlisted("/v1/chat/completions/../../admin", allowed) is None
|
||||
assert allowlisted(None, allowed) is None
|
||||
assert allowlisted(["/v1/chat/completions"], allowed) is None
|
||||
assert allowlisted({"provider": "openai"}, allowed) is None
|
||||
|
||||
|
||||
def test_build_bug_report_survives_unhashable_provider_from_request_data():
|
||||
report = build_bug_report(
|
||||
KeyError("missing"),
|
||||
surface="proxy",
|
||||
custom_llm_provider={"name": "openai"},
|
||||
)
|
||||
|
||||
assert report.custom_llm_provider is None
|
||||
|
||||
|
||||
def test_frames_are_capped_and_url_is_bounded():
|
||||
namespace: dict[str, object] = {}
|
||||
exec(
|
||||
compile(
|
||||
"def recurse(depth):\n if depth == 0:\n raise RuntimeError('deep')\n recurse(depth - 1)\n",
|
||||
str(Path(litellm.__file__).with_name("fake_deep_module.py")),
|
||||
"exec",
|
||||
),
|
||||
namespace,
|
||||
)
|
||||
recurse = cast(Callable[[int], None], namespace["recurse"])
|
||||
|
||||
with pytest.raises(RuntimeError) as raised:
|
||||
recurse(200)
|
||||
report = build_bug_report(raised.value, surface="proxy")
|
||||
|
||||
assert len(report.litellm_frames) == MAX_FRAMES
|
||||
assert all(frame.startswith("litellm/fake_deep_module.py:") for frame in report.litellm_frames)
|
||||
assert len(bug_report_issue_url(report)) <= MAX_URL_LENGTH
|
||||
|
||||
|
||||
def test_issue_url_builds_without_a_traceback():
|
||||
exc = RuntimeError("no traceback")
|
||||
assert exc.__traceback__ is None
|
||||
report = build_bug_report(exc, surface="proxy")
|
||||
|
||||
assert report.litellm_frames == ()
|
||||
assert bug_report_issue_url(report).startswith(ISSUE_URL_BASE)
|
||||
|
||||
|
||||
def test_bug_report_can_be_disabled(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv(DISABLE_ENV_VAR, "true")
|
||||
|
||||
assert bug_report_enabled() is False
|
||||
assert should_report_bug(RuntimeError("boom")) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"exc",
|
||||
[
|
||||
InternalServerError(message="upstream 500", llm_provider="openai", model="gpt-4"),
|
||||
APIConnectionError(
|
||||
message="connection reset",
|
||||
llm_provider="openai",
|
||||
model="gpt-4",
|
||||
request=httpx.Request(method="POST", url="https://api.openai.com/v1/"),
|
||||
),
|
||||
BadRequestError(message="bad input", llm_provider="openai", model="gpt-4"),
|
||||
"not an exception",
|
||||
],
|
||||
)
|
||||
def test_should_report_bug_skips_provider_and_network_errors(exc: object):
|
||||
assert should_report_bug(exc) is False
|
||||
|
||||
|
||||
def test_should_report_bug_accepts_plain_python_errors():
|
||||
assert should_report_bug(KeyError("missing")) is True
|
||||
|
||||
|
||||
def test_proxy_known_provider_uses_translation_domain():
|
||||
report = build_bug_report(
|
||||
RuntimeError("proxy failure"),
|
||||
surface="proxy",
|
||||
call_type="/v1/chat/completions",
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
query = parse_qs(urlparse(bug_report_issue_url(report)).query)
|
||||
|
||||
assert report.custom_llm_provider == "openai"
|
||||
assert query["domain"] == ["LLM translation: a specific provider's request or response"]
|
||||
assert "Endpoint / call: /v1/chat/completions" in query["description"][0]
|
||||
|
||||
|
||||
def test_strip_bug_report_notice():
|
||||
report = build_bug_report(RuntimeError("boom"), surface="sdk")
|
||||
notice = bug_report_notice(report)
|
||||
|
||||
assert strip_bug_report_notice(f"boom\n\n{notice}") == "boom\n"
|
||||
assert strip_bug_report_notice("boom") == "boom"
|
||||
|
|
@ -3,8 +3,6 @@ import openai
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
|
||||
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import (
|
||||
ExceptionCheckers,
|
||||
_get_body_error_code,
|
||||
|
|
@ -974,6 +972,31 @@ def test_an_unmapped_exception_with_no_model_or_provider_is_a_connection_error(q
|
|||
assert "boom" in raised.value.message
|
||||
|
||||
|
||||
def test_unmapped_sdk_exception_includes_bug_report_link(quiet_exception_mapping):
|
||||
with pytest.raises(litellm.APIConnectionError) as raised:
|
||||
exception_type(
|
||||
model="my-model",
|
||||
custom_llm_provider="minimax",
|
||||
original_exception=ValueError("boom"),
|
||||
)
|
||||
|
||||
assert "https://github.com/BerriAI/litellm/issues/new?" in str(raised.value)
|
||||
assert "ValueError" in str(raised.value)
|
||||
|
||||
|
||||
def test_unmapped_sdk_exception_bug_report_link_can_be_disabled(quiet_exception_mapping, monkeypatch):
|
||||
monkeypatch.setenv("LITELLM_DISABLE_BUG_REPORT_LINK", "true")
|
||||
|
||||
with pytest.raises(litellm.APIConnectionError) as raised:
|
||||
exception_type(
|
||||
model="my-model",
|
||||
custom_llm_provider="minimax",
|
||||
original_exception=ValueError("boom"),
|
||||
)
|
||||
|
||||
assert "https://github.com/BerriAI/litellm/issues/new?" not in str(raised.value)
|
||||
|
||||
|
||||
def _raise_and_map(model: str | None, original_exception: Exception, custom_llm_provider: str | None) -> None:
|
||||
"""Calls exception_type() from inside the except block, as litellm/main.py does,
|
||||
so traceback.format_exc() has a real stack."""
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import datetime
|
|||
import json
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import AsyncGenerator, Callable, Final, Iterator, Optional, Sequence
|
||||
from urllib.parse import unquote_plus
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -13,6 +14,12 @@ from fastapi.responses import JSONResponse, StreamingResponse
|
|||
|
||||
import litellm
|
||||
from litellm._uuid import uuid
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
DISABLE_ENV_VAR,
|
||||
ISSUE_URL_BASE,
|
||||
bug_report_notice,
|
||||
build_bug_report,
|
||||
)
|
||||
from litellm.constants import (
|
||||
CLIENT_REQUESTED_MODEL_SCOPE_KEY,
|
||||
MAX_LITELLM_CALL_ID_LENGTH,
|
||||
|
|
@ -4290,6 +4297,21 @@ class TestHandleLLMApiExceptionRetryAfter:
|
|||
proxy_exc = await self._invoke(ValueError("some other failure"))
|
||||
assert "retry-after" not in proxy_exc.headers
|
||||
|
||||
async def test_handle_llm_api_exception_strips_bug_report_notice_from_client_message(self, caplog):
|
||||
report = build_bug_report(RuntimeError("boom"), surface="sdk")
|
||||
notice = bug_report_notice(report)
|
||||
exc = litellm.APIConnectionError(
|
||||
message=f"boom\n{notice}",
|
||||
model="gpt-4o",
|
||||
llm_provider="openai",
|
||||
)
|
||||
|
||||
with caplog.at_level("ERROR"):
|
||||
proxy_exc = await self._invoke(exc)
|
||||
|
||||
assert ISSUE_URL_BASE not in proxy_exc.message
|
||||
assert ISSUE_URL_BASE in caplog.text
|
||||
|
||||
async def test_handle_llm_api_exception_retry_after_survives_callback_headers(self):
|
||||
from litellm.types.router import RouterRateLimitError
|
||||
|
||||
|
|
@ -9374,6 +9396,92 @@ async def test_handle_llm_api_exception_forwards_litellm_response_headers_when_r
|
|||
assert exc_info.value.headers["llm_provider-x-request-id"] == "req_openai_400"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_llm_api_exception_logs_bug_report_for_unmapped_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
):
|
||||
monkeypatch.delenv(DISABLE_ENV_VAR, raising=False)
|
||||
processor = ProxyBaseLLMRequestProcessing(
|
||||
data={
|
||||
"proxy_server_request": {"url": "https://example.test/v1/chat/completions?debug=true"},
|
||||
"model": "acme-prod-gpt4",
|
||||
"custom_llm_provider": "openai",
|
||||
}
|
||||
)
|
||||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
|
||||
proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={})
|
||||
|
||||
with caplog.at_level("ERROR", logger="LiteLLM Proxy"):
|
||||
with pytest.raises(ProxyException):
|
||||
await processor._handle_llm_api_exception(
|
||||
e=RuntimeError("unmapped for user@example.com"),
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"),
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
issue_url = next(word for word in caplog.text.split() if word.startswith(ISSUE_URL_BASE))
|
||||
assert "Endpoint / call: /v1/chat/completions" in unquote_plus(issue_url)
|
||||
assert "Provider: openai" in unquote_plus(issue_url)
|
||||
assert "acme-prod-gpt4" not in unquote_plus(issue_url)
|
||||
assert "user@example.com" not in unquote_plus(issue_url)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_llm_api_exception_bug_report_drops_unknown_route(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
):
|
||||
monkeypatch.delenv(DISABLE_ENV_VAR, raising=False)
|
||||
processor = ProxyBaseLLMRequestProcessing(
|
||||
data={"proxy_server_request": {"url": "https://example.test/v1/files/file-customer-123/content"}}
|
||||
)
|
||||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
|
||||
proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={})
|
||||
|
||||
with caplog.at_level("ERROR", logger="LiteLLM Proxy"):
|
||||
with pytest.raises(ProxyException):
|
||||
await processor._handle_llm_api_exception(
|
||||
e=RuntimeError("unmapped"),
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"),
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
issue_url = next(word for word in caplog.text.split() if word.startswith(ISSUE_URL_BASE))
|
||||
assert "Endpoint / call: unknown" in unquote_plus(issue_url)
|
||||
assert "file-customer-123" not in unquote_plus(issue_url)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_llm_api_exception_skips_bug_report_for_provider_status(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
):
|
||||
monkeypatch.delenv(DISABLE_ENV_VAR, raising=False)
|
||||
|
||||
class ProviderRateLimitError(Exception):
|
||||
def __init__(self, message: str):
|
||||
super().__init__(message)
|
||||
self.status_code = 429
|
||||
|
||||
processor = ProxyBaseLLMRequestProcessing(data={})
|
||||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
|
||||
proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={})
|
||||
|
||||
with caplog.at_level("ERROR", logger="LiteLLM Proxy"):
|
||||
with pytest.raises(ProxyException):
|
||||
await processor._handle_llm_api_exception(
|
||||
e=ProviderRateLimitError("rate limited"),
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"),
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
assert ISSUE_URL_BASE not in caplog.text
|
||||
|
||||
|
||||
class TestBackgroundResponseRetrievalGovernance:
|
||||
"""LIT-7175: retrieving a background Response attaches the model's post_call policy pipelines."""
|
||||
|
||||
|
|
|
|||
|
|
@ -6,10 +6,12 @@ import pytest
|
|||
from fastapi import HTTPException
|
||||
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.exceptions import InternalServerError
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.bug_report import ISSUE_URL_BASE
|
||||
from litellm.proxy._types import ProxyErrorTypes, UserAPIKeyAuth
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging, handle_exception_on_proxy
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
||||
|
||||
|
|
@ -2418,3 +2420,16 @@ def test_mcp_auth_policy_uses_original_request_model(monkeypatch, model, expecte
|
|||
synthetic = proxy_logging._convert_mcp_to_llm_format(proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs)
|
||||
assert ("model-rule" in synthetic["metadata"]["guardrails"]) is expected
|
||||
assert "request-rule" in synthetic["metadata"]["guardrails"]
|
||||
|
||||
|
||||
def test_handle_exception_on_proxy_logs_bug_report_only_for_unmapped_500(caplog):
|
||||
with caplog.at_level("ERROR", logger="LiteLLM Proxy"):
|
||||
provider_result = handle_exception_on_proxy(
|
||||
InternalServerError(message="upstream 500", llm_provider="openai", model="gpt-4")
|
||||
)
|
||||
assert ISSUE_URL_BASE not in caplog.text
|
||||
internal_result = handle_exception_on_proxy(KeyError("missing"))
|
||||
|
||||
assert provider_result.code == internal_result.code == "500"
|
||||
assert ISSUE_URL_BASE in caplog.text
|
||||
assert ISSUE_URL_BASE not in internal_result.message
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue