mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-24 00:52:24 +00:00
feat(proxy): opt-in litellm_call_id in JSON error bodies (#42391)
* feat(proxy): opt-in litellm_call_id in JSON error bodies Add general_settings.include_call_id_in_error_body. When true, the value already on the x-litellm-call-id response header is copied into JSON error bodies: as error.litellm_call_id on the OpenAI-shaped routes, /v1/messages, and streaming first-chunk errors, and as a top-level litellm_call_id on pass-through routes. Off by default, so error bodies stay byte-identical unless an admin opts in * chore(proxy): drop helper docstring and restore lazy OpenAPI snapshot --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
parent
b833e1fc4c
commit
8b33da7bb3
13 changed files with 433 additions and 8 deletions
|
|
@ -25,6 +25,7 @@ class AnthropicErrorDetail(TypedDict):
|
|||
type: AnthropicErrorType
|
||||
message: str
|
||||
provider_specific_fields: NotRequired[ReadOnly[Mapping[str, object]]]
|
||||
litellm_call_id: NotRequired[ReadOnly[str]]
|
||||
|
||||
|
||||
class AnthropicErrorResponse(TypedDict, total=False):
|
||||
|
|
|
|||
|
|
@ -2648,6 +2648,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
|
|||
None,
|
||||
description="opt-in to RFC 8628 verification_uri_complete for the CLI SSO device flow, pre-filling the user_code in the browser. Off by default; intended for same-host clients where the device that starts the flow and the browser run on the same machine",
|
||||
)
|
||||
include_call_id_in_error_body: bool | None = Field(
|
||||
None,
|
||||
description="opt-in to copy the x-litellm-call-id response header's value into JSON error bodies, as error.litellm_call_id on the OpenAI-shaped and /v1/messages routes and as a top-level litellm_call_id on pass-through routes, so an error a client prints names the request to look up. Off by default",
|
||||
)
|
||||
enable_claude_code_gateway: bool | None = Field(
|
||||
None,
|
||||
description="serve the Claude Code gateway protocol (https://code.claude.com/docs/en/claude-apps-gateway) under /claude_code_gateway: OAuth device-flow sign-in reusing proxy SSO, plus managed settings and OTLP telemetry ingestion. Off by default",
|
||||
|
|
|
|||
|
|
@ -8,7 +8,11 @@ from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
|||
from fastapi.responses import JSONResponse
|
||||
|
||||
import litellm
|
||||
from litellm.anthropic_interface.exceptions import AnthropicErrorResponse, AnthropicExceptionMapping
|
||||
from litellm.anthropic_interface.exceptions import (
|
||||
AnthropicErrorDetail,
|
||||
AnthropicErrorResponse,
|
||||
AnthropicExceptionMapping,
|
||||
)
|
||||
from litellm.integrations.custom_guardrail import ModifyResponseException
|
||||
from litellm.llms.anthropic.experimental_pass_through.context_management import (
|
||||
AnthropicContextManagementError,
|
||||
|
|
@ -25,8 +29,10 @@ from litellm.proxy.common_request_processing import (
|
|||
proxy_exception_from_http_exception,
|
||||
resolve_litellm_call_id,
|
||||
)
|
||||
from litellm.proxy.common_utils.error_body_call_id import error_body_call_id
|
||||
from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
|
||||
from litellm.proxy.common_utils.openai_error_payload import (
|
||||
LITELLM_CALL_ID_HEADER,
|
||||
error_status_code,
|
||||
openai_error_param,
|
||||
openai_error_type,
|
||||
|
|
@ -37,9 +43,29 @@ from litellm.types.utils import TokenCountResponse
|
|||
router: Final = APIRouter()
|
||||
|
||||
|
||||
def _with_provider_specific_fields(exc: ProxyException, detail: AnthropicErrorDetail) -> AnthropicErrorDetail:
|
||||
if not exc.provider_specific_fields:
|
||||
return detail
|
||||
with_fields: Final[AnthropicErrorDetail] = {**detail, "provider_specific_fields": exc.provider_specific_fields}
|
||||
return with_fields
|
||||
|
||||
|
||||
def _anthropic_error_detail(
|
||||
exc: ProxyException, detail: AnthropicErrorDetail, call_id: str | None
|
||||
) -> AnthropicErrorDetail:
|
||||
if call_id is None:
|
||||
return _with_provider_specific_fields(exc, detail)
|
||||
with_call_id: Final[AnthropicErrorDetail] = {
|
||||
**_with_provider_specific_fields(exc, detail),
|
||||
"litellm_call_id": call_id,
|
||||
}
|
||||
return with_call_id
|
||||
|
||||
|
||||
def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSONResponse:
|
||||
from litellm.proxy.proxy_server import (
|
||||
_close_dangling_otel_server_span, # pyright: ignore[reportPrivateUsage] # proxy_server keeps the span-close helper private; error JSONResponses returned by the route must stamp the OTel server span like the global ProxyException handler does
|
||||
general_settings_view,
|
||||
)
|
||||
|
||||
status_code: Final = int(exc.code) if exc.code is not None and exc.code.isdigit() else 500
|
||||
|
|
@ -49,11 +75,10 @@ def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSO
|
|||
raw_message=exc.message,
|
||||
request_id=request.headers.get("x-request-id"),
|
||||
)
|
||||
if not exc.provider_specific_fields:
|
||||
return JSONResponse(status_code=status_code, content=envelope, headers=exc.headers)
|
||||
body_call_id: Final = error_body_call_id(general_settings_view(), exc.headers.get(LITELLM_CALL_ID_HEADER))
|
||||
content: Final[AnthropicErrorResponse] = {
|
||||
**envelope,
|
||||
"error": {**envelope["error"], "provider_specific_fields": exc.provider_specific_fields},
|
||||
"error": _anthropic_error_detail(exc, envelope["error"], body_call_id),
|
||||
}
|
||||
return JSONResponse(status_code=status_code, content=content, headers=exc.headers)
|
||||
|
||||
|
|
|
|||
|
|
@ -75,11 +75,13 @@ from litellm.proxy.common_utils.callback_utils import (
|
|||
get_logging_caching_headers,
|
||||
get_remaining_tokens_and_requests_from_request_data,
|
||||
)
|
||||
from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id
|
||||
from litellm.proxy.common_utils.http_parsing_utils import (
|
||||
get_client_requested_model,
|
||||
get_tags_from_request_body,
|
||||
)
|
||||
from litellm.proxy.common_utils.openai_error_payload import (
|
||||
LITELLM_CALL_ID_HEADER,
|
||||
attribute_of,
|
||||
error_status_code,
|
||||
openai_error_param,
|
||||
|
|
@ -946,6 +948,9 @@ async def _resolve_stream_headers(
|
|||
return headers
|
||||
|
||||
|
||||
_NO_GENERAL_SETTINGS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
async def create_response(
|
||||
generator: AsyncGenerator[str, None],
|
||||
media_type: str,
|
||||
|
|
@ -953,6 +958,7 @@ async def create_response(
|
|||
default_status_code: int = status.HTTP_200_OK,
|
||||
request: Request | None = None,
|
||||
refresh_headers: Callable[[], Awaitable[Mapping[str, str]]] | None = None,
|
||||
general_settings: Mapping[str, object] = _NO_GENERAL_SETTINGS,
|
||||
) -> StreamingResponse | JSONResponse:
|
||||
"""
|
||||
Create streaming response, checking if the first chunk is an error.
|
||||
|
|
@ -960,7 +966,8 @@ async def create_response(
|
|||
Otherwise, return StreamingResponse and stream all content.
|
||||
|
||||
``refresh_headers`` is consulted once the first chunk has been buffered, for
|
||||
callers whose headers can only be known then.
|
||||
callers whose headers can only be known then. ``general_settings`` decides whether
|
||||
the first-chunk error body also carries the ``x-litellm-call-id`` header's value.
|
||||
"""
|
||||
first_chunk_value: str | None = None
|
||||
final_status_code = default_status_code
|
||||
|
|
@ -987,7 +994,10 @@ async def create_response(
|
|||
)
|
||||
|
||||
# Parse error content
|
||||
error_dict: Final = _extract_error_from_sse_chunk(first_chunk_value)
|
||||
error_dict: Final = with_call_id(
|
||||
JSON_OBJECT.validate_python(_extract_error_from_sse_chunk(first_chunk_value)),
|
||||
error_body_call_id(general_settings, resolved_headers.get(LITELLM_CALL_ID_HEADER)),
|
||||
)
|
||||
|
||||
# Consume and close generator (avoid resource leak)
|
||||
try:
|
||||
|
|
@ -2738,6 +2748,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
headers=custom_headers,
|
||||
request=request,
|
||||
refresh_headers=refresh_stream_headers,
|
||||
general_settings=general_settings,
|
||||
)
|
||||
|
||||
### CALL HOOKS ### - modify outgoing data
|
||||
|
|
|
|||
20
litellm/proxy/common_utils/error_body_call_id.py
Normal file
20
litellm/proxy/common_utils/error_body_call_id.py
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
INCLUDE_CALL_ID_IN_ERROR_BODY_SETTING: Final = "include_call_id_in_error_body"
|
||||
LITELLM_CALL_ID_BODY_KEY: Final = "litellm_call_id"
|
||||
JSON_OBJECT: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) # mutable-ok: JSONResponse input
|
||||
|
||||
|
||||
def error_body_call_id(general_settings: Mapping[str, object], call_id: str | None) -> str | None:
|
||||
if general_settings.get(INCLUDE_CALL_ID_IN_ERROR_BODY_SETTING) is not True:
|
||||
return None
|
||||
return call_id if call_id else None
|
||||
|
||||
|
||||
def with_call_id(error: dict[str, object], call_id: str | None) -> dict[str, object]: # mutable-ok: JSONResponse input
|
||||
if call_id is None:
|
||||
return error
|
||||
return {**error, LITELLM_CALL_ID_BODY_KEY: call_id} # mutable-ok: JSONResponse input
|
||||
|
|
@ -78,11 +78,13 @@ from litellm.proxy.common_request_processing import (
|
|||
open_sse_before_first_byte,
|
||||
resolve_litellm_call_id,
|
||||
)
|
||||
from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id
|
||||
from litellm.proxy.common_utils.http_parsing_utils import (
|
||||
_read_request_body,
|
||||
_safe_get_request_headers,
|
||||
)
|
||||
from litellm.proxy.common_utils.openai_error_payload import (
|
||||
LITELLM_CALL_ID_HEADER,
|
||||
error_status_code,
|
||||
litellm_call_id_headers,
|
||||
openai_error_param,
|
||||
|
|
@ -1127,6 +1129,9 @@ async def pass_through_request(
|
|||
from litellm.proxy.proxy_server import (
|
||||
general_settings as proxy_general_settings,
|
||||
)
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings_view,
|
||||
)
|
||||
|
||||
_managed_id_provider: Final = resolve_passthrough_managed_id_provider(custom_llm_provider)
|
||||
|
||||
|
|
@ -1656,11 +1661,24 @@ async def pass_through_request(
|
|||
headers=response.headers,
|
||||
custom_headers=custom_headers,
|
||||
)
|
||||
emitted_call_id: Final = (
|
||||
JSON_OBJECT.validate_python(response_headers).get(LITELLM_CALL_ID_HEADER)
|
||||
if response.status_code >= 400
|
||||
else None
|
||||
)
|
||||
error_call_id: Final = (
|
||||
error_body_call_id(general_settings_view(), emitted_call_id) if isinstance(emitted_call_id, str) else None
|
||||
)
|
||||
relayed_content: Final = (
|
||||
json.dumps(with_call_id(JSON_OBJECT.validate_python(response_body), error_call_id)).encode("utf-8")
|
||||
if error_call_id is not None and isinstance(response_body, dict)
|
||||
else content
|
||||
)
|
||||
if _content_modified:
|
||||
response_headers.pop("content-length", None)
|
||||
|
||||
return Response(
|
||||
content=content,
|
||||
content=relayed_content,
|
||||
status_code=response.status_code,
|
||||
headers=response_headers,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -393,6 +393,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
|
|||
decrypt_value_helper,
|
||||
encrypt_value_helper,
|
||||
)
|
||||
from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id
|
||||
from litellm.proxy.common_utils.healthy_model_filter import (
|
||||
get_hidden_unhealthy_model_names,
|
||||
is_healthy_only_listing_default,
|
||||
|
|
@ -418,6 +419,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import (
|
|||
remove_sensitive_info_from_deployment,
|
||||
)
|
||||
from litellm.proxy.common_utils.openai_error_payload import (
|
||||
LITELLM_CALL_ID_HEADER,
|
||||
headers_with_litellm_call_id,
|
||||
litellm_call_id_headers,
|
||||
with_litellm_call_id,
|
||||
|
|
@ -1813,7 +1815,10 @@ async def openai_exception_handler(request: Request, exc: ProxyException):
|
|||
# NOTE: DO NOT MODIFY THIS, its crucial to map to Openai exceptions
|
||||
_log_model_access_denial(exc)
|
||||
headers: Final = exc.headers
|
||||
error_dict: Final = exc.to_dict()
|
||||
error_dict: Final = with_call_id(
|
||||
JSON_OBJECT.validate_python(exc.to_dict()),
|
||||
error_body_call_id(general_settings_view(), headers.get(LITELLM_CALL_ID_HEADER)),
|
||||
)
|
||||
status_code: Final = int(exc.code) if exc.code else status.HTTP_500_INTERNAL_SERVER_ERROR
|
||||
_close_dangling_otel_server_span(request, status_code, exc=exc)
|
||||
return JSONResponse(
|
||||
|
|
@ -2477,6 +2482,13 @@ heuristic_v1_tuning_baselines: Mapping[str, str] | None = None
|
|||
# second ProxyConfig instance must not get its own independent lock over it.
|
||||
MODEL_RECONCILE_LOCK: Final = asyncio.Lock()
|
||||
general_settings: dict = {}
|
||||
_GENERAL_SETTINGS_VIEW: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def general_settings_view() -> Mapping[str, object]:
|
||||
return _GENERAL_SETTINGS_VIEW.validate_python(general_settings)
|
||||
|
||||
|
||||
config_passthrough_endpoints: list[dict[str, Any]] | None = None
|
||||
log_file: Final = "api_log.json"
|
||||
worker_config: Final = None
|
||||
|
|
|
|||
|
|
@ -198,6 +198,62 @@ class TestProxyExceptionAnthropicEnvelope:
|
|||
assert fallback.status_code == 500
|
||||
assert json.loads(fallback.body)["error"]["type"] == "api_error"
|
||||
|
||||
@staticmethod
|
||||
def _call_id_error_response(general_settings, provider_specific_fields=None):
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
||||
request = MagicMock()
|
||||
request.headers = {}
|
||||
exc = ProxyException(
|
||||
message="Rate limit exceeded",
|
||||
type="rate_limit_error",
|
||||
param=None,
|
||||
code=429,
|
||||
headers={"x-litellm-call-id": "call-8302"},
|
||||
provider_specific_fields=provider_specific_fields,
|
||||
)
|
||||
with patch("litellm.proxy.proxy_server.general_settings", general_settings):
|
||||
return ep._anthropic_error_json_response(exc, request)
|
||||
|
||||
def test_anthropic_error_copies_the_call_id_into_the_error_when_opted_in(self):
|
||||
"""With include_call_id_in_error_body on, error.litellm_call_id is byte-identical to
|
||||
the x-litellm-call-id header and lives inside the error object, which is what the
|
||||
Anthropic SDK keeps as e.body."""
|
||||
response = self._call_id_error_response({"include_call_id_in_error_body": True})
|
||||
|
||||
assert response.headers["x-litellm-call-id"] == "call-8302"
|
||||
assert json.loads(response.body) == {
|
||||
"type": "error",
|
||||
"error": {
|
||||
"type": "rate_limit_error",
|
||||
"message": "Rate limit exceeded",
|
||||
"litellm_call_id": "call-8302",
|
||||
},
|
||||
}
|
||||
|
||||
def test_anthropic_error_keeps_provider_specific_fields_next_to_the_call_id(self):
|
||||
response = self._call_id_error_response(
|
||||
{"include_call_id_in_error_body": True},
|
||||
provider_specific_fields={"guardrail": "keyword-block"},
|
||||
)
|
||||
|
||||
assert json.loads(response.body)["error"] == {
|
||||
"type": "rate_limit_error",
|
||||
"message": "Rate limit exceeded",
|
||||
"provider_specific_fields": {"guardrail": "keyword-block"},
|
||||
"litellm_call_id": "call-8302",
|
||||
}
|
||||
|
||||
def test_anthropic_error_leaves_the_envelope_alone_when_opted_out(self):
|
||||
response = self._call_id_error_response({})
|
||||
|
||||
assert response.headers["x-litellm-call-id"] == "call-8302"
|
||||
assert json.loads(response.body) == {
|
||||
"type": "error",
|
||||
"error": {"type": "rate_limit_error", "message": "Rate limit exceeded"},
|
||||
}
|
||||
|
||||
|
||||
class TestHttpExceptionDictDetail:
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
import pytest
|
||||
|
||||
from litellm.proxy._types import ConfigGeneralSettings
|
||||
from litellm.proxy.common_utils.error_body_call_id import error_body_call_id, with_call_id
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"general_settings, call_id, expected",
|
||||
[
|
||||
({"include_call_id_in_error_body": True}, "call-1", "call-1"),
|
||||
({"include_call_id_in_error_body": True}, None, None),
|
||||
({"include_call_id_in_error_body": True}, "", None),
|
||||
({"include_call_id_in_error_body": False}, "call-1", None),
|
||||
({"include_call_id_in_error_body": "true"}, "call-1", None),
|
||||
({}, "call-1", None),
|
||||
],
|
||||
)
|
||||
def test_only_the_boolean_opt_in_with_a_real_id_yields_a_body_call_id(general_settings, call_id, expected):
|
||||
"""The setting is off by default and only a literal true turns it on; without an id
|
||||
there is nothing to copy, so the body must never get a fabricated one."""
|
||||
assert error_body_call_id(general_settings, call_id) == expected
|
||||
|
||||
|
||||
def test_with_call_id_appends_the_key_without_touching_the_input():
|
||||
error = {"message": "bad input", "type": "invalid_request_error", "param": None, "code": "400"}
|
||||
|
||||
assert with_call_id(error, "call-1") == {**error, "litellm_call_id": "call-1"}
|
||||
assert with_call_id(error, None) == error
|
||||
assert "litellm_call_id" not in error
|
||||
|
||||
|
||||
def test_the_setting_name_is_a_config_general_settings_field():
|
||||
"""The yaml key the docs name and the key the runtime reads must be the same field."""
|
||||
assert ConfigGeneralSettings.model_validate({"include_call_id_in_error_body": True}).include_call_id_in_error_body
|
||||
assert ConfigGeneralSettings.model_validate({}).include_call_id_in_error_body is None
|
||||
|
|
@ -4651,6 +4651,90 @@ async def test_pass_through_request_upstream_error_body_stays_buffered():
|
|||
await fake_client.aclose()
|
||||
|
||||
|
||||
_UPSTREAM_JSON_ERROR: Final = b'{"error": {"message": "bad request", "type": "invalid_request_error"}}'
|
||||
|
||||
|
||||
async def _relay_upstream_through_pass_through_request(
|
||||
general_settings, status_code, content_type, body, callback_headers=None
|
||||
):
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
fake_client, cleanup = _inject_fake_passthrough_client(
|
||||
_FakeUpstreamTransport(
|
||||
status_code=status_code,
|
||||
headers={"content-type": content_type},
|
||||
stream=_RecordingUpstreamByteStream((body,)),
|
||||
),
|
||||
timeout=313.0,
|
||||
)
|
||||
try:
|
||||
with ExitStack() as stack:
|
||||
mock_proxy_logging, _ = _enter_relay_logging_mocks(stack, {})
|
||||
mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value=callback_headers)
|
||||
stack.enter_context(patch("litellm.proxy.proxy_server.general_settings", general_settings))
|
||||
return await pass_through_request(
|
||||
request=_relay_client_request(),
|
||||
target="http://upstream.test/v1/messages",
|
||||
custom_headers={},
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"),
|
||||
timeout=313.0,
|
||||
)
|
||||
finally:
|
||||
cleanup()
|
||||
await fake_client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pass_through_error_body_carries_the_call_id_when_opted_in():
|
||||
"""With include_call_id_in_error_body on, a buffered upstream JSON error gets a top-level
|
||||
litellm_call_id byte-identical to the x-litellm-call-id header, and content-length still
|
||||
matches the rewritten body."""
|
||||
response = await _relay_upstream_through_pass_through_request(
|
||||
{"include_call_id_in_error_body": True}, 400, "application/json", _UPSTREAM_JSON_ERROR
|
||||
)
|
||||
|
||||
call_id = response.headers["x-litellm-call-id"]
|
||||
assert response.status_code == 400
|
||||
assert json.loads(response.body) == {**json.loads(_UPSTREAM_JSON_ERROR), "litellm_call_id": call_id}
|
||||
assert int(response.headers["content-length"]) == len(response.body)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pass_through_error_body_call_id_follows_a_restamped_header():
|
||||
"""A post_call_response_headers_hook that rewrites x-litellm-call-id wins in the header, so the
|
||||
body copies the emitted header value rather than the id the proxy generated."""
|
||||
response = await _relay_upstream_through_pass_through_request(
|
||||
{"include_call_id_in_error_body": True},
|
||||
400,
|
||||
"application/json",
|
||||
_UPSTREAM_JSON_ERROR,
|
||||
callback_headers={"x-litellm-call-id": "restamped-by-hook"},
|
||||
)
|
||||
|
||||
assert response.headers["x-litellm-call-id"] == "restamped-by-hook"
|
||||
assert json.loads(response.body)["litellm_call_id"] == "restamped-by-hook"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"general_settings, status_code, content_type, body",
|
||||
[
|
||||
({}, 400, "application/json", _UPSTREAM_JSON_ERROR),
|
||||
({"include_call_id_in_error_body": True}, 502, "text/plain", b"upstream exploded"),
|
||||
({"include_call_id_in_error_body": True}, 200, "application/json", b'{"id": "msg_1", "type": "message"}'),
|
||||
],
|
||||
)
|
||||
async def test_pass_through_body_stays_byte_identical_outside_the_opt_in(
|
||||
general_settings, status_code, content_type, body
|
||||
):
|
||||
"""Opted out, a non-JSON error, or a success body: the upstream bytes are relayed as-is."""
|
||||
response = await _relay_upstream_through_pass_through_request(general_settings, status_code, content_type, body)
|
||||
|
||||
assert response.status_code == status_code
|
||||
assert response.body == body
|
||||
assert "x-litellm-call-id" in response.headers
|
||||
|
||||
|
||||
_PARTIAL_RELAY_WARNING_MARKER = "ended before upstream body was fully relayed"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -95,6 +95,71 @@ async def test_openai_exception_handler_invalid_empty_code_defaults_to_500():
|
|||
}
|
||||
|
||||
|
||||
def _call_id_exception(headers):
|
||||
return ProxyException(
|
||||
message="bad input",
|
||||
type="invalid_request_error",
|
||||
param="model",
|
||||
code=400,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_exception_handler_copies_the_call_id_into_the_error_when_opted_in(monkeypatch):
|
||||
"""With include_call_id_in_error_body on, error.litellm_call_id is byte-identical to the
|
||||
x-litellm-call-id header, so a pasted str(e) names the request to look up."""
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"include_call_id_in_error_body": True})
|
||||
exc = _call_id_exception({"x-litellm-call-id": "call-8302"})
|
||||
|
||||
response = await openai_exception_handler(request=_make_request(), exc=exc)
|
||||
body = json.loads(response.body)
|
||||
|
||||
assert response.headers["x-litellm-call-id"] == "call-8302"
|
||||
assert body == {
|
||||
"error": {
|
||||
"message": "bad input",
|
||||
"type": "invalid_request_error",
|
||||
"param": "model",
|
||||
"code": "400",
|
||||
"litellm_call_id": "call-8302",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_exception_handler_leaves_the_error_alone_when_opted_out(monkeypatch):
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {})
|
||||
exc = _call_id_exception({"x-litellm-call-id": "call-8302"})
|
||||
|
||||
response = await openai_exception_handler(request=_make_request(), exc=exc)
|
||||
body = json.loads(response.body)
|
||||
|
||||
assert response.headers["x-litellm-call-id"] == "call-8302"
|
||||
assert body == {
|
||||
"error": {
|
||||
"message": "bad input",
|
||||
"type": "invalid_request_error",
|
||||
"param": "model",
|
||||
"code": "400",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_exception_handler_never_fabricates_a_call_id(monkeypatch):
|
||||
"""An error raised before a call id exists (auth failures, say) carries no header,
|
||||
and the body must not invent one."""
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"include_call_id_in_error_body": True})
|
||||
exc = _call_id_exception({})
|
||||
|
||||
response = await openai_exception_handler(request=_make_request(), exc=exc)
|
||||
body = json.loads(response.body)
|
||||
|
||||
assert "x-litellm-call-id" not in response.headers
|
||||
assert "litellm_call_id" not in body["error"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _close_dangling_otel_server_span
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -2343,6 +2343,39 @@ class TestCommonRequestProcessingHelpers:
|
|||
assert isinstance(response, JSONResponse)
|
||||
assert response.headers["x-litellm-model-id"] == "fallback-deployment"
|
||||
|
||||
@staticmethod
|
||||
async def _first_chunk_error_response(**create_response_kwargs):
|
||||
async def mock_generator():
|
||||
yield 'data: {"error": {"code": 403, "message": "forbidden"}}\n\n'
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
return await create_response(
|
||||
mock_generator(),
|
||||
"text/event-stream",
|
||||
{"x-litellm-call-id": "call-8302"},
|
||||
**create_response_kwargs,
|
||||
)
|
||||
|
||||
async def test_create_response_first_chunk_error_carries_the_call_id_when_opted_in(self):
|
||||
"""A stream that fails on its first chunk answers as JSON, and with
|
||||
include_call_id_in_error_body on that JSON names the request like the
|
||||
non-streaming error path does, byte-identical to the header."""
|
||||
response = await self._first_chunk_error_response(general_settings={"include_call_id_in_error_body": True})
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 403
|
||||
assert response.headers["x-litellm-call-id"] == "call-8302"
|
||||
assert json.loads(response.body) == {
|
||||
"error": {"code": 403, "message": "forbidden", "litellm_call_id": "call-8302"}
|
||||
}
|
||||
|
||||
async def test_create_response_first_chunk_error_body_is_unchanged_by_default(self):
|
||||
response = await self._first_chunk_error_response()
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.headers["x-litellm-call-id"] == "call-8302"
|
||||
assert json.loads(response.body) == {"error": {"code": 403, "message": "forbidden"}}
|
||||
|
||||
async def test_create_streaming_response_disables_proxy_buffering(self):
|
||||
"""Regression for #28384: every StreamingResponse create_response returns
|
||||
must carry the headers that stop nginx/ingress/Envoy from buffering the
|
||||
|
|
@ -9122,6 +9155,62 @@ class TestStreamingResponseHeadersFollowFallback:
|
|||
assert result.status_code == 400
|
||||
assert result.headers["x-litellm-applied-guardrails"] == "stream-blocker"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_first_chunk_error_carries_the_call_id_when_opted_in(self, monkeypatch):
|
||||
"""The opt-in reaches the streaming path through base_process_llm_request, so a stream
|
||||
that fails on its first chunk answers with the call id inside its JSON error body,
|
||||
byte-identical to the x-litellm-call-id header."""
|
||||
|
||||
def select_data_generator(**kwargs):
|
||||
async def generator():
|
||||
yield 'data: {"error": {"code": 403, "message": "forbidden"}}\n\n'
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
return generator()
|
||||
|
||||
logging_obj = MagicMock()
|
||||
logging_obj.litellm_call_id = "lit-8302-call"
|
||||
logging_obj._defer_async_logging = False
|
||||
logging_obj._on_deferred_stream_complete = None
|
||||
logging_obj.cost_breakdown = None
|
||||
processor = ProxyBaseLLMRequestProcessing(
|
||||
data={"model": "oa", "stream": True, "litellm_logging_obj": logging_obj}
|
||||
)
|
||||
|
||||
proxy_logging_obj = MagicMock(spec=ProxyLogging)
|
||||
proxy_logging_obj.during_call_hook = AsyncMock(return_value=None)
|
||||
proxy_logging_obj.update_request_status = AsyncMock(return_value=None)
|
||||
proxy_logging_obj.post_call_success_hook = AsyncMock(
|
||||
side_effect=lambda data, user_api_key_dict, response: response
|
||||
)
|
||||
proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={})
|
||||
|
||||
async def fake_route_request(**kwargs):
|
||||
async def call():
|
||||
return SimpleNamespace(_hidden_params={}, fallback_headers_adopted=False)
|
||||
|
||||
return call()
|
||||
|
||||
monkeypatch.setattr(litellm.proxy.common_request_processing, "route_request", fake_route_request)
|
||||
|
||||
result = await processor.base_process_llm_request(
|
||||
request=Request(scope={"type": "http", "headers": []}),
|
||||
fastapi_response=Response(),
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"),
|
||||
route_type="acompletion",
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
general_settings={"include_call_id_in_error_body": True},
|
||||
proxy_config=MagicMock(spec=ProxyConfig),
|
||||
select_data_generator=select_data_generator,
|
||||
is_streaming_request=True,
|
||||
skip_pre_call_logic=True,
|
||||
)
|
||||
|
||||
assert isinstance(result, JSONResponse)
|
||||
assert result.status_code == 403
|
||||
assert result.headers["x-litellm-call-id"] == "lit-8302-call"
|
||||
assert json.loads(result.body)["error"]["litellm_call_id"] == "lit-8302-call"
|
||||
|
||||
|
||||
class _MessagesFallbackStream:
|
||||
def __init__(self) -> None:
|
||||
|
|
|
|||
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -27119,6 +27119,11 @@ export interface components {
|
|||
* @default false
|
||||
*/
|
||||
health_check_skip_disabled_background_models: boolean;
|
||||
/**
|
||||
* Include Call Id In Error Body
|
||||
* @description opt-in to copy the x-litellm-call-id response header's value into JSON error bodies, as error.litellm_call_id on the OpenAI-shaped and /v1/messages routes and as a top-level litellm_call_id on pass-through routes, so an error a client prints names the request to look up. Off by default
|
||||
*/
|
||||
include_call_id_in_error_body?: boolean | null;
|
||||
/**
|
||||
* Infer Model From Keys
|
||||
* @description for `/models` endpoint, infers available model based on environment keys (e.g. OPENAI_API_KEY)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue