mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-26 01:12:21 +00:00
f6882246d4
* ci: run the unit_selection.sh shard files on every event instead of only fork pull requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: rename fork-flag to unit-flag now that it applies on every event * test: move tests/test_litellm root and small trees into tests/unit Pure renames, no content changes. Follow-up commits in this PR fix references, merge the three files that already existed in tests/unit, keep live-provider tests in tests/test_litellm and wire CI. * test: carry tests/test_litellm conftest isolation into tests/unit Callback lists, routing fallbacks, cached HTTP clients, logger state, AWS, proxy-URL and keychain env, and session-end client cleanup now reset for unit tests too. The environment isolation owns its MonkeyPatch so a test's own monkeypatch is undone before the model-cost teardown runs. * test: merge, split and prune the moved root and small-tree tests Merge batches/test_batch_utils.py and the chat_completions and messages dispatch tests into the files that already existed in tests/unit. Keep the live Gemini interactions tests, the async image-fetch format test and the OpenAI embedding scorer test in tests/test_litellm since they need real network or keys. Put test_router.py under tests/unit/test_router so the existing package no longer shadows it. Delete eight tests the audit found superseded by stronger ones kept in this move. * ci: run the moved root and small-tree tests under their legacy flags Add the misc and responses-caching-types flags to unit_selection.sh and CircleCI, extend enterprise-routing and mcp-integration, and point the legacy GHA shards, Makefile, redis-compat workflow, merge smoke manifest and change classifier at the new paths. * test: make the new tests/unit directories packages tests/unit/test_package_layout.py requires every directory to carry an __init__.py, and without one the moved and retained test_litellm_responses_bridge.py modules collide on import. * test: scope the unit socket block to tests/unit in shared sessions The GHA shards collect the legacy test-path and the unit selection in one pytest session. The unit conftest's loopback-only block leaked into legacy modules that reach the network at import. The legacy conftest now lifts the restriction at collect and setup time, and the unit conftest re-applies it when collecting its own modules. * test: give the shard-script tests their own GITHUB_OUTPUT They only passed where the runner set it. The CircleCI unit job's env allowlist drops it, so the script's redirect failed there. * test: point the router and module-deletion checks at tests/unit router_code_coverage and code_qa_check_tests only searched tests/test_litellm, so the moved router tests no longer counted. The two silent-experiment tests the audit deleted were the only direct callers of those methods; they are replaced with tests that assert the forwarded shadow request and the recursion guard. --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
397 lines
15 KiB
Python
397 lines
15 KiB
Python
"""
|
|
Tests for exception header preservation.
|
|
|
|
These tests verify that when LLM providers return error responses with headers,
|
|
those headers are preserved in the exception and can be returned to clients.
|
|
|
|
This is important for debugging and observability - headers like x-request-id,
|
|
x-ms-region, rate limit headers, etc. should be available even when errors occur.
|
|
"""
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from litellm.exceptions import (
|
|
BadRequestError,
|
|
ContentPolicyViolationError,
|
|
ContextWindowExceededError,
|
|
ImageFetchError,
|
|
MidStreamFallbackError,
|
|
RateLimitError,
|
|
ServiceUnavailableError,
|
|
)
|
|
|
|
|
|
class TestExceptionHeaderPreservation:
|
|
"""Test that exception classes preserve headers from provider responses."""
|
|
|
|
@pytest.fixture
|
|
def mock_response_with_headers(self) -> httpx.Response:
|
|
"""Create a mock response with typical provider headers."""
|
|
return httpx.Response(
|
|
status_code=400,
|
|
headers={
|
|
"x-request-id": "req-abc123",
|
|
"x-ms-region": "eastus",
|
|
"x-ratelimit-remaining-requests": "99",
|
|
"x-ratelimit-remaining-tokens": "9999",
|
|
},
|
|
request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions"),
|
|
)
|
|
|
|
def test_bad_request_error_preserves_headers(
|
|
self, mock_response_with_headers: httpx.Response
|
|
):
|
|
"""BadRequestError should preserve headers from the provider response."""
|
|
error = BadRequestError(
|
|
message="Invalid request",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
response=mock_response_with_headers,
|
|
)
|
|
|
|
assert error.response is not None
|
|
assert error.response.headers.get("x-request-id") == "req-abc123"
|
|
assert error.response.headers.get("x-ms-region") == "eastus"
|
|
assert error.response.headers.get("x-ratelimit-remaining-requests") == "99"
|
|
|
|
def test_content_policy_violation_error_preserves_headers(
|
|
self, mock_response_with_headers: httpx.Response
|
|
):
|
|
"""ContentPolicyViolationError should preserve headers from the provider response."""
|
|
error = ContentPolicyViolationError(
|
|
message="Content policy violation",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
response=mock_response_with_headers,
|
|
)
|
|
|
|
assert error.response is not None
|
|
assert error.response.headers.get("x-request-id") == "req-abc123"
|
|
assert error.response.headers.get("x-ms-region") == "eastus"
|
|
|
|
def test_context_window_exceeded_error_preserves_headers(
|
|
self, mock_response_with_headers: httpx.Response
|
|
):
|
|
"""ContextWindowExceededError should preserve headers from the provider response."""
|
|
error = ContextWindowExceededError(
|
|
message="Context window exceeded",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
response=mock_response_with_headers,
|
|
)
|
|
|
|
assert error.response is not None
|
|
assert error.response.headers.get("x-request-id") == "req-abc123"
|
|
assert error.response.headers.get("x-ms-region") == "eastus"
|
|
|
|
def test_image_fetch_error_preserves_headers(
|
|
self, mock_response_with_headers: httpx.Response
|
|
):
|
|
"""ImageFetchError should preserve headers from the provider response."""
|
|
error = ImageFetchError(
|
|
message="Failed to fetch image",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
response=mock_response_with_headers,
|
|
)
|
|
|
|
assert error.response is not None
|
|
assert error.response.headers.get("x-request-id") == "req-abc123"
|
|
assert error.response.headers.get("x-ms-region") == "eastus"
|
|
|
|
def test_bad_request_error_handles_none_response(self):
|
|
"""BadRequestError should handle None response gracefully."""
|
|
error = BadRequestError(
|
|
message="Invalid request",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
response=None,
|
|
)
|
|
|
|
assert error.response is not None
|
|
# Headers should be empty but not cause an error
|
|
assert error.response.headers.get("x-request-id") is None
|
|
|
|
def test_content_policy_violation_error_handles_none_response(self):
|
|
"""ContentPolicyViolationError should handle None response gracefully."""
|
|
error = ContentPolicyViolationError(
|
|
message="Content policy violation",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
response=None,
|
|
)
|
|
|
|
assert error.response is not None
|
|
assert error.response.headers.get("x-request-id") is None
|
|
|
|
def test_context_window_exceeded_error_handles_none_response(self):
|
|
"""ContextWindowExceededError should handle None response gracefully."""
|
|
error = ContextWindowExceededError(
|
|
message="Context window exceeded",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
response=None,
|
|
)
|
|
|
|
assert error.response is not None
|
|
assert error.response.headers.get("x-request-id") is None
|
|
|
|
|
|
class TestExceptionMessageFormatting:
|
|
"""Test that exception messages are formatted correctly after refactoring."""
|
|
|
|
def test_bad_request_error_message_format(self):
|
|
"""BadRequestError should format message with litellm prefix."""
|
|
error = BadRequestError(
|
|
message="test error",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
)
|
|
|
|
assert "litellm.BadRequestError" in error.message
|
|
assert "test error" in error.message
|
|
|
|
def test_content_policy_violation_error_message_format(self):
|
|
"""ContentPolicyViolationError should format message with specific prefix."""
|
|
error = ContentPolicyViolationError(
|
|
message="test error",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
)
|
|
|
|
assert "litellm.ContentPolicyViolationError" in error.message
|
|
assert "test error" in error.message
|
|
|
|
def test_context_window_exceeded_error_message_format(self):
|
|
"""ContextWindowExceededError should format message with specific prefix."""
|
|
error = ContextWindowExceededError(
|
|
message="test error",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
)
|
|
|
|
assert "litellm.ContextWindowExceededError" in error.message
|
|
assert "test error" in error.message
|
|
|
|
|
|
class TestExceptionAttributes:
|
|
"""Test that exception attributes are set correctly."""
|
|
|
|
def test_content_policy_violation_error_provider_specific_fields(self):
|
|
"""ContentPolicyViolationError should preserve provider_specific_fields."""
|
|
provider_fields = {"innererror": {"code": "ResponsibleAIPolicyViolation"}}
|
|
|
|
error = ContentPolicyViolationError(
|
|
message="test error",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
provider_specific_fields=provider_fields,
|
|
)
|
|
|
|
assert error.provider_specific_fields == provider_fields
|
|
assert (
|
|
error.provider_specific_fields["innererror"]["code"]
|
|
== "ResponsibleAIPolicyViolation"
|
|
)
|
|
|
|
def test_bad_request_error_attributes(self):
|
|
"""BadRequestError should set all expected attributes."""
|
|
error = BadRequestError(
|
|
message="test error",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
litellm_debug_info="debug info",
|
|
max_retries=3,
|
|
num_retries=1,
|
|
)
|
|
|
|
assert error.model == "gpt-4"
|
|
assert error.llm_provider == "azure"
|
|
assert error.litellm_debug_info == "debug info"
|
|
assert error.max_retries == 3
|
|
assert error.num_retries == 1
|
|
assert error.status_code == 400
|
|
|
|
def test_midstream_fallback_error_status_code_propagation(self):
|
|
"""
|
|
MidStreamFallbackError should preserve the original status code and keep
|
|
message/request/response fields consistent after super().__init__().
|
|
"""
|
|
original_req = httpx.Request(
|
|
"POST", "https://api.openai.com/v1/chat/completions"
|
|
)
|
|
original_resp = httpx.Response(status_code=429, request=original_req)
|
|
|
|
rate_limit_error = RateLimitError(
|
|
message="Rate limit exceeded",
|
|
llm_provider="openai",
|
|
model="gpt-4o-mini",
|
|
response=original_resp,
|
|
)
|
|
|
|
midstream_error = MidStreamFallbackError(
|
|
message="stream broke",
|
|
model="gpt-4o-mini",
|
|
llm_provider="openai",
|
|
original_exception=rate_limit_error,
|
|
)
|
|
|
|
assert midstream_error.status_code == 429
|
|
assert midstream_error.response.status_code == 429
|
|
assert str(midstream_error.response.request.url) == "https://openai.com/v1/"
|
|
assert midstream_error.message == "litellm.MidStreamFallbackError: stream broke"
|
|
assert midstream_error.args == ("litellm.MidStreamFallbackError: stream broke",)
|
|
|
|
# With no original exception, should default to 503.
|
|
midstream_fallback = MidStreamFallbackError(
|
|
message="stream broke without original",
|
|
model="gpt-4o-mini",
|
|
llm_provider="openai",
|
|
original_exception=None,
|
|
)
|
|
|
|
assert midstream_fallback.status_code == 503
|
|
assert midstream_fallback.response.status_code == 503
|
|
assert str(midstream_fallback.response.request.url) == "https://openai.com/v1/"
|
|
|
|
|
|
class TestProxyHeaderExtraction:
|
|
"""Test that proxy correctly extracts headers from exceptions."""
|
|
|
|
def test_get_response_headers_adds_llm_provider_prefix(self):
|
|
"""get_response_headers should prefix non-OpenAI headers with llm_provider-."""
|
|
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
|
|
get_response_headers,
|
|
)
|
|
|
|
response_headers = {
|
|
"x-request-id": "req-abc123",
|
|
"x-ms-region": "eastus",
|
|
"x-ratelimit-remaining-requests": "99", # OpenAI header - should not be prefixed
|
|
}
|
|
|
|
result = get_response_headers(response_headers)
|
|
|
|
# OpenAI ratelimit headers should be preserved as-is
|
|
assert result.get("x-ratelimit-remaining-requests") == "99"
|
|
# Other headers should be prefixed with llm_provider-
|
|
assert result.get("llm_provider-x-request-id") == "req-abc123"
|
|
assert result.get("llm_provider-x-ms-region") == "eastus"
|
|
|
|
def test_proxy_can_extract_headers_from_exception_response(self):
|
|
"""Simulate how proxy extracts headers from exception.response.headers."""
|
|
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
|
|
get_response_headers,
|
|
)
|
|
|
|
# Create exception with headers in response
|
|
mock_response = httpx.Response(
|
|
status_code=400,
|
|
headers={
|
|
"x-request-id": "req-abc123",
|
|
"x-ms-region": "eastus",
|
|
},
|
|
request=httpx.Request("POST", "https://test.com"),
|
|
)
|
|
error = ContentPolicyViolationError(
|
|
message="test",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
response=mock_response,
|
|
)
|
|
|
|
# Simulate proxy header extraction logic
|
|
headers = getattr(error, "headers", None) or {}
|
|
if not headers:
|
|
_response = getattr(error, "response", None)
|
|
if _response is not None:
|
|
_response_headers = getattr(_response, "headers", None)
|
|
if _response_headers:
|
|
headers = get_response_headers(dict(_response_headers))
|
|
|
|
# Verify headers are extracted and prefixed correctly
|
|
assert headers.get("llm_provider-x-request-id") == "req-abc123"
|
|
assert headers.get("llm_provider-x-ms-region") == "eastus"
|
|
|
|
|
|
class TestBedrockErrorHeaders:
|
|
"""A BedrockError built with headers but no response still exposes them (LIT-5428)."""
|
|
|
|
def test_synthesized_response_carries_headers(self):
|
|
from litellm.llms.bedrock.common_utils import BedrockError
|
|
|
|
error = BedrockError(
|
|
status_code=500,
|
|
message="Amazon Bedrock is unable to process your request.",
|
|
headers={"x-amzn-RequestId": "req-base-500"},
|
|
)
|
|
|
|
assert error.response.headers["x-amzn-requestid"] == "req-base-500"
|
|
assert str(error.request.url) == str(BedrockError(status_code=500, message="boom").request.url)
|
|
assert str(error.response.request.url) == str(error.request.url)
|
|
|
|
def test_synthesized_response_without_headers_stays_empty(self):
|
|
from litellm.llms.bedrock.common_utils import BedrockError
|
|
|
|
error = BedrockError(status_code=500, message="boom")
|
|
|
|
assert dict(error.response.headers) == {}
|
|
|
|
def test_explicit_response_is_kept(self):
|
|
from litellm.llms.bedrock.common_utils import BedrockError
|
|
|
|
provider_response = httpx.Response(
|
|
status_code=500,
|
|
headers={"x-amzn-RequestId": "from-response"},
|
|
request=httpx.Request("POST", "https://bedrock-runtime.us-east-1.amazonaws.com/"),
|
|
)
|
|
error = BedrockError(
|
|
status_code=500,
|
|
message="boom",
|
|
headers={"x-amzn-RequestId": "from-headers"},
|
|
response=provider_response,
|
|
)
|
|
|
|
assert error.response is provider_response
|
|
|
|
def test_proxy_extraction_surfaces_bedrock_request_id(self):
|
|
"""End-to-end shape the proxy error handler returns to the caller."""
|
|
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
|
|
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
|
|
get_response_headers,
|
|
)
|
|
from litellm.llms.bedrock.common_utils import BedrockError
|
|
|
|
provider_response = httpx.Response(
|
|
status_code=500,
|
|
headers={"x-amzn-RequestId": "req-proxy-500"},
|
|
text='{"message":"Amazon Bedrock is unable to process your request."}',
|
|
request=httpx.Request("POST", "https://bedrock-runtime.us-east-1.amazonaws.com/"),
|
|
)
|
|
|
|
with pytest.raises(ServiceUnavailableError) as exc_info:
|
|
exception_type(
|
|
model="anthropic.claude-haiku-4-5-20251001-v1:0",
|
|
original_exception=BedrockError(
|
|
status_code=500,
|
|
message=provider_response.text,
|
|
headers=provider_response.headers,
|
|
response=provider_response,
|
|
),
|
|
custom_llm_provider="bedrock",
|
|
completion_kwargs={},
|
|
extra_kwargs={},
|
|
)
|
|
|
|
# Mirrors ProxyBaseLLMRequestProcessing._handle_llm_api_exception
|
|
error = exc_info.value
|
|
headers = getattr(error, "headers", None) or {}
|
|
if not headers:
|
|
_response = getattr(error, "response", None)
|
|
if _response is not None:
|
|
_response_headers = getattr(_response, "headers", None)
|
|
if _response_headers:
|
|
headers = get_response_headers(dict(_response_headers))
|
|
|
|
assert headers.get("llm_provider-x-amzn-requestid") == "req-proxy-500"
|