mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
fix(docs+tests): fix health_check_ignore_transient_errors doc section and test coverage
- Move health_check_ignore_transient_errors from router_settings to general_settings in config_settings.md (code reads it from general_settings) - Remove duplicate enable_health_check_routing / health_check_staleness_threshold entries that were incorrectly listed under router_settings - Replace TestHealthCheckEndpointExceptionPropagation tests with ones that exercise the real _perform_health_check code path via mocked ahealth_check, verifying exceptions appear in exceptions_by_model_id and NOT in endpoint dicts Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
763cf56c4f
commit
d000af1c34
2 changed files with 75 additions and 23 deletions
|
|
@ -279,6 +279,34 @@ router_settings:
|
|||
| forward_client_headers_to_llm_api | boolean | If true, forwards the client headers (any `x-` headers and `anthropic-beta` headers) to the backend LLM call |
|
||||
| maximum_spend_logs_retention_period | str | Used to set the max retention time for spend logs in the db, after which they will be auto-purged |
|
||||
| maximum_spend_logs_retention_interval | str | Used to set the interval in which the spend log cleanup task should run in. |
|
||||
| alert_type_config | dict | Configuration mapping alert types to their handler settings |
|
||||
| always_include_stream_usage | boolean | If true, includes usage metrics in every streaming response chunk |
|
||||
| auto_redirect_ui_login_to_sso | boolean | If true, automatically redirects UI login page to SSO provider |
|
||||
| control_plane_url | string | URL of the control plane for cross-instance state sharing |
|
||||
| custom_auth_run_common_checks | boolean | If true, runs standard auth validation checks alongside custom auth handlers |
|
||||
| custom_ui_sso_sign_in_handler | string | Custom handler for SSO sign-in logic in the UI |
|
||||
| database_connection_pool_timeout | integer | Database connection pool timeout in seconds |
|
||||
| disable_error_logs | boolean | If true, suppresses error tracking and storage in the database |
|
||||
| enable_health_check_routing | boolean | If true, enables health check-driven request routing to avoid unhealthy deployments |
|
||||
| health_check_ignore_transient_errors | boolean | If true, 429 (rate limit) and 408 (timeout) health check failures are ignored and do not affect routing or cooldown |
|
||||
| enable_mcp_registry | boolean | If true, enables access to the centralized MCP server registry |
|
||||
| enforce_rbac | boolean | If true, enables role-based access control (RBAC) for all proxy operations |
|
||||
| forward_llm_provider_auth_headers | boolean | If true, forwards provider-specific auth headers to LLM API calls |
|
||||
| health_check_concurrency | integer | Maximum number of concurrent health check operations |
|
||||
| health_check_staleness_threshold | integer | Maximum age in seconds for health check results before marking deployments as stale |
|
||||
| maximum_spend_logs_cleanup_cron | string | Cron expression for scheduling automatic spend log cleanup tasks |
|
||||
| mcp_client_side_auth_header_name | string | HTTP header name for client-side MCP server credentials |
|
||||
| mcp_internal_ip_ranges | list | CIDR ranges considered internal for non-public MCP server access control |
|
||||
| mcp_required_fields | list | List of required field names for MCP server submissions |
|
||||
| mcp_trusted_proxy_ranges | list | CIDR ranges of proxies trusted to forward X-Forwarded-For headers for MCP |
|
||||
| require_end_user_mcp_access_defined | boolean | If true, requires end users to have explicit MCP access permissions defined |
|
||||
| role_permissions | list | List of role-based permission configurations |
|
||||
| search_tools | list | List of search tool configurations for enabling web search capabilities |
|
||||
| token_rate_limit_type | string | Rate limit counting method: "total", "output", or "input" tokens |
|
||||
| use_redis_transaction_buffer | boolean | If true, buffers database transactions in Redis before writing |
|
||||
| use_shared_health_check | boolean | If true, uses Redis-backed shared health check state across multiple proxy instances |
|
||||
| user_header_mappings | dict | Map custom request headers to user IDs using lookup rules |
|
||||
| user_header_name | string | HTTP header name to extract user identity from requests |
|
||||
|
||||
### router_settings - Reference
|
||||
|
||||
|
|
@ -369,7 +397,6 @@ router_settings:
|
|||
| guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) |
|
||||
| enable_health_check_routing | boolean | If true, enables health check-driven deployment filtering to avoid routing requests to unhealthy deployments |
|
||||
| health_check_staleness_threshold | integer | Maximum age in seconds for cached health check results before marking deployments as stale |
|
||||
| health_check_ignore_transient_errors | boolean | If true, 429 (rate limit) and 408 (timeout) health check failures are ignored and do not affect routing or cooldown |
|
||||
|
||||
|
||||
### environment variables - Reference
|
||||
|
|
|
|||
|
|
@ -43,41 +43,66 @@ class TestAhealthCheckExceptionPreservation:
|
|||
|
||||
|
||||
class TestHealthCheckEndpointExceptionPropagation:
|
||||
"""Test that _perform_health_check propagates exception objects through to unhealthy_endpoints."""
|
||||
"""Test that _perform_health_check returns exceptions via exceptions_by_model_id."""
|
||||
|
||||
def test_unhealthy_endpoint_with_exception_dict(self):
|
||||
"""When health check returns {"error": ..., "exception": e}, exception should be in the endpoint."""
|
||||
from litellm.proxy.health_check import _clean_endpoint_data
|
||||
@pytest.mark.asyncio
|
||||
async def test_unhealthy_endpoint_dict_exception_in_map(self):
|
||||
"""When ahealth_check returns {"error": ..., "exception": e}, the exception
|
||||
must appear in exceptions_by_model_id keyed by model_id — not in the endpoint dict."""
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from litellm.proxy.health_check import _perform_health_check
|
||||
|
||||
auth_error = litellm.AuthenticationError(
|
||||
message="Invalid key", llm_provider="openai", model="gpt-4"
|
||||
)
|
||||
model_list = [
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
"litellm_params": {"model": "gpt-4", "api_key": "fake"},
|
||||
"model_info": {"id": "deploy-abc"},
|
||||
}
|
||||
]
|
||||
|
||||
# Simulate what _perform_health_check does for an unhealthy dict result
|
||||
is_healthy = {"error": "auth failed", "exception": auth_error}
|
||||
litellm_params = {"model": "gpt-4", "api_key": "fake"}
|
||||
cleaned = _clean_endpoint_data({**litellm_params, **is_healthy}, details=True)
|
||||
# Exception should be preserved after cleaning
|
||||
if "exception" in is_healthy:
|
||||
cleaned["exception"] = is_healthy["exception"]
|
||||
with patch(
|
||||
"litellm.proxy.health_check.litellm.ahealth_check",
|
||||
new=AsyncMock(return_value={"error": "auth failed", "exception": auth_error}),
|
||||
):
|
||||
healthy, unhealthy, exc_map = await _perform_health_check(model_list)
|
||||
|
||||
assert cleaned["exception"] is auth_error
|
||||
assert len(unhealthy) == 1
|
||||
assert "exception" not in unhealthy[0], "exception must not be in endpoint dict"
|
||||
assert exc_map.get("deploy-abc") is auth_error
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_exception_from_gather_in_map(self):
|
||||
"""When asyncio.gather returns a raw Exception, it must appear in
|
||||
exceptions_by_model_id — not in the endpoint dict."""
|
||||
from unittest.mock import patch
|
||||
|
||||
from litellm.proxy.health_check import _perform_health_check
|
||||
|
||||
def test_unhealthy_endpoint_raw_exception(self):
|
||||
"""When gather returns a raw Exception, it should be stored in the endpoint dict."""
|
||||
raw_exc = litellm.RateLimitError(
|
||||
message="Rate limited", llm_provider="openai", model="gpt-4"
|
||||
)
|
||||
model_list = [
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
"litellm_params": {"model": "gpt-4", "api_key": "fake"},
|
||||
"model_info": {"id": "deploy-xyz"},
|
||||
}
|
||||
]
|
||||
|
||||
# Simulate the else branch in _perform_health_check
|
||||
from litellm.proxy.health_check import _clean_endpoint_data
|
||||
# Simulate asyncio.gather returning a raw exception for this task
|
||||
with patch(
|
||||
"litellm.proxy.health_check._run_model_health_check",
|
||||
side_effect=raw_exc,
|
||||
):
|
||||
healthy, unhealthy, exc_map = await _perform_health_check(model_list)
|
||||
|
||||
litellm_params = {"model": "gpt-4"}
|
||||
cleaned = _clean_endpoint_data(litellm_params, details=True)
|
||||
if isinstance(raw_exc, Exception):
|
||||
cleaned["exception"] = raw_exc
|
||||
|
||||
assert cleaned["exception"] is raw_exc
|
||||
assert len(unhealthy) == 1
|
||||
assert "exception" not in unhealthy[0], "exception must not be in endpoint dict"
|
||||
assert exc_map.get("deploy-xyz") is raw_exc
|
||||
|
||||
|
||||
class TestGetAllowedFailsFromPolicyWithHealthCheckExceptions:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue