mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(router): count a failed deployment as tried when the caller supplies the key
With caller-supplied credentials the retry record's deployment_id is the generated id, so the web search preference never saw the configured deployment that failed and sent every retry back to it. Retry records now also carry the configured id.
This commit is contained in:
parent
a179118b3e
commit
27ec8d2adf
4 changed files with 27 additions and 8 deletions
|
|
@ -119,6 +119,7 @@ from litellm.litellm_core_utils.batch_utils import (
|
|||
)
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
as_str_mapping,
|
||||
coerce_token_limit,
|
||||
get_litellm_metadata_from_kwargs,
|
||||
get_metadata_variable_name_from_kwargs,
|
||||
|
|
@ -9880,14 +9881,15 @@ class Router:
|
|||
_metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"
|
||||
request_metadata: Final = cast("Mapping[str, object]", kwargs[_metadata_var]) # cast-ok: untyped kwargs
|
||||
model_group: Final = kwargs.get("model")
|
||||
model_info: Final = request_metadata.get("model_info")
|
||||
deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None
|
||||
model_info: Final = as_str_mapping(request_metadata.get("model_info")) or {}
|
||||
deployment_id: Final = model_info.get("id")
|
||||
configured_deployment_id: Final = model_info.get("original_model_id")
|
||||
base_model: Final = get_base_model_from_litellm_call_metadata(
|
||||
metadata=request_metadata
|
||||
) or self._deployment_params_base_model(deployment_id if isinstance(deployment_id, str) else None)
|
||||
deployment_model: Final = request_metadata.get("deployment")
|
||||
attempted_retries: Final = request_metadata.get("attempted_retries")
|
||||
attempt_record: Final[RetryAttemptRecord] = {
|
||||
attempt_fields: Final[RetryAttemptRecord] = {
|
||||
"model_group": model_group if isinstance(model_group, str) else None,
|
||||
"deployment_id": deployment_id if isinstance(deployment_id, str) else None,
|
||||
"model": deployment_model if isinstance(deployment_model, str) else None,
|
||||
|
|
@ -9896,6 +9898,11 @@ class Router:
|
|||
"exception_string": str(e),
|
||||
"attempted_retries": attempted_retries if type(attempted_retries) is int else None,
|
||||
}
|
||||
attempt_record: Final[RetryAttemptRecord] = (
|
||||
{**attempt_fields, "configured_deployment_id": configured_deployment_id}
|
||||
if isinstance(configured_deployment_id, str)
|
||||
else attempt_fields
|
||||
)
|
||||
earlier_breadcrumbs: Final = request_metadata.get("previous_models")
|
||||
kept_breadcrumbs: Final[tuple[object, ...]] = (
|
||||
tuple(earlier_breadcrumbs)[-(RETRY_BREADCRUMB_LIMIT - 1) :]
|
||||
|
|
|
|||
|
|
@ -515,7 +515,9 @@ def _deployment_ids_retried_in_request(request_kwargs: Mapping[str, object]) ->
|
|||
*_retry_breadcrumbs(request_kwargs.get("litellm_metadata")),
|
||||
)
|
||||
return tuple(
|
||||
deployment_id for breadcrumb in breadcrumbs if isinstance(deployment_id := breadcrumb.get("deployment_id"), str)
|
||||
deployment_id
|
||||
for breadcrumb in breadcrumbs
|
||||
if isinstance(deployment_id := breadcrumb.get("configured_deployment_id", breadcrumb.get("deployment_id")), str)
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
|||
import httpx
|
||||
from openai import Timeout as SDKTimeout
|
||||
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictInt, field_validator, model_validator
|
||||
from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable
|
||||
from typing_extensions import NotRequired, Protocol, ReadOnly, Required, TypedDict, runtime_checkable
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
|
|
@ -1197,6 +1197,7 @@ class RetryAttemptRecord(TypedDict):
|
|||
exception_type: ReadOnly[str]
|
||||
exception_string: ReadOnly[str]
|
||||
attempted_retries: ReadOnly[int | None]
|
||||
configured_deployment_id: ReadOnly[NotRequired[str]]
|
||||
|
||||
|
||||
VALID_LITELLM_ENVIRONMENTS = [
|
||||
|
|
|
|||
|
|
@ -735,8 +735,12 @@ async def test_web_search_preference_gives_way_once_the_capable_deployment_faile
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@_WEB_SEARCH_REQUESTS
|
||||
@pytest.mark.parametrize(
|
||||
"caller_credentials", [{}, {"api_key": "caller-key"}], ids=["configured-credentials", "caller-credentials"]
|
||||
)
|
||||
async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches_the_other_one(
|
||||
web_search_request: dict[str, object],
|
||||
caller_credentials: dict[str, str],
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
|
|
@ -752,7 +756,12 @@ async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches
|
|||
},
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi", "weight": 10**6},
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-y",
|
||||
"api_key": "fake-key",
|
||||
"mock_response": "from no-search",
|
||||
"weight": 10**6,
|
||||
},
|
||||
"model_info": {"id": "no-search", "supports_web_search": False},
|
||||
},
|
||||
],
|
||||
|
|
@ -760,9 +769,9 @@ async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches
|
|||
disable_cooldowns=True,
|
||||
)
|
||||
response: Final = await router.acompletion(
|
||||
model="mixed", messages=[{"role": "user", "content": "hi"}], **web_search_request
|
||||
model="mixed", messages=[{"role": "user", "content": "hi"}], **web_search_request, **caller_credentials
|
||||
)
|
||||
assert response._hidden_params["model_id"] == "no-search"
|
||||
assert response.choices[0].message.content == "from no-search"
|
||||
|
||||
|
||||
def _rate_limited_web_search_group(routing_strategy: str) -> Router:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue