mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-24 00:52:24 +00:00
fix(proxy): attribute provider and model_info on pre_call_hook rejections (#41077)
* fix(proxy): attribute provider and model info on pre-call rejected requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep pre-call rejections out of deployment cooldown and prometheus deployment state Stamp model_info only into the logging metadata so the router's failure callbacks do not count a key-level 429 or guardrail 403 against the deployment, treat a resolved plus an unresolved deployment as ambiguous provider attribution, and stop the prometheus deployment counters and deployment_state from treating a proxy-side reject as a selected deployment Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): skip deployment attribution when the rejected body's model is not a string Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(prometheus): bucket non-string request models as other instead of raising in failure hook Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): resolve team deployments and treat guardrail rejects as proxy-side in failure attribution Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(prometheus): flag pre-routing rejects instead of matching exception names Post-call GuardrailRaisedException failures kept their deployment labels on main but lost them on this branch because every GuardrailRaisedException was treated as a pre-routing reject. The proxy failure path now flags litellm_params with proxy_rejected_before_routing only when it adds deployment attribution itself, and the Prometheus logger keys deployment selection off that flag Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): key pre-routing reject flag off provider handoff, not caller metadata Caller-supplied metadata.model_info (kept for keys allowed to override pricing) no longer suppresses proxy_rejected_before_routing. The hook now checks the logging object's first_api_call_start_time, which only the provider handoff sets, so Prometheus never records a deployment failure for a request that was rejected before routing. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): poll for both served and rejected spend rows before asserting attribution Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: yucheng <yucheng@berri.ai>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
cd69342eef
commit
989d7b87b2
10 changed files with 734 additions and 12 deletions
|
|
@ -238,6 +238,9 @@ LOGS_GUARDRAIL_INFORMATION_MARKER: Final = "_litellm_logs_guardrail_information"
|
|||
# llm_provider stamped on proxy-side rate limit errors when the model resolves to no deployment
|
||||
PROXY_LLM_PROVIDER_FALLBACK: Final = "litellm_proxy"
|
||||
|
||||
# litellm_params flag on failure logs for requests the proxy rejected before routing to a deployment
|
||||
PROXY_REJECTED_BEFORE_ROUTING_KEY: Final = "proxy_rejected_before_routing"
|
||||
|
||||
# Generic fallback for unknown models
|
||||
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = int(
|
||||
os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET", 128)
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
|
||||
import litellm
|
||||
from litellm._logging import print_verbose, verbose_logger
|
||||
from litellm.constants import PROXY_LLM_PROVIDER_FALLBACK
|
||||
from litellm.constants import PROXY_LLM_PROVIDER_FALLBACK, PROXY_REJECTED_BEFORE_ROUTING_KEY
|
||||
from litellm.exceptions import (
|
||||
validate_rate_limit_category,
|
||||
validate_rate_limit_type,
|
||||
|
|
@ -214,17 +214,22 @@ def _get_proxy_llm_router() -> Router | None:
|
|||
return llm_router
|
||||
|
||||
|
||||
def _bounded_requested_model_label(requested_model: str | None, router_originated: bool = False) -> str | None:
|
||||
def _bounded_requested_model_label(requested_model: object, router_originated: bool = False) -> str | None:
|
||||
"""
|
||||
Bound ``requested_model`` label cardinality: names the router recognizes
|
||||
(model names, deployment ids, aliases, routing groups, team public model
|
||||
names) or matches via a global or team wildcard/pattern route keep their
|
||||
own label value; any other client-supplied string collapses into the
|
||||
single ``other`` bucket. With no proxy router to vouch for the string,
|
||||
client-supplied values collapse to ``other`` while ``router_originated``
|
||||
values (emitted by an SDK ``Router``'s own deployment failure and
|
||||
fallback events, where the proxy router never exists) pass through.
|
||||
single ``other`` bucket, as does any non-string request ``model`` value.
|
||||
With no proxy router to vouch for the string, client-supplied values
|
||||
collapse to ``other`` while ``router_originated`` values (emitted by an
|
||||
SDK ``Router``'s own deployment failure and fallback events, where the
|
||||
proxy router never exists) pass through.
|
||||
"""
|
||||
if requested_model is None:
|
||||
return None
|
||||
if not isinstance(requested_model, str):
|
||||
return UNRECOGNIZED_REQUESTED_MODEL_LABEL
|
||||
if not requested_model:
|
||||
return requested_model
|
||||
llm_router: Final = _get_proxy_llm_router()
|
||||
|
|
@ -2832,7 +2837,7 @@ class PrometheusLogger(CustomLogger):
|
|||
|
||||
# On LiteLLM-side rejects (no deployment picked), route request_kwargs["model"]
|
||||
# into requested_model and leave deployment-scoped labels empty.
|
||||
deployment_selected: Final = bool(model_id)
|
||||
deployment_selected: Final = bool(model_id) and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
|
||||
if deployment_selected:
|
||||
label_litellm_model_name = litellm_model_name
|
||||
label_model_id = model_id
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ from litellm.constants import (
|
|||
DEFAULT_MODEL_CREATED_AT_TIME,
|
||||
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL,
|
||||
MAX_TEAM_LIST_LIMIT,
|
||||
PROXY_REJECTED_BEFORE_ROUTING_KEY,
|
||||
REDIS_SPEND_LOGS_BUFFER_DEQUEUE_COUNT,
|
||||
SPEND_LOG_QUEUE_MAX_BYTES,
|
||||
SPEND_LOG_WRITE_BATCH_MAX_BYTES,
|
||||
|
|
@ -979,6 +980,92 @@ def _failure_usage_to_lift(
|
|||
_EMPTY_LIFT: Final = MappingProxyType({})
|
||||
|
||||
|
||||
def _stamp_deployment_attribution(
|
||||
litellm_params: dict[str, object], model_group: str | None, team_id: str | None, dispatched: bool
|
||||
) -> Mapping[str, object]:
|
||||
"""Stamp provider and logging-metadata attribution onto ``litellm_params`` and return it.
|
||||
``litellm_params["model_info"]`` stays unset: the router's cooldown and per-deployment rpm
|
||||
callbacks key off it and must not count a proxy-side reject against the deployment. A failure
|
||||
after the provider handoff keeps the metadata the router stamped; a request that never reached a
|
||||
provider is flagged ``PROXY_REJECTED_BEFORE_ROUTING_KEY`` (deployment metrics key off it) whatever
|
||||
its metadata says, since ``metadata.model_info`` can be caller supplied."""
|
||||
attribution: Final = _deployment_attribution_for_model_group(model_group, team_id)
|
||||
if "custom_llm_provider" in attribution:
|
||||
litellm_params["custom_llm_provider"] = attribution["custom_llm_provider"]
|
||||
if dispatched:
|
||||
return attribution
|
||||
litellm_params[PROXY_REJECTED_BEFORE_ROUTING_KEY] = True
|
||||
if "model_info" not in attribution:
|
||||
return attribution
|
||||
if litellm_params.get("metadata") is None:
|
||||
litellm_params["metadata"] = {} # mutable-ok: legacy logging payload is populated in place
|
||||
metadata: Final = litellm_params["metadata"]
|
||||
if not isinstance(metadata, dict):
|
||||
return attribution
|
||||
metadata.setdefault("model_info", attribution["model_info"])
|
||||
metadata.setdefault("deployment", attribution["deployment"])
|
||||
return attribution
|
||||
|
||||
|
||||
def _deployment_attribution_for_model_group(model_group: object, team_id: str | None) -> Mapping[str, object]:
|
||||
"""Provider fields the router would have stamped had it reached a deployment:
|
||||
``custom_llm_provider`` when every deployment in the group resolves to the same
|
||||
provider, plus ``model_info`` and ``deployment`` when the group has exactly one.
|
||||
``team_id`` picks the key's team deployments over a global group of the same public name."""
|
||||
if not isinstance(model_group, str):
|
||||
return _EMPTY_LIFT
|
||||
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
if llm_router is None:
|
||||
return _EMPTY_LIFT
|
||||
deployments: Final = llm_router.get_model_list(model_name=model_group, team_id=team_id)
|
||||
if not deployments:
|
||||
return _EMPTY_LIFT
|
||||
|
||||
def _provider_for_deployment(deployment: Mapping[str, object]) -> str | None:
|
||||
litellm_params: Final = cast( # cast-ok: router deployment parameters are mapping-shaped
|
||||
Mapping[str, object], deployment["litellm_params"]
|
||||
)
|
||||
try:
|
||||
provider: Final = litellm.get_llm_provider(
|
||||
model=cast(str, litellm_params["model"]), # cast-ok: router deployment model is a string
|
||||
custom_llm_provider=cast( # cast-ok: router deployment provider is optional
|
||||
str | None, litellm_params.get("custom_llm_provider")
|
||||
),
|
||||
)[1]
|
||||
return cast(str | None, provider) # cast-ok: provider resolver returns an optional provider string
|
||||
except Exception: # noqa: BLE001 # get_llm_provider raises for unmapped models
|
||||
return None
|
||||
|
||||
providers: Final = frozenset(_provider_for_deployment(deployment) for deployment in deployments)
|
||||
shared_provider: Final = next(iter(providers)) if len(providers) == 1 else None
|
||||
single_deployment: Final = deployments[0] if len(deployments) == 1 else None
|
||||
single_deployment_params: Final = (
|
||||
cast( # cast-ok: router deployment parameters are mapping-shaped
|
||||
Mapping[str, object], single_deployment["litellm_params"]
|
||||
)
|
||||
if single_deployment is not None
|
||||
else None
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
# mutable-ok: frozen immediately by the outer MappingProxyType
|
||||
**({"custom_llm_provider": shared_provider} if shared_provider is not None else {}),
|
||||
**(
|
||||
{ # mutable-ok: frozen immediately by the outer MappingProxyType
|
||||
"model_info": dict( # mutable-ok: preserve the router's mutable model-info payload
|
||||
single_deployment.get("model_info") or {}
|
||||
),
|
||||
"deployment": single_deployment_params["model"],
|
||||
}
|
||||
if single_deployment is not None and single_deployment_params is not None
|
||||
else {} # mutable-ok: frozen immediately by the outer MappingProxyType
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _call_type_for_route(route: str | None) -> str | None:
|
||||
"""The route's call type when it maps to a single operation (its async and sync variants);
|
||||
None for routes shared by several operations, since the method is not known here."""
|
||||
|
|
@ -3227,11 +3314,25 @@ class ProxyLogging:
|
|||
elif k not in ("model", "user", "litellm_logging_obj"):
|
||||
_optional_params[k] = v
|
||||
|
||||
attribution: Final = _stamp_deployment_attribution(
|
||||
_litellm_params,
|
||||
request_data.get("model"),
|
||||
user_api_key_dict.team_id,
|
||||
dispatched=litellm_logging_obj.model_call_details.get("first_api_call_start_time") is not None,
|
||||
)
|
||||
|
||||
litellm_logging_obj.update_environment_variables(
|
||||
model=request_data.get("model", ""),
|
||||
user=request_data.get("user", ""),
|
||||
optional_params=_optional_params,
|
||||
litellm_params=_litellm_params,
|
||||
**(
|
||||
{ # mutable-ok: frozen immediately by keyword expansion
|
||||
"custom_llm_provider": attribution["custom_llm_provider"]
|
||||
}
|
||||
if "custom_llm_provider" in attribution
|
||||
else {} # mutable-ok: frozen immediately by keyword expansion
|
||||
),
|
||||
)
|
||||
|
||||
input: list | str | dict = ""
|
||||
|
|
|
|||
|
|
@ -216,7 +216,7 @@ quota_management.<behavior>.<variant>.<assertion>
|
|||
| isolates_per_model | isolates_per_member | isolates_per_group | enforced_across_keys
|
||||
| routes_to_fallback | reseed_matches_db | reports_spend | logs_cost | zero_cost
|
||||
| matches_sum_of_logs | loses_no_spend | attributes_spend | writes_own_rows
|
||||
| writes_failure_row | returns_cost | keeps_total | joins_key | reports_alias_and_email
|
||||
| writes_failure_row | attributes_provider | returns_cost | keeps_total | joins_key | reports_alias_and_email
|
||||
| health_rows_keep_service_account | retrieve_batch_cost_joins_retrieving_key
|
||||
| poller_batch_cost_joins_creating_key | bills_under_request_session
|
||||
e.g. quota_management.ratelimit.rpm.blocks_over_limit exercised_on=[chat_completions, messages]
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@
|
|||
- {id: quota_management.spend_tracking.end_user.attributes_spend, module: quota_management, tier: P1, behavior: spend_tracking, variant: end_user, assertions: [attributes_spend], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "user= attribution lands the end-user id on the spend row"}
|
||||
- {id: quota_management.spend_tracking.per_model.writes_own_rows, module: quota_management, tier: P2, behavior: spend_tracking, variant: per_model, assertions: [writes_own_rows], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Each model on a shared key gets its own spend row"}
|
||||
- {id: quota_management.spend_tracking.failure.writes_failure_row, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_failure_row], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_log_error_logger.py", rationale: "A failed call writes a failure-status spend row"}
|
||||
- {id: quota_management.spend_tracking.failure.attributes_provider, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [attributes_provider], exercised_on: [chat_completions], source: "proxy/utils.py", rationale: "A request rejected in pre_call_hook (rate limit, guardrail) still lands its single deployment's provider and model_id on the failure spend row"}
|
||||
- {id: quota_management.spend_tracking.spend_calculate.returns_cost, module: quota_management, tier: P2, behavior: spend_tracking, variant: spend_calculate, assertions: [returns_cost], exercised_on: [spend_calculate], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "/spend/calculate prices a hypothetical request at nonzero cost"}
|
||||
- {id: quota_management.spend_tracking.pagination.keeps_total, module: quota_management, tier: P2, behavior: spend_tracking, variant: pagination, assertions: [keeps_total], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "Spend-logs v2 pagination caps page size without losing the total"}
|
||||
- {id: quota_management.spend_tracking.cache_write.bills_cache_creation_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_write, assertions: [bills_cache_creation_rate], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "OpenAI cache-write tokens land on the spend row as cache-creation tokens billed at the cache-creation rate, not silently at the input rate (#34046)"}
|
||||
|
|
|
|||
|
|
@ -952,6 +952,7 @@ class SpendLogRow(BaseModel):
|
|||
cache_hit: str | None = None
|
||||
call_type: str | None = None
|
||||
custom_llm_provider: str | None = None
|
||||
model_id: str | None = None
|
||||
team_id: str | None = None
|
||||
user: str | None = None
|
||||
end_user: str | None = None
|
||||
|
|
|
|||
|
|
@ -21,9 +21,9 @@ from math import isclose
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from e2e_http import Success
|
||||
from e2e_http import RateLimitedError, Success
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody, SpendLogs, SpendLogsParams
|
||||
from models import KeyGenerateBody, LiteLLMParamsBody, SpendLogs, SpendLogsParams
|
||||
from spend_e2e_client import SpendClient, SpendLogRow, is_ok, unique_marker, unwrap
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
|
@ -43,6 +43,7 @@ def _summarize(rows: list[SpendLogRow]) -> list[dict[str, object]]:
|
|||
"cache_hit",
|
||||
"call_type",
|
||||
"custom_llm_provider",
|
||||
"model_id",
|
||||
"prompt_tokens",
|
||||
"completion_tokens",
|
||||
"total_tokens",
|
||||
|
|
@ -498,6 +499,47 @@ def test_failure_call_writes_failure_status_row(
|
|||
assert (failure_row.spend or 0) == 0.0, "failed call must not be charged"
|
||||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.failure.attributes_provider")
|
||||
def test_pre_call_rejection_row_attributes_provider_and_model_id(
|
||||
client: SpendClient, resources: ResourceManager
|
||||
) -> None:
|
||||
"""A request the proxy rejects before the router picks a deployment (here the
|
||||
key's rpm limit, a pre_call_hook 429) never reaches the code that stamps the
|
||||
deployment onto the log. The failure row must still carry the provider and
|
||||
model_id of the model group's only deployment, so per-provider failure reports
|
||||
can count it."""
|
||||
model = f"e2e-spend-precall-{unique_marker()}"
|
||||
model_id = client.proxy.create_model(
|
||||
model, LiteLLMParamsBody(model="openai/gpt-5.5", api_key="os.environ/OPENAI_API_KEY")
|
||||
)
|
||||
resources.defer(lambda: client.proxy.delete_model(model_id))
|
||||
key = client.proxy.generate_key(KeyGenerateBody(models=[model], rpm_limit=1))
|
||||
resources.defer(lambda: client.proxy.delete_key(key))
|
||||
|
||||
unwrap(client.chat(key, model, f"reply with one word {unique_marker()}", max_tokens=8))
|
||||
rejected = client.chat(key, model, f"over the rpm limit {unique_marker()}", max_tokens=8)
|
||||
assert isinstance(rejected, RateLimitedError), (
|
||||
f"the second call on an rpm_limit=1 key must be rejected with 429 before routing, got {rejected}"
|
||||
)
|
||||
|
||||
rows = client.poll_logs_for_key(
|
||||
key,
|
||||
min_rows=2,
|
||||
predicate=lambda rs: {r.status for r in rs} >= {"success", "failure"},
|
||||
)
|
||||
success_row = _require_row(rows, lambda r: r.status == "success", "for the served call")
|
||||
failure_row = _require_row(rows, lambda r: r.status == "failure", "for the rate-limited call")
|
||||
|
||||
assert failure_row.custom_llm_provider == success_row.custom_llm_provider, (
|
||||
f"rejected call lost its provider: failure row {failure_row.custom_llm_provider!r} vs "
|
||||
f"served row {success_row.custom_llm_provider!r}; {_summarize(rows)}"
|
||||
)
|
||||
assert failure_row.model_id == model_id, (
|
||||
f"rejected call lost its deployment: failure row model_id {failure_row.model_id!r} vs "
|
||||
f"registered {model_id!r}; {_summarize(rows)}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.spend_calculate.returns_cost")
|
||||
def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None:
|
||||
cost = client.calculate_spend(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,101 @@
|
|||
"""
|
||||
LIT-7701 attributes a pre_call_hook rejection's failure log to the model
|
||||
group's single deployment (``model_id`` and ``custom_llm_provider``) and flags
|
||||
it with ``PROXY_REJECTED_BEFORE_ROUTING_KEY``. The deployment health metrics
|
||||
must keep treating such rejects as "no deployment picked": a key rate limit or
|
||||
guardrail block never reached the deployment, so it must not flip
|
||||
``litellm_deployment_state`` to partial outage or count as a deployment failure
|
||||
response. A failure raised after the router picked a deployment (a post-call
|
||||
guardrail block, a provider error) carries no flag and keeps its deployment labels.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from prometheus_client import REGISTRY
|
||||
|
||||
from litellm.constants import PROXY_REJECTED_BEFORE_ROUTING_KEY
|
||||
from litellm.exceptions import GuardrailRaisedException
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def cleanup_prometheus_registry():
|
||||
for collector in list(REGISTRY._collector_to_names.keys()):
|
||||
try:
|
||||
REGISTRY.unregister(collector)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
yield
|
||||
|
||||
for collector in list(REGISTRY._collector_to_names.keys()):
|
||||
try:
|
||||
REGISTRY.unregister(collector)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _attributed_failure_kwargs(exception: Exception, rejected_before_routing: bool) -> dict:
|
||||
return {
|
||||
"model": "openai/gpt-4.1",
|
||||
"litellm_params": {
|
||||
"custom_llm_provider": "openai",
|
||||
"metadata": {"model_info": {"id": "dep-1"}, "model_group": "internal-model"},
|
||||
**({PROXY_REJECTED_BEFORE_ROUTING_KEY: True} if rejected_before_routing else {}),
|
||||
},
|
||||
"standard_logging_object": {
|
||||
"model_id": "dep-1",
|
||||
"model_group": "internal-model",
|
||||
"api_base": "https://api.openai.com",
|
||||
"metadata": {},
|
||||
},
|
||||
"exception": exception,
|
||||
}
|
||||
|
||||
|
||||
def _model_id_values(metric) -> set[str]:
|
||||
index = metric._labelnames.index("model_id")
|
||||
return {sample_key[index] for sample_key in metric._metrics}
|
||||
|
||||
|
||||
class _ProviderError(Exception):
|
||||
status_code = 500
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rejection",
|
||||
[
|
||||
HTTPException(status_code=403, detail="guardrail blocked"),
|
||||
ProxyException(message="budget exceeded", type="budget_exceeded", param=None, code=400),
|
||||
ProxyRateLimitError(detail={"error": "key rpm limit"}),
|
||||
GuardrailRaisedException(guardrail_name="pii", message="blocked", status_code=403),
|
||||
],
|
||||
ids=["http_exception", "proxy_exception", "proxy_rate_limit", "guardrail_raised"],
|
||||
)
|
||||
def test_attributed_proxy_reject_leaves_deployment_healthy(rejection: Exception):
|
||||
logger = PrometheusLogger()
|
||||
|
||||
logger.set_llm_deployment_failure_metrics(_attributed_failure_kwargs(rejection, rejected_before_routing=True))
|
||||
|
||||
assert logger.litellm_deployment_state._metrics == {}
|
||||
assert _model_id_values(logger.litellm_deployment_failure_responses) == {""}
|
||||
assert _model_id_values(logger.litellm_deployment_total_requests) == {""}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"failure",
|
||||
[
|
||||
_ProviderError("upstream 500"),
|
||||
GuardrailRaisedException(guardrail_name="pii", message="response blocked", status_code=400),
|
||||
],
|
||||
ids=["provider_error", "post_call_guardrail"],
|
||||
)
|
||||
def test_failure_after_routing_still_marks_deployment_partial_outage(failure: Exception):
|
||||
logger = PrometheusLogger()
|
||||
|
||||
logger.set_llm_deployment_failure_metrics(_attributed_failure_kwargs(failure, rejected_before_routing=False))
|
||||
|
||||
assert _model_id_values(logger.litellm_deployment_state) == {"dep-1"}
|
||||
assert _model_id_values(logger.litellm_deployment_failure_responses) == {"dep-1"}
|
||||
|
|
@ -116,6 +116,22 @@ async def test_unknown_models_collapse_to_one_series_on_proxy_request_metrics(ro
|
|||
assert _total_value(metric) == 25
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model", [["gpt-4o-mini"], {"name": "gpt-4o-mini"}, 123])
|
||||
async def test_non_string_models_collapse_to_other_on_proxy_request_metrics(router, model: object):
|
||||
logger = PrometheusLogger()
|
||||
|
||||
with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
|
||||
await logger.async_post_call_failure_hook(
|
||||
request_data={"model": model, "metadata": {}, "proxy_server_request": {}},
|
||||
original_exception=_ClientSideError("'model' must be a string."),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key-1"),
|
||||
)
|
||||
|
||||
assert _requested_model_values(logger.litellm_proxy_failed_requests_metric) == {UNRECOGNIZED_REQUESTED_MODEL_LABEL}
|
||||
assert _total_value(logger.litellm_proxy_failed_requests_metric) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_known_alias_and_wildcard_models_keep_their_own_labels(router):
|
||||
logger = PrometheusLogger()
|
||||
|
|
|
|||
|
|
@ -5,16 +5,18 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from typing import Any, Final
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
import litellm
|
||||
from litellm.constants import PROXY_REJECTED_BEFORE_ROUTING_KEY
|
||||
from litellm.exceptions import GuardrailRaisedException
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.proxy._types import AlertType, ProxyErrorTypes, UserAPIKeyAuth
|
||||
from litellm.proxy._types import ProxyErrorTypes, UserAPIKeyAuth
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
|
||||
|
|
@ -99,6 +101,456 @@ async def test_post_call_failure_hook_no_callbacks_returns_none(
|
|||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_failure_hook_attributes_single_router_deployment(
|
||||
proxy_logging, make_user_api_key_auth, monkeypatch
|
||||
):
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
recorded: list[dict] = []
|
||||
|
||||
class _RecordingLogger(CustomLogger):
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
recorded.append(kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "internal-model",
|
||||
"litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"},
|
||||
"model_info": {"provider": "acme"},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()])
|
||||
proxy_logging.alert_types = []
|
||||
|
||||
await proxy_logging.post_call_failure_hook(
|
||||
request_data={"model": "internal-model", "messages": [{"role": "user", "content": "hi"}]},
|
||||
original_exception=HTTPException(status_code=403, detail="blocked"),
|
||||
user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"),
|
||||
route="/chat/completions",
|
||||
)
|
||||
|
||||
assert len(recorded) == 1
|
||||
kwargs = recorded[0]
|
||||
assert kwargs["custom_llm_provider"] == "openai"
|
||||
assert kwargs["litellm_params"]["custom_llm_provider"] == "openai"
|
||||
assert kwargs["litellm_params"]["metadata"]["model_info"]["provider"] == "acme"
|
||||
assert kwargs["litellm_params"]["metadata"]["deployment"] == "openai/gpt-4.1"
|
||||
assert kwargs["litellm_params"][PROXY_REJECTED_BEFORE_ROUTING_KEY] is True
|
||||
assert kwargs["standard_logging_object"]["custom_llm_provider"] == "openai"
|
||||
assert (
|
||||
kwargs["standard_logging_object"]["model_id"] == proxy_server.llm_router.get_model_list()[0]["model_info"]["id"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_failure_hook_keeps_router_stamped_metadata_for_post_call_failures(
|
||||
proxy_logging, make_user_api_key_auth, monkeypatch
|
||||
):
|
||||
"""A post-call guardrail block arrives after the provider handoff with the router's own
|
||||
``model_info`` in the request metadata. The pre-routing flag must stay off so deployment
|
||||
metrics keep attributing the failure to the deployment that actually served the call."""
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
recorded: list[dict] = []
|
||||
|
||||
class _RecordingLogger(CustomLogger):
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
recorded.append(kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "internal-model",
|
||||
"litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"},
|
||||
"model_info": {"id": "routed-deployment"},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()])
|
||||
proxy_logging.alert_types = []
|
||||
|
||||
request_data = {
|
||||
"litellm_call_id": "post-call-guardrail",
|
||||
"model": "internal-model",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"metadata": {"model_info": {"id": "routed-deployment", "served": True}},
|
||||
}
|
||||
logging_obj, request_data = litellm.utils.function_setup(
|
||||
original_function="acompletion", rules_obj=litellm.utils.Rules(), start_time=datetime.now(), **request_data
|
||||
)
|
||||
logging_obj.model_call_details["first_api_call_start_time"] = datetime.now()
|
||||
request_data["litellm_logging_obj"] = logging_obj
|
||||
|
||||
await proxy_logging.post_call_failure_hook(
|
||||
request_data=request_data,
|
||||
original_exception=GuardrailRaisedException(guardrail_name="g", message="response blocked"),
|
||||
user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"),
|
||||
route="/chat/completions",
|
||||
)
|
||||
|
||||
assert len(recorded) == 1
|
||||
kwargs = recorded[0]
|
||||
assert kwargs["litellm_params"]["metadata"]["model_info"] == {"id": "routed-deployment", "served": True}
|
||||
assert PROXY_REJECTED_BEFORE_ROUTING_KEY not in kwargs["litellm_params"]
|
||||
assert kwargs["standard_logging_object"]["model_id"] == "routed-deployment"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_failure_hook_flags_pre_routing_reject_despite_caller_model_info(
|
||||
proxy_logging, make_user_api_key_auth, monkeypatch
|
||||
):
|
||||
"""A key allowed to override pricing keeps caller-supplied ``metadata.model_info``. A reject
|
||||
before any provider handoff must still carry the pre-routing flag so deployment metrics do
|
||||
not record an outage for a deployment the request never reached."""
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
recorded: list[dict] = []
|
||||
|
||||
class _RecordingLogger(CustomLogger):
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
recorded.append(kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "internal-model",
|
||||
"litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"},
|
||||
"model_info": {"id": "real-deployment"},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()])
|
||||
proxy_logging.alert_types = []
|
||||
|
||||
await proxy_logging.post_call_failure_hook(
|
||||
request_data={
|
||||
"model": "internal-model",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"metadata": {"model_info": {"id": "spoofed-deployment"}},
|
||||
},
|
||||
original_exception=HTTPException(status_code=429, detail="key over limit"),
|
||||
user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"),
|
||||
route="/chat/completions",
|
||||
)
|
||||
|
||||
assert len(recorded) == 1
|
||||
kwargs = recorded[0]
|
||||
assert kwargs["litellm_params"][PROXY_REJECTED_BEFORE_ROUTING_KEY] is True
|
||||
assert kwargs["litellm_params"]["custom_llm_provider"] == "openai"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_failure_hook_attribution_does_not_count_against_the_deployment(
|
||||
proxy_logging, make_user_api_key_auth, monkeypatch
|
||||
):
|
||||
"""The router's failure callbacks run on this path too. A proxy-side reject must not
|
||||
bump the deployment's failure or rpm counters, or a key hitting its own limit
|
||||
could cool down the only deployment for everyone."""
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "internal-model",
|
||||
"litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test", "rpm": 100},
|
||||
}
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
proxy_logging.alert_types = []
|
||||
deployment_id = router.get_model_list()[0]["model_info"]["id"]
|
||||
|
||||
for status in (403, 429):
|
||||
await proxy_logging.post_call_failure_hook(
|
||||
request_data={"model": "internal-model", "messages": [{"role": "user", "content": "hi"}]},
|
||||
original_exception=HTTPException(status_code=status, detail="blocked"),
|
||||
user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"),
|
||||
route="/chat/completions",
|
||||
)
|
||||
pending = asyncio.all_tasks() - {asyncio.current_task()}
|
||||
await asyncio.gather(*pending, return_exceptions=True)
|
||||
|
||||
deployment_keys = [key for key in router.cache.in_memory_cache.cache_dict if deployment_id in key]
|
||||
assert deployment_keys == [], f"proxy reject was counted against the deployment: {deployment_keys}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_failure_hook_attributes_the_keys_team_deployment_over_the_global_group(
|
||||
proxy_logging, make_user_api_key_auth, monkeypatch
|
||||
):
|
||||
"""A team key requesting its team public model name must be attributed to the team's
|
||||
deployment, not to a global group that happens to share the public name."""
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
recorded: list[dict] = []
|
||||
|
||||
class _RecordingLogger(CustomLogger):
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
recorded.append(kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "shared-name",
|
||||
"litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"},
|
||||
"model_info": {"id": "global-deployment"},
|
||||
},
|
||||
{
|
||||
"model_name": "shared-name_test-team_deadbeef",
|
||||
"litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": "sk-test"},
|
||||
"model_info": {
|
||||
"id": "team-deployment",
|
||||
"team_id": "test-team",
|
||||
"team_public_model_name": "shared-name",
|
||||
},
|
||||
},
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()])
|
||||
proxy_logging.alert_types = []
|
||||
|
||||
await proxy_logging.post_call_failure_hook(
|
||||
request_data={"model": "shared-name", "messages": [{"role": "user", "content": "hi"}]},
|
||||
original_exception=HTTPException(status_code=429, detail="rate limited"),
|
||||
user_api_key_dict=make_user_api_key_auth(team_id="test-team", request_route="/chat/completions"),
|
||||
route="/chat/completions",
|
||||
)
|
||||
|
||||
assert len(recorded) == 1
|
||||
kwargs = recorded[0]
|
||||
assert kwargs["custom_llm_provider"] == "anthropic"
|
||||
assert kwargs["litellm_params"]["metadata"]["deployment"] == "anthropic/claude-sonnet-4-5"
|
||||
assert kwargs["standard_logging_object"]["model_id"] == "team-deployment"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_failure_hook_omits_provider_for_mixed_router_deployments(
|
||||
proxy_logging, make_user_api_key_auth, monkeypatch
|
||||
):
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
recorded: list[dict] = []
|
||||
|
||||
class _RecordingLogger(CustomLogger):
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
recorded.append(kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "internal-model",
|
||||
"litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"},
|
||||
},
|
||||
{
|
||||
"model_name": "internal-model",
|
||||
"litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": "sk-test"},
|
||||
},
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()])
|
||||
proxy_logging.alert_types = []
|
||||
|
||||
await proxy_logging.post_call_failure_hook(
|
||||
request_data={"model": "internal-model", "messages": [{"role": "user", "content": "hi"}]},
|
||||
original_exception=HTTPException(status_code=403, detail="blocked"),
|
||||
user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"),
|
||||
route="/chat/completions",
|
||||
)
|
||||
|
||||
assert len(recorded) == 1
|
||||
kwargs = recorded[0]
|
||||
assert kwargs.get("custom_llm_provider") is None
|
||||
assert "model_info" not in (kwargs["litellm_params"].get("metadata") or {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_failure_hook_omits_provider_when_a_deployment_does_not_resolve(
|
||||
proxy_logging, make_user_api_key_auth, monkeypatch
|
||||
):
|
||||
"""One deployment resolves to openai and its sibling resolves to nothing: the group
|
||||
is not known to be single-provider, so no provider is stamped on the failure."""
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
recorded: list[dict] = []
|
||||
|
||||
class _RecordingLogger(CustomLogger):
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
recorded.append(kwargs)
|
||||
|
||||
router = MagicMock()
|
||||
router.get_model_list.return_value = [
|
||||
{"model_name": "internal-model", "litellm_params": {"model": "openai/gpt-4.1"}},
|
||||
{"model_name": "internal-model", "litellm_params": {"model": "unmapped-model-with-no-provider"}},
|
||||
]
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()])
|
||||
proxy_logging.alert_types = []
|
||||
|
||||
await proxy_logging.post_call_failure_hook(
|
||||
request_data={"model": "internal-model", "messages": [{"role": "user", "content": "hi"}]},
|
||||
original_exception=HTTPException(status_code=403, detail="blocked"),
|
||||
user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"),
|
||||
route="/chat/completions",
|
||||
)
|
||||
|
||||
assert len(recorded) == 1
|
||||
kwargs = recorded[0]
|
||||
assert kwargs.get("custom_llm_provider") is None
|
||||
assert kwargs["litellm_params"].get("custom_llm_provider") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_logging_proxy_only_path_attributes_with_read_only_metadata(
|
||||
proxy_logging, make_user_api_key_auth, monkeypatch
|
||||
):
|
||||
"""With a logging object already on the request, its metadata is taken as given;
|
||||
a read-only mapping there must not crash the stamp, and the failure handler
|
||||
still receives the provider attribution."""
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "internal-model",
|
||||
"litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"},
|
||||
"model_info": {"provider": "acme"},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
logging_obj = MagicMock()
|
||||
logging_obj.call_type = "acompletion"
|
||||
logging_obj.model_call_details = {}
|
||||
logging_obj.async_failure_handler = AsyncMock()
|
||||
|
||||
await proxy_logging._handle_logging_proxy_only_error(
|
||||
request_data={
|
||||
"litellm_logging_obj": logging_obj,
|
||||
"model": "internal-model",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"metadata": MappingProxyType({"user_api_key_alias": "frozen"}),
|
||||
},
|
||||
user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"),
|
||||
route="/chat/completions",
|
||||
original_exception=HTTPException(status_code=403, detail="blocked"),
|
||||
)
|
||||
|
||||
assert logging_obj.async_failure_handler.called
|
||||
update_kwargs = logging_obj.update_environment_variables.call_args.kwargs
|
||||
assert update_kwargs["custom_llm_provider"] == "openai"
|
||||
assert update_kwargs["litellm_params"]["custom_llm_provider"] == "openai"
|
||||
assert update_kwargs["litellm_params"]["metadata"] == {"user_api_key_alias": "frozen"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_failure_hook_fires_without_router_attribution(
|
||||
proxy_logging, make_user_api_key_auth, monkeypatch
|
||||
):
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
recorded: list[dict] = []
|
||||
|
||||
class _RecordingLogger(CustomLogger):
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
recorded.append(kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "different-model",
|
||||
"litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()])
|
||||
proxy_logging.alert_types = []
|
||||
|
||||
await proxy_logging.post_call_failure_hook(
|
||||
request_data={"model": "internal-model", "messages": [{"role": "user", "content": "hi"}]},
|
||||
original_exception=HTTPException(status_code=403, detail="blocked"),
|
||||
user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"),
|
||||
route="/chat/completions",
|
||||
)
|
||||
|
||||
assert len(recorded) == 1
|
||||
kwargs = recorded[0]
|
||||
assert kwargs.get("custom_llm_provider") is None
|
||||
assert "model_info" not in (kwargs["litellm_params"].get("metadata") or {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model", [123, ["internal-model"], {"name": "internal-model"}, None])
|
||||
async def test_post_call_failure_hook_fires_for_non_string_model(
|
||||
proxy_logging, make_user_api_key_auth, monkeypatch, model: object
|
||||
):
|
||||
"""A body whose ``model`` is not a string is rejected by the proxy before routing; its
|
||||
failure callback must still fire, unattributed, instead of a TypeError escaping the hook."""
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
recorded: list[dict] = []
|
||||
|
||||
class _RecordingLogger(CustomLogger):
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
recorded.append(kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "internal-model",
|
||||
"litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()])
|
||||
proxy_logging.alert_types = []
|
||||
|
||||
await proxy_logging.post_call_failure_hook(
|
||||
request_data={"model": model, "messages": [{"role": "user", "content": "hi"}]},
|
||||
original_exception=HTTPException(status_code=400, detail="'model' must be a string."),
|
||||
user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"),
|
||||
route="/chat/completions",
|
||||
)
|
||||
|
||||
assert len(recorded) == 1
|
||||
kwargs = recorded[0]
|
||||
assert kwargs.get("custom_llm_provider") is None
|
||||
assert "model_info" not in (kwargs["litellm_params"].get("metadata") or {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_failure_hook_callback_returns_http_exception(
|
||||
proxy_logging, make_user_api_key_auth, monkeypatch
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue