From 59ba92ef1e5289742b6b9a6ba63391811403002d Mon Sep 17 00:00:00 2001 From: pengzh1 Date: Fri, 28 Aug 2026 01:53:35 +0800 Subject: [PATCH] fix(prometheus): never label deployment failures api_provider=None MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a resolved deployment failed without custom_llm_provider on litellm_params (e.g. a fallback target), the failure counter emitted the literal label value api_provider="None", creating a phantom ~100%-failure series for any alert grouped by provider. Fall back to the existing best-effort resolver (_extract_api_provider_from_request_data, already used by the client-side failure hook) and coalesce to "" — the same pattern the set_deployment_partial_outage call a few lines below already uses. Fixes #38531 --- litellm/integrations/prometheus.py | 13 ++- ...metheus_deployment_failure_api_provider.py | 83 +++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 tests/test_litellm/integrations/test_prometheus_deployment_failure_api_provider.py diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 467ec72dc4a..cb36c933102 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -2620,7 +2620,18 @@ class PrometheusLogger(CustomLogger): label_litellm_model_name = litellm_model_name label_model_id = model_id label_api_base = api_base - label_api_provider = llm_provider + # custom_llm_provider can be absent from litellm_params when a + # resolved deployment fails (e.g. fallbacks); fall back to the + # best-effort resolver and coalesce to "" so the label never + # serializes as the literal "None" (same pattern as + # set_deployment_partial_outage below). + label_api_provider = ( # pyright: ignore[reportUnknownVariableType] # bare-dict kwargs, same pattern as every label reader in this handler + llm_provider + or self._extract_api_provider_from_request_data( + request_kwargs # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] + ) + or "" + ) label_requested_model = model_group or litellm_model_name else: label_litellm_model_name = "" diff --git a/tests/test_litellm/integrations/test_prometheus_deployment_failure_api_provider.py b/tests/test_litellm/integrations/test_prometheus_deployment_failure_api_provider.py new file mode 100644 index 00000000000..2688077a70b --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_deployment_failure_api_provider.py @@ -0,0 +1,83 @@ +""" +Regression tests for #38531: the deployment-failure counter must never emit +api_provider="None". When a resolved deployment fails without +custom_llm_provider on litellm_params, fall back to the best-effort resolver +and coalesce to "" so alerting never sees a phantom all-failure series. +""" + +import pytest +from prometheus_client import REGISTRY + +from litellm.integrations.prometheus import PrometheusLogger + + +@pytest.fixture(scope="function") +def prometheus_logger(): + collectors = list(REGISTRY._collector_to_names.keys()) + for collector in collectors: + REGISTRY.unregister(collector) + return PrometheusLogger() + + +def _failure_samples(logger: PrometheusLogger): + """Collected samples of the deployment-failure counter, as (labels, value).""" + samples = [] + for metric in logger.litellm_deployment_failure_responses.collect(): + for sample in metric.samples: + if sample.name == "litellm_deployment_failure_responses_total": + samples.append((dict(sample.labels), sample.value)) + return samples + + +def _request_kwargs(model: str, standard_custom_provider: str | None = None) -> dict: + standard_logging_object = { + "model_id": "resolved-deployment-id", + "model_group": "my-model-group", + "api_base": "https://example.com", + "metadata": {}, + } + if standard_custom_provider is not None: + standard_logging_object["custom_llm_provider"] = standard_custom_provider + return { + "model": model, + "exception": Exception("upstream 503"), + "litellm_params": { + # no "custom_llm_provider" key — the #38531 scenario + "metadata": {}, + }, + "standard_logging_object": standard_logging_object, + } + + +def test_missing_provider_on_resolved_deployment_never_labels_none(prometheus_logger): + """custom_llm_provider absent and the model name is not provider-prefixed: + the label must be "" (not the literal "None").""" + + prometheus_logger.set_llm_deployment_failure_metrics(_request_kwargs("my-fallback-model")) + + samples = _failure_samples(prometheus_logger) + assert len(samples) == 1 + assert samples[0][0]["api_provider"] == "" + + +def test_missing_provider_infers_from_provider_prefixed_model(prometheus_logger): + """The resolver can still infer the provider from a prefixed model name.""" + + prometheus_logger.set_llm_deployment_failure_metrics(_request_kwargs("openai/gpt-4o-mini")) + + samples = _failure_samples(prometheus_logger) + assert len(samples) == 1 + assert samples[0][0]["api_provider"] == "openai" + + +def test_explicit_provider_still_wins(prometheus_logger): + """Behavior is unchanged when custom_llm_provider is present.""" + + kwargs = _request_kwargs("openai/gpt-4o-mini") + kwargs["litellm_params"]["custom_llm_provider"] = "azure" + + prometheus_logger.set_llm_deployment_failure_metrics(kwargs) + + samples = _failure_samples(prometheus_logger) + assert len(samples) == 1 + assert samples[0][0]["api_provider"] == "azure"