diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 2528f07f92c..80d01471390 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -2617,6 +2617,7 @@ class PrometheusLogger(CustomLogger): model_id: Final = _metadata.get("model_info", {}).get("id") or request_data.get("model_info", {}).get("id") rate_limit_category, rate_limit_type = self._extract_rate_limit_labels(original_exception) api_provider: Final = self._extract_api_provider_from_request_data(request_data) + _standard_metadata: Final = StandardLoggingPayloadSetup.get_standard_logging_metadata(metadata=_metadata) enum_values: Final = UserAPIKeyLabelValues( end_user=user_api_key_dict.end_user_id, user=user_api_key_dict.user_id, @@ -2640,6 +2641,9 @@ class PrometheusLogger(CustomLogger): model_id=model_id, api_provider=api_provider, stream=(str(request_data.get("stream")) if litellm.prometheus_emit_stream_label else None), + custom_metadata_labels=get_custom_labels_from_metadata( + metadata=_get_combined_custom_metadata_from_standard_logging_metadata(_standard_metadata) + ), ) _label_ctx: Final = PrometheusLabelFactoryContext(enum_values) PrometheusLogger._inc_labeled_counter( @@ -2867,6 +2871,9 @@ class PrometheusLogger(CustomLogger): tags=standard_logging_payload.get("request_tags", []), client_ip=client_ip, user_agent=user_agent, + custom_metadata_labels=get_custom_labels_from_metadata( + metadata=_get_combined_custom_metadata_from_standard_logging_payload(standard_logging_payload) + ), ) """ @@ -3417,6 +3424,9 @@ class PrometheusLogger(CustomLogger): exception_status=str(getattr(original_exception, "status_code", None)), exception_class=self._get_exception_class_name(original_exception), tags=_tags, + custom_metadata_labels=get_custom_labels_from_metadata( + metadata=_get_combined_custom_metadata_from_standard_logging_metadata(standard_metadata) + ), ) PrometheusLogger._inc_labeled_counter( self, @@ -3458,6 +3468,9 @@ class PrometheusLogger(CustomLogger): exception_status=str(getattr(original_exception, "status_code", None)), exception_class=self._get_exception_class_name(original_exception), tags=_tags, + custom_metadata_labels=get_custom_labels_from_metadata( + metadata=_get_combined_custom_metadata_from_standard_logging_metadata(standard_metadata) + ), ) PrometheusLogger._inc_labeled_counter( @@ -4519,7 +4532,7 @@ def get_custom_labels_from_metadata(metadata: dict) -> dict[str, str]: def _get_combined_custom_metadata_from_standard_logging_payload( - standard_logging_payload: dict | None, + standard_logging_payload: Mapping[str, object] | None, ) -> dict[str, object]: """ Combine the metadata sources that can supply custom Prometheus labels. @@ -4528,11 +4541,20 @@ def _get_combined_custom_metadata_from_standard_logging_payload( user_api_key_project_alias, user_api_key_team_alias) so they are accessible via custom_prometheus_metadata_labels configuration. """ - if not isinstance(standard_logging_payload, dict): - return {} + return _get_combined_custom_metadata_from_standard_logging_metadata( + standard_logging_payload.get("metadata") if isinstance(standard_logging_payload, Mapping) else None + ) - standard_logging_metadata: Final = standard_logging_payload.get("metadata") or {} - if not isinstance(standard_logging_metadata, dict): + +def _get_combined_custom_metadata_from_standard_logging_metadata( + standard_logging_metadata: Mapping[str, object] | None, +) -> dict[str, object]: + """ + Same combination, for emitters holding the standard logging metadata itself + rather than a whole payload: the fallback emitters and the proxy failure hook + build it with StandardLoggingPayloadSetup.get_standard_logging_metadata. + """ + if not isinstance(standard_logging_metadata, Mapping): return {} requester_metadata: Final = standard_logging_metadata.get("requester_metadata") diff --git a/tests/test_litellm/integrations/test_prometheus_custom_metadata_label_counts.py b/tests/test_litellm/integrations/test_prometheus_custom_metadata_label_counts.py index 99eb5abb7b5..28f5ff4ba98 100644 --- a/tests/test_litellm/integrations/test_prometheus_custom_metadata_label_counts.py +++ b/tests/test_litellm/integrations/test_prometheus_custom_metadata_label_counts.py @@ -157,3 +157,120 @@ def test_virtual_key_rate_limit_metrics_preserve_zero_remaining_values( assert any(sample.value == 0 for sample in token_samples) assert not any(sample.value == sys.maxsize for sample in request_samples) assert not any(sample.value == sys.maxsize for sample in token_samples) + + +def _request_metadata_with_custom_labels() -> dict: + """Raw request metadata carrying the configured labels under spend_logs_metadata.""" + return { + "spend_logs_metadata": { + "department": "engineering", + "environment": "production", + } + } + + +def _has_custom_labels(metric_name: str) -> bool: + return any( + sample.labels.get("metadata_department") == "engineering" + and sample.labels.get("metadata_environment") == "production" + for sample in _metric_samples(metric_name) + ) + + +def _deployment_failure_kwargs() -> dict: + payload = _standard_logging_payload_with_requester_metadata() + return { + "model": "gpt-4o-mini", + "exception": Exception("upstream 503"), + "litellm_params": {"custom_llm_provider": "openai", "metadata": {}}, + "standard_logging_object": payload, + } + + +def test_deployment_failure_metrics_accept_custom_metadata_labels( + monkeypatch: pytest.MonkeyPatch, +): + """ + set_llm_deployment_failure_metrics declared the configured labels but never + filled them, so both series it emits exported the literal string "None". + """ + prometheus_logger = _create_prometheus_logger_with_custom_labels(monkeypatch) + + prometheus_logger.set_llm_deployment_failure_metrics(_deployment_failure_kwargs()) + + assert _has_custom_labels("litellm_deployment_failure_responses_total") + assert _has_custom_labels("litellm_deployment_total_requests_total") + + +def test_deployment_failure_metrics_omit_custom_labels_when_absent( + monkeypatch: pytest.MonkeyPatch, +): + """Nothing supplies the labels, so none must be attributed.""" + prometheus_logger = _create_prometheus_logger_with_custom_labels(monkeypatch) + kwargs = _deployment_failure_kwargs() + kwargs["standard_logging_object"]["metadata"]["requester_metadata"] = None + + prometheus_logger.set_llm_deployment_failure_metrics(kwargs) + + assert not _has_custom_labels("litellm_deployment_failure_responses_total") + + +@pytest.mark.asyncio +async def test_success_fallback_event_accepts_custom_metadata_labels( + monkeypatch: pytest.MonkeyPatch, +): + prometheus_logger = _create_prometheus_logger_with_custom_labels(monkeypatch) + + await prometheus_logger.log_success_fallback_event( + original_model_group="gpt-4o-mini", + kwargs={ + "model": "gpt-4o-mini-fallback", + "metadata": _request_metadata_with_custom_labels(), + }, + original_exception=Exception("upstream 503"), + ) + + assert _has_custom_labels("litellm_deployment_successful_fallbacks_total") + + +@pytest.mark.asyncio +async def test_failure_fallback_event_accepts_custom_metadata_labels( + monkeypatch: pytest.MonkeyPatch, +): + prometheus_logger = _create_prometheus_logger_with_custom_labels(monkeypatch) + + await prometheus_logger.log_failure_fallback_event( + original_model_group="gpt-4o-mini", + kwargs={ + "model": "gpt-4o-mini-fallback", + "metadata": _request_metadata_with_custom_labels(), + }, + original_exception=Exception("upstream 503"), + ) + + assert _has_custom_labels("litellm_deployment_failed_fallbacks_total") + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_accepts_custom_metadata_labels( + monkeypatch: pytest.MonkeyPatch, +): + """ + This hook holds raw request metadata, which the payload-shaped helper does not + flatten on its own, so both proxy-level series it emits missed the labels. + """ + from litellm.proxy._types import UserAPIKeyAuth + + prometheus_logger = _create_prometheus_logger_with_custom_labels(monkeypatch) + + await prometheus_logger.async_post_call_failure_hook( + request_data={ + "model": "gpt-4o-mini", + "metadata": _request_metadata_with_custom_labels(), + }, + original_exception=Exception("upstream 503"), + user_api_key_dict=UserAPIKeyAuth(token="test_token"), + ) + + assert _has_custom_labels("litellm_proxy_failed_requests_metric_total") + assert _has_custom_labels("litellm_proxy_total_requests_metric_total")