From 2780e2f81e0a33f74de29f5c9685ec879db472aa Mon Sep 17 00:00:00 2001 From: shin-bot-litellm Date: Sat, 31 Jan 2026 10:09:35 -0800 Subject: [PATCH] litellm_fix(test): update Prometheus metric test assertions with new labels (#20162) This fixes the failing litellm_mapped_enterprise_tests (metrics/logging) job. Recent commits added new labels to several Prometheus metrics (model_id, client_ip, user_agent) but the test assertions weren't fully updated to expect these new labels. Tests fixed: - test_async_post_call_failure_hook - test_async_log_failure_event - test_increment_token_metrics - test_log_failure_fallback_event - test_set_latency_metrics - test_set_llm_deployment_success_metrics Labels added to test assertions: - model_id for token metrics (litellm_tokens_metric, litellm_input_tokens_metric, litellm_output_tokens_metric) - model_id for latency metrics (litellm_llm_api_latency_metric) - model_id for remaining requests/tokens metrics - model_id for fallback metrics - model_id for overhead latency metric - client_ip and user_agent for deployment failure/total/success responses - client_ip and user_agent for proxy failed/total requests metrics --- .../test_prometheus_logging_callbacks.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py index a479d1a9fc9..7309092dd50 100644 --- a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py +++ b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py @@ -230,6 +230,7 @@ def test_increment_token_metrics(prometheus_logger): team_alias="test_team_alias", requested_model=None, model="gpt-3.5-turbo", + model_id="model-123", ) prometheus_logger.litellm_tokens_metric.labels().inc.assert_called_once_with(100) @@ -243,6 +244,7 @@ def test_increment_token_metrics(prometheus_logger): team_alias="test_team_alias", requested_model=None, model="gpt-3.5-turbo", + model_id="model-123", ) prometheus_logger.litellm_input_tokens_metric.labels().inc.assert_called_once_with( 50 @@ -258,6 +260,7 @@ def test_increment_token_metrics(prometheus_logger): team_alias="test_team_alias", requested_model=None, model="gpt-3.5-turbo", + model_id="model-123", ) prometheus_logger.litellm_output_tokens_metric.labels().inc.assert_called_once_with( 50 @@ -435,6 +438,7 @@ def test_set_latency_metrics(prometheus_logger): team_alias="test_team_alias", requested_model="openai-gpt", model="gpt-3.5-turbo", + model_id="model-123", ) prometheus_logger.litellm_llm_api_latency_metric.labels().observe.assert_called_once_with( 1.5 @@ -656,6 +660,7 @@ async def test_async_log_failure_event(prometheus_logger): "test_team", "test_team_alias", "test_user", + "model-123", ) prometheus_logger.litellm_llm_api_failed_requests_metric.labels().inc.assert_called_once() @@ -680,6 +685,8 @@ async def test_async_log_failure_event(prometheus_logger): api_key_alias="test_alias", team="test_team", team_alias="test_team_alias", + client_ip="127.0.0.1", # from standard logging payload + user_agent=None, ) prometheus_logger.litellm_deployment_failure_responses.labels().inc.assert_called_once() @@ -694,6 +701,8 @@ async def test_async_log_failure_event(prometheus_logger): api_key_alias="test_alias", team="test_team", team_alias="test_team_alias", + client_ip="127.0.0.1", # from standard logging payload + user_agent=None, ) prometheus_logger.litellm_deployment_total_requests.labels().inc.assert_called_once() @@ -747,6 +756,8 @@ async def test_async_post_call_failure_hook(prometheus_logger): exception_class="Openai.RateLimitError", route=user_api_key_dict.request_route, model_id=None, + client_ip=None, + user_agent=None, ) prometheus_logger.litellm_proxy_failed_requests_metric.labels().inc.assert_called_once() @@ -763,6 +774,8 @@ async def test_async_post_call_failure_hook(prometheus_logger): user_email=None, route=user_api_key_dict.request_route, model_id=None, + client_ip=None, + user_agent=None, ) prometheus_logger.litellm_proxy_total_requests_metric.labels().inc.assert_called_once() @@ -810,6 +823,8 @@ async def test_async_post_call_success_hook(prometheus_logger): user_email=None, route=user_api_key_dict.request_route, model_id=None, + client_ip=None, + user_agent=None, ) prometheus_logger.litellm_proxy_total_requests_metric.labels().inc.assert_called_once() @@ -875,6 +890,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): litellm_model_name="gpt-3.5-turbo", # actual model used - litellm model name hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"], api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"], + model_id="model-123", ) prometheus_logger.litellm_remaining_requests_metric.labels().set.assert_called_once_with( @@ -889,6 +905,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"], litellm_model_name="gpt-3.5-turbo", model_group="my_custom_model_group", + model_id="model-123", ) prometheus_logger.litellm_remaining_tokens_metric.labels().set.assert_called_once_with( @@ -914,6 +931,8 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"], team=standard_logging_payload["metadata"]["user_api_key_team_id"], team_alias=standard_logging_payload["metadata"]["user_api_key_team_alias"], + client_ip=None, + user_agent=None, ) prometheus_logger.litellm_deployment_success_responses.labels().inc.assert_called_once() @@ -928,6 +947,8 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"], team=standard_logging_payload["metadata"]["user_api_key_team_id"], team_alias=standard_logging_payload["metadata"]["user_api_key_team_alias"], + client_ip=None, + user_agent=None, ) prometheus_logger.litellm_deployment_total_requests.labels().inc.assert_called_once() @@ -949,6 +970,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"], litellm_model_name="gpt-3.5-turbo", model_group="my_custom_model_group", + model_id="model-123", ) # Calculate expected latency per token (1 second / 10 tokens = 0.1 seconds per token) @@ -991,6 +1013,7 @@ async def test_log_success_fallback_event(prometheus_logger): team_alias="test_team_alias", exception_status="429", exception_class="Openai.RateLimitError", + model_id=None, ) prometheus_logger.litellm_deployment_successful_fallbacks.labels().inc.assert_called_once() @@ -1028,6 +1051,7 @@ async def test_log_failure_fallback_event(prometheus_logger): team_alias="test_team_alias", exception_status="429", exception_class="Openai.RateLimitError", + model_id=None, ) prometheus_logger.litellm_deployment_failed_fallbacks.labels().inc.assert_called_once()