scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 108

litellm_fix(test): update Prometheus metric test assertions with new labels (#20162)

Browse source 
This fixes the failing litellm_mapped_enterprise_tests (metrics/logging) job.

Recent commits added new labels to several Prometheus metrics (model_id, client_ip, user_agent)
but the test assertions weren't fully updated to expect these new labels.

Tests fixed:
- test_async_post_call_failure_hook
- test_async_log_failure_event
- test_increment_token_metrics
- test_log_failure_fallback_event
- test_set_latency_metrics
- test_set_llm_deployment_success_metrics

Labels added to test assertions:
- model_id for token metrics (litellm_tokens_metric, litellm_input_tokens_metric, litellm_output_tokens_metric)
- model_id for latency metrics (litellm_llm_api_latency_metric)
- model_id for remaining requests/tokens metrics
- model_id for fallback metrics
- model_id for overhead latency metric
- client_ip and user_agent for deployment failure/total/success responses
- client_ip and user_agent for proxy failed/total requests metrics
This commit is contained in:
shin-bot-litellm 2026-01-31 10:09:35 -08:00 • committed by GitHub
parent 6c497d2990
commit 2780e2f81e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 24 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py
View file

@ -230,6 +230,7 @@ def test_increment_token_metrics(prometheus_logger):
team_alias="test_team_alias",
requested_model=None,
model="gpt-3.5-turbo",
model_id="model-123",
)
prometheus_logger.litellm_tokens_metric.labels().inc.assert_called_once_with(100)
@ -243,6 +244,7 @@ def test_increment_token_metrics(prometheus_logger):
team_alias="test_team_alias",
requested_model=None,
model="gpt-3.5-turbo",
model_id="model-123",
)
prometheus_logger.litellm_input_tokens_metric.labels().inc.assert_called_once_with(
50
@ -258,6 +260,7 @@ def test_increment_token_metrics(prometheus_logger):
team_alias="test_team_alias",
requested_model=None,
model="gpt-3.5-turbo",
model_id="model-123",
)
prometheus_logger.litellm_output_tokens_metric.labels().inc.assert_called_once_with(
50
@ -435,6 +438,7 @@ def test_set_latency_metrics(prometheus_logger):
team_alias="test_team_alias",
requested_model="openai-gpt",
model="gpt-3.5-turbo",
model_id="model-123",
)
prometheus_logger.litellm_llm_api_latency_metric.labels().observe.assert_called_once_with(
1.5
@ -656,6 +660,7 @@ async def test_async_log_failure_event(prometheus_logger):
"test_team",
"test_team_alias",
"test_user",
"model-123",
)
prometheus_logger.litellm_llm_api_failed_requests_metric.labels().inc.assert_called_once()
@ -680,6 +685,8 @@ async def test_async_log_failure_event(prometheus_logger):
api_key_alias="test_alias",
team="test_team",
team_alias="test_team_alias",
client_ip="127.0.0.1", # from standard logging payload
user_agent=None,
)
prometheus_logger.litellm_deployment_failure_responses.labels().inc.assert_called_once()
@ -694,6 +701,8 @@ async def test_async_log_failure_event(prometheus_logger):
api_key_alias="test_alias",
team="test_team",
team_alias="test_team_alias",
client_ip="127.0.0.1", # from standard logging payload
user_agent=None,
)
prometheus_logger.litellm_deployment_total_requests.labels().inc.assert_called_once()
@ -747,6 +756,8 @@ async def test_async_post_call_failure_hook(prometheus_logger):
exception_class="Openai.RateLimitError",
route=user_api_key_dict.request_route,
model_id=None,
client_ip=None,
user_agent=None,
)
prometheus_logger.litellm_proxy_failed_requests_metric.labels().inc.assert_called_once()
@ -763,6 +774,8 @@ async def test_async_post_call_failure_hook(prometheus_logger):
user_email=None,
route=user_api_key_dict.request_route,
model_id=None,
client_ip=None,
user_agent=None,
)
prometheus_logger.litellm_proxy_total_requests_metric.labels().inc.assert_called_once()
@ -810,6 +823,8 @@ async def test_async_post_call_success_hook(prometheus_logger):
user_email=None,
route=user_api_key_dict.request_route,
model_id=None,
client_ip=None,
user_agent=None,
)
prometheus_logger.litellm_proxy_total_requests_metric.labels().inc.assert_called_once()
@ -875,6 +890,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
litellm_model_name="gpt-3.5-turbo", # actual model used - litellm model name
hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"],
api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"],
model_id="model-123",
)
prometheus_logger.litellm_remaining_requests_metric.labels().set.assert_called_once_with(
@ -889,6 +905,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"],
litellm_model_name="gpt-3.5-turbo",
model_group="my_custom_model_group",
model_id="model-123",
)
prometheus_logger.litellm_remaining_tokens_metric.labels().set.assert_called_once_with(
@ -914,6 +931,8 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"],
team=standard_logging_payload["metadata"]["user_api_key_team_id"],
team_alias=standard_logging_payload["metadata"]["user_api_key_team_alias"],
client_ip=None,
user_agent=None,
)
prometheus_logger.litellm_deployment_success_responses.labels().inc.assert_called_once()
@ -928,6 +947,8 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"],
team=standard_logging_payload["metadata"]["user_api_key_team_id"],
team_alias=standard_logging_payload["metadata"]["user_api_key_team_alias"],
client_ip=None,
user_agent=None,
)
prometheus_logger.litellm_deployment_total_requests.labels().inc.assert_called_once()
@ -949,6 +970,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"],
litellm_model_name="gpt-3.5-turbo",
model_group="my_custom_model_group",
model_id="model-123",
)
# Calculate expected latency per token (1 second / 10 tokens = 0.1 seconds per token)
@ -991,6 +1013,7 @@ async def test_log_success_fallback_event(prometheus_logger):
team_alias="test_team_alias",
exception_status="429",
exception_class="Openai.RateLimitError",
model_id=None,
)
prometheus_logger.litellm_deployment_successful_fallbacks.labels().inc.assert_called_once()
@ -1028,6 +1051,7 @@ async def test_log_failure_fallback_event(prometheus_logger):
team_alias="test_team_alias",
exception_status="429",
exception_class="Openai.RateLimitError",
model_id=None,
)
prometheus_logger.litellm_deployment_failed_fallbacks.labels().inc.assert_called_once()