scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-03 02:22:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 80

fix(prometheus): emit org_id and org_alias on litellm_llm_api_failed_requests_metric (#28493)

Browse source 
`async_log_failure_event` was extracting `user_api_key_org_id` from
the standard logging payload but not `user_api_key_org_alias`, and
neither value was passed to `UserAPIKeyLabelValues`.  As a result
`org_id` / `org_alias` were always empty on failure metrics.

Also adds `litellm_llm_api_failed_requests_metric` to the
`_org_label_metrics` frozenset so that `get_labels()` includes
the two org labels when building the Prometheus metric counter.

Fixes LIT-2825.

Co-authored-by: shin-berri <shin-laptop@berri.ai>
Co-authored-by: yuneng-jiang <yuneng@berri.ai>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
ishaan-berri 2026-05-21 11:43:15 -07:00 • committed by GitHub
commit f9b532142a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 120 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/integrations/prometheus.py
View file

@ -1637,6 +1637,9 @@ class PrometheusLogger(CustomLogger):
user_api_key_org_id = standard_logging_payload["metadata"].get(
"user_api_key_org_id"
)
user_api_key_org_alias = standard_logging_payload["metadata"].get(
"user_api_key_org_alias"
)
try:
enum_values = UserAPIKeyLabelValues(
@ -1646,6 +1649,8 @@ class PrometheusLogger(CustomLogger):
model=model,
team=user_api_team,
team_alias=user_api_team_alias,
org_id=user_api_key_org_id,
org_alias=user_api_key_org_alias,
user=user_id,
model_id=standard_logging_payload.get("model_id", ""),
custom_metadata_labels=get_custom_labels_from_metadata(

1
litellm/types/integrations/prometheus.py
View file

@ -654,6 +654,7 @@ class PrometheusMetricLabels:
"litellm_input_tokens_metric",
"litellm_total_tokens_metric",
"litellm_output_tokens_metric",
"litellm_llm_api_failed_requests_metric",
}
)

114
tests/test_litellm/integrations/test_prometheus_user_team_metrics.py
View file

@ -922,3 +922,117 @@ def test_custom_latency_buckets():
REGISTRY.unregister(collector)
except Exception:
pass
# ---------------------------------------------------------------------------
# Regression tests: org labels in async_log_failure_event
# ---------------------------------------------------------------------------
def _make_failure_standard_logging_payload(
org_id: str = "org-abc",
org_alias: str = "my-org",
) -> dict:
return {
"model_id": "model-123",
"model_group": "gpt-4o-mini",
"api_base": "https://api.openai.com",
"custom_llm_provider": "openai",
"metadata": {
"user_api_key_hash": "test-hash",
"user_api_key_alias": "test-alias",
"user_api_key_team_id": "test-team",
"user_api_key_team_alias": "test-team-alias",
"user_api_key_user_id": "test-user",
"user_api_key_user_email": "test@example.com",
"user_api_key_org_id": org_id,
"user_api_key_org_alias": org_alias,
"requester_metadata": None,
"user_api_key_auth_metadata": None,
"spend_logs_metadata": None,
},
"request_tags": [],
"completion_tokens": 0,
"total_tokens": 0,
"response_cost": 0,
}
@pytest.mark.asyncio
async def test_async_log_failure_event_emits_org_labels(prometheus_logger):
"""
async_log_failure_event must emit org_id and org_alias labels on
litellm_llm_api_failed_requests_metric so that org context is
available in Prometheus failure metrics (LIT-2825).
"""
prometheus_logger.litellm_llm_api_failed_requests_metric = MagicMock()
kwargs = {
"model": "gpt-4o-mini",
"litellm_params": {
"metadata": {
"user_api_key_end_user_id": "end-user-1",
}
},
"standard_logging_object": _make_failure_standard_logging_payload(
org_id="org-xyz",
org_alias="acme-corp",
),
}
with patch(
"litellm.integrations.prometheus.PrometheusLogger._set_org_budget_metrics_after_api_request",
AsyncMock(),
):
await prometheus_logger.async_log_failure_event(
kwargs=kwargs,
response_obj=None,
start_time=None,
end_time=None,
)
labels_call = prometheus_logger.litellm_llm_api_failed_requests_metric.labels
labels_call.assert_called_once()
actual_labels = labels_call.call_args.kwargs
assert (
actual_labels.get("org_id") == "org-xyz"
), f"Expected org_id='org-xyz', got {actual_labels.get('org_id')!r}"
assert (
actual_labels.get("org_alias") == "acme-corp"
), f"Expected org_alias='acme-corp', got {actual_labels.get('org_alias')!r}"
@pytest.mark.asyncio
async def test_async_log_failure_event_org_labels_none_when_no_org(prometheus_logger):
"""
When there is no org context (org_id=None, org_alias absent), the org labels
on litellm_llm_api_failed_requests_metric must be None/empty — not raise.
"""
prometheus_logger.litellm_llm_api_failed_requests_metric = MagicMock()
kwargs = {
"model": "gpt-4o-mini",
"litellm_params": {"metadata": {}},
"standard_logging_object": _make_failure_standard_logging_payload(
org_id=None,
org_alias=None,
),
}
with patch(
"litellm.integrations.prometheus.PrometheusLogger._set_org_budget_metrics_after_api_request",
AsyncMock(),
):
await prometheus_logger.async_log_failure_event(
kwargs=kwargs,
response_obj=None,
start_time=None,
end_time=None,
)
labels_call = prometheus_logger.litellm_llm_api_failed_requests_metric.labels
labels_call.assert_called_once()
actual_labels = labels_call.call_args.kwargs
# org labels must be present as keys (empty string after label factory normalization)
assert "org_id" in actual_labels
assert "org_alias" in actual_labels