From cbf4078c79e1a76a0262f2ca319388db42447fd5 Mon Sep 17 00:00:00 2001 From: 72004 Date: Mon, 24 Aug 2026 22:44:55 +0500 Subject: [PATCH 1/4] fix(prometheus): backfill team spend from DB when metadata spend is None _assemble_team_object fetched team_info but only backfilled max_budget, not spend. When the cached team object had no spend hydrated, spend was None -> treated as 0, so litellm_remaining_team_budget_metric reported the full max_budget instead of the true remaining budget (flapping between real value and max as different pods scraped). Backfill team_object.spend from team_info.spend (plus the request cost) when metadata spend is None, mirroring the existing max_budget fallback. --- litellm/integrations/prometheus.py | 2 + .../test_prometheus_user_team_metrics.py | 55 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 708a75aad99..dc53a39ea99 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -4250,6 +4250,8 @@ class PrometheusLogger(CustomLogger): team_object.budget_reset_at = team_info.budget_reset_at if team_object.max_budget is None and team_info.max_budget is not None: team_object.max_budget = team_info.max_budget + if spend is None and team_info.spend is not None: + team_object.spend = team_info.spend + response_cost return team_object diff --git a/tests/unit/integrations/test_prometheus_user_team_metrics.py b/tests/unit/integrations/test_prometheus_user_team_metrics.py index ab0fb67b52f..798b3e3b550 100644 --- a/tests/unit/integrations/test_prometheus_user_team_metrics.py +++ b/tests/unit/integrations/test_prometheus_user_team_metrics.py @@ -338,6 +338,61 @@ async def test_assemble_team_object_does_not_override_metadata_max_budget( ), "max_budget from metadata must not be replaced by the DB value" +async def test_assemble_team_object_uses_db_spend_when_metadata_is_none( + prometheus_logger, +): + """ + When spend is None in request metadata (cached team object not hydrated), + _assemble_team_object must backfill it from get_team_object, otherwise the + remaining-budget gauge reports the full max_budget as if nothing was spent. + """ + db_team = MagicMock() + db_team.max_budget = 1000.0 + db_team.spend = 902.60 + db_team.budget_reset_at = None + + with patch("litellm.proxy.auth.auth_checks.get_team_object") as mock_get_team: + mock_get_team.return_value = db_team + team_object = await prometheus_logger._assemble_team_object( + team_id="team-1", + team_alias="team-a", + spend=None, # simulates None coming from request metadata + max_budget=1000.0, + response_cost=0.5, + ) + + assert ( + team_object.spend == 902.60 + 0.5 + ), "spend should be populated from DB (plus request cost) when metadata value is None" + + +async def test_assemble_team_object_does_not_override_metadata_spend( + prometheus_logger, +): + """ + When spend IS present in request metadata, it must not be replaced by the + DB value. + """ + db_team = MagicMock() + db_team.max_budget = 1000.0 + db_team.spend = 9999.0 + db_team.budget_reset_at = None + + with patch("litellm.proxy.auth.auth_checks.get_team_object") as mock_get_team: + mock_get_team.return_value = db_team + team_object = await prometheus_logger._assemble_team_object( + team_id="team-1", + team_alias="team-a", + spend=50.0, # metadata has a real value + max_budget=1000.0, + response_cost=1.0, + ) + + assert ( + team_object.spend == 50.0 + 1.0 + ), "spend from metadata must not be replaced by the DB value" + + async def test_set_team_budget_metrics_after_api_request_no_inf_when_metadata_budget_none( prometheus_logger, ): From dcd3dc6724445317e373eb5b3d3c67a23e4b7143 Mon Sep 17 00:00:00 2001 From: 72004 Date: Mon, 24 Aug 2026 22:54:14 +0500 Subject: [PATCH 2/4] test(prometheus): suppress TQ008 on new spend-backfill tests The two new _assemble_team_object tests stub get_team_object the same way the sibling max_budget-fallback tests above them do; mark them test-quality-ok so the test-quality budget gate passes. --- tests/unit/integrations/test_prometheus_user_team_metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/integrations/test_prometheus_user_team_metrics.py b/tests/unit/integrations/test_prometheus_user_team_metrics.py index 798b3e3b550..e10caae1c96 100644 --- a/tests/unit/integrations/test_prometheus_user_team_metrics.py +++ b/tests/unit/integrations/test_prometheus_user_team_metrics.py @@ -351,7 +351,7 @@ async def test_assemble_team_object_uses_db_spend_when_metadata_is_none( db_team.spend = 902.60 db_team.budget_reset_at = None - with patch("litellm.proxy.auth.auth_checks.get_team_object") as mock_get_team: + with patch("litellm.proxy.auth.auth_checks.get_team_object") as mock_get_team: # test-quality-ok: stubbing the DB fetch, same pattern as the sibling max_budget fallback tests above mock_get_team.return_value = db_team team_object = await prometheus_logger._assemble_team_object( team_id="team-1", @@ -378,7 +378,7 @@ async def test_assemble_team_object_does_not_override_metadata_spend( db_team.spend = 9999.0 db_team.budget_reset_at = None - with patch("litellm.proxy.auth.auth_checks.get_team_object") as mock_get_team: + with patch("litellm.proxy.auth.auth_checks.get_team_object") as mock_get_team: # test-quality-ok: stubbing the DB fetch, same pattern as the sibling max_budget fallback tests above mock_get_team.return_value = db_team team_object = await prometheus_logger._assemble_team_object( team_id="team-1", From 578521d119503416ff5bb40b6e9fb39e072147f5 Mon Sep 17 00:00:00 2001 From: 72004 Date: Sat, 10 Oct 2026 18:46:14 +0500 Subject: [PATCH 3/4] fix(prometheus): resolve team spend/max_budget before constructing team object Compute the DB-fallback spend and max_budget values up front instead of mutating team_object after construction, so _assemble_team_object's result does not depend on statement order. The auth-carried snapshot path is unchanged. --- litellm/integrations/prometheus.py | 32 +++++++++---------- .../test_prometheus_user_team_metrics.py | 4 +-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index dc53a39ea99..50ff8cd6f32 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -4230,30 +4230,30 @@ class PrometheusLogger(CustomLogger): max_budget=max_budget if max_budget is not None else carried.max_budget, budget_reset_at=carried.budget_reset_at, ) - team_object: Final = LiteLLM_TeamTable( - team_id=team_id, - team_alias=team_alias, - spend=_total_team_spend, - max_budget=max_budget, - ) + team_info: LiteLLM_TeamTable | None = None try: - team_info: Final = await get_team_object( + team_info = await get_team_object( team_id=team_id, prisma_client=prisma_client, user_api_key_cache=user_api_key_cache, ) except Exception as e: verbose_logger.debug("[Non-Blocking] Prometheus: Error getting team info: %s", e) - return team_object - if team_info: - team_object.budget_reset_at = team_info.budget_reset_at - if team_object.max_budget is None and team_info.max_budget is not None: - team_object.max_budget = team_info.max_budget - if spend is None and team_info.spend is not None: - team_object.spend = team_info.spend + response_cost - - return team_object + # metadata spend of None means "unknown", not "nothing spent" -- fall back to the DB value + _base_spend: Final = ( + spend if spend is not None else (team_info.spend if team_info and team_info.spend is not None else 0) + ) + _resolved_max_budget: Final = ( + max_budget if max_budget is not None else (team_info.max_budget if team_info else None) + ) + return LiteLLM_TeamTable( + team_id=team_id, + team_alias=team_alias, + spend=_base_spend + response_cost, + max_budget=_resolved_max_budget, + budget_reset_at=team_info.budget_reset_at if team_info else None, + ) def _set_team_budget_metrics( self, diff --git a/tests/unit/integrations/test_prometheus_user_team_metrics.py b/tests/unit/integrations/test_prometheus_user_team_metrics.py index e10caae1c96..3b9007f21e3 100644 --- a/tests/unit/integrations/test_prometheus_user_team_metrics.py +++ b/tests/unit/integrations/test_prometheus_user_team_metrics.py @@ -356,7 +356,7 @@ async def test_assemble_team_object_uses_db_spend_when_metadata_is_none( team_object = await prometheus_logger._assemble_team_object( team_id="team-1", team_alias="team-a", - spend=None, # simulates None coming from request metadata + spend=None, max_budget=1000.0, response_cost=0.5, ) @@ -383,7 +383,7 @@ async def test_assemble_team_object_does_not_override_metadata_spend( team_object = await prometheus_logger._assemble_team_object( team_id="team-1", team_alias="team-a", - spend=50.0, # metadata has a real value + spend=50.0, max_budget=1000.0, response_cost=1.0, ) From 32026a62720c7ac09cc0906eea83338eff039325 Mon Sep 17 00:00:00 2001 From: 72004 Date: Sun, 11 Oct 2026 18:55:18 +0500 Subject: [PATCH 4/4] fix(prometheus): return early on team lookup failure instead of rebinding team_info --- litellm/integrations/prometheus.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 50ff8cd6f32..cf6ddb6a8eb 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -4230,15 +4230,20 @@ class PrometheusLogger(CustomLogger): max_budget=max_budget if max_budget is not None else carried.max_budget, budget_reset_at=carried.budget_reset_at, ) - team_info: LiteLLM_TeamTable | None = None try: - team_info = await get_team_object( + team_info: Final = await get_team_object( team_id=team_id, prisma_client=prisma_client, user_api_key_cache=user_api_key_cache, ) except Exception as e: verbose_logger.debug("[Non-Blocking] Prometheus: Error getting team info: %s", e) + return LiteLLM_TeamTable( + team_id=team_id, + team_alias=team_alias, + spend=_total_team_spend, + max_budget=max_budget, + ) # metadata spend of None means "unknown", not "nothing spent" -- fall back to the DB value _base_spend: Final = (