From f2d0827029fe89652340d788165fd789fb00d581 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 14 May 2026 23:09:12 +0000 Subject: [PATCH] test: fix CI failures from prom label cache + flaky time-window assertion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PrometheusLogger.get_labels_for_metric now caches the per-metric label set at first read so the labels passed to counter.labels(...) stay in lock step with the labels the counter was registered with. This broke two existing test patterns: - test_prometheus_labels.py: tests bind the real method onto a MagicMock, but MagicMock auto-creates a Mock for _cached_metric_labels whose .get(...) returns a truthy Mock — treated as a populated cache and returned as the label set, producing empty filtered labels and KeyError on labels["requested_model"] / ["route"]. Seed real {} containers for _cached_metric_labels and label_filters before binding. - test_prometheus_logging_callbacks.py::test_set_team_budget_metrics_with_custom_labels: the fixture builds the logger before the test monkeypatches litellm.custom_prometheus_metadata_labels, so the cached label set never picks up the new metadata labels. Clear the cache after the monkeypatch (same pattern already used for the rate-limit toggle in test_async_post_call_failure_hook). UI: view_logs/index.test.tsx "Last Minute" window assertion is off by one at the minute boundary. start_date is floored to the minute, so the dropped sub-minute fraction can push the truncated-seconds diff up to (minMinutes+1)*60 exactly when the click lands near a minute rollover. Switch the upper bound to toBeLessThanOrEqual. --- .../test_prometheus_logging_callbacks.py | 4 ++++ .../test_litellm/integrations/test_prometheus_labels.py | 8 ++++++++ .../src/components/view_logs/index.test.tsx | 9 +++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py index 1dd9f69351d..235acba7930 100644 --- a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py +++ b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py @@ -1978,6 +1978,10 @@ def test_set_team_budget_metrics_with_custom_labels(prometheus_logger, monkeypat # Set custom prometheus labels custom_labels = ["metadata.organization", "metadata.environment"] monkeypatch.setattr("litellm.custom_prometheus_metadata_labels", custom_labels) + # Logger caches each metric's label set at construction time (fixture + # runs before this monkeypatch), so invalidate so the cached label set + # picks up the freshly-configured custom metadata labels. + prometheus_logger._cached_metric_labels.clear() # Create test team with custom metadata team = MagicMock( diff --git a/tests/test_litellm/integrations/test_prometheus_labels.py b/tests/test_litellm/integrations/test_prometheus_labels.py index 1ba332a341b..c83d89e87c4 100644 --- a/tests/test_litellm/integrations/test_prometheus_labels.py +++ b/tests/test_litellm/integrations/test_prometheus_labels.py @@ -284,6 +284,12 @@ def test_prometheus_metrics_use_normalized_routes(): # Create a mock PrometheusLogger prometheus_logger = MagicMock() + # ``get_labels_for_metric`` reads ``_cached_metric_labels`` and + # ``label_filters`` off ``self``; default MagicMock attribute access + # returns Mocks that masquerade as a populated cache, so seed real + # containers before binding the real method. + prometheus_logger._cached_metric_labels = {} + prometheus_logger.label_filters = {} prometheus_logger.get_labels_for_metric = ( PrometheusLogger.get_labels_for_metric.__get__(prometheus_logger) ) @@ -327,6 +333,8 @@ def test_prometheus_label_value_sanitization(): from unittest.mock import MagicMock prometheus_logger = MagicMock() + prometheus_logger._cached_metric_labels = {} + prometheus_logger.label_filters = {} prometheus_logger.get_labels_for_metric = ( PrometheusLogger.get_labels_for_metric.__get__(prometheus_logger) ) diff --git a/ui/litellm-dashboard/src/components/view_logs/index.test.tsx b/ui/litellm-dashboard/src/components/view_logs/index.test.tsx index 7a9a541d3e0..a0571849d6e 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/index.test.tsx @@ -250,9 +250,14 @@ describe("SpendLogsTable", () => { diff = moment .utc(lastCall.end_date, "YYYY-MM-DD HH:mm:ss") .diff(moment.utc(lastCall.start_date, "YYYY-MM-DD HH:mm:ss"), "seconds"); - // start_date is rounded down to the minute boundary; end_date is current time + // start_date is rounded down to the minute boundary, end_date is the + // current wall-clock at queryFn time. The dropped sub-minute fraction + // on start_date can push the diff up to (minMinutes+1)*60 seconds + // exactly (e.g. click at HH:MM:59.9 → start floors to HH:MM:00 and + // queryFn fires just past HH:(MM+1):00), so allow equality on the + // upper bound. expect(diff).toBeGreaterThanOrEqual(minMinutes * 60); - expect(diff).toBeLessThan((minMinutes + 1) * 60); + expect(diff).toBeLessThanOrEqual((minMinutes + 1) * 60); }); return diff; };