From 10444df3a0173a99ab4e4858ce6777f31530e6c5 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:27:49 +0000 Subject: [PATCH] test: deflake the JEV classifier select and two router tests that inherited leaked state (#44840) * test(ui): wait for the classifier model popup before picking its option Base UI exposes its select option asynchronously, and the helper waits for the option and its positioner to become clickable before selection Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: isolate two router tests from work leaked by earlier tests Collect earlier tests' garbage before warning capture so their unawaited coroutines cannot be attributed to the target's warning assertion Filter success-event callbacks by the request's litellm_call_id so queued logging work cannot replace the current request's captured messages Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_prompt_caching_deployment_check.py | 19 +++++++++++++++---- tests/unit/test_router/test_router.py | 2 ++ .../JevClassifierConfig.integration.test.tsx | 5 ++--- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py b/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py index c412546153a..18b23a9a703 100644 --- a/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py @@ -1,6 +1,7 @@ import asyncio import copy import functools +import uuid from typing import Final, cast import pytest @@ -350,10 +351,13 @@ def _affinity_messages(messages: list[AllMessageValues]) -> list[AllMessageValue class _SentMessagesCapture(CustomLogger): - def __init__(self): + def __init__(self, litellm_call_id: str): + self.litellm_call_id = litellm_call_id self.messages: list[AllMessageValues] | None = None async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + if kwargs.get("litellm_call_id") != self.litellm_call_id: + return standard_logging_object = kwargs.get("standard_logging_object") if standard_logging_object is not None: self.messages = standard_logging_object["messages"] @@ -380,7 +384,8 @@ async def test_affinity_key_matches_the_messages_auto_caching_actually_sends(mon request was actually sent with, otherwise auto-injected caching gets no affinity at all. """ monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) - capture = _SentMessagesCapture() + call_id: Final = str(uuid.uuid4()) + capture = _SentMessagesCapture(call_id) monkeypatch.setattr(litellm, "callbacks", [capture]) messages = _auto_caching_messages() @@ -389,6 +394,7 @@ async def test_affinity_key_matches_the_messages_auto_caching_actually_sends(mon messages=copy.deepcopy(messages), mock_response="ok", api_key="sk-fake", + litellm_call_id=call_id, ) sent_messages = await _eventually(lambda: capture.messages) assert sent_messages is not None @@ -904,13 +910,18 @@ async def test_pin_matches_when_the_success_event_truncated_an_image_payload(mon replaced by size placeholders, while routing sees the raw request. Hashing the raw bytes on the read side would key every image-carrying session past its own pin. """ - capture = _SentMessagesCapture() + call_id: Final = str(uuid.uuid4()) + capture = _SentMessagesCapture(call_id) monkeypatch.setattr(litellm, "callbacks", [capture]) image = {"type": "image_url", "image_url": {"url": ONE_PIXEL_PNG}} turn_one = _turn({"role": "user", "content": [image, _marked(LONG_PROMPT)]}) await litellm.acompletion( - model=AUTO_CACHING_MODEL, messages=copy.deepcopy(turn_one), mock_response="ok", api_key="sk-fake" + model=AUTO_CACHING_MODEL, + messages=copy.deepcopy(turn_one), + mock_response="ok", + api_key="sk-fake", + litellm_call_id=call_id, ) logged = await _eventually(lambda: capture.messages) assert logged is not None diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 3d9ab07350e..114d7415840 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -18968,6 +18968,8 @@ async def test_router_embedding_path_rejects_past_max_parallel_requests_without_ }, ) + # Reap earlier tests' garbage first so only this test's coroutines are recorded + gc.collect() with respx.mock() as respx_mock, warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") route: Final = respx_mock.post("https://max-parallel-embed.local/v1/embeddings").mock(side_effect=upstream) diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx index 1beacef7cf4..99726a8f071 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx @@ -1,7 +1,7 @@ import React, { useState } from "react"; import userEvent from "@testing-library/user-event"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils"; +import { chooseSelectOption, fireEvent, renderWithProviders, screen } from "../../../tests/test-utils"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import ClassificationMethodConfig from "./ClassificationMethodConfig"; import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs"; @@ -124,8 +124,7 @@ describe("JEV classifier editor", () => { fireEvent.click(screen.getByRole("radio", { name: label })); if (provider === "bespoke") expect(screen.getByLabelText("Classifier Model")).toHaveTextContent("nimble-latest"); if (provider !== "jev") { - await userEvent.click(screen.getByLabelText("Classifier Model")); - await userEvent.click(screen.getByRole("option", { name: model })); + await chooseSelectOption(userEvent, screen.getByLabelText("Classifier Model"), model); } else { fireEvent.change(screen.getByLabelText("Classifier Model"), { target: { value: model } }); }