Merge pull request #40771 from BerriAI/litellm_regression_coverage_followup

test: tighten regression tests added in #37974

tests/test_litellm/litellm_core_utils/llm_cost_calc/test_openai_cache_write_cost.py

@@ -39,7 +39,7 @@ def test_openai_cache_write_tokens_billed_at_the_cache_creation_rate(local_model
     input_rate = rates["input_cost_per_token"]
     cache_write_rate = rates["cache_creation_input_token_cost"]
     output_rate = rates["output_cost_per_token"]
-    assert cache_write_rate == pytest.approx(input_rate * 1.25)
+    assert cache_write_rate > input_rate
 
     prompt_tokens = 12317
     cache_write_tokens = 12314

tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py

@@ -1246,7 +1246,7 @@ async def _flush_logging_worker(capture: "_SuccessPayloadCapture") -> None:
     await asyncio.sleep(0)
     try:
         await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0)
-    except (asyncio.TimeoutError, RuntimeError):
+    except asyncio.TimeoutError:
         pass
     deadline = asyncio.get_running_loop().time() + 10.0
     while not capture.payloads and asyncio.get_running_loop().time() < deadline:

tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py

@@ -1832,8 +1832,10 @@ class TestOpenAIPassthroughResponsesStreamingSpendLog:
     def setup_method(self):
         self.start_time = datetime.now()
         self.end_time = datetime.now()
+
+    def _expected_spend(self) -> float:
         rates = litellm.model_cost[self.MODEL_MAP_KEY]
-        self.expected_spend = (
+        return (
             self.INPUT_TOKENS * rates["input_cost_per_token"]
             + self.OUTPUT_TOKENS * rates["output_cost_per_token"]
         )
@@ -1914,7 +1916,7 @@ class TestOpenAIPassthroughResponsesStreamingSpendLog:
         logging_obj.model_call_details["custom_llm_provider"] = "openai"
         return logging_obj
 
-    def test_streamed_responses_passthrough_spend_log_is_priced(self):
+    def test_streamed_responses_passthrough_spend_log_is_priced(self, local_model_cost_map):
         """The spend row books the same tokens, spend and `resp_` id as the buffered call."""
         result = OpenAIPassthroughLoggingHandler._handle_logging_openai_collected_chunks(
             litellm_logging_obj=self._logging_obj(),
@@ -1942,7 +1944,7 @@ class TestOpenAIPassthroughResponsesStreamingSpendLog:
         assert spend_log_row["prompt_tokens"] == self.INPUT_TOKENS
         assert spend_log_row["completion_tokens"] == self.OUTPUT_TOKENS
         assert spend_log_row["total_tokens"] == self.INPUT_TOKENS + self.OUTPUT_TOKENS
-        assert spend_log_row["spend"] == self.expected_spend
+        assert spend_log_row["spend"] == pytest.approx(self._expected_spend())
         assert spend_log_row["request_id"] == self.RESPONSE_ID
         assert spend_log_row["model"] == "gpt-4o-mini"
 
@@ -1967,7 +1969,6 @@ class TestOpenAIPassthroughEmbeddingsSpendLog:
     def setup_method(self):
         self.start_time = datetime.now()
         self.end_time = datetime.now()
-        self.expected_spend = self.PROMPT_TOKENS * litellm.model_cost[self.MODEL]["input_cost_per_token"]
         self.response_body = {
             "object": "list",
             "data": [{"object": "embedding", "index": 0, "embedding": [0.0, 1.0]}],
@@ -1976,6 +1977,9 @@ class TestOpenAIPassthroughEmbeddingsSpendLog:
         }
         self.request_body = {"model": self.MODEL, "input": "hello"}
 
+    def _expected_spend(self) -> float:
+        return self.PROMPT_TOKENS * litellm.model_cost[self.MODEL]["input_cost_per_token"]
+
     def _create_mock_httpx_response(self) -> httpx.Response:
         mock_response = MagicMock(spec=httpx.Response)
         mock_response.status_code = 200
@@ -2001,7 +2005,7 @@ class TestOpenAIPassthroughEmbeddingsSpendLog:
         )
         return logging_obj
 
-    def test_embeddings_passthrough_spend_log_is_priced(self):
+    def test_embeddings_passthrough_spend_log_is_priced(self, local_model_cost_map):
         """The dispatched call books prompt tokens and cost onto the spend row."""
         dispatched = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload(
             httpx_response=self._create_mock_httpx_response(),
@@ -2020,7 +2024,7 @@ class TestOpenAIPassthroughEmbeddingsSpendLog:
         )
 
         assert dispatched["standard_logging_response_object"] is not None
-        assert dispatched["kwargs"]["response_cost"] == self.expected_spend
+        assert dispatched["kwargs"]["response_cost"] == pytest.approx(self._expected_spend())
 
         spend_log_row = get_logging_payload(
             kwargs=dispatched["kwargs"],
@@ -2031,7 +2035,7 @@ class TestOpenAIPassthroughEmbeddingsSpendLog:
 
         assert spend_log_row["prompt_tokens"] == self.PROMPT_TOKENS
         assert spend_log_row["total_tokens"] == self.PROMPT_TOKENS
-        assert spend_log_row["spend"] == self.expected_spend
+        assert spend_log_row["spend"] == pytest.approx(self._expected_spend())
         assert spend_log_row["model"] == self.MODEL
         assert spend_log_row["custom_llm_provider"] == "openai"
         assert spend_log_row["request_id"] == self.CALL_ID

tests/test_litellm/test_main.py

@@ -874,15 +874,25 @@ def test_responses_api_bridge_check_gpt_5_4_tools_with_default_reasoning_routes_
     assert model_info.get("mode") == "responses"
 
 
-@pytest.mark.parametrize("model_name", ["gpt-5.6-sol", "gpt-5.6-luna", "gpt-5.6-terra"])
+@pytest.mark.parametrize(
+    "model_name, expected_mode",
+    [
+        pytest.param("gpt-5.6-sol", "responses", id="above-boundary-bridges"),
+        pytest.param("gpt-5.1", None, id="below-boundary-stays-chat"),
+    ],
+)
 def test_responses_api_bridge_check_gpt_5_6_tools_with_default_reasoning_routes_to_responses(
-    monkeypatch, model_name
+    monkeypatch, model_name, expected_mode
 ):
     """
-    The whole gpt-5.6 family must bridge on function tools alone. The bridge used to
-    require an explicit reasoning_effort, so a gpt-5.6 call carrying tools and no effort
-    was rejected with "Function tools with reasoning_effort are not supported for
-    gpt-5.6-sol in /v1/chat/completions".
+    gpt-5.6 must bridge on function tools alone. The bridge used to require an explicit
+    reasoning_effort, so a gpt-5.6 call carrying tools and no effort was rejected with
+    "Function tools with reasoning_effort are not supported for gpt-5.6-sol in
+    /v1/chat/completions".
+
+    Paired with a model below the gpt-5.4 boundary, which must still stay on chat. The
+    gate parses the version and drops any suffix, so the family members bridge
+    identically and only the boundary distinguishes behaviour.
     """
     import litellm
     from litellm.main import responses_api_bridge_check
@@ -901,7 +911,7 @@ def test_responses_api_bridge_check_gpt_5_6_tools_with_default_reasoning_routes_
         )
 
     assert model == model_name
-    assert model_info.get("mode") == "responses"
+    assert model_info.get("mode") == expected_mode
 
 
 def test_responses_api_bridge_check_gpt_5_4_tools_with_reasoning_none_stays_chat():
@@ -3311,15 +3321,19 @@ def local_cost_map(monkeypatch):
     """The prices these tests assert are the checked-in ones. Setting the environment
     variable alone does not reload the map, so pin the map itself.
 
-    ``get_model_info`` is lru_cached, so pinning ``model_cost`` is not enough on its
-    own: a cached entry warmed against the network-fetched map keeps its old prices
-    and ``completion_cost`` bills at those while the assertions read the pinned map.
-    Clear on the way in and out so entries never leak across tests in either direction."""
+    Prices are read through two separate lru_caches, so pinning ``model_cost`` is not
+    enough on its own: an entry warmed against the network-fetched map keeps its old
+    prices and billing reads those while the assertions read the pinned map.
+    ``_invalidate_model_cost_lowercase_map`` clears both caches, where
+    ``get_model_info.cache_clear`` reaches only one. Invalidate on the way in and out
+    so entries never leak across tests in either direction."""
+    from litellm.utils import _invalidate_model_cost_lowercase_map
+
     monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
     monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
-    litellm.get_model_info.cache_clear()
+    _invalidate_model_cost_lowercase_map()
     yield
-    litellm.get_model_info.cache_clear()
+    _invalidate_model_cost_lowercase_map()
 
 
 def test_a_streamed_response_bills_the_usage_the_provider_reported(local_cost_map):