From fca17b5ffda311d3704d2a65dc86f471072d4688 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:00:55 -0700 Subject: [PATCH] fix(proxy): name the failure when a model cost map reload cannot be recorded in the database --- litellm/proxy/proxy_server.py | 26 ++++++-- .../test_routes_model_cost_map.py | 63 +++++++++++++++++++ 2 files changed, 85 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 42816e7db61..d472983f858 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5400,6 +5400,11 @@ def _swap_in_model_cost_map(new_model_cost_map: dict) -> int: return adopt_model_cost_map(new_model_cost_map) +def _exception_reason(error: Exception) -> str: + message: Final = str(error) + return f"{type(error).__name__}: {message}" if message else type(error).__name__ + + def _websearch_handler_params(stored: Mapping[str, object]) -> dict[str, object]: """ Translate stored web search interception settings into handler kwargs. @@ -21108,9 +21113,22 @@ async def reload_model_cost_map( # Publish a new revision so every other pod reloads on its next poll; this pod has # already served it, so adopt it here rather than reloading again a tick later - proxy_config.model_cost_map_applied_revision = await record_manual_reload( - prisma_client, MODEL_COST_MAP_RELOAD_PARAM_NAME, current_time - ) + try: + proxy_config.model_cost_map_applied_revision = await record_manual_reload( + prisma_client, MODEL_COST_MAP_RELOAD_PARAM_NAME, current_time + ) + except Exception as e: + verbose_proxy_logger.exception( + "Model cost map reloaded in this worker but the reload was not recorded: %s", e + ) + raise HTTPException( + status_code=500, + detail=( + f"Failed to record the model cost map reload in the database ({_exception_reason(e)}). " + "The proxy worker that handled this request now serves the new pricing data, but other workers " + "and pods may keep their current pricing data until the next successful reload." + ), + ) from e verbose_proxy_logger.info("Model cost map reloaded successfully in current pod. Models count: %s", models_count) @@ -21125,7 +21143,7 @@ async def reload_model_cost_map( raise except Exception as e: verbose_proxy_logger.exception("Failed to reload model cost map: %s", e) - raise HTTPException(status_code=500, detail=f"Failed to reload model cost map: {e}") + raise HTTPException(status_code=500, detail=f"Failed to reload model cost map: {_exception_reason(e)}") @router.post( diff --git a/tests/unit/proxy/proxy_server/test_routes_model_cost_map.py b/tests/unit/proxy/proxy_server/test_routes_model_cost_map.py index 31f69d686d4..9de6c8f029f 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_cost_map.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_cost_map.py @@ -14,6 +14,8 @@ import json from pathlib import Path from unittest.mock import AsyncMock, MagicMock +import pytest + from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map_provenance @@ -207,6 +209,67 @@ def test_reload_model_cost_map_no_db_500(client, auth_as, monkeypatch): assert "Database connection not available" in response.json().get("detail", "") +def test_reload_model_cost_map_db_write_failure_names_the_error_and_the_pricing_this_pod_serves( + client, auth_as, monkeypatch, mock_prisma +): + """A revision write that times out with an empty message used to answer a blank reason.""" + import httpx + + import litellm + from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _attach_litellm_config(mock_prisma) + table.upsert = AsyncMock(side_effect=httpx.ReadTimeout("")) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(ps.proxy_config, "model_cost_map_applied_revision", 7, raising=False) + + fetched_map = {"gpt-4": {"input_cost": 0.03}} + monkeypatch.setattr( + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + AsyncMock(return_value=ModelCostMapReloaded(model_cost_map=fetched_map)), + ) + monkeypatch.setattr("litellm.add_known_models", lambda model_cost_map=None: None) + monkeypatch.setattr("litellm.model_cost", {"existing-model": {"input_cost": 0.01}}, raising=False) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post("/reload/model_cost_map") + + assert response.status_code == 500 + assert response.json()["detail"] == ( + "Failed to record the model cost map reload in the database (ReadTimeout). " + "The proxy worker that handled this request now serves the new pricing data, but other workers " + "and pods may keep their current pricing data until the next successful reload." + ) + assert litellm.model_cost is fetched_map + assert ps.proxy_config.model_cost_map_applied_revision == 7 + + +@pytest.mark.parametrize( + ("error", "reason"), + [(RuntimeError(), "RuntimeError"), (RuntimeError("engine restarting"), "RuntimeError: engine restarting")], +) +def test_reload_model_cost_map_unexpected_error_names_its_class( + client, auth_as, monkeypatch, mock_prisma, error, reason +): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _attach_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr( + "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", + AsyncMock(side_effect=error), + ) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post("/reload/model_cost_map") + + assert response.status_code == 500 + assert response.json()["detail"] == f"Failed to reload model cost map: {reason}" + + # --------------------------------------------------------------------------- # POST /schedule/model_cost_map_reload # ---------------------------------------------------------------------------