fix(proxy): name the failure when a model cost map reload cannot be recorded in the database

This commit is contained in:
mateo-berri 2026-10-28 23:00:55 -07:00
parent 57f05e56f6
commit fca17b5ffd
2 changed files with 85 additions and 4 deletions

View file

@ -5400,6 +5400,11 @@ def _swap_in_model_cost_map(new_model_cost_map: dict) -> int:
return adopt_model_cost_map(new_model_cost_map)
def _exception_reason(error: Exception) -> str:
message: Final = str(error)
return f"{type(error).__name__}: {message}" if message else type(error).__name__
def _websearch_handler_params(stored: Mapping[str, object]) -> dict[str, object]:
"""
Translate stored web search interception settings into handler kwargs.
@ -21108,9 +21113,22 @@ async def reload_model_cost_map(
# Publish a new revision so every other pod reloads on its next poll; this pod has
# already served it, so adopt it here rather than reloading again a tick later
proxy_config.model_cost_map_applied_revision = await record_manual_reload(
prisma_client, MODEL_COST_MAP_RELOAD_PARAM_NAME, current_time
)
try:
proxy_config.model_cost_map_applied_revision = await record_manual_reload(
prisma_client, MODEL_COST_MAP_RELOAD_PARAM_NAME, current_time
)
except Exception as e:
verbose_proxy_logger.exception(
"Model cost map reloaded in this worker but the reload was not recorded: %s", e
)
raise HTTPException(
status_code=500,
detail=(
f"Failed to record the model cost map reload in the database ({_exception_reason(e)}). "
"The proxy worker that handled this request now serves the new pricing data, but other workers "
"and pods may keep their current pricing data until the next successful reload."
),
) from e
verbose_proxy_logger.info("Model cost map reloaded successfully in current pod. Models count: %s", models_count)
@ -21125,7 +21143,7 @@ async def reload_model_cost_map(
raise
except Exception as e:
verbose_proxy_logger.exception("Failed to reload model cost map: %s", e)
raise HTTPException(status_code=500, detail=f"Failed to reload model cost map: {e}")
raise HTTPException(status_code=500, detail=f"Failed to reload model cost map: {_exception_reason(e)}")
@router.post(

View file

@ -14,6 +14,8 @@ import json
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock
import pytest
from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map_provenance
@ -207,6 +209,67 @@ def test_reload_model_cost_map_no_db_500(client, auth_as, monkeypatch):
assert "Database connection not available" in response.json().get("detail", "")
def test_reload_model_cost_map_db_write_failure_names_the_error_and_the_pricing_this_pod_serves(
client, auth_as, monkeypatch, mock_prisma
):
"""A revision write that times out with an empty message used to answer a blank reason."""
import httpx
import litellm
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
from litellm.proxy import proxy_server as ps
from litellm.proxy._types import LitellmUserRoles
table = _attach_litellm_config(mock_prisma)
table.upsert = AsyncMock(side_effect=httpx.ReadTimeout(""))
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
monkeypatch.setattr(ps.proxy_config, "model_cost_map_applied_revision", 7, raising=False)
fetched_map = {"gpt-4": {"input_cost": 0.03}}
monkeypatch.setattr(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
AsyncMock(return_value=ModelCostMapReloaded(model_cost_map=fetched_map)),
)
monkeypatch.setattr("litellm.add_known_models", lambda model_cost_map=None: None)
monkeypatch.setattr("litellm.model_cost", {"existing-model": {"input_cost": 0.01}}, raising=False)
with auth_as(LitellmUserRoles.PROXY_ADMIN):
response = client.post("/reload/model_cost_map")
assert response.status_code == 500
assert response.json()["detail"] == (
"Failed to record the model cost map reload in the database (ReadTimeout). "
"The proxy worker that handled this request now serves the new pricing data, but other workers "
"and pods may keep their current pricing data until the next successful reload."
)
assert litellm.model_cost is fetched_map
assert ps.proxy_config.model_cost_map_applied_revision == 7
@pytest.mark.parametrize(
("error", "reason"),
[(RuntimeError(), "RuntimeError"), (RuntimeError("engine restarting"), "RuntimeError: engine restarting")],
)
def test_reload_model_cost_map_unexpected_error_names_its_class(
client, auth_as, monkeypatch, mock_prisma, error, reason
):
from litellm.proxy import proxy_server as ps
from litellm.proxy._types import LitellmUserRoles
_attach_litellm_config(mock_prisma)
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
monkeypatch.setattr(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
AsyncMock(side_effect=error),
)
with auth_as(LitellmUserRoles.PROXY_ADMIN):
response = client.post("/reload/model_cost_map")
assert response.status_code == 500
assert response.json()["detail"] == f"Failed to reload model cost map: {reason}"
# ---------------------------------------------------------------------------
# POST /schedule/model_cost_map_reload
# ---------------------------------------------------------------------------