mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(caching): log a cache hit with the deployment's vertex_location (#52150)
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
abd43804f6
commit
a14d17822b
2 changed files with 57 additions and 0 deletions
|
|
@ -1526,6 +1526,7 @@ class LLMCachingHandler:
|
|||
"proxy_server_request": kwargs.get("proxy_server_request", None),
|
||||
"stream_response": kwargs.get("stream_response", {}),
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
"vertex_location": kwargs.get("vertex_location") or kwargs.get("vertex_ai_location"),
|
||||
}
|
||||
|
||||
if litellm.cache is not None:
|
||||
|
|
|
|||
|
|
@ -289,6 +289,62 @@ async def test_existing_provider_response_cache_still_hits(copilot_response_cach
|
|||
assert hidden_params.get("cache_hit") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cache_hit_logs_the_vertex_location_the_deployment_is_configured_with(
|
||||
monkeypatch: pytest.MonkeyPatch, copilot_response_cache: Cache
|
||||
) -> None:
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
||||
logged_locations: Final[list[tuple[object, object]]] = []
|
||||
both_logged: Final = asyncio.Event()
|
||||
|
||||
class LocationRecorder(CustomLogger):
|
||||
async def async_log_success_event(
|
||||
self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
payload: Final = TypeAdapter(Mapping[str, object]).validate_python(kwargs["standard_logging_object"])
|
||||
breakdown: Final = TypeAdapter(Mapping[str, object]).validate_python(payload["cost_breakdown"])
|
||||
logged_locations.append((payload["cache_hit"], breakdown.get("vertex_location")))
|
||||
if len(logged_locations) == 2:
|
||||
both_logged.set()
|
||||
|
||||
monkeypatch.delenv("VERTEXAI_LOCATION", raising=False)
|
||||
monkeypatch.delenv("VERTEX_LOCATION", raising=False)
|
||||
monkeypatch.setattr(litellm, "vertex_location", None)
|
||||
monkeypatch.setattr(litellm, "success_callback", [])
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", [])
|
||||
monkeypatch.setattr(litellm, "input_callback", [])
|
||||
monkeypatch.setattr(litellm, "callbacks", [LocationRecorder()])
|
||||
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "global-gemini",
|
||||
"litellm_params": {
|
||||
"model": "vertex_ai/gemini-2.5-flash",
|
||||
"vertex_location": "global",
|
||||
"mock_response": "ok",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
for _ in range(2):
|
||||
request: Final[dict[str, object]] = {
|
||||
"model": "global-gemini",
|
||||
"messages": [{"role": "user", "content": "cache hit location"}],
|
||||
"caching": True,
|
||||
"litellm_call_id": str(uuid.uuid4()),
|
||||
}
|
||||
logging_obj, _ = litellm.utils.function_setup(
|
||||
original_function="acompletion", rules_obj=litellm.utils.Rules(), start_time=datetime(2026, 1, 1), **request
|
||||
)
|
||||
await router.acompletion(**request, litellm_logging_obj=logging_obj)
|
||||
await asyncio.gather(*_PENDING_CACHE_WRITES)
|
||||
|
||||
await asyncio.wait_for(both_logged.wait(), timeout=10)
|
||||
assert sorted(logged_locations, key=str) == [(None, "global"), (True, "global")], logged_locations
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cache_hit_reports_zero_response_cost(copilot_response_cache: Cache) -> None:
|
||||
messages: Final = [{"role": "user", "content": "cache cost repro"}]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue