mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-16 04:29:31 +00:00
test(caching): cover cache-hit marker on HiddenParams carriers through the caching handler
This commit is contained in:
parent
6910ea644d
commit
cbf6975384
2 changed files with 98 additions and 4 deletions
|
|
@ -2268,3 +2268,57 @@ async def test_partial_embedding_cache_hit_sends_only_misses_and_keeps_input_ord
|
|||
|
||||
assert len(embedder.provider_inputs) == 2, embedder.provider_inputs
|
||||
assert [item["embedding"] for item in repeat.data] == [[float(len(text))] for text in mixed_input]
|
||||
|
||||
|
||||
def test_text_completion_cache_hit_is_marked_and_stamped_zero_cost(monkeypatch):
|
||||
"""A cached text_completion dict rebuilds into a HiddenParams carrier that still gets the cache_hit marker and a 0 cost."""
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
deployment_id = "lit8679-text-completion-deployment"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
deployment_id: {
|
||||
"input_cost_per_token": 1e-06,
|
||||
"output_cost_per_token": 2e-06,
|
||||
"litellm_provider": "openai",
|
||||
"mode": "completion",
|
||||
}
|
||||
}
|
||||
)
|
||||
caching_handler = LLMCachingHandler(original_function=lambda: None, request_kwargs={}, start_time=datetime.now())
|
||||
logging_obj = LiteLLMLogging(
|
||||
litellm_call_id="lit8679-text-completion-hit",
|
||||
call_type=CallTypes.text_completion.value,
|
||||
model="gpt-3.5-turbo-instruct",
|
||||
messages="Hello",
|
||||
function_id=str(uuid.uuid4()),
|
||||
stream=False,
|
||||
start_time=datetime.now(),
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
model="gpt-3.5-turbo-instruct",
|
||||
litellm_params={"metadata": {"model_info": {"id": deployment_id}}},
|
||||
optional_params={},
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
result = caching_handler._convert_cached_result_to_model_response(
|
||||
cached_result={
|
||||
"id": "cmpl-lit8679",
|
||||
"object": "text_completion",
|
||||
"created": 1,
|
||||
"model": "gpt-3.5-turbo-instruct",
|
||||
"choices": [{"text": "Hello, how can I help you today?", "index": 0, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18},
|
||||
},
|
||||
call_type=CallTypes.text_completion.value,
|
||||
kwargs={"model_info": {"id": deployment_id}},
|
||||
logging_obj=logging_obj,
|
||||
model="gpt-3.5-turbo-instruct",
|
||||
args=(),
|
||||
)
|
||||
|
||||
assert isinstance(result, TextCompletionResponse)
|
||||
assert result._hidden_params.get("cache_hit") is True
|
||||
assert result._hidden_params.get("response_cost") == 0.0
|
||||
assert logging_obj.cost_breakdown is None
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
|
|||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
_extract_response_obj_and_hidden_params,
|
||||
_get_status_fields,
|
||||
get_standard_logging_object_payload,
|
||||
set_callbacks,
|
||||
)
|
||||
from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
|
||||
|
|
@ -9090,12 +9091,51 @@ def test_uncached_response_cost_recomputes_price_past_a_zero_stamp():
|
|||
litellm.model_cost.pop(deployment_id, None)
|
||||
|
||||
|
||||
def test_uncached_response_cost_leaves_no_pricing_state_on_the_call():
|
||||
"""The uncached price probe restores every field the priced path writes on the logging object."""
|
||||
deployment_id: Final = "lit8679-probe-state-deployment"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
deployment_id: {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 1e-06,
|
||||
"output_cost_per_token": 2e-06,
|
||||
}
|
||||
},
|
||||
persist_across_reloads=False,
|
||||
)
|
||||
try:
|
||||
logging_obj: Final = _lit8679_priced_logging_obj(deployment_id, "lit8679-probe-state")
|
||||
cached_response: Final = ModelResponse(
|
||||
model="gpt-5.4-nano",
|
||||
choices=[litellm.Choices(message=litellm.Message(role="assistant", content="cached"))],
|
||||
usage=litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30),
|
||||
)
|
||||
cached_response._hidden_params = {"cache_hit": True, "cache_key": "lit8679-key", "response_cost": 0.0}
|
||||
state_before: Final = (
|
||||
logging_obj.cost_breakdown,
|
||||
logging_obj.billed_token_rates,
|
||||
logging_obj.zero_cost_warned,
|
||||
"response_cost_failure_debug_information" in logging_obj.model_call_details,
|
||||
)
|
||||
|
||||
cost: Final = logging_obj._uncached_response_cost(result=cached_response)
|
||||
|
||||
assert cost is not None and cost > 0.0
|
||||
assert (
|
||||
logging_obj.cost_breakdown,
|
||||
logging_obj.billed_token_rates,
|
||||
logging_obj.zero_cost_warned,
|
||||
"response_cost_failure_debug_information" in logging_obj.model_call_details,
|
||||
) == state_before
|
||||
assert state_before == (None, None, False, False)
|
||||
finally:
|
||||
litellm.model_cost.pop(deployment_id, None)
|
||||
|
||||
|
||||
def test_payload_saved_cache_cost_keeps_uncached_price_when_result_stamped_zero():
|
||||
"""A cache-hit payload still reports what the call would have cost, not the 0.0 stamped on the served response."""
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
get_standard_logging_object_payload,
|
||||
)
|
||||
|
||||
deployment_id: Final = "lit8679-saved-cost-deployment"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue