mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
test(router): pin the cut point and ttl of a prefix with turns too deep to serialize
/live-pr-risk found two mutants of the round 16 cut that no test killed. One cut at the last turn the router cannot turn into JSON instead of the first, and one took the pin ttl from every readable turn instead of the cut prefix. These tests check that a pin written past two such turns lands on the mark before the first, and that a pin cut before such a turn lasts only as long as the marks it kept
This commit is contained in:
parent
42d381255d
commit
02acc70dcb
1 changed files with 51 additions and 0 deletions
|
|
@ -426,6 +426,57 @@ def test_a_marked_turn_too_deep_to_serialize_keeps_the_pin_of_the_mark_before_it
|
|||
assert cache.get_model_id(messages=first_turn, tools=None) == {"model_id": "deployment-2"}
|
||||
|
||||
|
||||
def _deep_marked_question(question: str) -> AllMessageValues:
|
||||
return cast(
|
||||
AllMessageValues,
|
||||
{"role": "user", "content": question, "cache_control": {"type": "ephemeral"}, "x": _DEEP_VALUE},
|
||||
)
|
||||
|
||||
|
||||
def test_a_pin_written_past_two_turns_too_deep_to_serialize_lands_on_the_mark_before_the_first() -> None:
|
||||
cache: Final = _pin_cache(_ManualClock())
|
||||
first_turn: Final = _system_block_messages({"type": "ephemeral"})
|
||||
messages: Final = [
|
||||
*first_turn,
|
||||
_deep_marked_question("Find the termination clause."),
|
||||
cast(
|
||||
AllMessageValues,
|
||||
{"role": "user", "content": "Now the renewal clause.", "cache_control": {"type": "ephemeral"}},
|
||||
),
|
||||
_deep_marked_question("And the governing law."),
|
||||
]
|
||||
|
||||
cache.add_model_id(model_id="deployment-1", messages=first_turn, tools=None)
|
||||
cache.add_model_id(model_id="deployment-2", messages=messages, tools=None)
|
||||
|
||||
assert cache.get_model_id(messages=first_turn, tools=None) == {"model_id": "deployment-2"}
|
||||
|
||||
|
||||
def test_a_pin_cut_before_a_turn_too_deep_to_serialize_lasts_only_as_long_as_the_marks_it_kept() -> None:
|
||||
clock: Final = _ManualClock()
|
||||
cache: Final = _pin_cache(clock)
|
||||
first_turn: Final = _system_block_messages({"type": "ephemeral"})
|
||||
messages: Final = cast(
|
||||
list[AllMessageValues],
|
||||
[
|
||||
*first_turn,
|
||||
{"role": "assistant", "content": None, "tool_calls": [{**_tool_call("call_1"), "x": _DEEP_VALUE}]},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "Section 9."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Find the renewal clause.",
|
||||
"cache_control": {"type": "ephemeral", "ttl": "1h"},
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
cache.add_model_id(model_id="deployment-1", messages=messages, tools=None, honors_one_hour_ttl=True)
|
||||
|
||||
assert cache.get_model_id(messages=first_turn, tools=None) == {"model_id": "deployment-1"}
|
||||
clock.advance(PROMPT_CACHE_PIN_TTL_SECONDS + 1)
|
||||
assert cache.get_model_id(messages=first_turn, tools=None) is None
|
||||
|
||||
|
||||
async def _first_non_none(read):
|
||||
while True:
|
||||
value = await read()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue