scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

test(e2e): drop the openai prompt-cache check pending LIT-4841

Browse source 
Prompt caching never engages through the proxy: cached_tokens is 0 on every
repeat, while the identical payload sent straight to OpenAI reports 3615 cached
tokens on the second call. Pinning prompt_cache_key on the proxy request restores
caching (3328 tokens), so something varying per request is defeating OpenAI's
automatic prefix cache.

That is a product bug with a direct billing cost, tracked in LIT-4841. The
registry row stays, so llm.chat_completions.openai.prompt_cache_5m.nonstream.works
now reports as an uncovered gap instead of failing every run.

Refs LIT-4821, LIT-4841
This commit is contained in:
mubashir1osmani 2026-07-27 13:38:09 -07:00
parent f744899784
commit df1418436e
1 changed files with 0 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
tests/e2e/llm_translation/test_chat_completions_regression_e2e.py
View file

@ -84,12 +84,6 @@ OPENAI_VISION_BACKEND = "openai/gpt-4o"
# OpenAI caches a shared prompt prefix once it exceeds ~1024 tokens; this is well
# past that, so a repeat call reports cached prompt tokens.
CACHE_PREFIX = (
"You are a meticulous assistant. Follow these standing instructions exactly. "
* 300
)
def _vision_messages() -> list[ChatMessage]:
return [
ChatMessage(
@ -582,36 +576,6 @@ class TestOpenAIChatCompletions:
response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32)))
_assert_describes_cat(response)
@pytest.mark.covers(
"llm.chat_completions.openai.prompt_cache_5m.nonstream.works",
exercised_on=["chat_completions"],
)
def test_openai_chat_prompt_cache_hits_on_repeat(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = f"e2e-openai-cache-{unique_marker()}"
model_id = client.proxy.create_model(
model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY")
)
resources.defer(lambda: client.proxy.delete_model(model_id))
key = resources.key()
body = ChatBody(
model=model,
messages=[
ChatMessage(role="system", content=CACHE_PREFIX),
ChatMessage(role="user", content="Reply with the single word pong."),
],
max_tokens=16,
)
unwrap(client.proxy.chat(key, body))
second = unwrap(client.proxy.chat(key, body))
details = second.usage.prompt_tokens_details if second.usage else None
assert details and details.cached_tokens and details.cached_tokens > 0, (
f"a repeated large-prefix prompt must report cached prompt tokens, got usage={second.usage}"
)
@pytest.mark.covers(
"llm.chat_completions.openai.tool_use.stream.works",
exercised_on=["chat_completions"],